@makerclay/core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +221 -0
- package/README.md +82 -0
- package/package.json +52 -0
- package/src/admin/listing.js +149 -0
- package/src/admin/routes.js +362 -0
- package/src/attic.js +168 -0
- package/src/auth/can.js +79 -0
- package/src/auth/csrf.js +74 -0
- package/src/auth/none.js +24 -0
- package/src/auth/password.js +281 -0
- package/src/auth/passwords.js +79 -0
- package/src/auth/rate-limit.js +66 -0
- package/src/auth/sessions.js +86 -0
- package/src/auth/token-lanes.js +72 -0
- package/src/boot.js +120 -0
- package/src/client.js +99 -0
- package/src/collections/index.js +397 -0
- package/src/collections/routes.js +166 -0
- package/src/create-host.js +351 -0
- package/src/derived/data-extractor.js +22 -0
- package/src/derived/index.js +121 -0
- package/src/documents/format-html.js +313 -0
- package/src/documents/replace.js +257 -0
- package/src/documents/root-attrs.js +171 -0
- package/src/documents/serve.js +137 -0
- package/src/documents/stale.js +20 -0
- package/src/index.js +12 -0
- package/src/inspect.js +136 -0
- package/src/json-errors.js +59 -0
- package/src/livesync.js +75 -0
- package/src/nodes/identity.js +14 -0
- package/src/nodes/names.js +57 -0
- package/src/nodes/ops.js +613 -0
- package/src/nodes/scanner.js +321 -0
- package/src/nodes/store.js +111 -0
- package/src/pages.js +166 -0
- package/src/paths.js +302 -0
- package/src/recovery/overlay.js +327 -0
- package/src/recovery/replay.js +236 -0
- package/src/recovery-ui.js +185 -0
- package/src/recovery.js +30 -0
- package/src/requests.js +73 -0
- package/src/routes/meta.js +62 -0
- package/src/routes/read.js +105 -0
- package/src/routes/save.js +102 -0
- package/src/routes/sync.js +118 -0
- package/src/routes/upload.js +128 -0
- package/src/share/index.js +207 -0
- package/src/share/save-tokens.js +65 -0
- package/src/spec/codes.js +42 -0
- package/src/spec/meta.js +52 -0
- package/src/spec/wire.js +115 -0
- package/src/store/index.js +29 -0
- package/src/store/migrations/001-init.sql +114 -0
- package/src/store/sqlite.js +540 -0
- package/src/templates.js +50 -0
- package/src/tenants/index.js +355 -0
- package/src/tenants/isolation.js +91 -0
- package/src/tenants/routes.js +131 -0
- package/src/ui.js +95 -0
- package/src/util/cookies.js +26 -0
- package/src/util/express.js +8 -0
- package/src/util/fsx.js +205 -0
- package/src/util/id.js +37 -0
- package/src/util/lockfile.js +52 -0
- package/src/util/locks.js +35 -0
- package/src/util/multipart.js +33 -0
- package/src/versions/files.js +307 -0
- package/src/versions/index.js +16 -0
- package/src/versions/naming.js +172 -0
- package/src/versions/routes.js +91 -0
- package/src/wire-compat.js +61 -0
- package/ui/app.css +164 -0
- package/ui/attic.html +198 -0
- package/ui/dashboard.html +456 -0
- package/ui/editor.html +156 -0
- package/ui/error.html +18 -0
- package/ui/login.html +58 -0
- package/ui/records.html +173 -0
- package/ui/recovery.html +152 -0
- package/ui/setup.html +61 -0
- package/ui/share-qr.html +44 -0
- package/ui/templates/blank.html +16 -0
- package/ui/templates/devlog.html +71 -0
- package/ui/templates/hackable-dashboard.html +145 -0
- package/ui/templates/kanban.html +85 -0
- package/ui/templates/landing.html +108 -0
- package/ui/templates/writer.html +50 -0
- package/ui/tenants.html +172 -0
- package/ui/trash.html +134 -0
- package/ui/versions.html +114 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import path from 'upath';
|
|
2
|
+
import { exists, rmrf } from '../util/fsx.js';
|
|
3
|
+
|
|
4
|
+
// Finishing or unwinding the multi-step operations that were in flight when the
|
|
5
|
+
// process died (§9.2 step 4).
|
|
6
|
+
//
|
|
7
|
+
// The whole method rests on one property of how ops.js writes: the filesystem
|
|
8
|
+
// half of every operation is a SINGLE atomic rename. So which side of it the
|
|
9
|
+
// crash landed on is a fact that can be READ OFF DISK, never a guess. The
|
|
10
|
+
// journal says what was intended; the disk says whether it happened; this
|
|
11
|
+
// module makes the database agree with the disk.
|
|
12
|
+
//
|
|
13
|
+
// When the two genuinely disagree in a way that cannot be resolved, the
|
|
14
|
+
// operation is reported ambiguous and the host goes to recovery mode rather
|
|
15
|
+
// than picking an answer.
|
|
16
|
+
|
|
17
|
+
export function createReplay({ paths, store, nodes, versions, clock, logger = console }) {
|
|
18
|
+
const realFor = (owner, relPath) => path.join(paths.realmRoot(owner || ''), relPath);
|
|
19
|
+
|
|
20
|
+
async function recoverOperations() {
|
|
21
|
+
if (!store) return { completed: 0, unwound: 0, ambiguous: [] };
|
|
22
|
+
const pending = store.listOperations();
|
|
23
|
+
if (!pending.length) return { completed: 0, unwound: 0, ambiguous: [] };
|
|
24
|
+
|
|
25
|
+
let completed = 0;
|
|
26
|
+
let unwound = 0;
|
|
27
|
+
const ambiguous = [];
|
|
28
|
+
|
|
29
|
+
for (const op of pending) {
|
|
30
|
+
try {
|
|
31
|
+
const verdict = await settle(op);
|
|
32
|
+
if (verdict === 'completed') completed += 1;
|
|
33
|
+
else if (verdict === 'unwound') unwound += 1;
|
|
34
|
+
else ambiguous.push({ id: op.id, kind: op.kind, payload: op.payload });
|
|
35
|
+
if (verdict !== 'ambiguous') store.deleteOperation(op.id);
|
|
36
|
+
} catch (error) {
|
|
37
|
+
logger.error?.('[makerclay] recovery failed for operation', op.id, error?.message || error);
|
|
38
|
+
ambiguous.push({ id: op.id, kind: op.kind, payload: op.payload, error: error?.message || String(error) });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (completed || unwound || ambiguous.length) {
|
|
43
|
+
logger.warn?.(
|
|
44
|
+
`[makerclay] recovered ${completed} interrupted operation(s), unwound ${unwound}` +
|
|
45
|
+
(ambiguous.length ? `, ${ambiguous.length} need attention` : ''),
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
return { completed, unwound, ambiguous };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function settle(op) {
|
|
52
|
+
const { kind, payload = {}, nodeId } = op;
|
|
53
|
+
const owner = payload.owner ?? '';
|
|
54
|
+
|
|
55
|
+
if (kind === 'mkdir' || kind === 'create' || kind === 'copy') {
|
|
56
|
+
// Nothing was destroyed either way. If the bytes landed the scanner will
|
|
57
|
+
// adopt them a moment from now; if they did not, the plan is just noise.
|
|
58
|
+
//
|
|
59
|
+
// A copy that died before its rename left its staging tree behind. It is
|
|
60
|
+
// dot-prefixed so nothing lists it and the scanner skips it, but it is
|
|
61
|
+
// still the whole tree's worth of bytes.
|
|
62
|
+
if (payload.staging) await rmrf(payload.staging);
|
|
63
|
+
return (await exists(realFor(owner, payload.path ?? payload.to))) ? 'completed' : 'unwound';
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (kind === 'move') {
|
|
67
|
+
const from = payload.fromReal || realFor(owner, payload.from);
|
|
68
|
+
const to = payload.toReal || realFor(owner, payload.to);
|
|
69
|
+
const [hasFrom, hasTo] = [await exists(from), await exists(to)];
|
|
70
|
+
if (hasTo && !hasFrom) {
|
|
71
|
+
if (!finishMove(op, owner, payload)) return 'ambiguous';
|
|
72
|
+
await repairMoveRecord(op, owner, payload);
|
|
73
|
+
return 'completed';
|
|
74
|
+
}
|
|
75
|
+
if (hasFrom && !hasTo) return 'unwound';
|
|
76
|
+
return 'ambiguous';
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (kind === 'trash') {
|
|
80
|
+
const from = payload.fromReal || realFor(owner, payload.from);
|
|
81
|
+
const [hasFrom, hasTrash] = [await exists(from), await exists(payload.to)];
|
|
82
|
+
if (hasTrash && !hasFrom) {
|
|
83
|
+
const children = descendants(owner, payload.from);
|
|
84
|
+
if (!finishTrash(op, owner, payload)) return 'ambiguous';
|
|
85
|
+
await patchMetaQuietly(store.getNodeById(op.nodeId), { trashed: true });
|
|
86
|
+
for (const child of children) await patchMetaQuietly(child, { trashed: true });
|
|
87
|
+
return 'completed';
|
|
88
|
+
}
|
|
89
|
+
if (hasFrom && !hasTrash) return 'unwound';
|
|
90
|
+
return 'ambiguous';
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (kind === 'restore') {
|
|
94
|
+
const to = payload.toReal || realFor(owner, payload.to);
|
|
95
|
+
const [hasTrash, hasTo] = [await exists(payload.from), await exists(to)];
|
|
96
|
+
if (hasTo && !hasTrash) {
|
|
97
|
+
if (!finishRestore(op, owner, payload)) return 'ambiguous';
|
|
98
|
+
await patchMetaQuietly(store.getNodeById(op.nodeId), { trashed: false, path: payload.to });
|
|
99
|
+
for (const child of descendants(owner, payload.to)) {
|
|
100
|
+
await patchMetaQuietly(child, { trashed: false, path: child.path });
|
|
101
|
+
}
|
|
102
|
+
return 'completed';
|
|
103
|
+
}
|
|
104
|
+
if (hasTrash && !hasTo) return 'unwound';
|
|
105
|
+
return 'ambiguous';
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (kind === 'purge') {
|
|
109
|
+
if (await exists(payload.trash)) return 'unwound';
|
|
110
|
+
// Captured before the transaction, because after it the rows are gone and
|
|
111
|
+
// `versions.purge` resolves the directory by node id.
|
|
112
|
+
const orphaned = nodeId
|
|
113
|
+
? [...descendants(owner, payload.path), store.getNodeById(nodeId)].filter(Boolean)
|
|
114
|
+
: [];
|
|
115
|
+
const done = finishPurge(nodeId, owner, payload);
|
|
116
|
+
// The ordinary purge lane removes the history along with the bytes. A
|
|
117
|
+
// purge interrupted after the bytes went and before the rows did used to
|
|
118
|
+
// leave the history orphaned under versions/, which was harmless while
|
|
119
|
+
// nothing read it. It stops being harmless the moment the overlay
|
|
120
|
+
// reconciles against durable records: the orphan still says trashed:true,
|
|
121
|
+
// so the next boot reads it as a live trashed document and puts the row
|
|
122
|
+
// back.
|
|
123
|
+
if (done) for (const row of orphaned) await versions.purge(row);
|
|
124
|
+
return done ? 'completed' : 'unwound';
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return 'ambiguous';
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function descendants(owner, prefix) {
|
|
131
|
+
return store.listNodes({ owner, prefix: `${prefix}/`, includeDeleted: true });
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// finish* puts the ROW right. The durable record is the other half, and the
|
|
135
|
+
// kill that stranded this operation could just as easily have landed between
|
|
136
|
+
// the rename and the record write, so the same patch ops.js makes inline is
|
|
137
|
+
// made here before the intent is thrown away. Without it the record keeps the
|
|
138
|
+
// old path, or a stale trashed marker, and a later database loss rebuilds the
|
|
139
|
+
// overlay from that.
|
|
140
|
+
//
|
|
141
|
+
// Best-effort for the same reason as ops.js's patchMetaQuietly: a repair that
|
|
142
|
+
// throws must not re-brick the boot it is part of.
|
|
143
|
+
async function patchMetaQuietly(node, patch) {
|
|
144
|
+
if (!node) return;
|
|
145
|
+
try {
|
|
146
|
+
await versions.patchMeta(node, patch);
|
|
147
|
+
} catch (error) {
|
|
148
|
+
logger.warn?.('[makerclay] could not update the durable record:', error?.message || error);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function repairMoveRecord(op, owner, payload) {
|
|
153
|
+
const row = store.getNodeById(op.nodeId);
|
|
154
|
+
if (!row) return;
|
|
155
|
+
// Read back after finishMove, so `row.kind` is the recomputed one and the
|
|
156
|
+
// record cannot end up describing the file as something the row does not.
|
|
157
|
+
await patchMetaQuietly(row, { path: payload.to, kind: row.kind });
|
|
158
|
+
// A directory move rewrote its descendants too, the way ops.move patches
|
|
159
|
+
// `movedChildren`. Read back at the new prefix, so these are the rows
|
|
160
|
+
// finishMove just wrote.
|
|
161
|
+
for (const child of descendants(owner, payload.to)) {
|
|
162
|
+
await patchMetaQuietly(child, { path: child.path });
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function finishMove(op, owner, payload) {
|
|
167
|
+
const row = store.getNodeById(op.nodeId);
|
|
168
|
+
if (!row) return true;
|
|
169
|
+
// A rename can change what the file IS, and kind is what decides both the
|
|
170
|
+
// serve lane and the sandbox, so it is recomputed here exactly as ops.move
|
|
171
|
+
// recomputes it rather than carried over.
|
|
172
|
+
const kind = row.kind === 'dir' ? 'dir' : nodes.kindFor(payload.to);
|
|
173
|
+
return store.tx(() => {
|
|
174
|
+
for (const child of descendants(owner, payload.from)) {
|
|
175
|
+
store.updateNode(child.id, {
|
|
176
|
+
path: `${payload.to}${child.path.slice(payload.from.length)}`,
|
|
177
|
+
updatedAt: clock.now(),
|
|
178
|
+
seq: store.nextSeq(),
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
store.updateNode(row.id, { path: payload.to, kind, updatedAt: clock.now(), seq: store.nextSeq() });
|
|
182
|
+
return true;
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function finishTrash(op, owner, payload) {
|
|
187
|
+
const row = store.getNodeById(op.nodeId);
|
|
188
|
+
if (!row) return true;
|
|
189
|
+
const at = clock.now();
|
|
190
|
+
return store.tx(() => {
|
|
191
|
+
for (const child of descendants(owner, payload.from)) {
|
|
192
|
+
store.tombstone(child.id, at);
|
|
193
|
+
store.deleteSaveTokensForNode(child.id);
|
|
194
|
+
if (store.getRecordAuth(child.id)) store.updateRecordAuth(child.id, { deletedAt: at });
|
|
195
|
+
}
|
|
196
|
+
store.tombstone(row.id, at);
|
|
197
|
+
store.deleteSaveTokensForNode(row.id);
|
|
198
|
+
if (store.getRecordAuth(row.id)) store.updateRecordAuth(row.id, { deletedAt: at });
|
|
199
|
+
return true;
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function finishRestore(op, owner, payload) {
|
|
204
|
+
const row = store.getNodeById(op.nodeId);
|
|
205
|
+
if (!row) return true;
|
|
206
|
+
return store.tx(() => {
|
|
207
|
+
for (const child of descendants(owner, row.path)) {
|
|
208
|
+
store.updateNode(child.id, {
|
|
209
|
+
path: `${payload.to}${child.path.slice(row.path.length)}`,
|
|
210
|
+
deletedAt: null,
|
|
211
|
+
updatedAt: clock.now(),
|
|
212
|
+
seq: store.nextSeq(),
|
|
213
|
+
});
|
|
214
|
+
const childAuth = store.getRecordAuth(child.id);
|
|
215
|
+
if (childAuth?.deletedAt) store.updateRecordAuth(child.id, { deletedAt: null });
|
|
216
|
+
}
|
|
217
|
+
store.updateNode(row.id, {
|
|
218
|
+
path: payload.to, deletedAt: null, updatedAt: clock.now(), seq: store.nextSeq(),
|
|
219
|
+
});
|
|
220
|
+
const auth = store.getRecordAuth(row.id);
|
|
221
|
+
if (auth?.deletedAt) store.updateRecordAuth(row.id, { deletedAt: null });
|
|
222
|
+
return true;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function finishPurge(nodeId, owner, payload) {
|
|
227
|
+
if (!nodeId) return true;
|
|
228
|
+
return store.tx(() => {
|
|
229
|
+
for (const child of descendants(owner, payload.path)) store.deleteNode(child.id);
|
|
230
|
+
store.deleteNode(nodeId);
|
|
231
|
+
return true;
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
return { recoverOperations };
|
|
236
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import crypto from 'node:crypto';
|
|
3
|
+
import path from 'upath';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { readFileTextIfExists, statIfExists } from './util/fsx.js';
|
|
6
|
+
import { renderTemplate } from './ui.js';
|
|
7
|
+
import { wrap } from './util/express.js';
|
|
8
|
+
|
|
9
|
+
// Recovery mode (§18.3).
|
|
10
|
+
//
|
|
11
|
+
// When migrations fail, the database fails its integrity check, or the
|
|
12
|
+
// operations journal is ambiguous, the host refuses to serve documents at all: a
|
|
13
|
+
// half-started server sitting on uncertain privacy metadata is worse than a
|
|
14
|
+
// clearly refused one. This is what it serves instead.
|
|
15
|
+
//
|
|
16
|
+
// Three rules, and each is the reason the mode is worth having:
|
|
17
|
+
//
|
|
18
|
+
// - The operator is at the machine. Nobody on the network gets a listing of
|
|
19
|
+
// every file plus a raw download lane just because the database is unhappy.
|
|
20
|
+
// Loopback alone does not say that: the deploy configs this repo ships
|
|
21
|
+
// `proxy_pass http://127.0.0.1`, so behind either of them every request on
|
|
22
|
+
// the internet arrives from 127.0.0.1. The boot key is what actually says
|
|
23
|
+
// it, because it is printed to the server's own console and nowhere else.
|
|
24
|
+
// - It never serves a document. Every download is octet-stream, attachment,
|
|
25
|
+
// nosniff. A page whose privacy metadata we do not currently trust must not
|
|
26
|
+
// render in a browser at this origin.
|
|
27
|
+
// - It reads and never writes. The whole point is that nothing has decided
|
|
28
|
+
// what the truth is yet.
|
|
29
|
+
//
|
|
30
|
+
// Its page comes from the PACKAGE, never from `<data>/ui/`: in recovery mode the
|
|
31
|
+
// first-run copy into the data directory has not happened, and one of the things
|
|
32
|
+
// that can be broken is exactly that directory.
|
|
33
|
+
|
|
34
|
+
const PACKAGE_UI = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'ui');
|
|
35
|
+
|
|
36
|
+
const LOOPBACK = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']);
|
|
37
|
+
|
|
38
|
+
export function isLoopback(req) {
|
|
39
|
+
const address = req.socket?.remoteAddress || req.connection?.remoteAddress || '';
|
|
40
|
+
return LOOPBACK.has(String(address));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// files/ and tenants/ are the documents; versions/ is their history; records/ is
|
|
44
|
+
// collection submissions, the one part of a backup nobody can re-create by hand.
|
|
45
|
+
const DOWNLOADABLE_DIRS = ['files', 'tenants', 'versions', 'trash', 'records'];
|
|
46
|
+
const DOWNLOADABLE_FILES = { db: 'makerclay.db', config: 'config.json' };
|
|
47
|
+
|
|
48
|
+
function walk(root, base = '', out = [], budget = { left: 5000 }) {
|
|
49
|
+
let entries;
|
|
50
|
+
try {
|
|
51
|
+
entries = fs.readdirSync(path.join(root, base), { withFileTypes: true });
|
|
52
|
+
} catch {
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
for (const entry of entries) {
|
|
56
|
+
if (budget.left <= 0) return out;
|
|
57
|
+
const rel = base ? `${base}/${entry.name}` : entry.name;
|
|
58
|
+
if (entry.isDirectory()) {
|
|
59
|
+
walk(root, rel, out, budget);
|
|
60
|
+
} else if (entry.isFile()) {
|
|
61
|
+
budget.left -= 1;
|
|
62
|
+
let size = null;
|
|
63
|
+
try { size = fs.statSync(path.join(root, rel)).size; } catch { size = null; }
|
|
64
|
+
out.push({ path: rel, size });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
export function mountRecoveryUi({ app, paths, store, recovery, stateOf}) {
|
|
72
|
+
const data = paths.dirs.data;
|
|
73
|
+
const key = crypto.randomBytes(16).toString('hex');
|
|
74
|
+
|
|
75
|
+
function notFound(res) {
|
|
76
|
+
// Not a 403: to anyone without the key there is nothing here to find.
|
|
77
|
+
return res.status(404).json({ ok: false, code: 'not-found', msg: 'Not found' });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function keyMatches(given) {
|
|
81
|
+
const offered = Buffer.from(String(given ?? ''));
|
|
82
|
+
const expected = Buffer.from(key);
|
|
83
|
+
return offered.length === expected.length && crypto.timingSafeEqual(offered, expected);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function guard(req, res, next) {
|
|
87
|
+
// Outside recovery mode there is no recovery UI at all: the route falls
|
|
88
|
+
// through and `/_/recover` is an ordinary 404.
|
|
89
|
+
if (stateOf().state !== 'recover') return next('route');
|
|
90
|
+
if (!isLoopback(req)) return notFound(res);
|
|
91
|
+
if (!keyMatches(req.query.key ?? req.get('x-recovery-key'))) return notFound(res);
|
|
92
|
+
res.setHeader('Cache-Control', 'no-store');
|
|
93
|
+
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
94
|
+
// The key is in the URL, so keep it out of anything that follows a link out.
|
|
95
|
+
res.setHeader('Referrer-Policy', 'no-referrer');
|
|
96
|
+
return next();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Everything the doctor report is built from, computed defensively: in this
|
|
100
|
+
// mode any one of these calls is allowed to be the thing that is broken.
|
|
101
|
+
async function report() {
|
|
102
|
+
const state = stateOf();
|
|
103
|
+
const safely = async (label, fn) => {
|
|
104
|
+
try { return await fn(); } catch (error) { return { error: `${label}: ${error?.message || error}` }; }
|
|
105
|
+
};
|
|
106
|
+
return {
|
|
107
|
+
state: state.state,
|
|
108
|
+
reason: state.reason ?? null,
|
|
109
|
+
error: state.error ?? null,
|
|
110
|
+
ambiguous: state.ambiguous ?? [],
|
|
111
|
+
data,
|
|
112
|
+
counts: await safely('counts', () => ({
|
|
113
|
+
nodes: store ? store.listAllNodes().length : 0,
|
|
114
|
+
trash: store ? store.listTrash().length : 0,
|
|
115
|
+
operations: store ? store.listOperations().length : 0,
|
|
116
|
+
})),
|
|
117
|
+
operations: await safely('operations', () => (store ? store.listOperations() : [])),
|
|
118
|
+
integrity: await safely('integrity', () => (store ? store.integrityCheck() : ['no database'])),
|
|
119
|
+
records: await safely('records', async () => (await recovery.readAllMeta()).length),
|
|
120
|
+
artifacts: Object.entries(DOWNLOADABLE_FILES).map(([key, name]) => {
|
|
121
|
+
const file = path.join(data, name);
|
|
122
|
+
const exists = fs.existsSync(file);
|
|
123
|
+
return { key, name, exists, size: exists ? fs.statSync(file).size : null };
|
|
124
|
+
}),
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
app.get('/_/recover', guard, wrap(async (req, res) => {
|
|
129
|
+
const source = await readFileTextIfExists(path.join(PACKAGE_UI, 'recovery.html'));
|
|
130
|
+
if (source == null) {
|
|
131
|
+
return res.status(500).type('text/plain').send('Recovery page missing. Run `makerclay doctor`.');
|
|
132
|
+
}
|
|
133
|
+
const state = stateOf();
|
|
134
|
+
res.type('text/html; charset=utf-8');
|
|
135
|
+
return res.send(renderTemplate(source, { reason: state.reason || 'unknown', data }));
|
|
136
|
+
}));
|
|
137
|
+
|
|
138
|
+
app.get('/_/recover/report', guard, wrap(async (req, res) => res.json({ ok: true, ...(await report()) })));
|
|
139
|
+
|
|
140
|
+
app.get('/_/recover/tree', guard, (req, res) => {
|
|
141
|
+
const trees = DOWNLOADABLE_DIRS
|
|
142
|
+
.filter((dir) => fs.existsSync(path.join(data, dir)))
|
|
143
|
+
.map((dir) => ({ dir, files: walk(path.join(data, dir)).map((f) => ({ ...f, path: `${dir}/${f.path}` })) }));
|
|
144
|
+
return res.json({ ok: true, trees });
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
// The download lane. `attachment` + octet-stream is what keeps the promise
|
|
148
|
+
// that this mode never serves a document: the browser saves the bytes, it
|
|
149
|
+
// does not run them.
|
|
150
|
+
app.get('/_/recover/raw', guard, wrap(async (req, res) => {
|
|
151
|
+
const wanted = String(req.query.path || '');
|
|
152
|
+
// `Object.hasOwn`, because a plain lookup finds `constructor` and `toString`
|
|
153
|
+
// on the prototype and hands them to path.join as the filename.
|
|
154
|
+
const named = Object.hasOwn(DOWNLOADABLE_FILES, wanted) ? DOWNLOADABLE_FILES[wanted] : null;
|
|
155
|
+
const lexical = path.resolve(path.join(data, named || wanted));
|
|
156
|
+
|
|
157
|
+
// The allowlist is read off the RESOLVED path, never off what was asked
|
|
158
|
+
// for: `files/../config.json` starts with an allowed directory name and
|
|
159
|
+
// means something else entirely.
|
|
160
|
+
const inside = lexical.startsWith(`${data}/`);
|
|
161
|
+
const top = inside ? lexical.slice(data.length + 1).split('/')[0] : '';
|
|
162
|
+
if (!inside || !(named || DOWNLOADABLE_DIRS.includes(top))) return notFound(res);
|
|
163
|
+
|
|
164
|
+
let real;
|
|
165
|
+
try {
|
|
166
|
+
real = path.resolve(fs.realpathSync(lexical));
|
|
167
|
+
} catch {
|
|
168
|
+
return notFound(res);
|
|
169
|
+
}
|
|
170
|
+
// A symlink out of the data directory is the one way this lane could hand
|
|
171
|
+
// over something that is not ours, so it is checked after resolution.
|
|
172
|
+
const stat = real === lexical ? await statIfExists(real) : null;
|
|
173
|
+
if (!stat?.isFile()) return notFound(res);
|
|
174
|
+
|
|
175
|
+
res.setHeader('Content-Type', 'application/octet-stream');
|
|
176
|
+
res.setHeader('Content-Security-Policy', "default-src 'none'; sandbox");
|
|
177
|
+
// A filename is whatever the filesystem allowed, including a newline, and a
|
|
178
|
+
// newline in a header value throws rather than being escaped.
|
|
179
|
+
res.setHeader('Content-Disposition',
|
|
180
|
+
`attachment; filename="${path.basename(real).replace(/[^\w.-]+/g, '_')}"`);
|
|
181
|
+
return res.sendFile(real);
|
|
182
|
+
}));
|
|
183
|
+
|
|
184
|
+
return { report, isLoopback, key, url: `/_/recover?key=${key}` };
|
|
185
|
+
}
|
package/src/recovery.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { createReplay } from './recovery/replay.js';
|
|
2
|
+
import { createOverlay } from './recovery/overlay.js';
|
|
3
|
+
|
|
4
|
+
// What happens between `kill -9` and the next boot.
|
|
5
|
+
//
|
|
6
|
+
// Two independent jobs, both run before the host serves a request, and each in
|
|
7
|
+
// its own file because they answer different questions from different evidence:
|
|
8
|
+
//
|
|
9
|
+
// recovery/replay.js finish or unwind the multi-step ops that were in
|
|
10
|
+
// flight (§9.2 step 4). Reads the journal, then reads
|
|
11
|
+
// the disk to see which side of the single atomic
|
|
12
|
+
// rename the crash landed on.
|
|
13
|
+
//
|
|
14
|
+
// recovery/overlay.js the store is an overlay, so losing it must not lose
|
|
15
|
+
// privacy or tenancy (§6.3). Reads the durable records
|
|
16
|
+
// under versions/, and rebuilds the database from them.
|
|
17
|
+
//
|
|
18
|
+
// They share no state, and neither calls the other. What they do share is the
|
|
19
|
+
// one rule: when the evidence is genuinely ambiguous, never guess. A clear
|
|
20
|
+
// degraded mode beats a half-working dashboard sitting on top of uncertain
|
|
21
|
+
// privacy metadata.
|
|
22
|
+
//
|
|
23
|
+
// boot.js runs them in order — replay first, because an operation left half
|
|
24
|
+
// applied would otherwise be assessed as if it were the settled truth.
|
|
25
|
+
|
|
26
|
+
export function createRecovery({ paths, store, nodes, versions, clock, logger = console }) {
|
|
27
|
+
const replay = createReplay({ paths, store, nodes, versions, clock, logger });
|
|
28
|
+
const overlay = createOverlay({ paths, store, nodes, clock, logger });
|
|
29
|
+
return { ...replay, ...overlay };
|
|
30
|
+
}
|
package/src/requests.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { PathError } from './paths.js';
|
|
2
|
+
import { can } from './auth/can.js';
|
|
3
|
+
import { jsonError } from './json-errors.js';
|
|
4
|
+
import { rawDocumentPath, resolveDocumentHref } from './spec/wire.js';
|
|
5
|
+
|
|
6
|
+
// Turning a request into a node, and refusing when it cannot.
|
|
7
|
+
//
|
|
8
|
+
// These travelled together through every route as separate arguments, which made
|
|
9
|
+
// them look like unrelated passengers rather than one service with one rule. The
|
|
10
|
+
// rule is the important part, and it is easier to hold in one place than to
|
|
11
|
+
// re-derive at each call site:
|
|
12
|
+
//
|
|
13
|
+
// A read refusal is a 404, shaped exactly like a missing document.
|
|
14
|
+
//
|
|
15
|
+
// Never a 403. A 403 tells a stranger that the document exists, which is the
|
|
16
|
+
// whole disclosure a private document is trying to avoid, so "you may not read
|
|
17
|
+
// this" and "there is nothing here" have to be indistinguishable from outside.
|
|
18
|
+
// That is why `resolveTarget` returns null for both and never throws: a
|
|
19
|
+
// traversal attempt and a typo get the same answer.
|
|
20
|
+
//
|
|
21
|
+
// It still catches PathError rather than calling `paths.tryResolveRead`, because
|
|
22
|
+
// what it wraps is `serve.resolveForRead`, which resolves a path AND runs the
|
|
23
|
+
// gate. Only a refusal from the path half is ordinary; a HostError raised by the
|
|
24
|
+
// gate is a real answer and must reach the client.
|
|
25
|
+
|
|
26
|
+
export function createRequestResolution({ serve, store, clock, gateCtx }) {
|
|
27
|
+
function notFoundDocument(req, res) {
|
|
28
|
+
if (req.path.startsWith('/_/') || req.accepts(['html', 'json']) === 'json') {
|
|
29
|
+
return jsonError(res, 'not-found', 'Not found');
|
|
30
|
+
}
|
|
31
|
+
res.status(404).type('text/html').send('<!DOCTYPE html><title>Not found</title><h1>Not found</h1>');
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function resolveHref(href) {
|
|
36
|
+
const { owner, rest } = serve.parseRealm(rawDocumentPath(href));
|
|
37
|
+
return { owner, relPath: resolveDocumentHref(rest) };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function resolveTarget(actor, urlPathOrHref, action = 'read') {
|
|
41
|
+
const urlPath = rawDocumentPath(urlPathOrHref);
|
|
42
|
+
try {
|
|
43
|
+
const found = await serve.resolveForRead(actor, urlPath);
|
|
44
|
+
if (!found) return null;
|
|
45
|
+
if (action !== 'read' && !can(actor, action, found.node, gateCtx)) return null;
|
|
46
|
+
return { ...found, actor };
|
|
47
|
+
} catch (error) {
|
|
48
|
+
if (error instanceof PathError) return null;
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function resolveSaveToken(rawToken) {
|
|
54
|
+
if (!store || !rawToken) return null;
|
|
55
|
+
const record = store.getSaveToken(String(rawToken));
|
|
56
|
+
if (!record || record.expiresAt <= clock.now()) return null;
|
|
57
|
+
const node = store.getNodeById(record.nodeId);
|
|
58
|
+
if (!node) return null;
|
|
59
|
+
if (record.shareHash) {
|
|
60
|
+
const share = store.getShare(record.shareHash);
|
|
61
|
+
if (!share || share.revokedAt || (share.expiresAt && share.expiresAt <= clock.now())) return null;
|
|
62
|
+
}
|
|
63
|
+
// Defence in depth behind revocation: a save token is a bearer credential in
|
|
64
|
+
// a URL, and this is the one place every use of one passes through.
|
|
65
|
+
if (record.principal?.startsWith('tenant:')) {
|
|
66
|
+
const tenant = store.getTenant(record.principal.slice('tenant:'.length));
|
|
67
|
+
if (!tenant || tenant.disabledAt) return null;
|
|
68
|
+
}
|
|
69
|
+
return { nodeId: record.nodeId, grantKind: record.grantKind, node, principal: record.principal };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return { notFoundDocument, resolveHref, resolveTarget, resolveSaveToken };
|
|
73
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { hostMeta, documentBlock } from '../spec/meta.js';
|
|
2
|
+
import { documentUrlHeader, documentEtag } from '../spec/wire.js';
|
|
3
|
+
import { readFileTextIfExists } from '../util/fsx.js';
|
|
4
|
+
import { wrap } from '../util/express.js';
|
|
5
|
+
|
|
6
|
+
// spec §5 discovery.
|
|
7
|
+
//
|
|
8
|
+
// Host scope is never gated and is CORS-open: the caller is a page a person
|
|
9
|
+
// just loaded, often anonymously, and this is the only place a client is
|
|
10
|
+
// permitted to learn what the host supports.
|
|
11
|
+
|
|
12
|
+
export function mountMetaRoutes({
|
|
13
|
+
system, config, serve, gateCtx, resolveTarget, resolveSaveToken, tryResolveRead,
|
|
14
|
+
}) {
|
|
15
|
+
async function metaResponse(req, res, { node, real, actor }) {
|
|
16
|
+
// Host scope is CORS-open, unless a tighter rule already answered: the
|
|
17
|
+
// token lane below pins this to `null`, and widening it back to `*` here
|
|
18
|
+
// would hand any origin a document block it holds no token for.
|
|
19
|
+
if (!res.getHeader('Access-Control-Allow-Origin')) {
|
|
20
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
21
|
+
}
|
|
22
|
+
res.setHeader('Vary', 'Origin');
|
|
23
|
+
const body = hostMeta(config.capabilities);
|
|
24
|
+
if (node && real) {
|
|
25
|
+
const stored = await readFileTextIfExists(real);
|
|
26
|
+
if (stored != null) {
|
|
27
|
+
body.document = documentBlock({
|
|
28
|
+
node,
|
|
29
|
+
etag: documentEtag(stored),
|
|
30
|
+
actor,
|
|
31
|
+
ctx: gateCtx,
|
|
32
|
+
limits: config.limits,
|
|
33
|
+
isolated: serve.isolationFor(node),
|
|
34
|
+
capabilities: config.capabilities,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return res.json(body);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
system.options('/meta', (req, res) => {
|
|
42
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
43
|
+
res.setHeader('Access-Control-Allow-Headers', 'Document-URL, Page-URL, If-Match');
|
|
44
|
+
res.setHeader('Vary', 'Origin');
|
|
45
|
+
res.status(204).end();
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
system.get('/meta', wrap(async (req, res) => {
|
|
49
|
+
const href = documentUrlHeader(req);
|
|
50
|
+
if (!href) return metaResponse(req, res, {});
|
|
51
|
+
const found = await resolveTarget(req.actor, href, 'read');
|
|
52
|
+
return metaResponse(req, res, found || {});
|
|
53
|
+
}));
|
|
54
|
+
|
|
55
|
+
system.get('/meta/:token', wrap(async (req, res) => {
|
|
56
|
+
const grant = await resolveSaveToken(req.params.token);
|
|
57
|
+
if (!grant) return metaResponse(req, res, {});
|
|
58
|
+
const actor = { ...req.actor, grants: { ...req.actor.grants, saveToken: grant } };
|
|
59
|
+
const real = await tryResolveRead(grant.node.owner, grant.node.path);
|
|
60
|
+
return metaResponse(req, res, real ? { node: grant.node, real, actor } : {});
|
|
61
|
+
}));
|
|
62
|
+
}
|