@optimystic/db-p2p-storage-fs 0.14.0 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -0
- package/dist/src/atomic-write.d.ts +22 -0
- package/dist/src/atomic-write.d.ts.map +1 -0
- package/dist/src/atomic-write.js +82 -0
- package/dist/src/atomic-write.js.map +1 -0
- package/dist/src/file-kv-store.d.ts.map +1 -1
- package/dist/src/file-kv-store.js +6 -3
- package/dist/src/file-kv-store.js.map +1 -1
- package/dist/src/file-storage.d.ts +52 -20
- package/dist/src/file-storage.d.ts.map +1 -1
- package/dist/src/file-storage.js +285 -69
- package/dist/src/file-storage.js.map +1 -1
- package/package.json +8 -4
- package/src/atomic-write.ts +82 -0
- package/src/file-kv-store.ts +6 -3
- package/src/file-storage.ts +304 -88
package/src/file-storage.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
import { promises as fs } from 'fs';
|
|
2
2
|
import * as path from 'path';
|
|
3
|
-
import type { BlockId,
|
|
4
|
-
import
|
|
3
|
+
import type { BlockId, ActionId } from "@optimystic/db-core";
|
|
4
|
+
import { KvRawStorage, type RawStoreDriver } from "@optimystic/db-p2p";
|
|
5
5
|
import { createLogger } from './logger.js';
|
|
6
|
+
import { atomicWriteFile } from './atomic-write.js';
|
|
6
7
|
|
|
7
8
|
const log = createLogger('storage:file');
|
|
8
9
|
|
|
10
|
+
const decoder = new TextDecoder();
|
|
11
|
+
|
|
9
12
|
// Colons are illegal in Windows filenames; encode them so action ids like
|
|
10
13
|
// `tx:<hash>` and `stamp:<hash>` round-trip safely on all platforms.
|
|
11
14
|
function encodeActionIdForFilename(actionId: ActionId): string {
|
|
@@ -16,106 +19,265 @@ function decodeFilenameToActionId(filename: string): ActionId {
|
|
|
16
19
|
return filename.replace(/%3A/g, ':') as ActionId;
|
|
17
20
|
}
|
|
18
21
|
|
|
19
|
-
|
|
22
|
+
// A torn write leaves valid-prefix JSON cut off mid-token — JSON.parse throws
|
|
23
|
+
// SyntaxError. Used as the "corrupt content → treat as missing" guard so a
|
|
24
|
+
// crash-truncated file reads as absent (letting recover() make progress) instead
|
|
25
|
+
// of surfacing a parse error forever. Only the JSON-valued stores use this; the
|
|
26
|
+
// revisions store holds a bare ActionId string (not JSON) and is never guarded.
|
|
27
|
+
// NOTE: every guarded read parses the JSON here (validate, discard) and the kernel's
|
|
28
|
+
// decodeJson parses the same bytes AGAIN to build the value — 2× parse per get. The
|
|
29
|
+
// guard is intrinsic to the driver's "corrupt→missing" contract (the kernel can't
|
|
30
|
+
// express it), so it can't simply be dropped. Fine at current read volumes; if a read
|
|
31
|
+
// path ever shows up hot, parse once and thread the parsed value through the driver.
|
|
32
|
+
function isParseableJson(bytes: Uint8Array): boolean {
|
|
33
|
+
try {
|
|
34
|
+
JSON.parse(decoder.decode(bytes));
|
|
35
|
+
return true;
|
|
36
|
+
} catch (err) {
|
|
37
|
+
if (err instanceof SyntaxError) return false;
|
|
38
|
+
throw err;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Filesystem {@link RawStoreDriver}: the five logical block-storage stores mapped
|
|
44
|
+
* to five subdirectories under `basePath/<blockId>/`
|
|
45
|
+
* (`{meta.json,revs/,pend/,actions/,blocks/}`). The directory tree is a
|
|
46
|
+
* deliberate, human-inspectable/debuggable layout — it is NOT flattened into
|
|
47
|
+
* encoded-filename KV keys.
|
|
48
|
+
*
|
|
49
|
+
* `KvRawStorage` now owns all JSON serialization, so this driver reads/writes raw
|
|
50
|
+
* `Uint8Array` bytes and never does `JSON.stringify/parse` on values. Everything
|
|
51
|
+
* else fs-specific lives here: atomic (temp-file + rename) writes, the
|
|
52
|
+
* corrupt-content-as-missing read guard, colon-encoded action-id filenames with
|
|
53
|
+
* the legacy raw-colon read fallback + win32 guards, and rename-based promote.
|
|
54
|
+
*/
|
|
55
|
+
export class FileStoreDriver implements RawStoreDriver {
|
|
20
56
|
constructor(private readonly basePath: string) {
|
|
21
57
|
// TODO: use https://www.npmjs.com/package/proper-lockfile to take a lock on the basePath, also introduce explicit dispose pattern
|
|
22
58
|
}
|
|
23
59
|
|
|
24
|
-
|
|
25
|
-
|
|
60
|
+
// --- metadata ---
|
|
61
|
+
|
|
62
|
+
async getMetadata(blockId: BlockId): Promise<Uint8Array | undefined> {
|
|
63
|
+
return this.readBytesIfExists(this.getMetadataPath(blockId), true);
|
|
26
64
|
}
|
|
27
65
|
|
|
28
|
-
async
|
|
29
|
-
await this.
|
|
30
|
-
this.getMetadataPath(blockId),
|
|
31
|
-
JSON.stringify(metadata)
|
|
32
|
-
);
|
|
66
|
+
async putMetadata(blockId: BlockId, value: Uint8Array): Promise<void> {
|
|
67
|
+
await atomicWriteFile(this.getMetadataPath(blockId), value);
|
|
33
68
|
}
|
|
34
69
|
|
|
35
|
-
|
|
36
|
-
|
|
70
|
+
// --- revisions ---
|
|
71
|
+
|
|
72
|
+
// The revisions store value is a bare ActionId string (kernel `encodeActionId`,
|
|
73
|
+
// NOT JSON), so it is read WITHOUT the JSON guard — any bytes are a valid string
|
|
74
|
+
// and `decodeActionId` never throws.
|
|
75
|
+
// NOTE: because there is no guard here, a *torn* revision file reads back as a wrong
|
|
76
|
+
// (truncated) ActionId rather than as missing — unlike the JSON stores, which torn-read
|
|
77
|
+
// as undefined. Atomic writes (temp+rename) make a new torn revision impossible; only a
|
|
78
|
+
// legacy pre-atomic-write torn rev could hit this, and recover() re-derives revisions
|
|
79
|
+
// from the actions store anyway. If revisions ever move to a non-atomic writer, add a
|
|
80
|
+
// checksum/length guard here.
|
|
81
|
+
async getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined> {
|
|
82
|
+
return this.readBytesIfExists(this.getRevisionPath(blockId, rev), false);
|
|
37
83
|
}
|
|
38
84
|
|
|
39
|
-
async
|
|
40
|
-
await this.
|
|
41
|
-
this.getRevisionPath(blockId, rev),
|
|
42
|
-
JSON.stringify(actionId)
|
|
43
|
-
);
|
|
85
|
+
async putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void> {
|
|
86
|
+
await atomicWriteFile(this.getRevisionPath(blockId, rev), value);
|
|
44
87
|
}
|
|
45
88
|
|
|
46
|
-
async
|
|
47
|
-
|
|
89
|
+
async *rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]> {
|
|
90
|
+
// The fs backend has no cursor: walk the bounded [lo, hi] range rev-by-rev,
|
|
91
|
+
// reading each present rev. The range is caller-bounded, so this avoids
|
|
92
|
+
// listing an unbounded revs/ directory. Drain into an array BEFORE yielding
|
|
93
|
+
// (drain-before-yield contract) — matches the memory/native drivers and keeps
|
|
94
|
+
// the consumer's interleaved awaits from straddling any in-flight read.
|
|
95
|
+
const results: [number, Uint8Array][] = [];
|
|
96
|
+
for (let rev = lo; rev <= hi; rev++) {
|
|
97
|
+
const value = await this.readBytesIfExists(this.getRevisionPath(blockId, rev), false);
|
|
98
|
+
if (value !== undefined) {
|
|
99
|
+
results.push([rev, value]);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (reverse) {
|
|
103
|
+
results.reverse();
|
|
104
|
+
}
|
|
105
|
+
for (const result of results) {
|
|
106
|
+
yield result;
|
|
107
|
+
}
|
|
48
108
|
}
|
|
49
109
|
|
|
50
|
-
|
|
51
|
-
|
|
110
|
+
// --- pending ---
|
|
111
|
+
|
|
112
|
+
async getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
113
|
+
return this.readActionScopedBytes(
|
|
52
114
|
this.getPendingActionPath(blockId, actionId),
|
|
53
|
-
|
|
115
|
+
this.getPendingActionPath(blockId, actionId, false)
|
|
54
116
|
);
|
|
55
117
|
}
|
|
56
118
|
|
|
57
|
-
async
|
|
119
|
+
async putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
120
|
+
await atomicWriteFile(this.getPendingActionPath(blockId, actionId), value);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async deletePending(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
58
124
|
const pendingPath = this.getPendingActionPath(blockId, actionId);
|
|
59
125
|
await fs.unlink(pendingPath)
|
|
60
126
|
.catch((err) => {
|
|
61
|
-
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') log('
|
|
127
|
+
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') log('deletePending unlink failed for %s/%s - %o', blockId, actionId, err);
|
|
62
128
|
});
|
|
129
|
+
await this.unlinkRawColon(pendingPath, this.getPendingActionPath(blockId, actionId, false));
|
|
63
130
|
}
|
|
64
131
|
|
|
65
|
-
async *
|
|
132
|
+
async *listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId> {
|
|
66
133
|
const pendingPath = path.join(this.getBlockPath(blockId), 'pend');
|
|
67
134
|
|
|
68
|
-
|
|
135
|
+
// Only a genuinely-absent directory (ENOENT) maps to "no pendings". Any other error
|
|
136
|
+
// (EACCES, EIO, ENOTDIR, ...) must surface — swallowing it here would make
|
|
137
|
+
// listPendingActionIds silently report an empty directory, so pend's conflict
|
|
138
|
+
// detection would be skipped. Mirrors directoryByteSize's ENOENT-vs-other discrimination.
|
|
139
|
+
const files = await fs.readdir(pendingPath).catch((err) => {
|
|
140
|
+
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return [] as string[];
|
|
141
|
+
log('listPendingActionIds readdir failed for %s - %o', blockId, err);
|
|
142
|
+
throw err;
|
|
143
|
+
});
|
|
144
|
+
// Drain into an array before yielding (drain-before-yield): readdir has already
|
|
145
|
+
// resolved the full listing, so this just decodes/filters up front.
|
|
146
|
+
const ids: ActionId[] = [];
|
|
69
147
|
for (const file of files) {
|
|
70
148
|
if (!file.endsWith('.json')) continue;
|
|
71
149
|
const actionId = decodeFilenameToActionId(file.slice(0, -5));
|
|
72
|
-
// Accept
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
|
|
150
|
+
// Accept every realistic action id: legacy UUIDs (`[0-9a-f-]`), consensus
|
|
151
|
+
// tx:/stamp: ids (base64url-encoded SHA-256, alphabet `[A-Za-z0-9_-]` — see
|
|
152
|
+
// db-core hashString, NOT lowercase hex), AND the bare-alphanumeric ids the
|
|
153
|
+
// cross-backend conformance suite uses (`a1`, `b1`, ...). This is deliberately
|
|
154
|
+
// broad-but-not-total: an id is any `[A-Za-z0-9_-]` string, optionally prefixed
|
|
155
|
+
// with `tx:`/`stamp:`. It is NOT a total accept — a file whose decoded name
|
|
156
|
+
// carries other punctuation (a dot, a space) is genuine junk in pend/ and is
|
|
157
|
+
// logged-and-skipped rather than surfaced as a phantom pending. The memory/db
|
|
158
|
+
// reference drivers key on the raw id and never see a filesystem name, so this
|
|
159
|
+
// filter is fs-only; it must not drop an id those backends would list, hence the
|
|
160
|
+
// widened class (an earlier hex-only class silently dropped real consensus ids —
|
|
161
|
+
// see `optimystic-filestorage-colon-actionid-windows`).
|
|
162
|
+
if (!/^(?:tx:|stamp:)?[A-Za-z0-9_-]+$/.test(actionId)) {
|
|
163
|
+
// Leave a breadcrumb rather than silently dropping: the .json + decode guard
|
|
164
|
+
// already excludes *.tmp orphans, so anything reaching here is an unexpected
|
|
165
|
+
// filename a maintainer should see.
|
|
166
|
+
log('listPendingActionIds skipping unrecognized action-id file %s for %s', file, blockId);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
ids.push(actionId);
|
|
170
|
+
}
|
|
171
|
+
for (const id of ids) {
|
|
172
|
+
yield id;
|
|
77
173
|
}
|
|
78
174
|
}
|
|
79
175
|
|
|
80
|
-
|
|
81
|
-
|
|
176
|
+
// --- transactions ---
|
|
177
|
+
|
|
178
|
+
async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
179
|
+
return this.readActionScopedBytes(
|
|
180
|
+
this.getActionPath(blockId, actionId),
|
|
181
|
+
this.getActionPath(blockId, actionId, false)
|
|
182
|
+
);
|
|
82
183
|
}
|
|
83
184
|
|
|
84
|
-
async
|
|
85
|
-
|
|
86
|
-
const actionId = await this.getRevision(blockId, rev);
|
|
87
|
-
if (actionId) {
|
|
88
|
-
yield { actionId, rev };
|
|
89
|
-
}
|
|
90
|
-
}
|
|
185
|
+
async putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
186
|
+
await atomicWriteFile(this.getActionPath(blockId, actionId), value);
|
|
91
187
|
}
|
|
92
188
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
189
|
+
// --- materialized ---
|
|
190
|
+
|
|
191
|
+
async getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
|
|
192
|
+
return this.readActionScopedBytes(
|
|
193
|
+
this.getMaterializedPath(blockId, actionId),
|
|
194
|
+
this.getMaterializedPath(blockId, actionId, false)
|
|
97
195
|
);
|
|
98
196
|
}
|
|
99
197
|
|
|
100
|
-
async
|
|
101
|
-
|
|
198
|
+
async putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
|
|
199
|
+
await atomicWriteFile(this.getMaterializedPath(blockId, actionId), value);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// The kernel owns the put-or-delete branch of `saveMaterializedBlock`, so the
|
|
203
|
+
// driver exposes delete as a separate op.
|
|
204
|
+
async deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
205
|
+
const matPath = this.getMaterializedPath(blockId, actionId);
|
|
206
|
+
await fs.unlink(matPath)
|
|
207
|
+
.catch((err) => {
|
|
208
|
+
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') log('deleteMaterialized unlink failed for %s/%s - %o', blockId, actionId, err);
|
|
209
|
+
});
|
|
210
|
+
await this.unlinkRawColon(matPath, this.getMaterializedPath(blockId, actionId, false));
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// --- promote (the only cross-key atomic op) ---
|
|
214
|
+
|
|
215
|
+
async promote(blockId: BlockId, actionId: ActionId): Promise<void> {
|
|
216
|
+
const pendingPath = this.getPendingActionPath(blockId, actionId);
|
|
217
|
+
const actionPath = this.getActionPath(blockId, actionId);
|
|
218
|
+
|
|
219
|
+
await fs.mkdir(path.dirname(actionPath), { recursive: true });
|
|
220
|
+
|
|
221
|
+
// This single rename IS the atomic move — it is why fs honors the kernel's
|
|
222
|
+
// promote contract without a WAL. A crash leaves either the pending or the
|
|
223
|
+
// committed file, never both/neither. Do NOT replace with read-write-delete.
|
|
224
|
+
return fs.rename(pendingPath, actionPath)
|
|
225
|
+
.catch(err => {
|
|
226
|
+
if (err.code === 'ENOENT') {
|
|
227
|
+
throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
|
|
228
|
+
}
|
|
229
|
+
log('promote rename failed for %s/%s - %o', blockId, actionId, err);
|
|
230
|
+
throw err;
|
|
231
|
+
});
|
|
102
232
|
}
|
|
103
233
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
234
|
+
// --- optional passthroughs ---
|
|
235
|
+
|
|
236
|
+
async *listBlockIds(): AsyncIterable<BlockId> {
|
|
237
|
+
// The block layout is `basePath/<blockId>/{meta.json,revs/,pend/,actions/,blocks/}`
|
|
238
|
+
// (see getBlockPath), so the direct children of basePath are the per-block directories
|
|
239
|
+
// and each directory NAME is the blockId (used raw, no encoding). Filter to directories
|
|
240
|
+
// so a stray file can't be mistaken for a block; `*.tmp` atomic-write orphans live inside
|
|
241
|
+
// block subdirs, never at basePath root, so the root is clean.
|
|
242
|
+
//
|
|
243
|
+
// A directory alone is NOT sufficient to call a block "durable owned": a block that was
|
|
244
|
+
// only PENDED (never committed) still creates `<blockId>/pend/` — hence a root directory
|
|
245
|
+
// entry — via atomicWriteFile's recursive mkdir, but has no meta.json. So we gate on
|
|
246
|
+
// meta.json existence: `meta.json` IS this backend's metadata store, and enumerating it
|
|
247
|
+
// yields exactly the blocks with a committed revision / persisted replica (the same
|
|
248
|
+
// "owned" population the live change feed tracks, and the same one the metadata-keyed
|
|
249
|
+
// backends — sqlite/leveldb/indexeddb — enumerate for free). Existence (fs.access), not
|
|
250
|
+
// parse, matches key-existence semantics: a torn/corrupt meta.json still counts as a key,
|
|
251
|
+
// exactly as a corrupt value would in the other backends.
|
|
252
|
+
//
|
|
253
|
+
// ENOENT (basePath not created yet, or a dir without meta.json) maps to "not owned" —
|
|
254
|
+
// same discrimination as directoryByteSize. Any OTHER readdir/access error must surface:
|
|
255
|
+
// swallowing it would make the seed falsely report an empty store and under-protect data
|
|
256
|
+
// already on disk.
|
|
257
|
+
// NOTE: reads the whole root listing up front + one meta.json stat per block dir; if a
|
|
258
|
+
// store ever grows to millions of block subdirs and this becomes a startup-latency
|
|
259
|
+
// problem, page it (e.g. opendir cursor) — fine at current scale.
|
|
260
|
+
const entries = await fs.readdir(this.basePath, { withFileTypes: true })
|
|
261
|
+
.catch((err) => {
|
|
262
|
+
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return [];
|
|
263
|
+
log('listBlockIds readdir failed for %s - %o', this.basePath, err);
|
|
264
|
+
throw err;
|
|
265
|
+
});
|
|
266
|
+
for (const entry of entries) {
|
|
267
|
+
if (!entry.isDirectory()) continue;
|
|
268
|
+
const blockId = entry.name as BlockId;
|
|
269
|
+
const hasMeta = await fs.access(this.getMetadataPath(blockId))
|
|
270
|
+
.then(() => true)
|
|
112
271
|
.catch((err) => {
|
|
113
|
-
if ((err as NodeJS.ErrnoException)?.code
|
|
272
|
+
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return false;
|
|
273
|
+
log('listBlockIds access failed for %s - %o', blockId, err);
|
|
274
|
+
throw err;
|
|
114
275
|
});
|
|
276
|
+
if (hasMeta) yield blockId;
|
|
115
277
|
}
|
|
116
278
|
}
|
|
117
279
|
|
|
118
|
-
async
|
|
280
|
+
async approximateBytesUsed(): Promise<number> {
|
|
119
281
|
return this.directoryByteSize(this.basePath);
|
|
120
282
|
}
|
|
121
283
|
|
|
@@ -146,21 +308,7 @@ export class FileRawStorage implements IRawStorage {
|
|
|
146
308
|
return total;
|
|
147
309
|
}
|
|
148
310
|
|
|
149
|
-
|
|
150
|
-
const pendingPath = this.getPendingActionPath(blockId, actionId);
|
|
151
|
-
const actionPath = this.getActionPath(blockId, actionId);
|
|
152
|
-
|
|
153
|
-
await fs.mkdir(path.dirname(actionPath), { recursive: true });
|
|
154
|
-
|
|
155
|
-
return fs.rename(pendingPath, actionPath)
|
|
156
|
-
.catch(err => {
|
|
157
|
-
if (err.code === 'ENOENT') {
|
|
158
|
-
throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
|
|
159
|
-
}
|
|
160
|
-
log('promotePendingTransaction rename failed for %s/%s - %o', blockId, actionId, err);
|
|
161
|
-
throw err;
|
|
162
|
-
});
|
|
163
|
-
}
|
|
311
|
+
// --- paths ---
|
|
164
312
|
|
|
165
313
|
private getBlockPath(blockId: BlockId): string {
|
|
166
314
|
return path.join(this.basePath, blockId);
|
|
@@ -174,29 +322,97 @@ export class FileRawStorage implements IRawStorage {
|
|
|
174
322
|
return path.join(this.getBlockPath(blockId), 'revs', `${rev}.json`);
|
|
175
323
|
}
|
|
176
324
|
|
|
177
|
-
|
|
178
|
-
|
|
325
|
+
// `encoded` controls colon handling: writes and canonical reads use the
|
|
326
|
+
// percent-encoded filename (encoded = true); the legacy raw-colon fallback
|
|
327
|
+
// (see readActionScopedBytes) passes encoded = false to reach pre-encode
|
|
328
|
+
// POSIX files like `actions/tx:<hash>.json`.
|
|
329
|
+
private getPendingActionPath(blockId: BlockId, actionId: ActionId, encoded = true): string {
|
|
330
|
+
const filename = encoded ? encodeActionIdForFilename(actionId) : actionId;
|
|
331
|
+
return path.join(this.getBlockPath(blockId), 'pend', `${filename}.json`);
|
|
179
332
|
}
|
|
180
333
|
|
|
181
|
-
private getActionPath(blockId: BlockId, actionId: ActionId): string {
|
|
182
|
-
|
|
334
|
+
private getActionPath(blockId: BlockId, actionId: ActionId, encoded = true): string {
|
|
335
|
+
const filename = encoded ? encodeActionIdForFilename(actionId) : actionId;
|
|
336
|
+
return path.join(this.getBlockPath(blockId), 'actions', `${filename}.json`);
|
|
183
337
|
}
|
|
184
338
|
|
|
185
|
-
private getMaterializedPath(blockId: BlockId, actionId: ActionId): string {
|
|
186
|
-
|
|
339
|
+
private getMaterializedPath(blockId: BlockId, actionId: ActionId, encoded = true): string {
|
|
340
|
+
const filename = encoded ? encodeActionIdForFilename(actionId) : actionId;
|
|
341
|
+
return path.join(this.getBlockPath(blockId), 'blocks', `${filename}.json`);
|
|
187
342
|
}
|
|
188
343
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
344
|
+
// Best-effort removal of a pre-encode raw-colon file after the encoded delete,
|
|
345
|
+
// so a deleted item cannot resurface via the read fallback in readActionScopedBytes.
|
|
346
|
+
// Skipped on win32 (raw-colon files cannot exist there) and when paths are identical
|
|
347
|
+
// (action id contains no colon — only one syscall needed). ENOENT is silently ignored.
|
|
348
|
+
private async unlinkRawColon(encodedPath: string, rawPath: string): Promise<void> {
|
|
349
|
+
if (process.platform === 'win32' || rawPath === encodedPath) return;
|
|
350
|
+
await fs.unlink(rawPath).catch((err) => {
|
|
351
|
+
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') log('unlinkRawColon failed for %s - %o', rawPath, err);
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// Reads an action-id-keyed file (JSON-valued) by its canonical (percent-encoded)
|
|
356
|
+
// path, falling back on a miss to the legacy raw-colon path written by pre-encode
|
|
357
|
+
// nodes (e.g. POSIX files literally named `actions/tx:<hash>.json`).
|
|
358
|
+
//
|
|
359
|
+
// Tradeoff: this reads legacy files in place and never renames them, so a
|
|
360
|
+
// store upgraded from a pre-encode node keeps mixed naming on disk. That is
|
|
361
|
+
// acceptable pre-1.0; a future migration sweep can normalize if desired. We
|
|
362
|
+
// deliberately do NOT migrate-on-read here — reads stay side-effect-free.
|
|
363
|
+
//
|
|
364
|
+
// win32 guard: a raw-colon path is not a benign miss on Windows — the colon
|
|
365
|
+
// is parsed as an NTFS alternate-data-stream separator and a read there can
|
|
366
|
+
// throw a non-ENOENT error rather than cleanly missing. Raw-colon files
|
|
367
|
+
// cannot have been written on win32 anyway, so we skip the fallback there
|
|
368
|
+
// (losing nothing) and swallow ALL fallback errors elsewhere, guaranteeing
|
|
369
|
+
// the fallback never surfaces a new throw to callers.
|
|
370
|
+
private async readActionScopedBytes(encodedPath: string, rawPath: string): Promise<Uint8Array | undefined> {
|
|
371
|
+
const hit = await this.readBytesIfExists(encodedPath, true);
|
|
372
|
+
if (hit !== undefined) return hit;
|
|
373
|
+
if (process.platform === 'win32' || rawPath === encodedPath) return undefined;
|
|
374
|
+
return fs.readFile(rawPath)
|
|
375
|
+
.then(bytes => (isParseableJson(bytes) ? bytes : undefined))
|
|
376
|
+
.catch(() => undefined);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// Reads a file's raw bytes. ENOENT → undefined. When `jsonGuard` is set, a
|
|
380
|
+
// present-but-corrupt file (JSON.parse fails — most likely a torn write from a
|
|
381
|
+
// crash before atomic writes existed) is treated as "missing" so recover() and
|
|
382
|
+
// normal reads make progress instead of rethrowing forever. A real I/O error
|
|
383
|
+
// (permissions, EIO, EISDIR, ...) still throws — it must not be masked.
|
|
384
|
+
private async readBytesIfExists(filePath: string, jsonGuard: boolean): Promise<Uint8Array | undefined> {
|
|
385
|
+
let bytes: Uint8Array;
|
|
386
|
+
try {
|
|
387
|
+
bytes = await fs.readFile(filePath);
|
|
388
|
+
} catch (err) {
|
|
389
|
+
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return undefined;
|
|
390
|
+
throw err;
|
|
391
|
+
}
|
|
392
|
+
if (jsonGuard && !isParseableJson(bytes)) {
|
|
393
|
+
log('readBytesIfExists: corrupt JSON at %s, treating as missing', filePath);
|
|
394
|
+
return undefined;
|
|
395
|
+
}
|
|
396
|
+
return bytes;
|
|
196
397
|
}
|
|
398
|
+
}
|
|
197
399
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
400
|
+
/**
|
|
401
|
+
* Filesystem-backed {@link IRawStorage}, now a thin shell over the shared
|
|
402
|
+
* {@link KvRawStorage} kernel driven by a {@link FileStoreDriver}. The public
|
|
403
|
+
* name/constructor (`new FileRawStorage(basePath)`) is unchanged so existing
|
|
404
|
+
* imports keep resolving; the kernel supplies the `IRawStorage` surface and the
|
|
405
|
+
* driver supplies fs behavior.
|
|
406
|
+
*
|
|
407
|
+
* `listBlockIds`/`getApproximateBytesUsed` are re-declared here as always-present
|
|
408
|
+
* (the fs driver always implements them, so the kernel constructor always wires
|
|
409
|
+
* them) — the base declares them optional, but every fs consumer relies on them.
|
|
410
|
+
*/
|
|
411
|
+
export class FileRawStorage extends KvRawStorage {
|
|
412
|
+
declare listBlockIds: () => AsyncIterable<BlockId>;
|
|
413
|
+
declare getApproximateBytesUsed: () => Promise<number>;
|
|
414
|
+
|
|
415
|
+
constructor(basePath: string) {
|
|
416
|
+
super(new FileStoreDriver(basePath));
|
|
201
417
|
}
|
|
202
418
|
}
|