@frockbot/workspace-store 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +25 -6
- package/src/bucket.ts +80 -0
- package/src/index.ts +3 -0
- package/src/keys.ts +64 -0
- package/src/store.test.ts +984 -0
- package/src/store.ts +929 -0
- package/src/testing.ts +180 -0
- package/tsconfig.json +15 -0
- package/README.md +0 -3
package/src/store.ts
ADDED
|
@@ -0,0 +1,929 @@
|
|
|
1
|
+
// `WorkspaceFilesV1` over object storage, with every generation recorded in
|
|
2
|
+
// the owning Durable Object.
|
|
3
|
+
//
|
|
4
|
+
// This is the object-storage half of ADR 0013. The Computer-side half —
|
|
5
|
+
// `FlyWorkspaceFiles` in `@frockbot/plugin-fly-sprite` — implements the same
|
|
6
|
+
// interface over a Sprite's filesystem, and the two must answer the same way,
|
|
7
|
+
// because the same durable root is reachable through both: a refusal here is a
|
|
8
|
+
// refusal there, `unavailable` is an ordinary answer on both, and a losing
|
|
9
|
+
// conditional write is preserved on both rather than dropped.
|
|
10
|
+
//
|
|
11
|
+
// It is not kernel code. The kernel declares `WorkspaceFilesV1`; this package
|
|
12
|
+
// implements it and imports nothing but that declaration.
|
|
13
|
+
//
|
|
14
|
+
// Four constitutional rules are enforced here rather than described:
|
|
15
|
+
//
|
|
16
|
+
// - "every write to a durable root records its writer" — a write mints a
|
|
17
|
+
// generation, stores it beside the bytes, and records it in the Durable
|
|
18
|
+
// Object that owns the root. An `unattributed` writer is refused: it is a
|
|
19
|
+
// reader's answer about a file nobody recorded, never a writer a caller may
|
|
20
|
+
// present.
|
|
21
|
+
// - "a write that would overwrite a generation its writer has not seen is
|
|
22
|
+
// preserved as a conflicting generation and surfaced, never merged or
|
|
23
|
+
// dropped" — every `put` is conditional (`If-Match` on the current object's
|
|
24
|
+
// ETag, `If-None-Match: *` when the writer asserts absence), and the loser
|
|
25
|
+
// is written to its own conflict key, recorded as a conflicting generation,
|
|
26
|
+
// and returned to the caller with both generations.
|
|
27
|
+
// - "within a shared root each Bot's shard is written only on that Bot's
|
|
28
|
+
// behalf" — a Bot writer may write only under `by-agent/<its own id>/`,
|
|
29
|
+
// decided by `writerOwnsMemoryPathV1` and nowhere else.
|
|
30
|
+
// - "a Bot's instruction root and Bot Memory root are writable only by that
|
|
31
|
+
// Bot or its User" — a first-party Package is neither, exactly as the Fly
|
|
32
|
+
// implementation has it. The User-global instruction root (ADR 0016) is
|
|
33
|
+
// writable by that User or any of their Bots, and by nothing else; this is
|
|
34
|
+
// its only writer, because the Computer presents it read-only.
|
|
35
|
+
//
|
|
36
|
+
// A delete leaves a durable tombstone. Object storage forgets a deleted key
|
|
37
|
+
// entirely, so without one, "this file is gone, deliberately, and here is who
|
|
38
|
+
// removed it" would exist nowhere in durable state after the Durable Object is
|
|
39
|
+
// evicted — which is the same hole `unattributed` closes for arriving files.
|
|
40
|
+
// The delete is itself conditional, because object storage offers no
|
|
41
|
+
// conditional delete: it overwrites the file with an empty tombstone marker
|
|
42
|
+
// under `If-Match`, and the marker *stays* as the object, so a write racing a
|
|
43
|
+
// delete is preserved rather than destroyed. `delete` below has the detail,
|
|
44
|
+
// and `gcTombstoneMarkersV1` collects markers old enough that no create can
|
|
45
|
+
// still be conditioned on one.
|
|
46
|
+
import {
|
|
47
|
+
WORKSPACE_MAX_FILE_BYTES,
|
|
48
|
+
WORKSPACE_MAX_LIST_ENTRIES,
|
|
49
|
+
isWorkspaceComputerReadOnlyRootV1,
|
|
50
|
+
isWorkspaceMemoryRootV1,
|
|
51
|
+
normalizeWorkspaceRelativePathV1,
|
|
52
|
+
workspaceWriterMayWriteV1,
|
|
53
|
+
writerOwnsMemoryPathV1,
|
|
54
|
+
decodeWorkspaceGenerationV1,
|
|
55
|
+
type WorkspaceDeleteRequestV1,
|
|
56
|
+
type WorkspaceEntryV1,
|
|
57
|
+
type WorkspaceFailureV1,
|
|
58
|
+
type WorkspaceFilesV1,
|
|
59
|
+
type WorkspaceGenerationRecordV1,
|
|
60
|
+
type WorkspaceGenerationV1,
|
|
61
|
+
type WorkspaceGenerationsV1,
|
|
62
|
+
type WorkspaceListOutcomeV1,
|
|
63
|
+
type WorkspaceListRequestV1,
|
|
64
|
+
type WorkspacePathV1,
|
|
65
|
+
type WorkspaceReadOutcomeV1,
|
|
66
|
+
type WorkspaceRootV1,
|
|
67
|
+
type WorkspaceStatOutcomeV1,
|
|
68
|
+
type WorkspaceWriteOutcomeV1,
|
|
69
|
+
type WorkspaceWriteRequestV1,
|
|
70
|
+
type WorkspaceWriterV1,
|
|
71
|
+
} from "@frockbot/kernel-contracts";
|
|
72
|
+
import type {
|
|
73
|
+
ObjectBucketV1,
|
|
74
|
+
ObjectConditionsV1,
|
|
75
|
+
ObjectHeadV1,
|
|
76
|
+
} from "./bucket.js";
|
|
77
|
+
import {
|
|
78
|
+
isWorkspaceConflictKeyV1,
|
|
79
|
+
workspaceConflictKeyV1,
|
|
80
|
+
workspaceObjectKeyV1,
|
|
81
|
+
workspaceObjectPrefixV1,
|
|
82
|
+
workspaceRelativeFromKeyV1,
|
|
83
|
+
WORKSPACE_OBJECT_PREFIX,
|
|
84
|
+
} from "./keys.js";
|
|
85
|
+
|
|
86
|
+
/** The sha-256 of no bytes; a deletion tombstone's content address. */
|
|
87
|
+
export const WORKSPACE_EMPTY_SHA256 =
|
|
88
|
+
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
89
|
+
/** Where a generation rides beside its bytes in the object store. */
|
|
90
|
+
export const WORKSPACE_GENERATION_METADATA_KEY = "frockbot-generation";
|
|
91
|
+
/**
|
|
92
|
+
* Marks the empty object a delete leaves in the file's place.
|
|
93
|
+
*
|
|
94
|
+
* R2 has no conditional delete, so a delete fences with a conditional
|
|
95
|
+
* *overwrite* — see `delete` below. `read`, `stat`, and `list` treat a marker
|
|
96
|
+
* as absence, so the file reads as deleted rather than as an empty one, and
|
|
97
|
+
* the next create replaces the marker under `If-Match` on the marker's own
|
|
98
|
+
* ETag.
|
|
99
|
+
*/
|
|
100
|
+
export const WORKSPACE_TOMBSTONE_METADATA_KEY = "frockbot-tombstone";
|
|
101
|
+
/** Beyond this the generation is recorded durably but not mirrored on the object. */
|
|
102
|
+
const MAX_METADATA_BYTES = 1800;
|
|
103
|
+
const DEFAULT_LIST_LIMIT = 100;
|
|
104
|
+
const DEFAULT_MEDIA_TYPE = "application/octet-stream";
|
|
105
|
+
|
|
106
|
+
function failure(
|
|
107
|
+
status: WorkspaceFailureV1["status"],
|
|
108
|
+
reason: string,
|
|
109
|
+
): WorkspaceFailureV1 {
|
|
110
|
+
return { status, reason: reason.slice(0, 512) };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function digestV1(bytes: Uint8Array): Promise<string> {
|
|
114
|
+
const buffer = await crypto.subtle.digest("SHA-256", bytes as BufferSource);
|
|
115
|
+
return [...new Uint8Array(buffer)]
|
|
116
|
+
.map((byte) => byte.toString(16).padStart(2, "0"))
|
|
117
|
+
.join("");
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** True for the empty marker a delete leaves while it sweeps the key. */
|
|
121
|
+
function isTombstoneMarkerV1(head: ObjectHeadV1): boolean {
|
|
122
|
+
return head.customMetadata?.[WORKSPACE_TOMBSTONE_METADATA_KEY] !== undefined;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* `"kernel"` is the surface the kernel consumes: it refuses a write to every
|
|
127
|
+
* Memory root, because "The Memory Package is the single writer of Memory
|
|
128
|
+
* roots". `"memory"` is the Memory Package's own surface: it serves Memory
|
|
129
|
+
* roots and nothing else. Nothing accepts both — the same split the Fly
|
|
130
|
+
* Workspace makes.
|
|
131
|
+
*
|
|
132
|
+
* `"sync"` is the Computer-side durable-root sync of ADR 0013, and it is a
|
|
133
|
+
* mirror rather than an author. It reads every root — a Memory root has to be
|
|
134
|
+
* readable for the sync to present it read-only on the Computer — and writes
|
|
135
|
+
* every root except a Memory one, because pushing a Memory root would give
|
|
136
|
+
* that root a second writer. It is also the one surface that accepts an
|
|
137
|
+
* `unattributed` writer, and only on a non-Memory root: the file it is
|
|
138
|
+
* mirroring was written by a shell on the Computer, so nothing recorded who
|
|
139
|
+
* wrote it, and the choice is between recording that truthfully and losing a
|
|
140
|
+
* durable-root file at the next image rebuild. `unattributed` carries no
|
|
141
|
+
* authority — `isLoadableSkillSourceV1` refuses it — so the mirrored file is
|
|
142
|
+
* data the Bot can read and never an instruction it loads.
|
|
143
|
+
*/
|
|
144
|
+
export type WorkspaceStoreSurfaceV1 = "kernel" | "memory" | "sync";
|
|
145
|
+
|
|
146
|
+
export interface ObjectWorkspaceFilesOptionsV1 {
|
|
147
|
+
bucket: ObjectBucketV1;
|
|
148
|
+
/** The owning Durable Object's generation ledger. */
|
|
149
|
+
generations: WorkspaceGenerationsV1;
|
|
150
|
+
clock?: () => Date;
|
|
151
|
+
/** The User whose durable roots this store serves, when it serves one. */
|
|
152
|
+
owner?: { userId: string };
|
|
153
|
+
surface?: WorkspaceStoreSurfaceV1;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
class ObjectWorkspaceFiles implements WorkspaceFilesV1 {
|
|
157
|
+
private readonly bucket: ObjectBucketV1;
|
|
158
|
+
private readonly generations: WorkspaceGenerationsV1;
|
|
159
|
+
private readonly clock: () => Date;
|
|
160
|
+
private readonly owner: { userId: string } | undefined;
|
|
161
|
+
private readonly surface: WorkspaceStoreSurfaceV1;
|
|
162
|
+
|
|
163
|
+
constructor(options: ObjectWorkspaceFilesOptionsV1) {
|
|
164
|
+
this.bucket = options.bucket;
|
|
165
|
+
this.generations = options.generations;
|
|
166
|
+
this.clock = options.clock ?? (() => new Date());
|
|
167
|
+
this.owner = options.owner;
|
|
168
|
+
this.surface = options.surface ?? "kernel";
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* The one place a root is admitted for reading. Refusal is an ordinary
|
|
173
|
+
* outcome of this interface, so it answers a failure rather than throwing.
|
|
174
|
+
*/
|
|
175
|
+
private admit(root: WorkspaceRootV1): WorkspaceFailureV1 | undefined {
|
|
176
|
+
if (this.owner && root.userId !== this.owner.userId) {
|
|
177
|
+
return failure(
|
|
178
|
+
"refused",
|
|
179
|
+
"This store serves a different User's durable roots",
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
if (this.surface === "memory" && !isWorkspaceMemoryRootV1(root)) {
|
|
183
|
+
return failure("refused", "The Memory writer accepts Memory roots only");
|
|
184
|
+
}
|
|
185
|
+
return undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private admitWrite(
|
|
189
|
+
path: WorkspacePathV1,
|
|
190
|
+
writer: WorkspaceWriterV1,
|
|
191
|
+
): WorkspaceFailureV1 | undefined {
|
|
192
|
+
const root = path.root;
|
|
193
|
+
const refused = this.admit(root);
|
|
194
|
+
if (refused) return refused;
|
|
195
|
+
// The sync mirrors a file whose writer the Computer did not record. It is
|
|
196
|
+
// the only caller that may carry `unattributed`, and never into a Memory
|
|
197
|
+
// root, which it does not write at all.
|
|
198
|
+
const mirroring =
|
|
199
|
+
this.surface === "sync" &&
|
|
200
|
+
writer.kind === "unattributed" &&
|
|
201
|
+
!isWorkspaceComputerReadOnlyRootV1(root);
|
|
202
|
+
if (!workspaceWriterMayWriteV1(writer) && !mirroring) {
|
|
203
|
+
return failure(
|
|
204
|
+
"refused",
|
|
205
|
+
"Every write to a durable root records its writer; an unattributed writer records none",
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (isWorkspaceMemoryRootV1(root)) {
|
|
209
|
+
if (this.surface !== "memory") {
|
|
210
|
+
return failure(
|
|
211
|
+
"refused",
|
|
212
|
+
"The Workspace presents Memory roots read-only; the Memory Package is their only writer",
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
if (!writerOwnsMemoryPathV1(path, writer)) {
|
|
216
|
+
return failure(
|
|
217
|
+
"refused",
|
|
218
|
+
`Only the Bot that owns this Memory shard, or User "${root.userId}", may write "${path.path}"`,
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
if (root.kind === "bot-instructions") {
|
|
224
|
+
const byBot = writer.kind === "bot" && writer.botId === root.botId;
|
|
225
|
+
const byUser = writer.kind === "user" && writer.userId === root.userId;
|
|
226
|
+
if (!byBot && !byUser && !mirroring) {
|
|
227
|
+
return failure(
|
|
228
|
+
"refused",
|
|
229
|
+
`Only Bot "${root.botId}" or its User may write this root`,
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
if (root.kind === "user-instructions") {
|
|
234
|
+
// The User-global instruction root is shared by every Bot this User
|
|
235
|
+
// owns (ADR 0016), so any `bot` writer is admitted — `this.admit` above
|
|
236
|
+
// has already confined the store to one User's roots, and a Bot writes
|
|
237
|
+
// through the Durable Object that holds its own identity, so a recorded
|
|
238
|
+
// `bot` generation here is a Bot of this User. A `first-party` writer is
|
|
239
|
+
// not; nor is the sync, which never pushes into this root at all.
|
|
240
|
+
const byBot = writer.kind === "bot";
|
|
241
|
+
const byUser = writer.kind === "user" && writer.userId === root.userId;
|
|
242
|
+
if (!byBot && !byUser) {
|
|
243
|
+
return failure(
|
|
244
|
+
"refused",
|
|
245
|
+
`Only User "${root.userId}" or one of their Bots may write this root`,
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return undefined;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
private relative(path: WorkspacePathV1): WorkspaceFailureV1 | string {
|
|
253
|
+
try {
|
|
254
|
+
return normalizeWorkspaceRelativePathV1(path.path);
|
|
255
|
+
} catch (error) {
|
|
256
|
+
return failure(
|
|
257
|
+
"refused",
|
|
258
|
+
error instanceof Error ? error.message : String(error),
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
private async recordOf(
|
|
264
|
+
root: WorkspaceRootV1,
|
|
265
|
+
path: string,
|
|
266
|
+
): Promise<WorkspaceGenerationRecordV1 | undefined> {
|
|
267
|
+
try {
|
|
268
|
+
return await this.generations.current(root, path);
|
|
269
|
+
} catch {
|
|
270
|
+
return undefined;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
private encodeMetadata(
|
|
275
|
+
generation: WorkspaceGenerationV1,
|
|
276
|
+
): Record<string, string> | undefined {
|
|
277
|
+
const encoded = JSON.stringify(generation);
|
|
278
|
+
if (encoded.length > MAX_METADATA_BYTES) return undefined;
|
|
279
|
+
return { [WORKSPACE_GENERATION_METADATA_KEY]: encoded };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
private decodeMetadata(
|
|
283
|
+
head: ObjectHeadV1,
|
|
284
|
+
): WorkspaceGenerationV1 | undefined {
|
|
285
|
+
const encoded = head.customMetadata?.[WORKSPACE_GENERATION_METADATA_KEY];
|
|
286
|
+
if (!encoded) return undefined;
|
|
287
|
+
try {
|
|
288
|
+
return decodeWorkspaceGenerationV1(JSON.parse(encoded));
|
|
289
|
+
} catch {
|
|
290
|
+
return undefined;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Records what the bytes already say, when the ledger does not say it.
|
|
296
|
+
*
|
|
297
|
+
* A tombstone marker is never reconciled: it is an absence being swept, not
|
|
298
|
+
* a file, and the deletion has its own recorded generation. Nor is a ledger
|
|
299
|
+
* record ever moved backwards — the head may be one this caller read before
|
|
300
|
+
* a concurrent write landed, and generation ids sort, so a record naming a
|
|
301
|
+
* later generation stands.
|
|
302
|
+
*/
|
|
303
|
+
private async reconcile(
|
|
304
|
+
root: WorkspaceRootV1,
|
|
305
|
+
path: string,
|
|
306
|
+
head: ObjectHeadV1,
|
|
307
|
+
generation: WorkspaceGenerationV1,
|
|
308
|
+
recorded: WorkspaceGenerationRecordV1 | undefined,
|
|
309
|
+
): Promise<void> {
|
|
310
|
+
if (isTombstoneMarkerV1(head)) return;
|
|
311
|
+
if (
|
|
312
|
+
recorded &&
|
|
313
|
+
recorded.generation.generationId >= generation.generationId
|
|
314
|
+
) {
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
try {
|
|
318
|
+
await this.generations.record({
|
|
319
|
+
schemaVersion: 1,
|
|
320
|
+
root,
|
|
321
|
+
path,
|
|
322
|
+
generation,
|
|
323
|
+
etag: head.etag,
|
|
324
|
+
});
|
|
325
|
+
} catch {
|
|
326
|
+
// The ledger is briefly unreachable. The generation still rides beside
|
|
327
|
+
// the bytes, so the next read repairs it rather than wedging the file.
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Recovers the generation of one object.
|
|
333
|
+
*
|
|
334
|
+
* The Durable Object is the authority, so its record wins whenever it still
|
|
335
|
+
* describes these bytes — that is what `etag` proves. Otherwise the
|
|
336
|
+
* generation the writer stored beside the bytes is used, which is how a file
|
|
337
|
+
* written straight into object storage by the Computer-side sync keeps its
|
|
338
|
+
* writer. A file with neither is `unattributed`: nobody recorded who wrote
|
|
339
|
+
* it, so it is data the Bot can read and never an instruction it loads.
|
|
340
|
+
*
|
|
341
|
+
* Falling back to the metadata also *repairs* the ledger. `record` runs
|
|
342
|
+
* after the `put` that produced its etag, so a `record` that fails leaves
|
|
343
|
+
* bytes whose generation exists only beside them. Without the repair the
|
|
344
|
+
* ledger would answer "no current generation" forever, and every later
|
|
345
|
+
* conditional write on that file would be treated as unseen — a file no
|
|
346
|
+
* authorized writer could ever overwrite. The repair is best-effort: if the
|
|
347
|
+
* ledger is still unreachable the generation is still returned, and the next
|
|
348
|
+
* read tries again.
|
|
349
|
+
*/
|
|
350
|
+
private async generationOf(
|
|
351
|
+
root: WorkspaceRootV1,
|
|
352
|
+
path: string,
|
|
353
|
+
head: ObjectHeadV1,
|
|
354
|
+
bytes?: Uint8Array,
|
|
355
|
+
): Promise<WorkspaceGenerationV1> {
|
|
356
|
+
const recorded = await this.recordOf(root, path);
|
|
357
|
+
if (recorded && !recorded.deleted && recorded.etag === head.etag) {
|
|
358
|
+
return recorded.generation;
|
|
359
|
+
}
|
|
360
|
+
const beside = this.decodeMetadata(head);
|
|
361
|
+
if (beside) {
|
|
362
|
+
await this.reconcile(root, path, head, beside, recorded);
|
|
363
|
+
return beside;
|
|
364
|
+
}
|
|
365
|
+
const body = bytes ?? (await (await this.bucket.get(head.key))?.bytes());
|
|
366
|
+
return {
|
|
367
|
+
schemaVersion: 1,
|
|
368
|
+
generationId: `${head.uploaded.getTime().toString().padStart(15, "0")}-object`,
|
|
369
|
+
contentHash: body ? await digestV1(body) : WORKSPACE_EMPTY_SHA256,
|
|
370
|
+
size: Math.min(head.size, WORKSPACE_MAX_FILE_BYTES),
|
|
371
|
+
writer: { kind: "unattributed" },
|
|
372
|
+
writtenAt: head.uploaded.toISOString(),
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
async read(path: WorkspacePathV1): Promise<WorkspaceReadOutcomeV1> {
|
|
377
|
+
const refused = this.admit(path.root);
|
|
378
|
+
if (refused) return refused;
|
|
379
|
+
const relative = this.relative(path);
|
|
380
|
+
if (typeof relative !== "string") return relative;
|
|
381
|
+
const key = workspaceObjectKeyV1(path.root, relative);
|
|
382
|
+
let object;
|
|
383
|
+
try {
|
|
384
|
+
object = await this.bucket.get(key);
|
|
385
|
+
} catch (error) {
|
|
386
|
+
return failure(
|
|
387
|
+
"unavailable",
|
|
388
|
+
error instanceof Error ? error.message : String(error),
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
if (!object || isTombstoneMarkerV1(object)) {
|
|
392
|
+
return failure("not-found", `No such Workspace file: ${relative}`);
|
|
393
|
+
}
|
|
394
|
+
if (object.size > WORKSPACE_MAX_FILE_BYTES) {
|
|
395
|
+
return failure(
|
|
396
|
+
"refused",
|
|
397
|
+
`Workspace file exceeds ${WORKSPACE_MAX_FILE_BYTES} bytes`,
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
const bytes = await object.bytes();
|
|
401
|
+
return {
|
|
402
|
+
status: "ok",
|
|
403
|
+
file: {
|
|
404
|
+
path: { root: path.root, path: relative },
|
|
405
|
+
generation: await this.generationOf(path.root, relative, object, bytes),
|
|
406
|
+
bytes,
|
|
407
|
+
},
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
async stat(path: WorkspacePathV1): Promise<WorkspaceStatOutcomeV1> {
|
|
412
|
+
const refused = this.admit(path.root);
|
|
413
|
+
if (refused) return refused;
|
|
414
|
+
const relative = this.relative(path);
|
|
415
|
+
if (typeof relative !== "string") return relative;
|
|
416
|
+
let head;
|
|
417
|
+
try {
|
|
418
|
+
head = await this.bucket.head(workspaceObjectKeyV1(path.root, relative));
|
|
419
|
+
} catch (error) {
|
|
420
|
+
return failure(
|
|
421
|
+
"unavailable",
|
|
422
|
+
error instanceof Error ? error.message : String(error),
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
if (!head || isTombstoneMarkerV1(head)) {
|
|
426
|
+
return failure("not-found", `No such Workspace file: ${relative}`);
|
|
427
|
+
}
|
|
428
|
+
return {
|
|
429
|
+
status: "ok",
|
|
430
|
+
entry: {
|
|
431
|
+
path: { root: path.root, path: relative },
|
|
432
|
+
generation: await this.generationOf(path.root, relative, head),
|
|
433
|
+
},
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Lists one durable root.
|
|
439
|
+
*
|
|
440
|
+
* A shared Memory root is sharded per writing Bot, so a listing with no
|
|
441
|
+
* prefix returns every Bot's shard merged into one page — "readers merge
|
|
442
|
+
* shards" — while a prefix naming one shard returns that shard alone.
|
|
443
|
+
* Preserved losing writes are never listed as files; they are read through
|
|
444
|
+
* the generation ledger, which is where a conflict is surfaced.
|
|
445
|
+
*/
|
|
446
|
+
async list(request: WorkspaceListRequestV1): Promise<WorkspaceListOutcomeV1> {
|
|
447
|
+
const refused = this.admit(request.root);
|
|
448
|
+
if (refused) return refused;
|
|
449
|
+
let prefix = "";
|
|
450
|
+
if (request.prefix !== undefined) {
|
|
451
|
+
const normalized = this.relative({
|
|
452
|
+
root: request.root,
|
|
453
|
+
path: request.prefix,
|
|
454
|
+
});
|
|
455
|
+
if (typeof normalized !== "string") return normalized;
|
|
456
|
+
prefix = normalized;
|
|
457
|
+
}
|
|
458
|
+
const limit = Math.max(
|
|
459
|
+
1,
|
|
460
|
+
Math.min(request.limit ?? DEFAULT_LIST_LIMIT, WORKSPACE_MAX_LIST_ENTRIES),
|
|
461
|
+
);
|
|
462
|
+
let page;
|
|
463
|
+
try {
|
|
464
|
+
page = await this.bucket.list({
|
|
465
|
+
prefix: `${workspaceObjectPrefixV1(request.root)}${prefix}`,
|
|
466
|
+
limit,
|
|
467
|
+
...(request.cursor ? { cursor: request.cursor } : {}),
|
|
468
|
+
});
|
|
469
|
+
} catch (error) {
|
|
470
|
+
return failure(
|
|
471
|
+
"unavailable",
|
|
472
|
+
error instanceof Error ? error.message : String(error),
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
const entries: WorkspaceEntryV1[] = [];
|
|
476
|
+
for (const object of page.objects) {
|
|
477
|
+
if (isWorkspaceConflictKeyV1(object.key)) continue;
|
|
478
|
+
// A tombstone marker is a delete mid-sweep, not a file.
|
|
479
|
+
if (isTombstoneMarkerV1(object)) continue;
|
|
480
|
+
const relative = workspaceRelativeFromKeyV1(request.root, object.key);
|
|
481
|
+
if (relative === undefined) continue;
|
|
482
|
+
// A raw key prefix would also match a sibling whose name merely starts
|
|
483
|
+
// with it — `by-agent/bot-1` must not list `by-agent/bot-10/`.
|
|
484
|
+
if (prefix && relative !== prefix && !relative.startsWith(`${prefix}/`)) {
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
let path: WorkspacePathV1;
|
|
488
|
+
try {
|
|
489
|
+
path = {
|
|
490
|
+
root: request.root,
|
|
491
|
+
path: normalizeWorkspaceRelativePathV1(relative),
|
|
492
|
+
};
|
|
493
|
+
} catch {
|
|
494
|
+
continue;
|
|
495
|
+
}
|
|
496
|
+
entries.push({
|
|
497
|
+
path,
|
|
498
|
+
generation: await this.generationOf(request.root, relative, object),
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
return {
|
|
502
|
+
status: "ok",
|
|
503
|
+
entries,
|
|
504
|
+
...(page.truncated && page.cursor ? { cursor: page.cursor } : {}),
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* The conditional one write is sent under, or `undefined` when the writer
|
|
510
|
+
* has not seen what the store holds and must therefore lose.
|
|
511
|
+
*
|
|
512
|
+
* A tombstone marker is an absence being swept: a writer asserting absence
|
|
513
|
+
* conditions on the marker's own ETag rather than on `If-None-Match`, so a
|
|
514
|
+
* delete that could not sweep its marker never blocks the next create.
|
|
515
|
+
*/
|
|
516
|
+
private async precondition(
|
|
517
|
+
root: WorkspaceRootV1,
|
|
518
|
+
relative: string,
|
|
519
|
+
head: ObjectHeadV1 | null,
|
|
520
|
+
expectedGenerationId: string | null,
|
|
521
|
+
): Promise<ObjectConditionsV1 | undefined> {
|
|
522
|
+
if (!head) {
|
|
523
|
+
return expectedGenerationId === null
|
|
524
|
+
? { etagDoesNotMatch: "*" }
|
|
525
|
+
: undefined;
|
|
526
|
+
}
|
|
527
|
+
if (isTombstoneMarkerV1(head)) {
|
|
528
|
+
const removed = this.decodeMetadata(head);
|
|
529
|
+
return expectedGenerationId === null ||
|
|
530
|
+
expectedGenerationId === removed?.generationId
|
|
531
|
+
? { etagMatches: head.etag }
|
|
532
|
+
: undefined;
|
|
533
|
+
}
|
|
534
|
+
const holder = await this.generationOf(root, relative, head);
|
|
535
|
+
return holder.generationId === expectedGenerationId
|
|
536
|
+
? { etagMatches: head.etag }
|
|
537
|
+
: undefined;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
async write(
|
|
541
|
+
request: WorkspaceWriteRequestV1,
|
|
542
|
+
): Promise<WorkspaceWriteOutcomeV1> {
|
|
543
|
+
const refused = this.admitWrite(request.path, request.writer);
|
|
544
|
+
if (refused) return refused;
|
|
545
|
+
const relative = this.relative(request.path);
|
|
546
|
+
if (typeof relative !== "string") return relative;
|
|
547
|
+
if (request.bytes.byteLength > WORKSPACE_MAX_FILE_BYTES) {
|
|
548
|
+
return failure(
|
|
549
|
+
"refused",
|
|
550
|
+
`Workspace file exceeds ${WORKSPACE_MAX_FILE_BYTES} bytes`,
|
|
551
|
+
);
|
|
552
|
+
}
|
|
553
|
+
const root = request.path.root;
|
|
554
|
+
const at = this.clock();
|
|
555
|
+
const key = workspaceObjectKeyV1(root, relative);
|
|
556
|
+
const current = await this.recordOf(root, relative);
|
|
557
|
+
let generation: WorkspaceGenerationV1;
|
|
558
|
+
try {
|
|
559
|
+
generation = {
|
|
560
|
+
schemaVersion: 1,
|
|
561
|
+
generationId: await this.generations.mint(at, root),
|
|
562
|
+
contentHash: await digestV1(request.bytes),
|
|
563
|
+
size: request.bytes.byteLength,
|
|
564
|
+
writer: request.writer,
|
|
565
|
+
writtenAt: at.toISOString(),
|
|
566
|
+
};
|
|
567
|
+
} catch (error) {
|
|
568
|
+
return failure(
|
|
569
|
+
"unavailable",
|
|
570
|
+
error instanceof Error ? error.message : String(error),
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
// The caller's `expectedGenerationId` is mapped to the ETag that
|
|
574
|
+
// generation produced. `null` asserts absence, which is `If-None-Match: *`.
|
|
575
|
+
//
|
|
576
|
+
// The mapping is derived from what object storage actually holds, not from
|
|
577
|
+
// the ledger alone. `generationOf` prefers the Durable Object's record
|
|
578
|
+
// whenever that record still describes these bytes, and falls back to the
|
|
579
|
+
// generation stored beside them — so a writer that passes exactly the
|
|
580
|
+
// generation `read` or `stat` handed it wins, even when the ledger has no
|
|
581
|
+
// record at all because a `record` failed after its `put` or the object
|
|
582
|
+
// was mirrored with metadata only. Without that, an unrecorded file could
|
|
583
|
+
// never be overwritten by anyone: the writer would be judged unseen, and
|
|
584
|
+
// `null` would fail `If-None-Match`.
|
|
585
|
+
let head: ObjectHeadV1 | null;
|
|
586
|
+
try {
|
|
587
|
+
head = await this.bucket.head(key);
|
|
588
|
+
} catch (error) {
|
|
589
|
+
return failure(
|
|
590
|
+
"unavailable",
|
|
591
|
+
error instanceof Error ? error.message : String(error),
|
|
592
|
+
);
|
|
593
|
+
}
|
|
594
|
+
const seen = await this.precondition(
|
|
595
|
+
root,
|
|
596
|
+
relative,
|
|
597
|
+
head,
|
|
598
|
+
request.expectedGenerationId,
|
|
599
|
+
);
|
|
600
|
+
if (seen) {
|
|
601
|
+
const metadata = this.encodeMetadata(generation);
|
|
602
|
+
let written: ObjectHeadV1 | null;
|
|
603
|
+
try {
|
|
604
|
+
written = await this.bucket.put(key, request.bytes, {
|
|
605
|
+
onlyIf: seen,
|
|
606
|
+
contentType: request.mediaType ?? DEFAULT_MEDIA_TYPE,
|
|
607
|
+
...(metadata ? { customMetadata: metadata } : {}),
|
|
608
|
+
});
|
|
609
|
+
} catch (error) {
|
|
610
|
+
return failure(
|
|
611
|
+
"unavailable",
|
|
612
|
+
error instanceof Error ? error.message : String(error),
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
if (written) {
|
|
616
|
+
try {
|
|
617
|
+
await this.generations.record({
|
|
618
|
+
schemaVersion: 1,
|
|
619
|
+
root,
|
|
620
|
+
path: relative,
|
|
621
|
+
generation,
|
|
622
|
+
etag: written.etag,
|
|
623
|
+
});
|
|
624
|
+
} catch (error) {
|
|
625
|
+
return failure(
|
|
626
|
+
"unavailable",
|
|
627
|
+
error instanceof Error ? error.message : String(error),
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
return { status: "ok", generation };
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
return this.preserve(root, relative, request.bytes, generation, current);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Preserves a losing write. ADR 0013: the loser is stored under its own
|
|
638
|
+
* conflict key and recorded as a conflicting generation, so both sides
|
|
639
|
+
* survive and the caller is handed both — never merged, never dropped.
|
|
640
|
+
*/
|
|
641
|
+
private async preserve(
|
|
642
|
+
root: WorkspaceRootV1,
|
|
643
|
+
relative: string,
|
|
644
|
+
bytes: Uint8Array,
|
|
645
|
+
generation: WorkspaceGenerationV1,
|
|
646
|
+
current: WorkspaceGenerationRecordV1 | undefined,
|
|
647
|
+
): Promise<WorkspaceWriteOutcomeV1> {
|
|
648
|
+
let head: ObjectHeadV1 | null = null;
|
|
649
|
+
try {
|
|
650
|
+
head = await this.bucket.head(workspaceObjectKeyV1(root, relative));
|
|
651
|
+
} catch {
|
|
652
|
+
head = null;
|
|
653
|
+
}
|
|
654
|
+
const holder = head
|
|
655
|
+
? await this.generationOf(root, relative, head)
|
|
656
|
+
: current && !current.deleted
|
|
657
|
+
? current.generation
|
|
658
|
+
: undefined;
|
|
659
|
+
const preserved: WorkspaceGenerationV1 = {
|
|
660
|
+
...generation,
|
|
661
|
+
...(holder ? { conflictsWith: holder.generationId } : {}),
|
|
662
|
+
};
|
|
663
|
+
const conflictKey = workspaceConflictKeyV1(
|
|
664
|
+
root,
|
|
665
|
+
relative,
|
|
666
|
+
preserved.generationId,
|
|
667
|
+
);
|
|
668
|
+
try {
|
|
669
|
+
const metadata = this.encodeMetadata(preserved);
|
|
670
|
+
const stored = await this.bucket.put(conflictKey, bytes, {
|
|
671
|
+
contentType: DEFAULT_MEDIA_TYPE,
|
|
672
|
+
...(metadata ? { customMetadata: metadata } : {}),
|
|
673
|
+
});
|
|
674
|
+
await this.generations.conflict({
|
|
675
|
+
schemaVersion: 1,
|
|
676
|
+
root,
|
|
677
|
+
path: relative,
|
|
678
|
+
generation: preserved,
|
|
679
|
+
conflictKey,
|
|
680
|
+
...(stored ? { etag: stored.etag } : {}),
|
|
681
|
+
});
|
|
682
|
+
} catch (error) {
|
|
683
|
+
return failure(
|
|
684
|
+
"unavailable",
|
|
685
|
+
error instanceof Error ? error.message : String(error),
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
return {
|
|
689
|
+
status: "conflict",
|
|
690
|
+
reason: `Workspace file changed since the writer last saw it: ${relative}`,
|
|
691
|
+
...(holder ? { current: holder } : {}),
|
|
692
|
+
preserved,
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Deletes a file, fenced by a conditional overwrite.
|
|
698
|
+
*
|
|
699
|
+
* Object storage has no conditional delete, so `head` then `delete` would
|
|
700
|
+
* destroy a write that landed in between — last-writer-wins, which ADR 0013
|
|
701
|
+
* names as the one outcome that is prohibited. The delete therefore *writes*
|
|
702
|
+
* first: an empty object carrying `frockbot-tombstone` and the tombstone
|
|
703
|
+
* generation replaces the file under `If-Match` on the ETag the deleter saw.
|
|
704
|
+
* That put is the fence. A racing write either won before it — in which case
|
|
705
|
+
* the `If-Match` fails and the deletion is preserved as a conflicting
|
|
706
|
+
* generation, so both generations survive and the caller is handed both — or
|
|
707
|
+
* it arrives after, and then its own `If-Match` on the file's old ETag fails
|
|
708
|
+
* and it is preserved instead.
|
|
709
|
+
*
|
|
710
|
+
* The marker is *not* swept. An unconditional `delete` after the fence would
|
|
711
|
+
* reopen the race from the other side: a create that read the marker and
|
|
712
|
+
* conditioned on its ETag would land between the fence and the sweep, and
|
|
713
|
+
* the sweep would erase bytes that had already won their precondition —
|
|
714
|
+
* last-writer-wins, from the deleter this time. So the marker stays as the
|
|
715
|
+
* object: `read`, `stat`, and `list` treat it as absence, the next create
|
|
716
|
+
* replaces it under `If-Match` on the marker's ETag, and
|
|
717
|
+
* `gcTombstoneMarkersV1` collects markers only once they are old enough that
|
|
718
|
+
* no create can still be racing one.
|
|
719
|
+
*/
|
|
720
|
+
async delete(
|
|
721
|
+
request: WorkspaceDeleteRequestV1,
|
|
722
|
+
): Promise<WorkspaceWriteOutcomeV1> {
|
|
723
|
+
const refused = this.admitWrite(request.path, request.writer);
|
|
724
|
+
if (refused) return refused;
|
|
725
|
+
const relative = this.relative(request.path);
|
|
726
|
+
if (typeof relative !== "string") return relative;
|
|
727
|
+
const root = request.path.root;
|
|
728
|
+
const key = workspaceObjectKeyV1(root, relative);
|
|
729
|
+
let head: ObjectHeadV1 | null;
|
|
730
|
+
try {
|
|
731
|
+
head = await this.bucket.head(key);
|
|
732
|
+
} catch (error) {
|
|
733
|
+
return failure(
|
|
734
|
+
"unavailable",
|
|
735
|
+
error instanceof Error ? error.message : String(error),
|
|
736
|
+
);
|
|
737
|
+
}
|
|
738
|
+
if (!head || isTombstoneMarkerV1(head)) {
|
|
739
|
+
return failure("not-found", `No such Workspace file: ${relative}`);
|
|
740
|
+
}
|
|
741
|
+
const holder = await this.generationOf(root, relative, head);
|
|
742
|
+
if (holder.generationId !== request.expectedGenerationId) {
|
|
743
|
+
return {
|
|
744
|
+
status: "conflict",
|
|
745
|
+
reason: `Workspace file changed since the writer last saw it: ${relative}`,
|
|
746
|
+
current: holder,
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
const at = this.clock();
|
|
750
|
+
let tombstone: WorkspaceGenerationV1;
|
|
751
|
+
try {
|
|
752
|
+
tombstone = {
|
|
753
|
+
schemaVersion: 1,
|
|
754
|
+
generationId: await this.generations.mint(at, root),
|
|
755
|
+
contentHash: WORKSPACE_EMPTY_SHA256,
|
|
756
|
+
size: 0,
|
|
757
|
+
writer: request.writer,
|
|
758
|
+
writtenAt: at.toISOString(),
|
|
759
|
+
};
|
|
760
|
+
} catch (error) {
|
|
761
|
+
return failure(
|
|
762
|
+
"unavailable",
|
|
763
|
+
error instanceof Error ? error.message : String(error),
|
|
764
|
+
);
|
|
765
|
+
}
|
|
766
|
+
const empty = new Uint8Array(0);
|
|
767
|
+
let fenced: ObjectHeadV1 | null;
|
|
768
|
+
try {
|
|
769
|
+
fenced = await this.bucket.put(key, empty, {
|
|
770
|
+
onlyIf: { etagMatches: head.etag },
|
|
771
|
+
contentType: DEFAULT_MEDIA_TYPE,
|
|
772
|
+
customMetadata: {
|
|
773
|
+
...(this.encodeMetadata(tombstone) ?? {}),
|
|
774
|
+
[WORKSPACE_TOMBSTONE_METADATA_KEY]: "1",
|
|
775
|
+
},
|
|
776
|
+
});
|
|
777
|
+
} catch (error) {
|
|
778
|
+
return failure(
|
|
779
|
+
"unavailable",
|
|
780
|
+
error instanceof Error ? error.message : String(error),
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
if (!fenced) {
|
|
784
|
+
// A write landed between the head and the fence. It holds the file; the
|
|
785
|
+
// deletion is the loser, preserved as a conflicting generation like any
|
|
786
|
+
// other losing write, with both generations returned to the caller.
|
|
787
|
+
return this.preserve(
|
|
788
|
+
root,
|
|
789
|
+
relative,
|
|
790
|
+
empty,
|
|
791
|
+
tombstone,
|
|
792
|
+
await this.recordOf(root, relative),
|
|
793
|
+
);
|
|
794
|
+
}
|
|
795
|
+
try {
|
|
796
|
+
// The ledger tombstone is the durable evidence that the file was
|
|
797
|
+
// removed, by whom, and when: the marker is an absence in object
|
|
798
|
+
// storage, and object storage forgets a key entirely once the marker is
|
|
799
|
+
// collected. Its ETag is recorded too, so the record still describes
|
|
800
|
+
// exactly the object that stands in the file's place.
|
|
801
|
+
await this.generations.tombstone({
|
|
802
|
+
schemaVersion: 1,
|
|
803
|
+
root,
|
|
804
|
+
path: relative,
|
|
805
|
+
generation: tombstone,
|
|
806
|
+
etag: fenced.etag,
|
|
807
|
+
deleted: true,
|
|
808
|
+
});
|
|
809
|
+
return { status: "ok", generation: tombstone };
|
|
810
|
+
} catch (error) {
|
|
811
|
+
return failure(
|
|
812
|
+
"unavailable",
|
|
813
|
+
error instanceof Error ? error.message : String(error),
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
/**
|
|
820
|
+
* `WorkspaceFilesV1` over object storage. The kernel consumes Memory roots
|
|
821
|
+
* through `workspaceMemoryProjectionV1` of the result, which has no `write`
|
|
822
|
+
* and no `delete` to call.
|
|
823
|
+
*/
|
|
824
|
+
export function createObjectWorkspaceFilesV1(
|
|
825
|
+
options: ObjectWorkspaceFilesOptionsV1,
|
|
826
|
+
): WorkspaceFilesV1 {
|
|
827
|
+
return new ObjectWorkspaceFiles(options);
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
export interface GcTombstoneMarkersOptionsV1 {
|
|
831
|
+
bucket: ObjectBucketV1;
|
|
832
|
+
/**
|
|
833
|
+
* Markers uploaded at or after this instant are left alone. It is the
|
|
834
|
+
* declared age at which a marker is assumed to be racing nobody — longer
|
|
835
|
+
* than any single create can take between reading a marker and writing over
|
|
836
|
+
* it.
|
|
837
|
+
*/
|
|
838
|
+
olderThan: Date;
|
|
839
|
+
/** Object keys to sweep. Defaults to every durable root. */
|
|
840
|
+
prefix?: string;
|
|
841
|
+
/** Most objects examined in one run. */
|
|
842
|
+
limit?: number;
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/** What one collection run looked at, and what it removed. */
|
|
846
|
+
export interface GcTombstoneMarkersReportV1 {
|
|
847
|
+
scanned: number;
|
|
848
|
+
collected: number;
|
|
849
|
+
/** Markers left alone: too young, or no longer a marker when re-read. */
|
|
850
|
+
skipped: number;
|
|
851
|
+
/** True when the scan bound stopped the run before the listing ended. */
|
|
852
|
+
truncated: boolean;
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
/** Most objects one collection run examines when the caller names no bound. */
|
|
856
|
+
const DEFAULT_GC_SCAN_LIMIT = 1000;
|
|
857
|
+
const GC_PAGE_LIMIT = 200;
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* Collects tombstone markers old enough that no create can still be racing
|
|
861
|
+
* one.
|
|
862
|
+
*
|
|
863
|
+
* A delete leaves its marker in place (see `delete`), because sweeping it
|
|
864
|
+
* would erase a create that had already won its `If-Match` against the
|
|
865
|
+
* marker's ETag. Markers are therefore collected out of band, and only under
|
|
866
|
+
* two conditions checked immediately before the removal: `head` still answers
|
|
867
|
+
* with the tombstone metadata — so an object that has since become a real file
|
|
868
|
+
* is never touched — and its `uploaded` is older than the declared threshold.
|
|
869
|
+
*
|
|
870
|
+
* The residual: object storage has no conditional delete, so a create that
|
|
871
|
+
* lands in the round trip between that `head` and the `delete` is still lost.
|
|
872
|
+
* `olderThan` is what shrinks it to nothing in practice — a create racing a
|
|
873
|
+
* marker that has stood untouched for the threshold is not a race anyone is
|
|
874
|
+
* running — and either way a create that arrives *after* the collection is at
|
|
875
|
+
* worst refused, its `If-Match` on the now-absent marker failing, which the
|
|
876
|
+
* store preserves as a conflicting generation rather than dropping.
|
|
877
|
+
*
|
|
878
|
+
* It is not a `WorkspaceFilesV1` operation and mints no generation: the
|
|
879
|
+
* removal it performs is of an absence, and the deletion's own generation was
|
|
880
|
+
* recorded in the Durable Object when the marker was written.
|
|
881
|
+
*/
|
|
882
|
+
export async function gcTombstoneMarkersV1(
|
|
883
|
+
options: GcTombstoneMarkersOptionsV1,
|
|
884
|
+
): Promise<GcTombstoneMarkersReportV1> {
|
|
885
|
+
const bucket = options.bucket;
|
|
886
|
+
const threshold = options.olderThan.getTime();
|
|
887
|
+
const scanLimit = Math.max(1, options.limit ?? DEFAULT_GC_SCAN_LIMIT);
|
|
888
|
+
const report: GcTombstoneMarkersReportV1 = {
|
|
889
|
+
scanned: 0,
|
|
890
|
+
collected: 0,
|
|
891
|
+
skipped: 0,
|
|
892
|
+
truncated: false,
|
|
893
|
+
};
|
|
894
|
+
let cursor: string | undefined;
|
|
895
|
+
for (;;) {
|
|
896
|
+
const page = await bucket.list({
|
|
897
|
+
prefix: options.prefix ?? `${WORKSPACE_OBJECT_PREFIX}/`,
|
|
898
|
+
limit: Math.min(GC_PAGE_LIMIT, scanLimit - report.scanned),
|
|
899
|
+
...(cursor ? { cursor } : {}),
|
|
900
|
+
});
|
|
901
|
+
for (const object of page.objects) {
|
|
902
|
+
report.scanned += 1;
|
|
903
|
+
// Only a marker is a candidate; a file, and a preserved losing write,
|
|
904
|
+
// are data this must never touch.
|
|
905
|
+
if (!isTombstoneMarkerV1(object)) continue;
|
|
906
|
+
if (object.uploaded.getTime() >= threshold) {
|
|
907
|
+
report.skipped += 1;
|
|
908
|
+
continue;
|
|
909
|
+
}
|
|
910
|
+
const current = await bucket.head(object.key);
|
|
911
|
+
if (
|
|
912
|
+
!current ||
|
|
913
|
+
!isTombstoneMarkerV1(current) ||
|
|
914
|
+
current.uploaded.getTime() >= threshold
|
|
915
|
+
) {
|
|
916
|
+
report.skipped += 1;
|
|
917
|
+
continue;
|
|
918
|
+
}
|
|
919
|
+
await bucket.delete(object.key);
|
|
920
|
+
report.collected += 1;
|
|
921
|
+
}
|
|
922
|
+
if (!page.truncated || !page.cursor) return report;
|
|
923
|
+
if (report.scanned >= scanLimit) {
|
|
924
|
+
report.truncated = true;
|
|
925
|
+
return report;
|
|
926
|
+
}
|
|
927
|
+
cursor = page.cursor;
|
|
928
|
+
}
|
|
929
|
+
}
|