@cloudflare/workers-response-store 0.1.0-beta.0 → 0.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +238 -46
- package/dist/binding.d.ts +20 -0
- package/dist/binding.js +429 -71
- package/dist/index.d.ts +2 -0
- package/dist/index.js +6 -3
- package/dist/isolate-negative-cache.d.ts +11 -0
- package/dist/isolate-negative-cache.js +34 -0
- package/dist/metadata-do.d.ts +39 -6
- package/dist/metadata-do.js +245 -93
- package/package.json +1 -1
package/dist/metadata-do.js
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import { DurableObject } from "cloudflare:workers";
|
|
2
2
|
//#region src/metadata-do.ts
|
|
3
3
|
const MAX_SQL_PARAMETERS = 100;
|
|
4
|
-
const R2_DELETE_BATCH_SIZE = 1e3;
|
|
5
4
|
const ORPHAN_RETENTION_MS = 36e5;
|
|
6
5
|
const ORPHAN_CLEANUP_LIMIT = 100;
|
|
7
6
|
const ORPHAN_CLEANUP_RETRY_MS = 6e4;
|
|
7
|
+
const MAX_R2_CAS_ATTEMPTS = 3;
|
|
8
|
+
function metadataInteger(value) {
|
|
9
|
+
if (value === void 0 || !/^\d+$/.test(value)) return void 0;
|
|
10
|
+
const parsed = Number(value);
|
|
11
|
+
return Number.isSafeInteger(parsed) ? parsed : void 0;
|
|
12
|
+
}
|
|
8
13
|
function normalizeTags(tags) {
|
|
9
14
|
const normalized = tags.map((tag) => tag.trim().toLowerCase()).filter(Boolean);
|
|
10
15
|
return [...new Set(normalized)];
|
|
@@ -61,6 +66,8 @@ var CacheMetadata = class extends DurableObject {
|
|
|
61
66
|
tombstoned INTEGER NOT NULL DEFAULT 0
|
|
62
67
|
);
|
|
63
68
|
CREATE INDEX IF NOT EXISTS entries_cache_key ON entries(cache_key);
|
|
69
|
+
CREATE INDEX IF NOT EXISTS entries_active_object_key
|
|
70
|
+
ON entries(object_key) WHERE tombstoned = 0;
|
|
64
71
|
CREATE TABLE IF NOT EXISTS revalidation_claims (
|
|
65
72
|
key_hash TEXT PRIMARY KEY,
|
|
66
73
|
active_revision INTEGER NOT NULL,
|
|
@@ -82,6 +89,10 @@ var CacheMetadata = class extends DurableObject {
|
|
|
82
89
|
tag_invalidation_sequence INTEGER NOT NULL
|
|
83
90
|
);
|
|
84
91
|
INSERT OR IGNORE INTO metadata_state (singleton, tag_invalidation_sequence) VALUES (1, 0);
|
|
92
|
+
CREATE TABLE IF NOT EXISTS key_invalidations (
|
|
93
|
+
key_hash TEXT PRIMARY KEY,
|
|
94
|
+
invalidation_sequence INTEGER NOT NULL
|
|
95
|
+
) WITHOUT ROWID;
|
|
85
96
|
CREATE TABLE IF NOT EXISTS pending_objects (
|
|
86
97
|
object_key TEXT PRIMARY KEY,
|
|
87
98
|
invalidation_sequence INTEGER NOT NULL DEFAULT 0,
|
|
@@ -89,12 +100,21 @@ var CacheMetadata = class extends DurableObject {
|
|
|
89
100
|
created_at INTEGER NOT NULL
|
|
90
101
|
);
|
|
91
102
|
CREATE INDEX IF NOT EXISTS pending_objects_created_at ON pending_objects(created_at);
|
|
103
|
+
CREATE TABLE IF NOT EXISTS pending_r2_tombstones (
|
|
104
|
+
key_hash TEXT PRIMARY KEY,
|
|
105
|
+
cache_key TEXT NOT NULL,
|
|
106
|
+
object_key TEXT NOT NULL,
|
|
107
|
+
revision INTEGER NOT NULL,
|
|
108
|
+
r2_complete INTEGER NOT NULL DEFAULT 0,
|
|
109
|
+
edge_purge_complete INTEGER NOT NULL DEFAULT 0
|
|
110
|
+
) WITHOUT ROWID;
|
|
92
111
|
`);
|
|
93
112
|
ctx.storage.transactionSync(() => {
|
|
94
|
-
const migrations = new Set(ctx.storage.sql.exec("SELECT version FROM metadata_schema_migrations WHERE version IN (2, 3)").toArray().map(({ version }) => version));
|
|
95
|
-
if (migrations.size ===
|
|
113
|
+
const migrations = new Set(ctx.storage.sql.exec("SELECT version FROM metadata_schema_migrations WHERE version IN (2, 3, 4)").toArray().map(({ version }) => version));
|
|
114
|
+
if (migrations.size === 3) return;
|
|
96
115
|
const schemas = ctx.storage.sql.exec(`SELECT name, sql FROM sqlite_schema
|
|
97
|
-
WHERE type = 'table'
|
|
116
|
+
WHERE type = 'table'
|
|
117
|
+
AND name IN ('tag_invalidations', 'pending_objects', 'pending_r2_tombstones')`).toArray();
|
|
98
118
|
if (!migrations.has(2) && !schemas.find(({ name }) => name === "tag_invalidations")?.sql.includes("invalidation_sequence")) ctx.storage.sql.exec("ALTER TABLE tag_invalidations ADD COLUMN invalidation_sequence INTEGER NOT NULL DEFAULT 0");
|
|
99
119
|
if (!migrations.has(2) && !schemas.find(({ name }) => name === "pending_objects")?.sql.includes("invalidation_sequence")) ctx.storage.sql.exec("ALTER TABLE pending_objects ADD COLUMN invalidation_sequence INTEGER NOT NULL DEFAULT 0");
|
|
100
120
|
if (!migrations.has(2)) ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (2)");
|
|
@@ -102,26 +122,37 @@ var CacheMetadata = class extends DurableObject {
|
|
|
102
122
|
if (!schemas.find(({ name }) => name === "pending_objects")?.sql.includes("publishable")) ctx.storage.sql.exec("ALTER TABLE pending_objects ADD COLUMN publishable INTEGER NOT NULL DEFAULT 1");
|
|
103
123
|
ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (3)");
|
|
104
124
|
}
|
|
125
|
+
if (!migrations.has(4)) {
|
|
126
|
+
const tombstones = schemas.find(({ name }) => name === "pending_r2_tombstones")?.sql;
|
|
127
|
+
if (!tombstones?.includes("r2_complete")) ctx.storage.sql.exec("ALTER TABLE pending_r2_tombstones ADD COLUMN r2_complete INTEGER NOT NULL DEFAULT 0");
|
|
128
|
+
if (!tombstones?.includes("edge_purge_complete")) ctx.storage.sql.exec("ALTER TABLE pending_r2_tombstones ADD COLUMN edge_purge_complete INTEGER NOT NULL DEFAULT 0");
|
|
129
|
+
ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (4)");
|
|
130
|
+
}
|
|
105
131
|
});
|
|
106
132
|
});
|
|
107
133
|
}
|
|
108
134
|
async ensureCleanupAlarm(createdAt) {
|
|
109
135
|
if (this.cleanupAlarmKnown) return;
|
|
110
|
-
|
|
136
|
+
await this.scheduleCleanupAlarm(createdAt + ORPHAN_RETENTION_MS);
|
|
137
|
+
}
|
|
138
|
+
async scheduleCleanupAlarm(scheduledTime) {
|
|
139
|
+
const current = await this.ctx.storage.getAlarm();
|
|
140
|
+
if (current === null || current > scheduledTime) await this.ctx.storage.setAlarm(scheduledTime);
|
|
111
141
|
this.cleanupAlarmKnown = true;
|
|
112
142
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
143
|
+
logCleanupFailure(error) {
|
|
144
|
+
console.error(JSON.stringify({
|
|
145
|
+
message: "Workers Response Store cleanup scheduling failed",
|
|
146
|
+
error: error instanceof Error ? error.message : String(error)
|
|
147
|
+
}));
|
|
148
|
+
}
|
|
149
|
+
getR2TombstoneEdgePurger() {
|
|
150
|
+
const factory = Reflect.get(this.ctx.exports, "ResponseStoreBinding");
|
|
151
|
+
if (typeof factory !== "function") throw new Error("The ResponseStoreBinding entrypoint is not exported");
|
|
152
|
+
return factory({ props: {} });
|
|
120
153
|
}
|
|
121
154
|
findMatchingEntryRows(options, includePending = false) {
|
|
122
|
-
const rows = this.ctx.storage.sql.exec(includePending ?
|
|
123
|
-
WHERE (tombstoned = 0 AND active_revision IS NOT NULL)
|
|
124
|
-
OR active_revision IS NULL OR latest_revision > active_revision` : "SELECT * FROM entries WHERE tombstoned = 0 AND active_revision IS NOT NULL").toArray();
|
|
155
|
+
const rows = this.ctx.storage.sql.exec(includePending ? "SELECT * FROM entries" : "SELECT * FROM entries WHERE tombstoned = 0 AND active_revision IS NOT NULL").toArray();
|
|
125
156
|
if (options.purgeEverything) return rows;
|
|
126
157
|
const tags = new Set(normalizeTags(options.tags ?? []));
|
|
127
158
|
const prefixes = options.pathPrefixes ?? [];
|
|
@@ -143,7 +174,7 @@ var CacheMetadata = class extends DurableObject {
|
|
|
143
174
|
}
|
|
144
175
|
releaseWrite(keyHash, objectKey, claimId) {
|
|
145
176
|
this.ctx.storage.transactionSync(() => {
|
|
146
|
-
this.ctx.storage.sql.exec("
|
|
177
|
+
this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", objectKey);
|
|
147
178
|
if (claimId) this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ? AND claim_id = ?", keyHash, claimId);
|
|
148
179
|
});
|
|
149
180
|
}
|
|
@@ -153,29 +184,6 @@ var CacheMetadata = class extends DurableObject {
|
|
|
153
184
|
for (const batch of batches(objectKeys, MAX_SQL_PARAMETERS)) this.ctx.storage.sql.exec(`DELETE FROM pending_objects WHERE object_key IN (${batch.map(() => "?").join(", ")})`, ...batch);
|
|
154
185
|
});
|
|
155
186
|
}
|
|
156
|
-
async deleteTrackedObjects(objectKeys) {
|
|
157
|
-
for (const batch of batches(objectKeys, R2_DELETE_BATCH_SIZE)) try {
|
|
158
|
-
await this.env.CACHE_BODIES.delete(batch);
|
|
159
|
-
this.finishPendingObjects(batch);
|
|
160
|
-
} catch (error) {
|
|
161
|
-
for (const retryBatch of batches(batch, MAX_SQL_PARAMETERS)) this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_objects
|
|
162
|
-
(object_key, created_at, publishable) VALUES ${retryBatch.map(() => "(?, 0, 0)").join(", ")}`, ...retryBatch);
|
|
163
|
-
console.error(JSON.stringify({
|
|
164
|
-
message: "Workers Response Store R2 cleanup failed",
|
|
165
|
-
objectKeys: batch,
|
|
166
|
-
error: error instanceof Error ? error.message : String(error)
|
|
167
|
-
}));
|
|
168
|
-
try {
|
|
169
|
-
await this.ctx.storage.setAlarm(Date.now() + ORPHAN_CLEANUP_RETRY_MS);
|
|
170
|
-
this.cleanupAlarmKnown = true;
|
|
171
|
-
} catch (alarmError) {
|
|
172
|
-
console.error(JSON.stringify({
|
|
173
|
-
message: "Workers Response Store cleanup retry scheduling failed",
|
|
174
|
-
error: alarmError instanceof Error ? alarmError.message : String(alarmError)
|
|
175
|
-
}));
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
187
|
listExpiredPendingObjects(cutoff, limit) {
|
|
180
188
|
return this.ctx.storage.sql.exec(`SELECT pending_objects.object_key FROM pending_objects
|
|
181
189
|
LEFT JOIN entries
|
|
@@ -187,7 +195,6 @@ var CacheMetadata = class extends DurableObject {
|
|
|
187
195
|
async sweepExpiredPendingObjects(cutoff = Date.now() - ORPHAN_RETENTION_MS) {
|
|
188
196
|
this.cleanupAlarmKnown = false;
|
|
189
197
|
const rows = this.ctx.storage.sql.exec(`SELECT pending_objects.object_key, pending_objects.created_at,
|
|
190
|
-
pending_objects.publishable,
|
|
191
198
|
entries.object_key IS NOT NULL AS active
|
|
192
199
|
FROM pending_objects
|
|
193
200
|
LEFT JOIN entries
|
|
@@ -196,27 +203,22 @@ var CacheMetadata = class extends DurableObject {
|
|
|
196
203
|
LIMIT ?`, 101).toArray();
|
|
197
204
|
const batch = rows.slice(0, ORPHAN_CLEANUP_LIMIT);
|
|
198
205
|
const activeObjectKeys = batch.filter(({ active }) => active).map(({ object_key }) => object_key);
|
|
199
|
-
const
|
|
200
|
-
|
|
201
|
-
const cleanupObjectKeys = expired.filter(({ publishable }) => publishable === 0).map(({ object_key }) => object_key);
|
|
202
|
-
const fencedAt = Date.now();
|
|
203
|
-
const expiredObjectKeys = expired.map(({ object_key }) => object_key);
|
|
204
|
-
this.finishPendingObjects(activeObjectKeys);
|
|
205
|
-
for (const reservationBatch of batches(reservations, 99)) this.ctx.storage.sql.exec(`UPDATE pending_objects SET created_at = ?, publishable = 0
|
|
206
|
-
WHERE object_key IN (${reservationBatch.map(() => "?").join(", ")})`, fencedAt, ...reservationBatch.map(({ object_key }) => object_key));
|
|
207
|
-
if (expiredObjectKeys.length) {
|
|
208
|
-
await this.env.CACHE_BODIES.delete(expiredObjectKeys);
|
|
209
|
-
this.finishPendingObjects(cleanupObjectKeys);
|
|
210
|
-
}
|
|
206
|
+
const expiredObjectKeys = batch.filter(({ active, created_at }) => !active && created_at <= cutoff).map(({ object_key }) => object_key);
|
|
207
|
+
this.finishPendingObjects([...activeObjectKeys, ...expiredObjectKeys]);
|
|
211
208
|
const next = batch.find(({ active, created_at }) => !active && created_at > cutoff);
|
|
212
209
|
if (!next && rows.length > ORPHAN_CLEANUP_LIMIT) {
|
|
213
210
|
await this.ctx.storage.setAlarm(Date.now());
|
|
214
211
|
this.cleanupAlarmKnown = true;
|
|
215
|
-
} else if (next
|
|
212
|
+
} else if (next) await this.ensureCleanupAlarm(next.created_at);
|
|
216
213
|
return expiredObjectKeys.length;
|
|
217
214
|
}
|
|
218
215
|
async alarm() {
|
|
219
216
|
try {
|
|
217
|
+
if (await this.retryPendingTombstones()) {
|
|
218
|
+
await this.ctx.storage.setAlarm(Date.now());
|
|
219
|
+
this.cleanupAlarmKnown = true;
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
220
222
|
await this.sweepExpiredPendingObjects();
|
|
221
223
|
} catch (error) {
|
|
222
224
|
console.error(JSON.stringify({
|
|
@@ -227,15 +229,27 @@ var CacheMetadata = class extends DurableObject {
|
|
|
227
229
|
this.cleanupAlarmKnown = true;
|
|
228
230
|
}
|
|
229
231
|
}
|
|
232
|
+
async retryPendingTombstones() {
|
|
233
|
+
const drained = await this.drainPendingTombstones(400);
|
|
234
|
+
if (drained.failures.length) throw new AggregateError(drained.failures.map((failure) => new Error(failure)), "R2 tombstone cleanup failed");
|
|
235
|
+
const edgeEntries = this.listPendingEdgePurges(400);
|
|
236
|
+
if (edgeEntries.length) {
|
|
237
|
+
if (!await this.getR2TombstoneEdgePurger().purgeR2TombstoneEdges(edgeEntries)) throw new Error("Workers Response Store cache purge is unavailable");
|
|
238
|
+
this.markTombstonesEdgePurged(edgeEntries);
|
|
239
|
+
}
|
|
240
|
+
return this.ctx.storage.sql.exec("SELECT COUNT(*) AS count FROM pending_r2_tombstones").one().count > 0;
|
|
241
|
+
}
|
|
230
242
|
async claimRevalidation(keyHash, activeRevision, cacheKey, objectKeyPrefix, now, leaseMs) {
|
|
231
243
|
const claim = this.ctx.storage.transactionSync(() => {
|
|
232
|
-
const entry = this.ctx.storage.sql.exec(`SELECT entries
|
|
244
|
+
const entry = this.ctx.storage.sql.exec(`SELECT entries.*,
|
|
233
245
|
revalidation_claims.active_revision AS claim_active_revision,
|
|
234
246
|
revalidation_claims.expires_at AS claim_expires_at
|
|
235
247
|
FROM entries
|
|
236
248
|
LEFT JOIN revalidation_claims ON revalidation_claims.key_hash = entries.key_hash
|
|
237
249
|
WHERE entries.key_hash = ? AND entries.cache_key = ?`, keyHash, cacheKey).toArray()[0];
|
|
238
250
|
if (entry?.tombstoned || entry?.active_revision !== activeRevision) return null;
|
|
251
|
+
const storedEntry = storedEntryFromRow(entry);
|
|
252
|
+
if (!storedEntry?.revalidator) return null;
|
|
239
253
|
if (entry.claim_active_revision === activeRevision && (entry.claim_expires_at ?? 0) > now) return null;
|
|
240
254
|
const revision = entry.latest_revision + 1;
|
|
241
255
|
const claimId = crypto.randomUUID();
|
|
@@ -249,6 +263,7 @@ var CacheMetadata = class extends DurableObject {
|
|
|
249
263
|
VALUES (?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))`, objectKey, now);
|
|
250
264
|
return {
|
|
251
265
|
claimId,
|
|
266
|
+
entry: storedEntry,
|
|
252
267
|
objectKey,
|
|
253
268
|
revision
|
|
254
269
|
};
|
|
@@ -272,14 +287,59 @@ var CacheMetadata = class extends DurableObject {
|
|
|
272
287
|
VALUES (?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))`, objectKey, createdAt);
|
|
273
288
|
return {
|
|
274
289
|
objectKey,
|
|
290
|
+
r2ObjectAbsent: current === void 0,
|
|
275
291
|
revision
|
|
276
292
|
};
|
|
277
293
|
});
|
|
278
294
|
await this.ensureCleanupAlarm(createdAt);
|
|
279
295
|
return reservation;
|
|
280
296
|
}
|
|
281
|
-
async
|
|
282
|
-
const
|
|
297
|
+
async replaceFailedWrite(keyHash, cacheKey, objectKeyPrefix, failedObjectKey, fenceTags, createdAt) {
|
|
298
|
+
const reservation = this.ctx.storage.transactionSync(() => {
|
|
299
|
+
const pending = this.ctx.storage.sql.exec("SELECT invalidation_sequence, publishable FROM pending_objects WHERE object_key = ?", failedObjectKey).toArray()[0];
|
|
300
|
+
const currentInvalidationSequence = this.ctx.storage.sql.exec("SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1").one().tag_invalidation_sequence;
|
|
301
|
+
const keyInvalidationSequence = this.ctx.storage.sql.exec("SELECT invalidation_sequence FROM key_invalidations WHERE key_hash = ?", keyHash).toArray()[0]?.invalidation_sequence;
|
|
302
|
+
this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", failedObjectKey);
|
|
303
|
+
if (!pending || pending.publishable !== 1 || (keyInvalidationSequence ?? 0) > pending.invalidation_sequence || currentInvalidationSequence > pending.invalidation_sequence && this.getTagInvalidationMaximum(fenceTags, "invalidation_sequence") > pending.invalidation_sequence) return null;
|
|
304
|
+
const current = this.ctx.storage.sql.exec("SELECT active_revision, latest_revision FROM entries WHERE key_hash = ?", keyHash).toArray()[0];
|
|
305
|
+
const revision = this.reserveRevision(keyHash, cacheKey, current);
|
|
306
|
+
const objectKey = `${objectKeyPrefix}/${revision}`;
|
|
307
|
+
this.ctx.storage.sql.exec(`INSERT INTO pending_objects
|
|
308
|
+
(object_key, created_at, invalidation_sequence) VALUES (?, ?, ?)`, objectKey, createdAt, currentInvalidationSequence);
|
|
309
|
+
return {
|
|
310
|
+
objectKey,
|
|
311
|
+
r2ObjectAbsent: false,
|
|
312
|
+
revision
|
|
313
|
+
};
|
|
314
|
+
});
|
|
315
|
+
if (reservation) await this.ensureCleanupAlarm(createdAt);
|
|
316
|
+
return reservation;
|
|
317
|
+
}
|
|
318
|
+
async reserveRegeneration(keyHash, cacheKey, objectKeyPrefix, createdAt) {
|
|
319
|
+
const result = this.ctx.storage.transactionSync(() => {
|
|
320
|
+
const current = this.ctx.storage.sql.exec("SELECT * FROM entries WHERE key_hash = ? AND cache_key = ?", keyHash, cacheKey).toArray()[0];
|
|
321
|
+
const entry = current ? storedEntryFromRow(current) : null;
|
|
322
|
+
if (!entry) return null;
|
|
323
|
+
if (!entry.revalidator) return { entry };
|
|
324
|
+
const revision = this.reserveRevision(keyHash, cacheKey, current);
|
|
325
|
+
const objectKey = `${objectKeyPrefix}/${revision}`;
|
|
326
|
+
this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_objects
|
|
327
|
+
(object_key, created_at, invalidation_sequence)
|
|
328
|
+
VALUES (?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))`, objectKey, createdAt);
|
|
329
|
+
return {
|
|
330
|
+
entry,
|
|
331
|
+
reservation: {
|
|
332
|
+
objectKey,
|
|
333
|
+
r2ObjectAbsent: false,
|
|
334
|
+
revision
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
});
|
|
338
|
+
if (result?.reservation) await this.ensureCleanupAlarm(createdAt);
|
|
339
|
+
return result;
|
|
340
|
+
}
|
|
341
|
+
async publish(keyHash, revision, metadata, claimId, reservationObjectKey = metadata.objectKey) {
|
|
342
|
+
return this.ctx.storage.transactionSync(() => {
|
|
283
343
|
const current = this.ctx.storage.sql.exec(`SELECT entries.*,
|
|
284
344
|
revalidation_claims.active_revision AS claim_active_revision,
|
|
285
345
|
revalidation_claims.claim_id AS claim_id,
|
|
@@ -291,15 +351,13 @@ var CacheMetadata = class extends DurableObject {
|
|
|
291
351
|
CROSS JOIN metadata_state
|
|
292
352
|
LEFT JOIN pending_objects ON pending_objects.object_key = ?
|
|
293
353
|
LEFT JOIN revalidation_claims ON revalidation_claims.key_hash = entries.key_hash
|
|
294
|
-
WHERE entries.key_hash = ?`,
|
|
354
|
+
WHERE entries.key_hash = ?`, reservationObjectKey, keyHash).toArray()[0];
|
|
295
355
|
if (!current || current.pending_invalidation_sequence === null || current.pending_invalidation_sequence === void 0 || current.pending_publishable !== 1 || revision > current.latest_revision || current.active_revision !== null && revision <= current.active_revision || claimId !== void 0 && (current.claim_id !== claimId || current.claim_revision !== revision || current.claim_active_revision !== current.active_revision) || metadata.fenceTags.length > 0 && current.current_invalidation_sequence > current.pending_invalidation_sequence && this.getTagInvalidationMaximum(metadata.fenceTags, "invalidation_sequence") > current.pending_invalidation_sequence) {
|
|
296
356
|
if (claimId) this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ? AND claim_id = ?", keyHash, claimId);
|
|
357
|
+
this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", reservationObjectKey);
|
|
297
358
|
return {
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
entry: current ? storedEntryFromRow(current) : null,
|
|
301
|
-
published: false
|
|
302
|
-
}
|
|
359
|
+
entry: current ? storedEntryFromRow(current) : null,
|
|
360
|
+
published: false
|
|
303
361
|
};
|
|
304
362
|
}
|
|
305
363
|
const published = this.ctx.storage.sql.exec(`UPDATE entries SET
|
|
@@ -307,10 +365,9 @@ var CacheMetadata = class extends DurableObject {
|
|
|
307
365
|
fresh_until = ?, swr_until = ?,
|
|
308
366
|
revalidator_id = ?, revalidator_args = ?, cache_tags = ?, tombstoned = 0
|
|
309
367
|
WHERE key_hash = ? AND latest_revision >= ?
|
|
310
|
-
AND (active_revision IS NULL OR active_revision < ?)
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
(object_key, created_at, publishable) VALUES (?, ?, 0)`, current.object_key, Date.now());
|
|
368
|
+
AND (active_revision IS NULL OR active_revision < ?)
|
|
369
|
+
RETURNING key_hash`, revision, metadata.objectKey, metadata.statusText, JSON.stringify(metadata.responseHeaders), metadata.freshUntil, metadata.swrUntil, metadata.revalidator?.id ?? null, metadata.revalidator ? JSON.stringify(metadata.revalidator.args) : null, JSON.stringify(metadata.cacheTags), keyHash, revision, revision).toArray().length === 1;
|
|
370
|
+
this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", reservationObjectKey);
|
|
314
371
|
if (claimId) this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ? AND claim_id = ?", keyHash, claimId);
|
|
315
372
|
const entry = {
|
|
316
373
|
keyHash,
|
|
@@ -326,18 +383,33 @@ var CacheMetadata = class extends DurableObject {
|
|
|
326
383
|
cacheTags: metadata.cacheTags
|
|
327
384
|
};
|
|
328
385
|
return {
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
entry: published ? entry : null,
|
|
332
|
-
published
|
|
333
|
-
}
|
|
386
|
+
entry: published ? entry : null,
|
|
387
|
+
published
|
|
334
388
|
};
|
|
335
389
|
});
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
390
|
+
}
|
|
391
|
+
async invalidatePublishedRevision(keyHash, revision) {
|
|
392
|
+
if (!this.ctx.storage.transactionSync(() => {
|
|
393
|
+
const current = this.ctx.storage.sql.exec("SELECT * FROM entries WHERE key_hash = ?", keyHash).toArray()[0];
|
|
394
|
+
if (!current || current.active_revision !== revision || current.object_key === null) return false;
|
|
395
|
+
const tombstoneRevision = current.latest_revision + 1;
|
|
396
|
+
this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_r2_tombstones
|
|
397
|
+
(key_hash, cache_key, object_key, revision) VALUES (?, ?, ?, ?)`, keyHash, current.cache_key, current.object_key, tombstoneRevision);
|
|
398
|
+
this.ctx.storage.sql.exec(`UPDATE entries SET
|
|
399
|
+
latest_revision = ?, active_revision = ?, object_key = NULL,
|
|
400
|
+
status_text = NULL, response_headers = NULL, fresh_until = NULL,
|
|
401
|
+
swr_until = NULL, revalidator_id = NULL, revalidator_args = NULL,
|
|
402
|
+
cache_tags = NULL, tombstoned = 1
|
|
403
|
+
WHERE key_hash = ? AND active_revision = ?`, tombstoneRevision, tombstoneRevision, keyHash, revision);
|
|
404
|
+
this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ?", keyHash);
|
|
405
|
+
return true;
|
|
406
|
+
})) return {
|
|
407
|
+
failures: [],
|
|
408
|
+
pending: [],
|
|
409
|
+
purged: []
|
|
410
|
+
};
|
|
411
|
+
await this.scheduleCleanupAlarm(Date.now() + ORPHAN_CLEANUP_RETRY_MS).catch((error) => this.logCleanupFailure(error));
|
|
412
|
+
return this.drainPendingTombstones(1, keyHash);
|
|
341
413
|
}
|
|
342
414
|
getEntry(keyHash) {
|
|
343
415
|
const row = this.ctx.storage.sql.exec("SELECT * FROM entries WHERE key_hash = ?", keyHash).toArray()[0];
|
|
@@ -391,13 +463,15 @@ var CacheMetadata = class extends DurableObject {
|
|
|
391
463
|
if (candidates.some(({ reservation }) => reservation)) await this.ensureCleanupAlarm(createdAt);
|
|
392
464
|
return candidates;
|
|
393
465
|
}
|
|
394
|
-
async purgeMatching(options) {
|
|
395
|
-
const
|
|
396
|
-
const matches = this.ctx.storage.transactionSync(() => {
|
|
466
|
+
async purgeMatching(options, invalidatedAt = Date.now()) {
|
|
467
|
+
const reservation = this.ctx.storage.transactionSync(() => {
|
|
397
468
|
const matches = this.findMatchingEntryRows(options, true);
|
|
398
469
|
const tags = normalizeTags(options.tags ?? []);
|
|
399
|
-
|
|
400
|
-
|
|
470
|
+
this.ctx.storage.sql.exec(`UPDATE metadata_state SET tag_invalidation_sequence = tag_invalidation_sequence + 1
|
|
471
|
+
WHERE singleton = 1`);
|
|
472
|
+
for (const batch of batches(matches, MAX_SQL_PARAMETERS)) this.ctx.storage.sql.exec(`INSERT INTO key_invalidations (key_hash, invalidation_sequence) VALUES ${batch.map(() => "(?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))").join(", ")}
|
|
473
|
+
ON CONFLICT(key_hash) DO UPDATE SET invalidation_sequence =
|
|
474
|
+
MAX(key_invalidations.invalidation_sequence, excluded.invalidation_sequence)`, ...batch.map((row) => row.key_hash));
|
|
401
475
|
for (const batch of batches(tags, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`INSERT INTO tag_invalidations
|
|
402
476
|
(tag, invalidated_at, invalidation_sequence) VALUES ${batch.map(() => "(?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))").join(", ")}
|
|
403
477
|
ON CONFLICT(tag) DO UPDATE SET invalidated_at =
|
|
@@ -407,11 +481,16 @@ var CacheMetadata = class extends DurableObject {
|
|
|
407
481
|
excluded.invalidation_sequence
|
|
408
482
|
)`, ...batch.flatMap((tag) => [tag, invalidatedAt]));
|
|
409
483
|
for (const batch of batches(matches, 99)) {
|
|
484
|
+
const active = batch.filter((row) => row.object_key !== null);
|
|
485
|
+
for (const entryBatch of batches(active, MAX_SQL_PARAMETERS / 4)) this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_r2_tombstones
|
|
486
|
+
(key_hash, cache_key, object_key, revision) VALUES ${entryBatch.map(() => "(?, ?, ?, ?)").join(", ")}`, ...entryBatch.flatMap((row) => [
|
|
487
|
+
row.key_hash,
|
|
488
|
+
row.cache_key,
|
|
489
|
+
row.object_key,
|
|
490
|
+
row.latest_revision + 1
|
|
491
|
+
]));
|
|
410
492
|
const keyHashes = batch.map((row) => row.key_hash);
|
|
411
493
|
const placeholders = keyHashes.map(() => "?").join(", ");
|
|
412
|
-
this.ctx.storage.sql.exec(`INSERT OR IGNORE INTO pending_objects (object_key, created_at, publishable)
|
|
413
|
-
SELECT object_key, ?, 0 FROM entries
|
|
414
|
-
WHERE key_hash IN (${placeholders}) AND object_key IS NOT NULL`, invalidatedAt, ...keyHashes);
|
|
415
494
|
this.ctx.storage.sql.exec(`UPDATE entries SET
|
|
416
495
|
latest_revision = latest_revision + 1,
|
|
417
496
|
active_revision = latest_revision + 1,
|
|
@@ -427,17 +506,90 @@ var CacheMetadata = class extends DurableObject {
|
|
|
427
506
|
WHERE key_hash IN (${placeholders})`, ...keyHashes);
|
|
428
507
|
this.ctx.storage.sql.exec(`DELETE FROM revalidation_claims WHERE key_hash IN (${placeholders})`, ...keyHashes);
|
|
429
508
|
}
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
}
|
|
509
|
+
const pendingTombstones = this.ctx.storage.sql.exec("SELECT COUNT(*) AS count FROM pending_r2_tombstones").one().count;
|
|
510
|
+
return {
|
|
511
|
+
backingStoreUpdated: matches.length > 0 || tags.length > 0,
|
|
512
|
+
pendingTombstones
|
|
513
|
+
};
|
|
435
514
|
});
|
|
436
|
-
if (
|
|
437
|
-
|
|
438
|
-
|
|
515
|
+
if (reservation.pendingTombstones > 0) await this.scheduleCleanupAlarm(Date.now() + ORPHAN_CLEANUP_RETRY_MS).catch((error) => this.logCleanupFailure(error));
|
|
516
|
+
return reservation;
|
|
517
|
+
}
|
|
518
|
+
async writeR2Tombstone(entry) {
|
|
519
|
+
let etag;
|
|
520
|
+
for (let attempt = 0; attempt < MAX_R2_CAS_ATTEMPTS; attempt++) {
|
|
521
|
+
const current = await this.env.CACHE_BODIES.head(entry.objectKey);
|
|
522
|
+
const currentRevision = metadataInteger(current?.customMetadata?.latestRevision);
|
|
523
|
+
if (currentRevision !== void 0 && currentRevision >= entry.revision) return;
|
|
524
|
+
etag = current?.etag ?? null;
|
|
525
|
+
if (await this.env.CACHE_BODIES.put(entry.objectKey, /* @__PURE__ */ new Uint8Array(), {
|
|
526
|
+
onlyIf: etag === null ? { etagDoesNotMatch: "*" } : { etagMatches: etag },
|
|
527
|
+
customMetadata: {
|
|
528
|
+
latestRevision: String(entry.revision),
|
|
529
|
+
tombstoned: "1"
|
|
530
|
+
}
|
|
531
|
+
})) return;
|
|
439
532
|
}
|
|
440
|
-
|
|
533
|
+
const currentRevision = metadataInteger((await this.env.CACHE_BODIES.head(entry.objectKey))?.customMetadata?.latestRevision);
|
|
534
|
+
if (currentRevision !== void 0 && currentRevision >= entry.revision) return;
|
|
535
|
+
throw new Error(`R2 revision ${entry.revision} could not be tombstoned after concurrent writes`);
|
|
536
|
+
}
|
|
537
|
+
finishCompletedTombstones() {
|
|
538
|
+
this.ctx.storage.sql.exec("DELETE FROM pending_r2_tombstones WHERE r2_complete = 1 AND edge_purge_complete = 1");
|
|
539
|
+
}
|
|
540
|
+
async drainPendingTombstones(limit, keyHash, afterKeyHash) {
|
|
541
|
+
const filters = ["r2_complete = 0"];
|
|
542
|
+
const parameters = [];
|
|
543
|
+
if (keyHash !== void 0) {
|
|
544
|
+
filters.push("key_hash = ?");
|
|
545
|
+
parameters.push(keyHash);
|
|
546
|
+
}
|
|
547
|
+
if (afterKeyHash !== void 0) {
|
|
548
|
+
filters.push("key_hash > ?");
|
|
549
|
+
parameters.push(afterKeyHash);
|
|
550
|
+
}
|
|
551
|
+
parameters.push(limit);
|
|
552
|
+
const rows = this.ctx.storage.sql.exec(`SELECT key_hash, cache_key, object_key, revision, r2_complete, edge_purge_complete
|
|
553
|
+
FROM pending_r2_tombstones
|
|
554
|
+
WHERE ${filters.join(" AND ")}
|
|
555
|
+
ORDER BY key_hash LIMIT ?`, ...parameters).toArray();
|
|
556
|
+
const pending = rows.map((row) => ({
|
|
557
|
+
keyHash: row.key_hash,
|
|
558
|
+
cacheKey: row.cache_key,
|
|
559
|
+
objectKey: row.object_key,
|
|
560
|
+
revision: row.revision
|
|
561
|
+
}));
|
|
562
|
+
const settled = await Promise.allSettled(pending.map(async (entry) => {
|
|
563
|
+
await this.writeR2Tombstone(entry);
|
|
564
|
+
return entry;
|
|
565
|
+
}));
|
|
566
|
+
const purged = settled.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
|
567
|
+
this.ctx.storage.transactionSync(() => {
|
|
568
|
+
for (const batch of batches(purged, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`UPDATE pending_r2_tombstones SET r2_complete = 1 WHERE ${batch.map(() => "(key_hash = ? AND revision = ?)").join(" OR ")}`, ...batch.flatMap((entry) => [entry.keyHash, entry.revision]));
|
|
569
|
+
this.finishCompletedTombstones();
|
|
570
|
+
});
|
|
571
|
+
return {
|
|
572
|
+
...rows.length ? { cursor: rows.at(-1).key_hash } : {},
|
|
573
|
+
failures: settled.flatMap((result) => result.status === "rejected" ? [result.reason instanceof Error ? result.reason.message : String(result.reason)] : []),
|
|
574
|
+
pending,
|
|
575
|
+
purged
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
listPendingEdgePurges(limit) {
|
|
579
|
+
return this.ctx.storage.sql.exec(`SELECT key_hash, cache_key, object_key, revision, r2_complete, edge_purge_complete
|
|
580
|
+
FROM pending_r2_tombstones
|
|
581
|
+
WHERE r2_complete = 1 AND edge_purge_complete = 0 ORDER BY key_hash LIMIT ?`, limit).toArray().map((row) => ({
|
|
582
|
+
keyHash: row.key_hash,
|
|
583
|
+
cacheKey: row.cache_key,
|
|
584
|
+
objectKey: row.object_key,
|
|
585
|
+
revision: row.revision
|
|
586
|
+
}));
|
|
587
|
+
}
|
|
588
|
+
markTombstonesEdgePurged(entries) {
|
|
589
|
+
this.ctx.storage.transactionSync(() => {
|
|
590
|
+
for (const batch of batches(entries, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`UPDATE pending_r2_tombstones SET edge_purge_complete = 1 WHERE ${batch.map(() => "(key_hash = ? AND revision = ?)").join(" OR ")}`, ...batch.flatMap((entry) => [entry.keyHash, entry.revision]));
|
|
591
|
+
this.finishCompletedTombstones();
|
|
592
|
+
});
|
|
441
593
|
}
|
|
442
594
|
inspect() {
|
|
443
595
|
return storedEntriesFromRows(this.ctx.storage.sql.exec("SELECT * FROM entries ORDER BY cache_key").toArray());
|
package/package.json
CHANGED