@cloudflare/workers-response-store 0.1.0-beta.0 → 0.1.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,16 @@
1
+ import { mapSettledWithR2Concurrency } from "./r2-concurrency.js";
1
2
  import { DurableObject } from "cloudflare:workers";
2
3
  //#region src/metadata-do.ts
3
4
  const MAX_SQL_PARAMETERS = 100;
4
- const R2_DELETE_BATCH_SIZE = 1e3;
5
5
  const ORPHAN_RETENTION_MS = 36e5;
6
6
  const ORPHAN_CLEANUP_LIMIT = 100;
7
7
  const ORPHAN_CLEANUP_RETRY_MS = 6e4;
8
+ const MAX_R2_CAS_ATTEMPTS = 3;
9
+ function metadataInteger(value) {
10
+ if (value === void 0 || !/^\d+$/.test(value)) return void 0;
11
+ const parsed = Number(value);
12
+ return Number.isSafeInteger(parsed) ? parsed : void 0;
13
+ }
8
14
  function normalizeTags(tags) {
9
15
  const normalized = tags.map((tag) => tag.trim().toLowerCase()).filter(Boolean);
10
16
  return [...new Set(normalized)];
@@ -61,6 +67,14 @@ var CacheMetadata = class extends DurableObject {
61
67
  tombstoned INTEGER NOT NULL DEFAULT 0
62
68
  );
63
69
  CREATE INDEX IF NOT EXISTS entries_cache_key ON entries(cache_key);
70
+ CREATE INDEX IF NOT EXISTS entries_active_object_key
71
+ ON entries(object_key) WHERE tombstoned = 0;
72
+ CREATE TABLE IF NOT EXISTS entry_tags (
73
+ tag TEXT NOT NULL,
74
+ key_hash TEXT NOT NULL,
75
+ PRIMARY KEY (tag, key_hash)
76
+ ) WITHOUT ROWID;
77
+ CREATE INDEX IF NOT EXISTS entry_tags_key_hash ON entry_tags(key_hash);
64
78
  CREATE TABLE IF NOT EXISTS revalidation_claims (
65
79
  key_hash TEXT PRIMARY KEY,
66
80
  active_revision INTEGER NOT NULL,
@@ -79,9 +93,14 @@ var CacheMetadata = class extends DurableObject {
79
93
  );
80
94
  CREATE TABLE IF NOT EXISTS metadata_state (
81
95
  singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
82
- tag_invalidation_sequence INTEGER NOT NULL
96
+ tag_invalidation_sequence INTEGER NOT NULL,
97
+ tombstone_sequence INTEGER NOT NULL DEFAULT 0
83
98
  );
84
99
  INSERT OR IGNORE INTO metadata_state (singleton, tag_invalidation_sequence) VALUES (1, 0);
100
+ CREATE TABLE IF NOT EXISTS key_invalidations (
101
+ key_hash TEXT PRIMARY KEY,
102
+ invalidation_sequence INTEGER NOT NULL
103
+ ) WITHOUT ROWID;
85
104
  CREATE TABLE IF NOT EXISTS pending_objects (
86
105
  object_key TEXT PRIMARY KEY,
87
106
  invalidation_sequence INTEGER NOT NULL DEFAULT 0,
@@ -89,12 +108,22 @@ var CacheMetadata = class extends DurableObject {
89
108
  created_at INTEGER NOT NULL
90
109
  );
91
110
  CREATE INDEX IF NOT EXISTS pending_objects_created_at ON pending_objects(created_at);
111
+ CREATE TABLE IF NOT EXISTS pending_r2_tombstones (
112
+ key_hash TEXT PRIMARY KEY,
113
+ cache_key TEXT NOT NULL,
114
+ object_key TEXT NOT NULL,
115
+ revision INTEGER NOT NULL,
116
+ r2_complete INTEGER NOT NULL DEFAULT 0,
117
+ edge_purge_complete INTEGER NOT NULL DEFAULT 0,
118
+ tombstone_sequence INTEGER NOT NULL DEFAULT 0
119
+ ) WITHOUT ROWID;
92
120
  `);
93
121
  ctx.storage.transactionSync(() => {
94
- const migrations = new Set(ctx.storage.sql.exec("SELECT version FROM metadata_schema_migrations WHERE version IN (2, 3)").toArray().map(({ version }) => version));
95
- if (migrations.size === 2) return;
122
+ const migrations = new Set(ctx.storage.sql.exec("SELECT version FROM metadata_schema_migrations WHERE version IN (2, 3, 4, 5, 6)").toArray().map(({ version }) => version));
123
+ if (migrations.size === 5) return;
96
124
  const schemas = ctx.storage.sql.exec(`SELECT name, sql FROM sqlite_schema
97
- WHERE type = 'table' AND name IN ('tag_invalidations', 'pending_objects')`).toArray();
125
+ WHERE type = 'table'
126
+ AND name IN ('tag_invalidations', 'metadata_state', 'pending_objects', 'pending_r2_tombstones')`).toArray();
98
127
  if (!migrations.has(2) && !schemas.find(({ name }) => name === "tag_invalidations")?.sql.includes("invalidation_sequence")) ctx.storage.sql.exec("ALTER TABLE tag_invalidations ADD COLUMN invalidation_sequence INTEGER NOT NULL DEFAULT 0");
99
128
  if (!migrations.has(2) && !schemas.find(({ name }) => name === "pending_objects")?.sql.includes("invalidation_sequence")) ctx.storage.sql.exec("ALTER TABLE pending_objects ADD COLUMN invalidation_sequence INTEGER NOT NULL DEFAULT 0");
100
129
  if (!migrations.has(2)) ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (2)");
@@ -102,30 +131,98 @@ var CacheMetadata = class extends DurableObject {
102
131
  if (!schemas.find(({ name }) => name === "pending_objects")?.sql.includes("publishable")) ctx.storage.sql.exec("ALTER TABLE pending_objects ADD COLUMN publishable INTEGER NOT NULL DEFAULT 1");
103
132
  ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (3)");
104
133
  }
134
+ if (!migrations.has(4)) {
135
+ const tombstones = schemas.find(({ name }) => name === "pending_r2_tombstones")?.sql;
136
+ if (!tombstones?.includes("r2_complete")) ctx.storage.sql.exec("ALTER TABLE pending_r2_tombstones ADD COLUMN r2_complete INTEGER NOT NULL DEFAULT 0");
137
+ if (!tombstones?.includes("edge_purge_complete")) ctx.storage.sql.exec("ALTER TABLE pending_r2_tombstones ADD COLUMN edge_purge_complete INTEGER NOT NULL DEFAULT 0");
138
+ ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (4)");
139
+ }
140
+ if (!migrations.has(5)) {
141
+ const metadataState = schemas.find(({ name }) => name === "metadata_state")?.sql;
142
+ const tombstones = schemas.find(({ name }) => name === "pending_r2_tombstones")?.sql;
143
+ if (!metadataState?.includes("tombstone_sequence")) ctx.storage.sql.exec("ALTER TABLE metadata_state ADD COLUMN tombstone_sequence INTEGER NOT NULL DEFAULT 0");
144
+ if (!tombstones?.includes("tombstone_sequence")) ctx.storage.sql.exec("ALTER TABLE pending_r2_tombstones ADD COLUMN tombstone_sequence INTEGER NOT NULL DEFAULT 0");
145
+ ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (5)");
146
+ }
147
+ if (!migrations.has(6)) {
148
+ ctx.storage.sql.exec("DELETE FROM entry_tags");
149
+ const entries = ctx.storage.sql.exec("SELECT key_hash, cache_tags FROM entries WHERE cache_tags IS NOT NULL");
150
+ for (const entry of entries) this.replaceEntryTags(entry.key_hash, JSON.parse(entry.cache_tags));
151
+ ctx.storage.sql.exec("INSERT INTO metadata_schema_migrations (version) VALUES (6)");
152
+ }
105
153
  });
154
+ ctx.storage.sql.exec(`
155
+ CREATE INDEX IF NOT EXISTS pending_r2_tombstones_r2_pending
156
+ ON pending_r2_tombstones(key_hash) WHERE r2_complete = 0;
157
+ CREATE INDEX IF NOT EXISTS pending_r2_tombstones_edge_pending
158
+ ON pending_r2_tombstones(key_hash)
159
+ WHERE r2_complete = 1 AND edge_purge_complete = 0;
160
+ CREATE INDEX IF NOT EXISTS pending_r2_tombstones_r2_sequence
161
+ ON pending_r2_tombstones(tombstone_sequence, key_hash)
162
+ WHERE r2_complete = 0;
163
+ CREATE INDEX IF NOT EXISTS pending_r2_tombstones_edge_sequence
164
+ ON pending_r2_tombstones(tombstone_sequence, key_hash)
165
+ WHERE r2_complete = 1 AND edge_purge_complete = 0;
166
+ `);
106
167
  });
107
168
  }
108
169
  async ensureCleanupAlarm(createdAt) {
109
170
  if (this.cleanupAlarmKnown) return;
110
- if (await this.ctx.storage.getAlarm() === null) await this.ctx.storage.setAlarm(createdAt + ORPHAN_RETENTION_MS);
171
+ await this.scheduleCleanupAlarm(createdAt + ORPHAN_RETENTION_MS);
172
+ }
173
+ async scheduleCleanupAlarm(scheduledTime) {
174
+ const current = await this.ctx.storage.getAlarm();
175
+ if (current === null || current > scheduledTime) await this.ctx.storage.setAlarm(scheduledTime);
111
176
  this.cleanupAlarmKnown = true;
112
177
  }
113
- async maintainCleanupAlarm(createdAt) {
114
- await this.ensureCleanupAlarm(createdAt).catch((error) => {
115
- console.error(JSON.stringify({
116
- message: "Workers Response Store cleanup alarm update failed",
117
- error: error instanceof Error ? error.message : String(error)
118
- }));
119
- });
178
+ logCleanupFailure(error) {
179
+ console.error(JSON.stringify({
180
+ message: "Workers Response Store cleanup scheduling failed",
181
+ error: error instanceof Error ? error.message : String(error)
182
+ }));
183
+ }
184
+ tombstoneSequence(increment = false) {
185
+ const statement = increment ? `UPDATE metadata_state SET tombstone_sequence = tombstone_sequence + 1
186
+ WHERE singleton = 1 RETURNING tombstone_sequence` : "SELECT tombstone_sequence FROM metadata_state WHERE singleton = 1";
187
+ return this.ctx.storage.sql.exec(statement).one().tombstone_sequence;
188
+ }
189
+ getR2TombstoneEdgePurger() {
190
+ const factory = Reflect.get(this.ctx.exports, "ResponseStoreBinding");
191
+ if (typeof factory !== "function") throw new Error("The ResponseStoreBinding entrypoint is not exported");
192
+ return factory({ props: {} });
193
+ }
194
+ findMatchingEntryRows(options, projection = "entry") {
195
+ const selectors = [];
196
+ const parameters = [];
197
+ if (!options.purgeEverything) {
198
+ const tags = normalizeTags(options.tags ?? []);
199
+ if (tags.length) {
200
+ selectors.push(`entries.key_hash IN (
201
+ SELECT key_hash FROM entry_tags
202
+ WHERE tag IN (SELECT value FROM json_each(?))
203
+ )`);
204
+ parameters.push(JSON.stringify(tags));
205
+ }
206
+ if (options.pathPrefixes?.length) {
207
+ selectors.push(`entries.key_hash IN (
208
+ SELECT path_entry.key_hash FROM json_each(?) AS path_prefix
209
+ JOIN entries AS path_entry
210
+ ON path_entry.cache_key >= path_prefix.value
211
+ AND path_entry.cache_key < path_prefix.value || char(127)
212
+ )`);
213
+ parameters.push(JSON.stringify(options.pathPrefixes));
214
+ }
215
+ }
216
+ if (!options.purgeEverything && !selectors.length) return [];
217
+ const conditions = [];
218
+ if (projection !== "purge") conditions.push("tombstoned = 0 AND active_revision IS NOT NULL");
219
+ if (projection === "refresh") conditions.push("object_key IS NOT NULL AND response_headers IS NOT NULL AND fresh_until IS NOT NULL AND swr_until IS NOT NULL");
220
+ if (!options.purgeEverything) conditions.push(`(${selectors.join(" OR ")})`);
221
+ return this.ctx.storage.sql.exec(`SELECT ${projection === "purge" ? "key_hash, cache_key, latest_revision, object_key" : projection === "refresh" ? "key_hash, cache_key, active_revision, latest_revision, revalidator_id IS NOT NULL AS has_revalidator" : "*"} FROM entries ${conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""}`, ...parameters).toArray();
120
222
  }
121
- findMatchingEntryRows(options, includePending = false) {
122
- const rows = this.ctx.storage.sql.exec(includePending ? `SELECT * FROM entries
123
- WHERE (tombstoned = 0 AND active_revision IS NOT NULL)
124
- OR active_revision IS NULL OR latest_revision > active_revision` : "SELECT * FROM entries WHERE tombstoned = 0 AND active_revision IS NOT NULL").toArray();
125
- if (options.purgeEverything) return rows;
126
- const tags = new Set(normalizeTags(options.tags ?? []));
127
- const prefixes = options.pathPrefixes ?? [];
128
- return rows.filter((row) => prefixes.some((prefix) => row.cache_key.startsWith(prefix)) || normalizeTags(JSON.parse(row.cache_tags ?? "[]")).some((tag) => tags.has(tag)));
223
+ replaceEntryTags(keyHash, tags) {
224
+ this.ctx.storage.sql.exec("DELETE FROM entry_tags WHERE key_hash = ?", keyHash);
225
+ for (const batch of batches(normalizeTags(tags), MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`INSERT INTO entry_tags (tag, key_hash) VALUES ${batch.map(() => "(?, ?)").join(", ")}`, ...batch.flatMap((tag) => [tag, keyHash]));
129
226
  }
130
227
  async trackPendingObjects(objectKeys, createdAt) {
131
228
  if (!objectKeys.length) return;
@@ -143,7 +240,7 @@ var CacheMetadata = class extends DurableObject {
143
240
  }
144
241
  releaseWrite(keyHash, objectKey, claimId) {
145
242
  this.ctx.storage.transactionSync(() => {
146
- this.ctx.storage.sql.exec("UPDATE pending_objects SET created_at = 0, publishable = 0 WHERE object_key = ?", objectKey);
243
+ this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", objectKey);
147
244
  if (claimId) this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ? AND claim_id = ?", keyHash, claimId);
148
245
  });
149
246
  }
@@ -153,29 +250,6 @@ var CacheMetadata = class extends DurableObject {
153
250
  for (const batch of batches(objectKeys, MAX_SQL_PARAMETERS)) this.ctx.storage.sql.exec(`DELETE FROM pending_objects WHERE object_key IN (${batch.map(() => "?").join(", ")})`, ...batch);
154
251
  });
155
252
  }
156
- async deleteTrackedObjects(objectKeys) {
157
- for (const batch of batches(objectKeys, R2_DELETE_BATCH_SIZE)) try {
158
- await this.env.CACHE_BODIES.delete(batch);
159
- this.finishPendingObjects(batch);
160
- } catch (error) {
161
- for (const retryBatch of batches(batch, MAX_SQL_PARAMETERS)) this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_objects
162
- (object_key, created_at, publishable) VALUES ${retryBatch.map(() => "(?, 0, 0)").join(", ")}`, ...retryBatch);
163
- console.error(JSON.stringify({
164
- message: "Workers Response Store R2 cleanup failed",
165
- objectKeys: batch,
166
- error: error instanceof Error ? error.message : String(error)
167
- }));
168
- try {
169
- await this.ctx.storage.setAlarm(Date.now() + ORPHAN_CLEANUP_RETRY_MS);
170
- this.cleanupAlarmKnown = true;
171
- } catch (alarmError) {
172
- console.error(JSON.stringify({
173
- message: "Workers Response Store cleanup retry scheduling failed",
174
- error: alarmError instanceof Error ? alarmError.message : String(alarmError)
175
- }));
176
- }
177
- }
178
- }
179
253
  listExpiredPendingObjects(cutoff, limit) {
180
254
  return this.ctx.storage.sql.exec(`SELECT pending_objects.object_key FROM pending_objects
181
255
  LEFT JOIN entries
@@ -187,7 +261,6 @@ var CacheMetadata = class extends DurableObject {
187
261
  async sweepExpiredPendingObjects(cutoff = Date.now() - ORPHAN_RETENTION_MS) {
188
262
  this.cleanupAlarmKnown = false;
189
263
  const rows = this.ctx.storage.sql.exec(`SELECT pending_objects.object_key, pending_objects.created_at,
190
- pending_objects.publishable,
191
264
  entries.object_key IS NOT NULL AS active
192
265
  FROM pending_objects
193
266
  LEFT JOIN entries
@@ -196,27 +269,22 @@ var CacheMetadata = class extends DurableObject {
196
269
  LIMIT ?`, 101).toArray();
197
270
  const batch = rows.slice(0, ORPHAN_CLEANUP_LIMIT);
198
271
  const activeObjectKeys = batch.filter(({ active }) => active).map(({ object_key }) => object_key);
199
- const expired = batch.filter(({ active, created_at }) => !active && created_at <= cutoff);
200
- const reservations = expired.filter(({ publishable }) => publishable === 1);
201
- const cleanupObjectKeys = expired.filter(({ publishable }) => publishable === 0).map(({ object_key }) => object_key);
202
- const fencedAt = Date.now();
203
- const expiredObjectKeys = expired.map(({ object_key }) => object_key);
204
- this.finishPendingObjects(activeObjectKeys);
205
- for (const reservationBatch of batches(reservations, 99)) this.ctx.storage.sql.exec(`UPDATE pending_objects SET created_at = ?, publishable = 0
206
- WHERE object_key IN (${reservationBatch.map(() => "?").join(", ")})`, fencedAt, ...reservationBatch.map(({ object_key }) => object_key));
207
- if (expiredObjectKeys.length) {
208
- await this.env.CACHE_BODIES.delete(expiredObjectKeys);
209
- this.finishPendingObjects(cleanupObjectKeys);
210
- }
272
+ const expiredObjectKeys = batch.filter(({ active, created_at }) => !active && created_at <= cutoff).map(({ object_key }) => object_key);
273
+ this.finishPendingObjects([...activeObjectKeys, ...expiredObjectKeys]);
211
274
  const next = batch.find(({ active, created_at }) => !active && created_at > cutoff);
212
275
  if (!next && rows.length > ORPHAN_CLEANUP_LIMIT) {
213
276
  await this.ctx.storage.setAlarm(Date.now());
214
277
  this.cleanupAlarmKnown = true;
215
- } else if (next || reservations.length) await this.ensureCleanupAlarm(Math.min(next?.created_at ?? Number.POSITIVE_INFINITY, fencedAt));
278
+ } else if (next) await this.ensureCleanupAlarm(next.created_at);
216
279
  return expiredObjectKeys.length;
217
280
  }
218
281
  async alarm() {
219
282
  try {
283
+ if (await this.retryPendingTombstones()) {
284
+ await this.ctx.storage.setAlarm(Date.now());
285
+ this.cleanupAlarmKnown = true;
286
+ return;
287
+ }
220
288
  await this.sweepExpiredPendingObjects();
221
289
  } catch (error) {
222
290
  console.error(JSON.stringify({
@@ -227,15 +295,27 @@ var CacheMetadata = class extends DurableObject {
227
295
  this.cleanupAlarmKnown = true;
228
296
  }
229
297
  }
298
+ async retryPendingTombstones() {
299
+ const drained = await this.drainPendingTombstones(400);
300
+ if (drained.failures.length) throw new AggregateError(drained.failures.map((failure) => new Error(failure)), "R2 tombstone cleanup failed");
301
+ const edgeEntries = this.listPendingEdgePurges(400);
302
+ if (edgeEntries.length) {
303
+ if (!await this.getR2TombstoneEdgePurger().purgeR2TombstoneEdges(edgeEntries)) throw new Error("Workers Response Store cache purge is unavailable");
304
+ this.markTombstonesEdgePurged(edgeEntries);
305
+ }
306
+ return this.ctx.storage.sql.exec("SELECT EXISTS(SELECT 1 FROM pending_r2_tombstones) AS pending").one().pending === 1;
307
+ }
230
308
  async claimRevalidation(keyHash, activeRevision, cacheKey, objectKeyPrefix, now, leaseMs) {
231
309
  const claim = this.ctx.storage.transactionSync(() => {
232
- const entry = this.ctx.storage.sql.exec(`SELECT entries.active_revision, entries.latest_revision, entries.tombstoned,
310
+ const entry = this.ctx.storage.sql.exec(`SELECT entries.*,
233
311
  revalidation_claims.active_revision AS claim_active_revision,
234
312
  revalidation_claims.expires_at AS claim_expires_at
235
313
  FROM entries
236
314
  LEFT JOIN revalidation_claims ON revalidation_claims.key_hash = entries.key_hash
237
315
  WHERE entries.key_hash = ? AND entries.cache_key = ?`, keyHash, cacheKey).toArray()[0];
238
316
  if (entry?.tombstoned || entry?.active_revision !== activeRevision) return null;
317
+ const storedEntry = storedEntryFromRow(entry);
318
+ if (!storedEntry?.revalidator) return null;
239
319
  if (entry.claim_active_revision === activeRevision && (entry.claim_expires_at ?? 0) > now) return null;
240
320
  const revision = entry.latest_revision + 1;
241
321
  const claimId = crypto.randomUUID();
@@ -249,6 +329,7 @@ var CacheMetadata = class extends DurableObject {
249
329
  VALUES (?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))`, objectKey, now);
250
330
  return {
251
331
  claimId,
332
+ entry: storedEntry,
252
333
  objectKey,
253
334
  revision
254
335
  };
@@ -272,14 +353,60 @@ var CacheMetadata = class extends DurableObject {
272
353
  VALUES (?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))`, objectKey, createdAt);
273
354
  return {
274
355
  objectKey,
356
+ r2ObjectAbsent: current === void 0,
275
357
  revision
276
358
  };
277
359
  });
278
360
  await this.ensureCleanupAlarm(createdAt);
279
361
  return reservation;
280
362
  }
281
- async publish(keyHash, revision, metadata, claimId) {
282
- const { cleanupObjectKey, result } = this.ctx.storage.transactionSync(() => {
363
+ async replaceFailedWrite(keyHash, cacheKey, objectKeyPrefix, failedObjectKey, fenceTags, createdAt) {
364
+ const reservation = this.ctx.storage.transactionSync(() => {
365
+ const pending = this.ctx.storage.sql.exec("SELECT invalidation_sequence, publishable FROM pending_objects WHERE object_key = ?", failedObjectKey).toArray()[0];
366
+ const currentInvalidationSequence = this.ctx.storage.sql.exec("SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1").one().tag_invalidation_sequence;
367
+ const keyInvalidationSequence = this.ctx.storage.sql.exec("SELECT invalidation_sequence FROM key_invalidations WHERE key_hash = ?", keyHash).toArray()[0]?.invalidation_sequence;
368
+ this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", failedObjectKey);
369
+ if (!pending || pending.publishable !== 1 || (keyInvalidationSequence ?? 0) > pending.invalidation_sequence || currentInvalidationSequence > pending.invalidation_sequence && this.getTagInvalidationMaximum(fenceTags, "invalidation_sequence") > pending.invalidation_sequence) return null;
370
+ const current = this.ctx.storage.sql.exec("SELECT active_revision, latest_revision FROM entries WHERE key_hash = ?", keyHash).toArray()[0];
371
+ const revision = this.reserveRevision(keyHash, cacheKey, current);
372
+ const objectKey = `${objectKeyPrefix}/${revision}`;
373
+ this.ctx.storage.sql.exec(`INSERT INTO pending_objects
374
+ (object_key, created_at, invalidation_sequence) VALUES (?, ?, ?)`, objectKey, createdAt, currentInvalidationSequence);
375
+ return {
376
+ objectKey,
377
+ r2ObjectAbsent: false,
378
+ revision
379
+ };
380
+ });
381
+ if (reservation) await this.ensureCleanupAlarm(createdAt);
382
+ return reservation;
383
+ }
384
+ async reserveRegeneration(keyHash, cacheKey, objectKeyPrefix, createdAt, expectedActiveRevision, expectedLatestRevision) {
385
+ const result = this.ctx.storage.transactionSync(() => {
386
+ const current = this.ctx.storage.sql.exec("SELECT * FROM entries WHERE key_hash = ? AND cache_key = ?", keyHash, cacheKey).toArray()[0];
387
+ const entry = current ? storedEntryFromRow(current) : null;
388
+ if (!entry) return null;
389
+ if (expectedActiveRevision !== void 0 && entry.activeRevision !== expectedActiveRevision || expectedLatestRevision !== void 0 && entry.latestRevision !== expectedLatestRevision) return null;
390
+ if (!entry.revalidator) return { entry };
391
+ const revision = this.reserveRevision(keyHash, cacheKey, current);
392
+ const objectKey = `${objectKeyPrefix}/${revision}`;
393
+ this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_objects
394
+ (object_key, created_at, invalidation_sequence)
395
+ VALUES (?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))`, objectKey, createdAt);
396
+ return {
397
+ entry,
398
+ reservation: {
399
+ objectKey,
400
+ r2ObjectAbsent: false,
401
+ revision
402
+ }
403
+ };
404
+ });
405
+ if (result?.reservation) await this.ensureCleanupAlarm(createdAt);
406
+ return result;
407
+ }
408
+ async publish(keyHash, revision, metadata, claimId, reservationObjectKey = metadata.objectKey) {
409
+ return this.ctx.storage.transactionSync(() => {
283
410
  const current = this.ctx.storage.sql.exec(`SELECT entries.*,
284
411
  revalidation_claims.active_revision AS claim_active_revision,
285
412
  revalidation_claims.claim_id AS claim_id,
@@ -291,15 +418,14 @@ var CacheMetadata = class extends DurableObject {
291
418
  CROSS JOIN metadata_state
292
419
  LEFT JOIN pending_objects ON pending_objects.object_key = ?
293
420
  LEFT JOIN revalidation_claims ON revalidation_claims.key_hash = entries.key_hash
294
- WHERE entries.key_hash = ?`, metadata.objectKey, keyHash).toArray()[0];
421
+ WHERE entries.key_hash = ?`, reservationObjectKey, keyHash).toArray()[0];
295
422
  if (!current || current.pending_invalidation_sequence === null || current.pending_invalidation_sequence === void 0 || current.pending_publishable !== 1 || revision > current.latest_revision || current.active_revision !== null && revision <= current.active_revision || claimId !== void 0 && (current.claim_id !== claimId || current.claim_revision !== revision || current.claim_active_revision !== current.active_revision) || metadata.fenceTags.length > 0 && current.current_invalidation_sequence > current.pending_invalidation_sequence && this.getTagInvalidationMaximum(metadata.fenceTags, "invalidation_sequence") > current.pending_invalidation_sequence) {
296
423
  if (claimId) this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ? AND claim_id = ?", keyHash, claimId);
424
+ this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", reservationObjectKey);
297
425
  return {
298
- cleanupObjectKey: current?.object_key === metadata.objectKey ? void 0 : metadata.objectKey,
299
- result: {
300
- entry: current ? storedEntryFromRow(current) : null,
301
- published: false
302
- }
426
+ edgePurgeRequired: false,
427
+ entry: current ? storedEntryFromRow(current) : null,
428
+ published: false
303
429
  };
304
430
  }
305
431
  const published = this.ctx.storage.sql.exec(`UPDATE entries SET
@@ -307,10 +433,10 @@ var CacheMetadata = class extends DurableObject {
307
433
  fresh_until = ?, swr_until = ?,
308
434
  revalidator_id = ?, revalidator_args = ?, cache_tags = ?, tombstoned = 0
309
435
  WHERE key_hash = ? AND latest_revision >= ?
310
- AND (active_revision IS NULL OR active_revision < ?)`, revision, metadata.objectKey, metadata.statusText, JSON.stringify(metadata.responseHeaders), metadata.freshUntil, metadata.swrUntil, metadata.revalidator?.id ?? null, metadata.revalidator ? JSON.stringify(metadata.revalidator.args) : null, JSON.stringify(metadata.cacheTags), keyHash, revision, revision).rowsWritten === 1;
311
- if (published) this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", metadata.objectKey);
312
- if (published && current.object_key && current.object_key !== metadata.objectKey) this.ctx.storage.sql.exec(`INSERT OR IGNORE INTO pending_objects
313
- (object_key, created_at, publishable) VALUES (?, ?, 0)`, current.object_key, Date.now());
436
+ AND (active_revision IS NULL OR active_revision < ?)
437
+ RETURNING key_hash`, revision, metadata.objectKey, metadata.statusText, JSON.stringify(metadata.responseHeaders), metadata.freshUntil, metadata.swrUntil, metadata.revalidator?.id ?? null, metadata.revalidator ? JSON.stringify(metadata.revalidator.args) : null, JSON.stringify(metadata.cacheTags), keyHash, revision, revision).toArray().length === 1;
438
+ if (published) this.replaceEntryTags(keyHash, metadata.cacheTags);
439
+ this.ctx.storage.sql.exec("DELETE FROM pending_objects WHERE object_key = ?", reservationObjectKey);
314
440
  if (claimId) this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ? AND claim_id = ?", keyHash, claimId);
315
441
  const entry = {
316
442
  keyHash,
@@ -326,18 +452,37 @@ var CacheMetadata = class extends DurableObject {
326
452
  cacheTags: metadata.cacheTags
327
453
  };
328
454
  return {
329
- cleanupObjectKey: published ? current.object_key && current.object_key !== metadata.objectKey ? current.object_key : void 0 : metadata.objectKey,
330
- result: {
331
- entry: published ? entry : null,
332
- published
333
- }
455
+ edgePurgeRequired: published && current.active_revision !== null,
456
+ entry: published ? entry : null,
457
+ published
334
458
  };
335
459
  });
336
- if (cleanupObjectKey) {
337
- await this.maintainCleanupAlarm(Date.now());
338
- await this.deleteTrackedObjects([cleanupObjectKey]);
339
- }
340
- return result;
460
+ }
461
+ async invalidatePublishedRevision(keyHash, revision) {
462
+ if (!this.ctx.storage.transactionSync(() => {
463
+ const current = this.ctx.storage.sql.exec("SELECT * FROM entries WHERE key_hash = ?", keyHash).toArray()[0];
464
+ if (!current || current.active_revision !== revision || current.object_key === null) return false;
465
+ const tombstoneRevision = current.latest_revision + 1;
466
+ const tombstoneSequence = this.tombstoneSequence(true);
467
+ this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_r2_tombstones
468
+ (key_hash, cache_key, object_key, revision, tombstone_sequence)
469
+ VALUES (?, ?, ?, ?, ?)`, keyHash, current.cache_key, current.object_key, tombstoneRevision, tombstoneSequence);
470
+ this.ctx.storage.sql.exec(`UPDATE entries SET
471
+ latest_revision = ?, active_revision = ?, object_key = NULL,
472
+ status_text = NULL, response_headers = NULL, fresh_until = NULL,
473
+ swr_until = NULL, revalidator_id = NULL, revalidator_args = NULL,
474
+ cache_tags = NULL, tombstoned = 1
475
+ WHERE key_hash = ? AND active_revision = ?`, tombstoneRevision, tombstoneRevision, keyHash, revision);
476
+ this.ctx.storage.sql.exec("DELETE FROM revalidation_claims WHERE key_hash = ?", keyHash);
477
+ this.ctx.storage.sql.exec("DELETE FROM entry_tags WHERE key_hash = ?", keyHash);
478
+ return true;
479
+ })) return {
480
+ failures: [],
481
+ pending: [],
482
+ purged: []
483
+ };
484
+ await this.scheduleCleanupAlarm(Date.now() + ORPHAN_CLEANUP_RETRY_MS).catch((error) => this.logCleanupFailure(error));
485
+ return this.drainPendingTombstones(1, keyHash);
341
486
  }
342
487
  getEntry(keyHash) {
343
488
  const row = this.ctx.storage.sql.exec("SELECT * FROM entries WHERE key_hash = ?", keyHash).toArray()[0];
@@ -357,47 +502,26 @@ var CacheMetadata = class extends DurableObject {
357
502
  getTagExpiration(tags) {
358
503
  return this.getTagInvalidationMaximum(tags, "invalidated_at");
359
504
  }
360
- async reserveRefresh(options, objectKeyRoot, createdAt) {
361
- const candidates = this.ctx.storage.transactionSync(() => {
362
- const matches = this.findMatchingEntryRows(options);
363
- const reservations = matches.flatMap((row) => {
364
- return storedEntryFromRow(row)?.revalidator ? [{
365
- keyHash: row.key_hash,
366
- objectKey: `${objectKeyRoot}/${row.key_hash}/${row.latest_revision + 1}`,
367
- revision: row.latest_revision + 1
368
- }] : [];
369
- });
370
- for (const batch of batches(reservations, MAX_SQL_PARAMETERS)) {
371
- const keyHashes = batch.map(({ keyHash }) => keyHash);
372
- this.ctx.storage.sql.exec(`UPDATE entries SET latest_revision = latest_revision + 1
373
- WHERE key_hash IN (${keyHashes.map(() => "?").join(", ")})`, ...keyHashes);
374
- }
375
- for (const batch of batches(reservations, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_objects
376
- (object_key, created_at, invalidation_sequence) VALUES ${batch.map(() => "(?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))").join(", ")}`, ...batch.flatMap(({ objectKey }) => [objectKey, createdAt]));
377
- const byKey = new Map(reservations.map((reservation) => [reservation.keyHash, reservation]));
378
- return matches.flatMap((row) => {
379
- const entry = storedEntryFromRow(row);
380
- if (!entry) return [];
381
- const reservation = byKey.get(row.key_hash);
382
- return [{
383
- entry,
384
- ...reservation ? { reservation: {
385
- objectKey: reservation.objectKey,
386
- revision: reservation.revision
387
- } } : {}
388
- }];
389
- });
390
- });
391
- if (candidates.some(({ reservation }) => reservation)) await this.ensureCleanupAlarm(createdAt);
392
- return candidates;
505
+ findRefreshCandidates(options, projection) {
506
+ if (projection === void 0) return storedEntriesFromRows(this.findMatchingEntryRows(options));
507
+ return this.findMatchingEntryRows(options, "refresh").map((row) => ({
508
+ keyHash: row.key_hash,
509
+ cacheKey: row.cache_key,
510
+ activeRevision: row.active_revision,
511
+ latestRevision: row.latest_revision,
512
+ hasRevalidator: row.has_revalidator === 1
513
+ }));
393
514
  }
394
- async purgeMatching(options) {
395
- const invalidatedAt = Date.now();
396
- const matches = this.ctx.storage.transactionSync(() => {
397
- const matches = this.findMatchingEntryRows(options, true);
515
+ async purgeMatching(options, invalidatedAt = Date.now()) {
516
+ const reservation = this.ctx.storage.transactionSync(() => {
517
+ const matches = this.findMatchingEntryRows(options, "purge");
518
+ const tombstoneSequence = this.tombstoneSequence(matches.some((row) => row.object_key !== null));
398
519
  const tags = normalizeTags(options.tags ?? []);
399
- if (tags.length) this.ctx.storage.sql.exec(`UPDATE metadata_state SET tag_invalidation_sequence = tag_invalidation_sequence + 1
400
- WHERE singleton = 1`);
520
+ this.ctx.storage.sql.exec(`UPDATE metadata_state SET tag_invalidation_sequence = tag_invalidation_sequence + 1
521
+ WHERE singleton = 1`);
522
+ for (const batch of batches(matches, MAX_SQL_PARAMETERS)) this.ctx.storage.sql.exec(`INSERT INTO key_invalidations (key_hash, invalidation_sequence) VALUES ${batch.map(() => "(?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))").join(", ")}
523
+ ON CONFLICT(key_hash) DO UPDATE SET invalidation_sequence =
524
+ MAX(key_invalidations.invalidation_sequence, excluded.invalidation_sequence)`, ...batch.map((row) => row.key_hash));
401
525
  for (const batch of batches(tags, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`INSERT INTO tag_invalidations
402
526
  (tag, invalidated_at, invalidation_sequence) VALUES ${batch.map(() => "(?, ?, (SELECT tag_invalidation_sequence FROM metadata_state WHERE singleton = 1))").join(", ")}
403
527
  ON CONFLICT(tag) DO UPDATE SET invalidated_at =
@@ -407,11 +531,17 @@ var CacheMetadata = class extends DurableObject {
407
531
  excluded.invalidation_sequence
408
532
  )`, ...batch.flatMap((tag) => [tag, invalidatedAt]));
409
533
  for (const batch of batches(matches, 99)) {
534
+ const active = batch.filter((row) => row.object_key !== null);
535
+ for (const entryBatch of batches(active, MAX_SQL_PARAMETERS / 5)) this.ctx.storage.sql.exec(`INSERT OR REPLACE INTO pending_r2_tombstones
536
+ (key_hash, cache_key, object_key, revision, tombstone_sequence) VALUES ${entryBatch.map(() => "(?, ?, ?, ?, ?)").join(", ")}`, ...entryBatch.flatMap((row) => [
537
+ row.key_hash,
538
+ row.cache_key,
539
+ row.object_key,
540
+ row.latest_revision + 1,
541
+ tombstoneSequence
542
+ ]));
410
543
  const keyHashes = batch.map((row) => row.key_hash);
411
544
  const placeholders = keyHashes.map(() => "?").join(", ");
412
- this.ctx.storage.sql.exec(`INSERT OR IGNORE INTO pending_objects (object_key, created_at, publishable)
413
- SELECT object_key, ?, 0 FROM entries
414
- WHERE key_hash IN (${placeholders}) AND object_key IS NOT NULL`, invalidatedAt, ...keyHashes);
415
545
  this.ctx.storage.sql.exec(`UPDATE entries SET
416
546
  latest_revision = latest_revision + 1,
417
547
  active_revision = latest_revision + 1,
@@ -426,18 +556,103 @@ var CacheMetadata = class extends DurableObject {
426
556
  tombstoned = 1
427
557
  WHERE key_hash IN (${placeholders})`, ...keyHashes);
428
558
  this.ctx.storage.sql.exec(`DELETE FROM revalidation_claims WHERE key_hash IN (${placeholders})`, ...keyHashes);
559
+ this.ctx.storage.sql.exec(`DELETE FROM entry_tags WHERE key_hash IN (${placeholders})`, ...keyHashes);
429
560
  }
430
- return matches.flatMap((row) => row.object_key === null ? [] : [{
431
- keyHash: row.key_hash,
432
- cacheKey: row.cache_key,
433
- objectKey: row.object_key
434
- }]);
561
+ return {
562
+ backingStoreUpdated: matches.length > 0 || tags.length > 0,
563
+ pendingTombstones: matches.filter((row) => row.object_key !== null).length,
564
+ tombstoneSequence
565
+ };
435
566
  });
436
- if (matches.length) {
437
- await this.maintainCleanupAlarm(invalidatedAt);
438
- await this.deleteTrackedObjects(matches.map((entry) => entry.objectKey));
567
+ if (reservation.pendingTombstones > 0) await this.scheduleCleanupAlarm(Date.now() + ORPHAN_CLEANUP_RETRY_MS).catch((error) => this.logCleanupFailure(error));
568
+ return reservation;
569
+ }
570
+ async writeR2Tombstone(entry) {
571
+ let etag;
572
+ for (let attempt = 0; attempt < MAX_R2_CAS_ATTEMPTS; attempt++) {
573
+ const current = await this.env.CACHE_BODIES.head(entry.objectKey);
574
+ const currentRevision = metadataInteger(current?.customMetadata?.latestRevision);
575
+ if (currentRevision !== void 0 && currentRevision >= entry.revision) return;
576
+ etag = current?.etag ?? null;
577
+ if (await this.env.CACHE_BODIES.put(entry.objectKey, /* @__PURE__ */ new Uint8Array(), {
578
+ onlyIf: etag === null ? { etagDoesNotMatch: "*" } : { etagMatches: etag },
579
+ customMetadata: {
580
+ latestRevision: String(entry.revision),
581
+ tombstoned: "1"
582
+ }
583
+ })) return;
584
+ }
585
+ const currentRevision = metadataInteger((await this.env.CACHE_BODIES.head(entry.objectKey))?.customMetadata?.latestRevision);
586
+ if (currentRevision !== void 0 && currentRevision >= entry.revision) return;
587
+ throw new Error(`R2 revision ${entry.revision} could not be tombstoned after concurrent writes`);
588
+ }
589
+ finishCompletedTombstones(entries) {
590
+ for (const batch of batches(entries, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`DELETE FROM pending_r2_tombstones
591
+ WHERE r2_complete = 1 AND edge_purge_complete = 1 AND (${batch.map(() => "(key_hash = ? AND revision = ?)").join(" OR ")})`, ...batch.flatMap((entry) => [entry.keyHash, entry.revision]));
592
+ }
593
+ async drainPendingTombstones(limit, keyHash, afterKeyHash, tombstoneSequence) {
594
+ const filters = ["r2_complete = 0"];
595
+ const parameters = [];
596
+ if (keyHash !== void 0) {
597
+ filters.push("key_hash = ?");
598
+ parameters.push(keyHash);
439
599
  }
440
- return matches;
600
+ if (afterKeyHash !== void 0) {
601
+ filters.push("key_hash > ?");
602
+ parameters.push(afterKeyHash);
603
+ }
604
+ if (tombstoneSequence !== void 0) {
605
+ filters.push("tombstone_sequence = ?");
606
+ parameters.push(tombstoneSequence);
607
+ }
608
+ parameters.push(limit);
609
+ const rows = this.ctx.storage.sql.exec(`SELECT key_hash, cache_key, object_key, revision, r2_complete, edge_purge_complete
610
+ FROM pending_r2_tombstones
611
+ WHERE ${filters.join(" AND ")}
612
+ ORDER BY key_hash LIMIT ?`, ...parameters).toArray();
613
+ const pending = rows.map((row) => ({
614
+ keyHash: row.key_hash,
615
+ cacheKey: row.cache_key,
616
+ objectKey: row.object_key,
617
+ revision: row.revision
618
+ }));
619
+ const settled = await mapSettledWithR2Concurrency(pending, async (entry) => {
620
+ await this.writeR2Tombstone(entry);
621
+ return entry;
622
+ });
623
+ const purged = settled.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
624
+ this.ctx.storage.transactionSync(() => {
625
+ for (const batch of batches(purged, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`UPDATE pending_r2_tombstones SET r2_complete = 1 WHERE ${batch.map(() => "(key_hash = ? AND revision = ?)").join(" OR ")}`, ...batch.flatMap((entry) => [entry.keyHash, entry.revision]));
626
+ this.finishCompletedTombstones(purged);
627
+ });
628
+ return {
629
+ ...rows.length ? { cursor: rows.at(-1).key_hash } : {},
630
+ failures: settled.flatMap((result) => result.status === "rejected" ? [result.reason instanceof Error ? result.reason.message : String(result.reason)] : []),
631
+ pending,
632
+ purged
633
+ };
634
+ }
635
+ listPendingEdgePurges(limit, tombstoneSequence) {
636
+ const sequenceFilter = tombstoneSequence === void 0 ? "" : " AND tombstone_sequence = ?";
637
+ return this.ctx.storage.sql.exec(`SELECT key_hash, cache_key, object_key, revision, r2_complete, edge_purge_complete
638
+ FROM pending_r2_tombstones
639
+ WHERE r2_complete = 1 AND edge_purge_complete = 0${sequenceFilter}
640
+ ORDER BY key_hash LIMIT ?`, ...tombstoneSequence === void 0 ? [] : [tombstoneSequence], limit).toArray().map((row) => ({
641
+ keyHash: row.key_hash,
642
+ cacheKey: row.cache_key,
643
+ objectKey: row.object_key,
644
+ revision: row.revision
645
+ }));
646
+ }
647
+ markTombstonesEdgePurged(entries) {
648
+ this.ctx.storage.transactionSync(() => {
649
+ for (const batch of batches(entries, MAX_SQL_PARAMETERS / 2)) this.ctx.storage.sql.exec(`UPDATE pending_r2_tombstones SET edge_purge_complete = 1 WHERE ${batch.map(() => "(key_hash = ? AND revision = ?)").join(" OR ")}`, ...batch.flatMap((entry) => [entry.keyHash, entry.revision]));
650
+ this.finishCompletedTombstones(entries);
651
+ });
652
+ }
653
+ markTombstonesEdgePurgedThrough(tombstoneSequence) {
654
+ this.ctx.storage.sql.exec(`DELETE FROM pending_r2_tombstones
655
+ WHERE r2_complete = 1 AND edge_purge_complete = 0 AND tombstone_sequence <= ?`, tombstoneSequence);
441
656
  }
442
657
  inspect() {
443
658
  return storedEntriesFromRows(this.ctx.storage.sql.exec("SELECT * FROM entries ORDER BY cache_key").toArray());