@ixo/oracle-runtime-workers 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ixo/oracle-runtime-workers",
3
- "version": "0.9.0",
3
+ "version": "0.10.0",
4
4
  "description": "QiForge oracle runtime for Cloudflare Workers — Durable Object per user (SQLite in WASM over DO storage), Matrix E2EE gateway Durable Object, Hono HTTP/SSE shell.",
5
5
  "private": false,
6
6
  "publishConfig": {
package/src/core/env.ts CHANGED
@@ -76,6 +76,15 @@ export const baseEnvSchema = z.object({
76
76
  /** UCAN store worker base URL (defaults per NETWORK). */
77
77
  UCAN_STORE_URL: z.string().url().optional(),
78
78
 
79
+ /**
80
+ * R2 page tier knobs (the bucket itself is a binding, `TIER_BUCKET`, not
81
+ * a var). Hot-set target in bytes (or `16m`), periods of inactivity before
82
+ * a chunk is evicted, and the period length in ms (a day; tests shorten it).
83
+ */
84
+ TIER_HOT_BUDGET_BYTES: z.string().min(1).optional(),
85
+ TIER_EVICT_AFTER_PERIODS: z.coerce.number().int().min(1).optional(),
86
+ TIER_PERIOD_MS: z.coerce.number().int().min(1000).optional(),
87
+
79
88
  // --- llm ----------------------------------------------------------------
80
89
  OPEN_ROUTER_API_KEY: z.string().min(1),
81
90
  /**
@@ -1,6 +1,7 @@
1
1
  import type { BotStatus, EncryptedFileInfo } from '@ixo/matrix-bot-workers-sdk';
2
2
  import type { AttachmentInput } from '../attachments/types';
3
3
  import type { RealtimeStatus } from '../realtime/realtime-endpoint';
4
+ import type { TierFlushResult, TierStatus } from '../sqlite/do-vfs';
4
5
  /**
5
6
  * Cross-object contracts for the Workers runtime.
6
7
  *
@@ -82,6 +83,20 @@ export interface OracleWorkerEnv {
82
83
  * for the hit/miss counters that show what a budget buys.
83
84
  */
84
85
  CHUNK_CACHE_BYTES?: string;
86
+ /**
87
+ * R2 page tier for the users' working copies (see `sqlite/page-tier.ts`):
88
+ * chunks no turn touched for `TIER_EVICT_AFTER_PERIODS` periods move
89
+ * from DO SQLite ($0.20/GB-month, 10 GB cap per object) into 1 MiB R2
90
+ * segment objects ($0.015/GB-month, no cap) under a per-object prefix.
91
+ * Absent = every chunk stays in DO storage (the pre-tier behaviour).
92
+ */
93
+ TIER_BUCKET?: R2Bucket;
94
+ /** Soft target for hot bytes per user object (bytes or `16m`; default 16 MiB). */
95
+ TIER_HOT_BUDGET_BYTES?: string;
96
+ /** Periods (of `TIER_PERIOD_MS`) a chunk must go untouched before eviction (default 2). */
97
+ TIER_EVICT_AFTER_PERIODS?: string;
98
+ /** Length of one access-tracking period in ms (default one day; tests shorten it). */
99
+ TIER_PERIOD_MS?: string;
85
100
  /** IXO VFS worker base URL when `OWNER_STORE=vfs` (defaults per NETWORK). */
86
101
  VFS_BASE_URL?: string;
87
102
  /** UCAN store worker base URL when `OWNER_STORE=vfs` (defaults per NETWORK). */
@@ -284,6 +299,8 @@ export interface StorageStatus {
284
299
  freelistCount?: number;
285
300
  /** SQLite's own page cache (`PRAGMA cache_size`; negative = KiB). */
286
301
  sqliteCacheSize?: number;
302
+ /** The R2 page tier of this object's working copy (see sqlite/page-tier.ts). */
303
+ tier?: TierStatus & { lastPassAt?: number };
287
304
  /** The DO VFS clean-chunk cache and its storage counters (isolate lifetime). */
288
305
  chunkCache?: {
289
306
  budgetBytes: number;
@@ -356,6 +373,11 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
356
373
  * the "user deleted their file upstream" path.
357
374
  */
358
375
  resetWorkingCopy(): Promise<{ reloadedFromOwnerStore: boolean }>;
376
+ /** Run one R2 page-tier eviction pass now (debug routes; `force` ignores recency). */
377
+ tierFlush(opts?: {
378
+ force?: boolean;
379
+ maxSegments?: number;
380
+ }): Promise<TierFlushResult>;
359
381
  /** Raw upstream schema of the memory search tool as fetched right now. */
360
382
  debugMemorySchema(userDid: string): Promise<MemorySchemaDebug>;
361
383
  /** Forcibly reset the object like a platform host drain (debug routes only). The call itself rejects. */
@@ -83,12 +83,17 @@ import {
83
83
  finishCompaction,
84
84
  type CompactCursors,
85
85
  } from '../sqlite/blob-compactor';
86
- import { createHash } from 'node:crypto';
87
86
  import {
88
87
  cachePagesForBytes,
88
+ parseByteSize,
89
89
  parseChunkCacheBytes,
90
90
  } from '../sqlite/cache-config';
91
- import { DoSqliteDatabase } from '../sqlite/database';
91
+ import { DoSqliteDatabase, type DoSqliteOpenOptions } from '../sqlite/database';
92
+ import type { TierFlushResult } from '../sqlite/do-vfs';
93
+ import {
94
+ DEFAULT_TIER_HOT_BUDGET_BYTES,
95
+ type PageTierOptions,
96
+ } from '../sqlite/page-tier';
92
97
  import {
93
98
  shouldVacuum,
94
99
  vacuumWanted,
@@ -107,6 +112,7 @@ import {
107
112
  } from '../sqlite/sessions-store';
108
113
  import { createAttachmentDecryptor } from '@ixo/matrix-bot-workers-sdk';
109
114
  import { MatrixMediaOwnerStore } from '../owner-store/matrix-media-store';
115
+ import { measureForSave } from '../owner-store/measure';
110
116
  import { MigratingOwnerStore } from '../owner-store/migrating-store';
111
117
  import type { OwnerCopy } from '../owner-store/types';
112
118
  import {
@@ -224,6 +230,14 @@ interface MatrixUserIdCache {
224
230
  /** Consecutive flush failures (cleared on success). */
225
231
  const META_FLUSH_FAILURES = 'meta:flushFailures';
226
232
  const META_LAST_VACUUM = 'meta:lastVacuumAt';
233
+ /** When the last R2 page-tier eviction pass ran (see `tierTick`). */
234
+ const META_LAST_TIER_PASS = 'meta:lastTierPassAt';
235
+ /**
236
+ * Eviction passes at most this often per object. The horizon is two daily
237
+ * periods, so four checks a day catch chunks the moment they age out
238
+ * without re-reading the hot set more than that.
239
+ */
240
+ const TIER_PASS_INTERVAL_MS = 6 * 60 * 60_000;
227
241
  /**
228
242
  * Persisted twin of the in-memory `dirty` flag. The flag alone dies with an
229
243
  * eviction, and evictions are routine — without the marker an alarm waking a
@@ -314,22 +328,6 @@ interface FlushResult {
314
328
  }
315
329
 
316
330
  /** Hex SHA-256 of a byte stream, hashed as it flows. */
317
- async function sha256OfStream(
318
- stream: ReadableStream<Uint8Array>,
319
- ): Promise<string> {
320
- const hash = createHash('sha256');
321
- const reader = stream.getReader();
322
- try {
323
- for (;;) {
324
- const { done, value } = await reader.read();
325
- if (done) break;
326
- hash.update(value);
327
- }
328
- } finally {
329
- reader.releaseLock();
330
- }
331
- return hash.digest('hex');
332
- }
333
331
 
334
332
  /** Does this adapter expose provider config + role resolution (platform adapters do)? */
335
333
  function isProviderAdapter(llm: LlmAdapter): llm is OpenRouterLlmAdapter {
@@ -1131,7 +1129,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1131
1129
  if (db) {
1132
1130
  await db.close().catch(() => undefined);
1133
1131
  }
1134
- await DoSqliteDatabase.wipe(this.ctx, DB_FILE);
1132
+ const tier = this.tierOptions();
1133
+ await DoSqliteDatabase.wipe(this.ctx, DB_FILE, tier ? { tier } : {});
1135
1134
  await this.ctx.storage.delete([
1136
1135
  META_OWNER_ETAG,
1137
1136
  META_LAST_CHECKSUM,
@@ -1142,12 +1141,40 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1142
1141
  ]);
1143
1142
  }
1144
1143
 
1145
- /** `CHUNK_CACHE_BYTES` → the VFS clean-chunk budget for this object. */
1146
- private sqliteOpenOptions(): { cachePages: number } {
1144
+ /**
1145
+ * `CHUNK_CACHE_BYTES` → the VFS clean-chunk budget for this object, and
1146
+ * the R2 page tier when the script binds `TIER_BUCKET` (the object's id
1147
+ * is its key prefix; the knobs are documented in `contracts.ts`).
1148
+ */
1149
+ private sqliteOpenOptions(): DoSqliteOpenOptions {
1147
1150
  const bytes = parseChunkCacheBytes(this.env.CHUNK_CACHE_BYTES, (m) =>
1148
1151
  console.warn(`[user-do] ${m}`),
1149
1152
  );
1150
- return { cachePages: cachePagesForBytes(bytes) };
1153
+ const tier = this.tierOptions();
1154
+ return { cachePages: cachePagesForBytes(bytes), ...(tier && { tier }) };
1155
+ }
1156
+
1157
+ private tierOptions(): PageTierOptions | undefined {
1158
+ const bucket = this.env.TIER_BUCKET;
1159
+ if (!bucket) return undefined;
1160
+ const warn = (m: string): void => console.warn(`[user-do] ${m}`);
1161
+ const hotBudgetBytes = parseByteSize(this.env.TIER_HOT_BUDGET_BYTES, {
1162
+ name: 'TIER_HOT_BUDGET_BYTES',
1163
+ fallback: DEFAULT_TIER_HOT_BUDGET_BYTES,
1164
+ min: 1024 * 1024,
1165
+ max: 1024 * 1024 * 1024,
1166
+ warn,
1167
+ });
1168
+ const periods = Number(this.env.TIER_EVICT_AFTER_PERIODS);
1169
+ const periodMs = Number(this.env.TIER_PERIOD_MS);
1170
+ return {
1171
+ bucket,
1172
+ prefix: this.ctx.id.toString(),
1173
+ hotBudgetBytes,
1174
+ ...(Number.isInteger(periods) &&
1175
+ periods >= 1 && { evictAfterPeriods: periods }),
1176
+ ...(Number.isFinite(periodMs) && periodMs >= 1000 && { periodMs }),
1177
+ };
1151
1178
  }
1152
1179
 
1153
1180
  /**
@@ -1333,6 +1360,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1333
1360
  if (next !== null) deadlines.push(next);
1334
1361
  }
1335
1362
 
1363
+ // R2 page tier: move chunks no turn touched for a couple of days out
1364
+ // of DO storage (see sqlite/page-tier.ts). After the flush above (the
1365
+ // upload reads hot rows, not R2) and never over an in-flight one (it
1366
+ // holds a snapshot). A pass is capped; more work re-arms soon.
1367
+ if (this.db?.tierEnabled && !this.flushInFlight) {
1368
+ const next = await this.tierTick(this.db, now);
1369
+ if (next !== null) deadlines.push(next);
1370
+ }
1371
+
1336
1372
  // Idle housekeeping: a user silent for a day loses the cached pages —
1337
1373
  // their file is safe upstream, and we stop paying to store a copy. Never
1338
1374
  // evict while tasks are scheduled: their runs ARE activity. The flush
@@ -1361,6 +1397,71 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1361
1397
  );
1362
1398
  }
1363
1399
 
1400
+ /**
1401
+ * One page-tier eviction pass, at most once per `TIER_PASS_INTERVAL_MS`
1402
+ * (a pass that hit its segment cap re-arms in a minute). Returns the next
1403
+ * time a pass is due, or null when nothing is pending.
1404
+ */
1405
+ private async tierTick(
1406
+ db: DoSqliteDatabase,
1407
+ now: number,
1408
+ ): Promise<number | null> {
1409
+ const last =
1410
+ (await this.ctx.storage.get<number>(META_LAST_TIER_PASS)) ?? 0;
1411
+ if (now - last < TIER_PASS_INTERVAL_MS)
1412
+ return last + TIER_PASS_INTERVAL_MS;
1413
+ try {
1414
+ const pass = await db.tierFlush();
1415
+ await this.ctx.storage.put(META_LAST_TIER_PASS, now);
1416
+ if (pass.skipped) {
1417
+ if (pass.skipped !== 'no-tier')
1418
+ console.log(
1419
+ `[user-do] tier pass skipped for ${this.userDid}: ${pass.skipped}`,
1420
+ );
1421
+ return now + 60_000;
1422
+ }
1423
+ if (pass.evictedChunks > 0 || pass.remaining > 0) {
1424
+ console.log(
1425
+ `[user-do] tier pass for ${this.userDid}: ${pass.evictedChunks} chunks → R2 in ${pass.segmentsRewritten} segment(s), ${pass.hotRows} hot rows (${Math.round(pass.hotBytes / 1024 / 1024)} MB) kept, ${pass.remaining} segment(s) pending`,
1426
+ );
1427
+ }
1428
+ const budget = this.tierOptions()?.hotBudgetBytes;
1429
+ if (
1430
+ budget !== undefined &&
1431
+ pass.remaining === 0 &&
1432
+ pass.hotBytes > budget
1433
+ ) {
1434
+ // Everything left is recent: the user's real working set exceeds
1435
+ // the target — a sizing signal, not a failure.
1436
+ console.warn(
1437
+ `[user-do] tier hot set of ${this.userDid} is ${Math.round(pass.hotBytes / 1024 / 1024)} MB, over the ${Math.round(budget / 1024 / 1024)} MB target (all of it touched within the eviction horizon)`,
1438
+ );
1439
+ }
1440
+ return pass.remaining > 0 ? now + 60_000 : now + TIER_PASS_INTERVAL_MS;
1441
+ } catch (err) {
1442
+ console.error(
1443
+ `[user-do] tier pass failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
1444
+ );
1445
+ return now + FLUSH_RETRY_DELAY_MS;
1446
+ }
1447
+ }
1448
+
1449
+ /** Operator/test entry: run an eviction pass now (debug routes). */
1450
+ async tierFlush(
1451
+ opts: { force?: boolean; maxSegments?: number } = {},
1452
+ ): Promise<TierFlushResult> {
1453
+ if (!this.db) {
1454
+ const userDid =
1455
+ this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
1456
+ if (userDid) await this.ready({ userDid });
1457
+ }
1458
+ if (!this.db) throw new Error('no working copy');
1459
+ if (this.flushInFlight) await this.flushInFlight.catch(() => undefined);
1460
+ const pass = await this.db.tierFlush(opts);
1461
+ await this.ctx.storage.put(META_LAST_TIER_PASS, Date.now());
1462
+ return pass;
1463
+ }
1464
+
1364
1465
  /**
1365
1466
  * One VACUUM policy check. Returns the time of a follow-up check when a
1366
1467
  * rebuild is wanted but the object is not quiet yet, else null.
@@ -2015,7 +2116,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2015
2116
 
2016
2117
  const snapshot = await db.snapshot();
2017
2118
  try {
2018
- const checksum = await sha256OfStream(snapshot.open());
2119
+ // One pass yields the change gate (hash) AND the length the store
2120
+ // needs to upload; the upload is the only other read of the file.
2121
+ const { sha256Hex: checksum, gzippedLength } = await measureForSave(
2122
+ snapshot.open(),
2123
+ );
2019
2124
  const last = await this.ctx.storage.get<string>(META_LAST_CHECKSUM);
2020
2125
  if (last === checksum) {
2021
2126
  await this.ctx.storage.put(META_UPLOADED_GEN, snapshot.generation);
@@ -2030,7 +2135,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2030
2135
 
2031
2136
  let saved: { etag: string; bytes: number };
2032
2137
  try {
2033
- saved = await store.save(snapshot);
2138
+ saved = await store.save(snapshot, { gzippedLength });
2034
2139
  } catch (err) {
2035
2140
  // Never diverted anywhere else: the working copy in Durable Object
2036
2141
  // storage is durable, stays dirty, and is retried. Loud from the
@@ -2295,6 +2400,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2295
2400
  rowsWritten: vfs.rowsWritten,
2296
2401
  }
2297
2402
  : undefined,
2403
+ tier: db
2404
+ ? {
2405
+ ...db.tierStatus(),
2406
+ lastPassAt:
2407
+ await this.ctx.storage.get<number>(META_LAST_TIER_PASS),
2408
+ }
2409
+ : undefined,
2298
2410
  };
2299
2411
  }
2300
2412
 
@@ -3235,6 +3347,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3235
3347
  await sessions.setTitle(sessionId, title, { onlyIfUntitled: true });
3236
3348
  }
3237
3349
  this.markDirty();
3350
+ // Which chunks this turn touched, for the page tier's eviction policy
3351
+ // (one row write; a no-op without a tier).
3352
+ try {
3353
+ this.db?.recordAccess();
3354
+ } catch (err) {
3355
+ console.warn(
3356
+ `[user-do] tier access record failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
3357
+ );
3358
+ }
3238
3359
  }
3239
3360
 
3240
3361
  private async generateTitle(
@@ -50,6 +50,7 @@ import {
50
50
  type OwnerCopy,
51
51
  type OwnerStore,
52
52
  type SaveResult,
53
+ type SaveHints,
53
54
  } from './types';
54
55
 
55
56
  const VFS_RESOURCE = 'ixo:filesystem';
@@ -233,9 +234,11 @@ export class IxoVfsOwnerStore implements OwnerStore {
233
234
  * Atomic replace, always delete + re-create, never a new version:
234
235
  *
235
236
  * 1. drop stale `.uploading-*` temp files a crashed flush left behind;
236
- * 2. measure the gzipped length in a counting pass (the VFS needs
237
- * `Upload-Length` at session creation — `Upload-Defer-Length` is not
238
- * supported — and gzip output is deterministic for identical input);
237
+ * 2. take the gzipped length from `hints` (the object measured it in the
238
+ * same pass as its checksum) or, without a hint, count it in a pass
239
+ * of its own (the VFS needs `Upload-Length` at session creation —
240
+ * `Upload-Defer-Length` is not supported — and gzip output is
241
+ * deterministic for identical input);
239
242
  * 3. upload gzip → a TEMP path next to the real one: one `POST /files`
240
243
  * when it fits in a single part, else a tus session in 5 MiB parts
241
244
  * (`tusUpload`, resumable per part);
@@ -247,7 +250,10 @@ export class IxoVfsOwnerStore implements OwnerStore {
247
250
  * file is cleaned up then). Peak memory is one part (5 MiB) plus fixed
248
251
  * overhead, whatever the file size.
249
252
  */
250
- async save(snapshot: FileSnapshot): Promise<SaveResult> {
253
+ async save(
254
+ snapshot: FileSnapshot,
255
+ hints: SaveHints = {},
256
+ ): Promise<SaveResult> {
251
257
  const listing = await this.listOracleFiles();
252
258
  const existing = listing.filter((f) => f.path === this.path);
253
259
  const stale = listing.filter((f) =>
@@ -261,7 +267,8 @@ export class IxoVfsOwnerStore implements OwnerStore {
261
267
  });
262
268
  }
263
269
 
264
- const gzLength = await countStream(gzipStream(snapshot.open()));
270
+ const gzLength =
271
+ hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
265
272
  const tempPath = `${this.path}${TEMP_PATH_INFIX}${Date.now().toString(36)}`;
266
273
  const uploaded =
267
274
  gzLength <= SINGLE_SHOT_MAX_BYTES
@@ -22,6 +22,7 @@ import {
22
22
  type OwnerCopy,
23
23
  type OwnerStore,
24
24
  type SaveResult,
25
+ type SaveHints,
25
26
  } from './types';
26
27
 
27
28
  /**
@@ -89,19 +90,23 @@ export class MatrixMediaOwnerStore implements OwnerStore {
89
90
  }
90
91
 
91
92
  /**
92
- * Streamed as well: the snapshot is opened three times — a header peek, a
93
- * gzip pass that only counts (the homeserver needs the exact ciphertext
94
- * length up front; gzip output is deterministic for identical input, the
95
- * VFS store relies on the same property) and the gzip pass that is
96
- * uploaded — and never held whole.
93
+ * Streamed as well: the snapshot is opened for a header peek, for a gzip
94
+ * pass that only counts unless the caller measured the length already
95
+ * (`hints`; the homeserver needs the exact ciphertext length up front and
96
+ * gzip output is deterministic for identical input) and for the gzip pass
97
+ * that is uploaded — and never held whole.
97
98
  */
98
- async save(snapshot: FileSnapshot): Promise<SaveResult> {
99
+ async save(
100
+ snapshot: FileSnapshot,
101
+ hints: SaveHints = {},
102
+ ): Promise<SaveResult> {
99
103
  const checked = await assertSqliteStream(
100
104
  snapshot.open(),
101
105
  `MatrixMediaOwnerStore: refusing to save — the working copy (${snapshot.size} bytes)`,
102
106
  );
103
107
  await checked.cancel().catch(() => undefined);
104
- const gzLength = await countStream(gzipStream(snapshot.open()));
108
+ const gzLength =
109
+ hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
105
110
  const { eventId } = await this.gateway.uploadUserSnapshotStream(
106
111
  this.userDid,
107
112
  this.storageKey,
@@ -0,0 +1,39 @@
1
+ /**
2
+ * One streamed pass over a snapshot that yields BOTH what a flush needs
3
+ * before it uploads: the SHA-256 of the raw SQLite bytes (the "did anything
4
+ * change" gate against the last upload) and the exact gzipped length (the
5
+ * VFS needs `Upload-Length` when the tus session is created). Before this
6
+ * helper the object hashed in one pass and the store counted the gzip in a
7
+ * second one; with the R2 page tier every pass over a cold file is one R2
8
+ * GET per segment, so the two are folded together and the upload itself is
9
+ * the only other read.
10
+ *
11
+ * The length is exact for the store's own gzip: `gzipStream` is the same
12
+ * `CompressionStream('gzip')` the stores pipe the upload through, and its
13
+ * output is deterministic for identical input.
14
+ */
15
+ import { createHash } from 'node:crypto';
16
+ import { countStream, gzipStream } from './types';
17
+
18
+ export interface SnapshotMeasure {
19
+ /** Hex SHA-256 of the raw (uncompressed) bytes. */
20
+ sha256Hex: string;
21
+ /** Byte length of `gzipStream(<the same bytes>)`. */
22
+ gzippedLength: number;
23
+ }
24
+
25
+ export async function measureForSave(
26
+ stream: ReadableStream<Uint8Array>,
27
+ ): Promise<SnapshotMeasure> {
28
+ const hash = createHash('sha256');
29
+ const tapped = stream.pipeThrough(
30
+ new TransformStream<Uint8Array, Uint8Array>({
31
+ transform(chunk, controller) {
32
+ hash.update(chunk);
33
+ controller.enqueue(chunk);
34
+ },
35
+ }),
36
+ );
37
+ const gzippedLength = await countStream(gzipStream(tapped));
38
+ return { sha256Hex: hash.digest('hex'), gzippedLength };
39
+ }
@@ -20,7 +20,13 @@
20
20
  * Matrix copy so no second copy of the user's history lingers.
21
21
  */
22
22
  import { VfsNoDelegationError } from './ixo-vfs-store';
23
- import type { FileSnapshot, OwnerCopy, OwnerStore, SaveResult } from './types';
23
+ import type {
24
+ FileSnapshot,
25
+ OwnerCopy,
26
+ OwnerStore,
27
+ SaveHints,
28
+ SaveResult,
29
+ } from './types';
24
30
 
25
31
  export interface MigratingOwnerStoreOptions {
26
32
  /** System of record (IXO VFS). */
@@ -104,8 +110,8 @@ export class MigratingOwnerStore implements OwnerStore {
104
110
  * "not on VFS yet" (no delegation) — so the object keeps the copy dirty
105
111
  * and retries; Matrix media is never written.
106
112
  */
107
- async save(snapshot: FileSnapshot): Promise<SaveResult> {
108
- return this.primary.save(snapshot);
113
+ async save(snapshot: FileSnapshot, hints?: SaveHints): Promise<SaveResult> {
114
+ return this.primary.save(snapshot, hints);
109
115
  }
110
116
 
111
117
  /**
@@ -15,9 +15,13 @@
15
15
  * Both directions STREAM: `load()` hands back the raw (already gunzipped)
16
16
  * SQLite bytes as a `ReadableStream` the object writes into its VFS chunk by
17
17
  * chunk, and `save()` takes a `FileSnapshot` — a re-openable, consistent
18
- * view of the working copy — that the store reads as many times as it needs
19
- * (the VFS store measures the gzipped length in one pass and uploads in a
20
- * second). Nothing on this path ever holds the whole file in memory — the
18
+ * view of the working copy — that the store reads as many times as it needs.
19
+ * The object measures the hash and the gzipped length in ONE pass
20
+ * (`measureForSave`) and hands the length in as a `SaveHints`, so a store
21
+ * only reads the snapshot once more, for the upload itself; a store called
22
+ * without the hint counts the gzip in a pass of its own. That matters with
23
+ * the R2 page tier, where every pass over a cold file is one R2 GET per
24
+ * segment. Nothing on this path ever holds the whole file in memory — the
21
25
  * legacy Matrix media path included: the gateway hands the media over as a
22
26
  * stream and decryption, gunzip and the SQLite header check run chunk by
23
27
  * chunk in the object. Compression is the store's concern. `etag` lets the object skip a reload
@@ -26,7 +30,7 @@
26
30
  export interface OwnerStore {
27
31
  readonly kind: 'matrix' | 'vfs';
28
32
  load(): Promise<OwnerCopy | null>;
29
- save(snapshot: FileSnapshot): Promise<SaveResult>;
33
+ save(snapshot: FileSnapshot, hints?: SaveHints): Promise<SaveResult>;
30
34
  /** Cheap upstream freshness probe — null when unsupported/unknown. */
31
35
  head(): Promise<{ etag: string } | null>;
32
36
  /** Remove the durable copy (user asked to be forgotten). */
@@ -60,6 +64,12 @@ export interface OwnerCopy {
60
64
  fromLegacy?: true;
61
65
  }
62
66
 
67
+ /** What the caller already measured about the snapshot (see `measureForSave`). */
68
+ export interface SaveHints {
69
+ /** Byte length of `gzipStream(snapshot.open())`; spares the store its counting pass. */
70
+ gzippedLength?: number;
71
+ }
72
+
63
73
  export interface SaveResult {
64
74
  etag: string;
65
75
  /** Bytes actually sent upstream (compressed size on the VFS path). */
package/src/shell/app.ts CHANGED
@@ -471,6 +471,18 @@ export function createShell(
471
471
  app.post('/debug/storage/flush', async (c) =>
472
472
  c.json(await userStub(c.env, c.get('auth').userDid).flushToOwnerStore()),
473
473
  );
474
+ app.post('/debug/storage/tier-flush', async (c) => {
475
+ const body: unknown = await c.req.json().catch(() => ({}));
476
+ const rec = typeof body === 'object' && body !== null ? body : {};
477
+ const force = Reflect.get(rec, 'force') === true;
478
+ const max = Number(Reflect.get(rec, 'maxSegments'));
479
+ return c.json(
480
+ await userStub(c.env, c.get('auth').userDid).tierFlush({
481
+ force,
482
+ ...(Number.isInteger(max) && max > 0 && { maxSegments: max }),
483
+ }),
484
+ );
485
+ });
474
486
  app.post('/debug/storage/reset', async (c) =>
475
487
  c.json(await userStub(c.env, c.get('auth').userDid).resetWorkingCopy()),
476
488
  );
@@ -19,22 +19,43 @@ export function parseChunkCacheBytes(
19
19
  raw: string | undefined,
20
20
  warn: (message: string) => void = () => undefined,
21
21
  ): number {
22
- if (raw === undefined || raw.trim() === '') return DEFAULT_CHUNK_CACHE_BYTES;
22
+ return parseByteSize(raw, {
23
+ name: 'CHUNK_CACHE_BYTES',
24
+ fallback: DEFAULT_CHUNK_CACHE_BYTES,
25
+ min: MIN_CHUNK_CACHE_BYTES,
26
+ max: MAX_CHUNK_CACHE_BYTES,
27
+ warn,
28
+ });
29
+ }
30
+
31
+ /** Same grammar for any byte-sized env knob (`TIER_HOT_BUDGET_BYTES`, ...). */
32
+ export function parseByteSize(
33
+ raw: string | undefined,
34
+ options: {
35
+ name: string;
36
+ fallback: number;
37
+ min: number;
38
+ max: number;
39
+ warn?: (message: string) => void;
40
+ },
41
+ ): number {
42
+ const warn = options.warn ?? (() => undefined);
43
+ if (raw === undefined || raw.trim() === '') return options.fallback;
23
44
  const match = /^\s*(\d+)\s*([kKmM]?)\s*$/.exec(raw);
24
45
  if (!match) {
25
46
  warn(
26
- `CHUNK_CACHE_BYTES=${JSON.stringify(raw)} is not a byte count — using the default ${DEFAULT_CHUNK_CACHE_BYTES}`,
47
+ `${options.name}=${JSON.stringify(raw)} is not a byte count — using the default ${options.fallback}`,
27
48
  );
28
- return DEFAULT_CHUNK_CACHE_BYTES;
49
+ return options.fallback;
29
50
  }
30
51
  const unit = match[2]?.toLowerCase();
31
52
  const multiplier = unit === 'k' ? 1024 : unit === 'm' ? 1024 * 1024 : 1;
32
53
  const bytes = Number(match[1]) * multiplier;
33
- if (bytes < MIN_CHUNK_CACHE_BYTES || bytes > MAX_CHUNK_CACHE_BYTES) {
54
+ if (bytes < options.min || bytes > options.max) {
34
55
  warn(
35
- `CHUNK_CACHE_BYTES=${raw} is outside ${MIN_CHUNK_CACHE_BYTES}..${MAX_CHUNK_CACHE_BYTES} — using the default ${DEFAULT_CHUNK_CACHE_BYTES}`,
56
+ `${options.name}=${raw} is outside ${options.min}..${options.max} — using the default ${options.fallback}`,
36
57
  );
37
- return DEFAULT_CHUNK_CACHE_BYTES;
58
+ return options.fallback;
38
59
  }
39
60
  return bytes;
40
61
  }
@@ -0,0 +1,14 @@
1
+ /**
2
+ * The storage grain shared by the DO VFS and the R2 page tier. Kept in its
3
+ * own module so `page-tier.ts` can import it without a cycle through
4
+ * `do-vfs.ts`.
5
+ */
6
+
7
+ /** Bytes per SQLite page. Matches `PRAGMA page_size=4096`. */
8
+ export const VFS_PAGE_SIZE = 4096;
9
+
10
+ /** SQLite pages packed into one stored row (billing grain). */
11
+ export const PAGES_PER_CHUNK = 16;
12
+
13
+ /** Bytes per stored chunk row (64 KiB — well under the 2 MB DO value cap). */
14
+ export const CHUNK_SIZE = VFS_PAGE_SIZE * PAGES_PER_CHUNK;