@ixo/oracle-runtime-workers 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/core/env.ts +9 -0
- package/src/do/contracts.ts +22 -0
- package/src/do/user-oracle-do.ts +145 -24
- package/src/owner-store/ixo-vfs-store.ts +12 -5
- package/src/owner-store/matrix-media-store.ts +12 -7
- package/src/owner-store/measure.ts +39 -0
- package/src/owner-store/migrating-store.ts +9 -3
- package/src/owner-store/types.ts +14 -4
- package/src/shell/app.ts +12 -0
- package/src/sqlite/cache-config.ts +27 -6
- package/src/sqlite/chunk-layout.ts +14 -0
- package/src/sqlite/database.ts +188 -19
- package/src/sqlite/do-vfs.ts +963 -99
- package/src/sqlite/index.ts +13 -0
- package/src/sqlite/page-tier.ts +668 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ixo/oracle-runtime-workers",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "QiForge oracle runtime for Cloudflare Workers — Durable Object per user (SQLite in WASM over DO storage), Matrix E2EE gateway Durable Object, Hono HTTP/SSE shell.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"publishConfig": {
|
package/src/core/env.ts
CHANGED
|
@@ -76,6 +76,15 @@ export const baseEnvSchema = z.object({
|
|
|
76
76
|
/** UCAN store worker base URL (defaults per NETWORK). */
|
|
77
77
|
UCAN_STORE_URL: z.string().url().optional(),
|
|
78
78
|
|
|
79
|
+
/**
|
|
80
|
+
* R2 page tier knobs (the bucket itself is a binding, `TIER_BUCKET`, not
|
|
81
|
+
* a var). Hot-set target in bytes (or `16m`), periods of inactivity before
|
|
82
|
+
* a chunk is evicted, and the period length in ms (a day; tests shorten it).
|
|
83
|
+
*/
|
|
84
|
+
TIER_HOT_BUDGET_BYTES: z.string().min(1).optional(),
|
|
85
|
+
TIER_EVICT_AFTER_PERIODS: z.coerce.number().int().min(1).optional(),
|
|
86
|
+
TIER_PERIOD_MS: z.coerce.number().int().min(1000).optional(),
|
|
87
|
+
|
|
79
88
|
// --- llm ----------------------------------------------------------------
|
|
80
89
|
OPEN_ROUTER_API_KEY: z.string().min(1),
|
|
81
90
|
/**
|
package/src/do/contracts.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BotStatus, EncryptedFileInfo } from '@ixo/matrix-bot-workers-sdk';
|
|
2
2
|
import type { AttachmentInput } from '../attachments/types';
|
|
3
3
|
import type { RealtimeStatus } from '../realtime/realtime-endpoint';
|
|
4
|
+
import type { TierFlushResult, TierStatus } from '../sqlite/do-vfs';
|
|
4
5
|
/**
|
|
5
6
|
* Cross-object contracts for the Workers runtime.
|
|
6
7
|
*
|
|
@@ -82,6 +83,20 @@ export interface OracleWorkerEnv {
|
|
|
82
83
|
* for the hit/miss counters that show what a budget buys.
|
|
83
84
|
*/
|
|
84
85
|
CHUNK_CACHE_BYTES?: string;
|
|
86
|
+
/**
|
|
87
|
+
* R2 page tier for the users' working copies (see `sqlite/page-tier.ts`):
|
|
88
|
+
* chunks no turn touched for `TIER_EVICT_AFTER_PERIODS` periods move
|
|
89
|
+
* from DO SQLite ($0.20/GB-month, 10 GB cap per object) into 1 MiB R2
|
|
90
|
+
* segment objects ($0.015/GB-month, no cap) under a per-object prefix.
|
|
91
|
+
* Absent = every chunk stays in DO storage (the pre-tier behaviour).
|
|
92
|
+
*/
|
|
93
|
+
TIER_BUCKET?: R2Bucket;
|
|
94
|
+
/** Soft target for hot bytes per user object (bytes or `16m`; default 16 MiB). */
|
|
95
|
+
TIER_HOT_BUDGET_BYTES?: string;
|
|
96
|
+
/** Periods (of `TIER_PERIOD_MS`) a chunk must go untouched before eviction (default 2). */
|
|
97
|
+
TIER_EVICT_AFTER_PERIODS?: string;
|
|
98
|
+
/** Length of one access-tracking period in ms (default one day; tests shorten it). */
|
|
99
|
+
TIER_PERIOD_MS?: string;
|
|
85
100
|
/** IXO VFS worker base URL when `OWNER_STORE=vfs` (defaults per NETWORK). */
|
|
86
101
|
VFS_BASE_URL?: string;
|
|
87
102
|
/** UCAN store worker base URL when `OWNER_STORE=vfs` (defaults per NETWORK). */
|
|
@@ -284,6 +299,8 @@ export interface StorageStatus {
|
|
|
284
299
|
freelistCount?: number;
|
|
285
300
|
/** SQLite's own page cache (`PRAGMA cache_size`; negative = KiB). */
|
|
286
301
|
sqliteCacheSize?: number;
|
|
302
|
+
/** The R2 page tier of this object's working copy (see sqlite/page-tier.ts). */
|
|
303
|
+
tier?: TierStatus & { lastPassAt?: number };
|
|
287
304
|
/** The DO VFS clean-chunk cache and its storage counters (isolate lifetime). */
|
|
288
305
|
chunkCache?: {
|
|
289
306
|
budgetBytes: number;
|
|
@@ -356,6 +373,11 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
|
|
|
356
373
|
* the "user deleted their file upstream" path.
|
|
357
374
|
*/
|
|
358
375
|
resetWorkingCopy(): Promise<{ reloadedFromOwnerStore: boolean }>;
|
|
376
|
+
/** Run one R2 page-tier eviction pass now (debug routes; `force` ignores recency). */
|
|
377
|
+
tierFlush(opts?: {
|
|
378
|
+
force?: boolean;
|
|
379
|
+
maxSegments?: number;
|
|
380
|
+
}): Promise<TierFlushResult>;
|
|
359
381
|
/** Raw upstream schema of the memory search tool as fetched right now. */
|
|
360
382
|
debugMemorySchema(userDid: string): Promise<MemorySchemaDebug>;
|
|
361
383
|
/** Forcibly reset the object like a platform host drain (debug routes only). The call itself rejects. */
|
package/src/do/user-oracle-do.ts
CHANGED
|
@@ -83,12 +83,17 @@ import {
|
|
|
83
83
|
finishCompaction,
|
|
84
84
|
type CompactCursors,
|
|
85
85
|
} from '../sqlite/blob-compactor';
|
|
86
|
-
import { createHash } from 'node:crypto';
|
|
87
86
|
import {
|
|
88
87
|
cachePagesForBytes,
|
|
88
|
+
parseByteSize,
|
|
89
89
|
parseChunkCacheBytes,
|
|
90
90
|
} from '../sqlite/cache-config';
|
|
91
|
-
import { DoSqliteDatabase } from '../sqlite/database';
|
|
91
|
+
import { DoSqliteDatabase, type DoSqliteOpenOptions } from '../sqlite/database';
|
|
92
|
+
import type { TierFlushResult } from '../sqlite/do-vfs';
|
|
93
|
+
import {
|
|
94
|
+
DEFAULT_TIER_HOT_BUDGET_BYTES,
|
|
95
|
+
type PageTierOptions,
|
|
96
|
+
} from '../sqlite/page-tier';
|
|
92
97
|
import {
|
|
93
98
|
shouldVacuum,
|
|
94
99
|
vacuumWanted,
|
|
@@ -107,6 +112,7 @@ import {
|
|
|
107
112
|
} from '../sqlite/sessions-store';
|
|
108
113
|
import { createAttachmentDecryptor } from '@ixo/matrix-bot-workers-sdk';
|
|
109
114
|
import { MatrixMediaOwnerStore } from '../owner-store/matrix-media-store';
|
|
115
|
+
import { measureForSave } from '../owner-store/measure';
|
|
110
116
|
import { MigratingOwnerStore } from '../owner-store/migrating-store';
|
|
111
117
|
import type { OwnerCopy } from '../owner-store/types';
|
|
112
118
|
import {
|
|
@@ -224,6 +230,14 @@ interface MatrixUserIdCache {
|
|
|
224
230
|
/** Consecutive flush failures (cleared on success). */
|
|
225
231
|
const META_FLUSH_FAILURES = 'meta:flushFailures';
|
|
226
232
|
const META_LAST_VACUUM = 'meta:lastVacuumAt';
|
|
233
|
+
/** When the last R2 page-tier eviction pass ran (see `tierTick`). */
|
|
234
|
+
const META_LAST_TIER_PASS = 'meta:lastTierPassAt';
|
|
235
|
+
/**
|
|
236
|
+
* Eviction passes at most this often per object. The horizon is two daily
|
|
237
|
+
* periods, so four checks a day catch chunks the moment they age out
|
|
238
|
+
* without re-reading the hot set more than that.
|
|
239
|
+
*/
|
|
240
|
+
const TIER_PASS_INTERVAL_MS = 6 * 60 * 60_000;
|
|
227
241
|
/**
|
|
228
242
|
* Persisted twin of the in-memory `dirty` flag. The flag alone dies with an
|
|
229
243
|
* eviction, and evictions are routine — without the marker an alarm waking a
|
|
@@ -314,22 +328,6 @@ interface FlushResult {
|
|
|
314
328
|
}
|
|
315
329
|
|
|
316
330
|
/** Hex SHA-256 of a byte stream, hashed as it flows. */
|
|
317
|
-
async function sha256OfStream(
|
|
318
|
-
stream: ReadableStream<Uint8Array>,
|
|
319
|
-
): Promise<string> {
|
|
320
|
-
const hash = createHash('sha256');
|
|
321
|
-
const reader = stream.getReader();
|
|
322
|
-
try {
|
|
323
|
-
for (;;) {
|
|
324
|
-
const { done, value } = await reader.read();
|
|
325
|
-
if (done) break;
|
|
326
|
-
hash.update(value);
|
|
327
|
-
}
|
|
328
|
-
} finally {
|
|
329
|
-
reader.releaseLock();
|
|
330
|
-
}
|
|
331
|
-
return hash.digest('hex');
|
|
332
|
-
}
|
|
333
331
|
|
|
334
332
|
/** Does this adapter expose provider config + role resolution (platform adapters do)? */
|
|
335
333
|
function isProviderAdapter(llm: LlmAdapter): llm is OpenRouterLlmAdapter {
|
|
@@ -1131,7 +1129,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1131
1129
|
if (db) {
|
|
1132
1130
|
await db.close().catch(() => undefined);
|
|
1133
1131
|
}
|
|
1134
|
-
|
|
1132
|
+
const tier = this.tierOptions();
|
|
1133
|
+
await DoSqliteDatabase.wipe(this.ctx, DB_FILE, tier ? { tier } : {});
|
|
1135
1134
|
await this.ctx.storage.delete([
|
|
1136
1135
|
META_OWNER_ETAG,
|
|
1137
1136
|
META_LAST_CHECKSUM,
|
|
@@ -1142,12 +1141,40 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1142
1141
|
]);
|
|
1143
1142
|
}
|
|
1144
1143
|
|
|
1145
|
-
/**
|
|
1146
|
-
|
|
1144
|
+
/**
|
|
1145
|
+
* `CHUNK_CACHE_BYTES` → the VFS clean-chunk budget for this object, and
|
|
1146
|
+
* the R2 page tier when the script binds `TIER_BUCKET` (the object's id
|
|
1147
|
+
* is its key prefix; the knobs are documented in `contracts.ts`).
|
|
1148
|
+
*/
|
|
1149
|
+
private sqliteOpenOptions(): DoSqliteOpenOptions {
|
|
1147
1150
|
const bytes = parseChunkCacheBytes(this.env.CHUNK_CACHE_BYTES, (m) =>
|
|
1148
1151
|
console.warn(`[user-do] ${m}`),
|
|
1149
1152
|
);
|
|
1150
|
-
|
|
1153
|
+
const tier = this.tierOptions();
|
|
1154
|
+
return { cachePages: cachePagesForBytes(bytes), ...(tier && { tier }) };
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
private tierOptions(): PageTierOptions | undefined {
|
|
1158
|
+
const bucket = this.env.TIER_BUCKET;
|
|
1159
|
+
if (!bucket) return undefined;
|
|
1160
|
+
const warn = (m: string): void => console.warn(`[user-do] ${m}`);
|
|
1161
|
+
const hotBudgetBytes = parseByteSize(this.env.TIER_HOT_BUDGET_BYTES, {
|
|
1162
|
+
name: 'TIER_HOT_BUDGET_BYTES',
|
|
1163
|
+
fallback: DEFAULT_TIER_HOT_BUDGET_BYTES,
|
|
1164
|
+
min: 1024 * 1024,
|
|
1165
|
+
max: 1024 * 1024 * 1024,
|
|
1166
|
+
warn,
|
|
1167
|
+
});
|
|
1168
|
+
const periods = Number(this.env.TIER_EVICT_AFTER_PERIODS);
|
|
1169
|
+
const periodMs = Number(this.env.TIER_PERIOD_MS);
|
|
1170
|
+
return {
|
|
1171
|
+
bucket,
|
|
1172
|
+
prefix: this.ctx.id.toString(),
|
|
1173
|
+
hotBudgetBytes,
|
|
1174
|
+
...(Number.isInteger(periods) &&
|
|
1175
|
+
periods >= 1 && { evictAfterPeriods: periods }),
|
|
1176
|
+
...(Number.isFinite(periodMs) && periodMs >= 1000 && { periodMs }),
|
|
1177
|
+
};
|
|
1151
1178
|
}
|
|
1152
1179
|
|
|
1153
1180
|
/**
|
|
@@ -1333,6 +1360,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1333
1360
|
if (next !== null) deadlines.push(next);
|
|
1334
1361
|
}
|
|
1335
1362
|
|
|
1363
|
+
// R2 page tier: move chunks no turn touched for a couple of days out
|
|
1364
|
+
// of DO storage (see sqlite/page-tier.ts). After the flush above (the
|
|
1365
|
+
// upload reads hot rows, not R2) and never over an in-flight one (it
|
|
1366
|
+
// holds a snapshot). A pass is capped; more work re-arms soon.
|
|
1367
|
+
if (this.db?.tierEnabled && !this.flushInFlight) {
|
|
1368
|
+
const next = await this.tierTick(this.db, now);
|
|
1369
|
+
if (next !== null) deadlines.push(next);
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1336
1372
|
// Idle housekeeping: a user silent for a day loses the cached pages —
|
|
1337
1373
|
// their file is safe upstream, and we stop paying to store a copy. Never
|
|
1338
1374
|
// evict while tasks are scheduled: their runs ARE activity. The flush
|
|
@@ -1361,6 +1397,71 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1361
1397
|
);
|
|
1362
1398
|
}
|
|
1363
1399
|
|
|
1400
|
+
/**
|
|
1401
|
+
* One page-tier eviction pass, at most once per `TIER_PASS_INTERVAL_MS`
|
|
1402
|
+
* (a pass that hit its segment cap re-arms in a minute). Returns the next
|
|
1403
|
+
* time a pass is due, or null when nothing is pending.
|
|
1404
|
+
*/
|
|
1405
|
+
private async tierTick(
|
|
1406
|
+
db: DoSqliteDatabase,
|
|
1407
|
+
now: number,
|
|
1408
|
+
): Promise<number | null> {
|
|
1409
|
+
const last =
|
|
1410
|
+
(await this.ctx.storage.get<number>(META_LAST_TIER_PASS)) ?? 0;
|
|
1411
|
+
if (now - last < TIER_PASS_INTERVAL_MS)
|
|
1412
|
+
return last + TIER_PASS_INTERVAL_MS;
|
|
1413
|
+
try {
|
|
1414
|
+
const pass = await db.tierFlush();
|
|
1415
|
+
await this.ctx.storage.put(META_LAST_TIER_PASS, now);
|
|
1416
|
+
if (pass.skipped) {
|
|
1417
|
+
if (pass.skipped !== 'no-tier')
|
|
1418
|
+
console.log(
|
|
1419
|
+
`[user-do] tier pass skipped for ${this.userDid}: ${pass.skipped}`,
|
|
1420
|
+
);
|
|
1421
|
+
return now + 60_000;
|
|
1422
|
+
}
|
|
1423
|
+
if (pass.evictedChunks > 0 || pass.remaining > 0) {
|
|
1424
|
+
console.log(
|
|
1425
|
+
`[user-do] tier pass for ${this.userDid}: ${pass.evictedChunks} chunks → R2 in ${pass.segmentsRewritten} segment(s), ${pass.hotRows} hot rows (${Math.round(pass.hotBytes / 1024 / 1024)} MB) kept, ${pass.remaining} segment(s) pending`,
|
|
1426
|
+
);
|
|
1427
|
+
}
|
|
1428
|
+
const budget = this.tierOptions()?.hotBudgetBytes;
|
|
1429
|
+
if (
|
|
1430
|
+
budget !== undefined &&
|
|
1431
|
+
pass.remaining === 0 &&
|
|
1432
|
+
pass.hotBytes > budget
|
|
1433
|
+
) {
|
|
1434
|
+
// Everything left is recent: the user's real working set exceeds
|
|
1435
|
+
// the target — a sizing signal, not a failure.
|
|
1436
|
+
console.warn(
|
|
1437
|
+
`[user-do] tier hot set of ${this.userDid} is ${Math.round(pass.hotBytes / 1024 / 1024)} MB, over the ${Math.round(budget / 1024 / 1024)} MB target (all of it touched within the eviction horizon)`,
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
return pass.remaining > 0 ? now + 60_000 : now + TIER_PASS_INTERVAL_MS;
|
|
1441
|
+
} catch (err) {
|
|
1442
|
+
console.error(
|
|
1443
|
+
`[user-do] tier pass failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1444
|
+
);
|
|
1445
|
+
return now + FLUSH_RETRY_DELAY_MS;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
/** Operator/test entry: run an eviction pass now (debug routes). */
|
|
1450
|
+
async tierFlush(
|
|
1451
|
+
opts: { force?: boolean; maxSegments?: number } = {},
|
|
1452
|
+
): Promise<TierFlushResult> {
|
|
1453
|
+
if (!this.db) {
|
|
1454
|
+
const userDid =
|
|
1455
|
+
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
1456
|
+
if (userDid) await this.ready({ userDid });
|
|
1457
|
+
}
|
|
1458
|
+
if (!this.db) throw new Error('no working copy');
|
|
1459
|
+
if (this.flushInFlight) await this.flushInFlight.catch(() => undefined);
|
|
1460
|
+
const pass = await this.db.tierFlush(opts);
|
|
1461
|
+
await this.ctx.storage.put(META_LAST_TIER_PASS, Date.now());
|
|
1462
|
+
return pass;
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1364
1465
|
/**
|
|
1365
1466
|
* One VACUUM policy check. Returns the time of a follow-up check when a
|
|
1366
1467
|
* rebuild is wanted but the object is not quiet yet, else null.
|
|
@@ -2015,7 +2116,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2015
2116
|
|
|
2016
2117
|
const snapshot = await db.snapshot();
|
|
2017
2118
|
try {
|
|
2018
|
-
|
|
2119
|
+
// One pass yields the change gate (hash) AND the length the store
|
|
2120
|
+
// needs to upload; the upload is the only other read of the file.
|
|
2121
|
+
const { sha256Hex: checksum, gzippedLength } = await measureForSave(
|
|
2122
|
+
snapshot.open(),
|
|
2123
|
+
);
|
|
2019
2124
|
const last = await this.ctx.storage.get<string>(META_LAST_CHECKSUM);
|
|
2020
2125
|
if (last === checksum) {
|
|
2021
2126
|
await this.ctx.storage.put(META_UPLOADED_GEN, snapshot.generation);
|
|
@@ -2030,7 +2135,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2030
2135
|
|
|
2031
2136
|
let saved: { etag: string; bytes: number };
|
|
2032
2137
|
try {
|
|
2033
|
-
saved = await store.save(snapshot);
|
|
2138
|
+
saved = await store.save(snapshot, { gzippedLength });
|
|
2034
2139
|
} catch (err) {
|
|
2035
2140
|
// Never diverted anywhere else: the working copy in Durable Object
|
|
2036
2141
|
// storage is durable, stays dirty, and is retried. Loud from the
|
|
@@ -2295,6 +2400,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2295
2400
|
rowsWritten: vfs.rowsWritten,
|
|
2296
2401
|
}
|
|
2297
2402
|
: undefined,
|
|
2403
|
+
tier: db
|
|
2404
|
+
? {
|
|
2405
|
+
...db.tierStatus(),
|
|
2406
|
+
lastPassAt:
|
|
2407
|
+
await this.ctx.storage.get<number>(META_LAST_TIER_PASS),
|
|
2408
|
+
}
|
|
2409
|
+
: undefined,
|
|
2298
2410
|
};
|
|
2299
2411
|
}
|
|
2300
2412
|
|
|
@@ -3235,6 +3347,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3235
3347
|
await sessions.setTitle(sessionId, title, { onlyIfUntitled: true });
|
|
3236
3348
|
}
|
|
3237
3349
|
this.markDirty();
|
|
3350
|
+
// Which chunks this turn touched, for the page tier's eviction policy
|
|
3351
|
+
// (one row write; a no-op without a tier).
|
|
3352
|
+
try {
|
|
3353
|
+
this.db?.recordAccess();
|
|
3354
|
+
} catch (err) {
|
|
3355
|
+
console.warn(
|
|
3356
|
+
`[user-do] tier access record failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
3357
|
+
);
|
|
3358
|
+
}
|
|
3238
3359
|
}
|
|
3239
3360
|
|
|
3240
3361
|
private async generateTitle(
|
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
type OwnerCopy,
|
|
51
51
|
type OwnerStore,
|
|
52
52
|
type SaveResult,
|
|
53
|
+
type SaveHints,
|
|
53
54
|
} from './types';
|
|
54
55
|
|
|
55
56
|
const VFS_RESOURCE = 'ixo:filesystem';
|
|
@@ -233,9 +234,11 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
233
234
|
* Atomic replace, always delete + re-create, never a new version:
|
|
234
235
|
*
|
|
235
236
|
* 1. drop stale `.uploading-*` temp files a crashed flush left behind;
|
|
236
|
-
* 2.
|
|
237
|
-
*
|
|
238
|
-
*
|
|
237
|
+
* 2. take the gzipped length from `hints` (the object measured it in the
|
|
238
|
+
* same pass as its checksum) or, without a hint, count it in a pass
|
|
239
|
+
* of its own (the VFS needs `Upload-Length` at session creation —
|
|
240
|
+
* `Upload-Defer-Length` is not supported — and gzip output is
|
|
241
|
+
* deterministic for identical input);
|
|
239
242
|
* 3. upload gzip → a TEMP path next to the real one: one `POST /files`
|
|
240
243
|
* when it fits in a single part, else a tus session in 5 MiB parts
|
|
241
244
|
* (`tusUpload`, resumable per part);
|
|
@@ -247,7 +250,10 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
247
250
|
* file is cleaned up then). Peak memory is one part (5 MiB) plus fixed
|
|
248
251
|
* overhead, whatever the file size.
|
|
249
252
|
*/
|
|
250
|
-
async save(
|
|
253
|
+
async save(
|
|
254
|
+
snapshot: FileSnapshot,
|
|
255
|
+
hints: SaveHints = {},
|
|
256
|
+
): Promise<SaveResult> {
|
|
251
257
|
const listing = await this.listOracleFiles();
|
|
252
258
|
const existing = listing.filter((f) => f.path === this.path);
|
|
253
259
|
const stale = listing.filter((f) =>
|
|
@@ -261,7 +267,8 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
261
267
|
});
|
|
262
268
|
}
|
|
263
269
|
|
|
264
|
-
const gzLength =
|
|
270
|
+
const gzLength =
|
|
271
|
+
hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
|
|
265
272
|
const tempPath = `${this.path}${TEMP_PATH_INFIX}${Date.now().toString(36)}`;
|
|
266
273
|
const uploaded =
|
|
267
274
|
gzLength <= SINGLE_SHOT_MAX_BYTES
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
type OwnerCopy,
|
|
23
23
|
type OwnerStore,
|
|
24
24
|
type SaveResult,
|
|
25
|
+
type SaveHints,
|
|
25
26
|
} from './types';
|
|
26
27
|
|
|
27
28
|
/**
|
|
@@ -89,19 +90,23 @@ export class MatrixMediaOwnerStore implements OwnerStore {
|
|
|
89
90
|
}
|
|
90
91
|
|
|
91
92
|
/**
|
|
92
|
-
* Streamed as well: the snapshot is opened
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
* uploaded — and never held whole.
|
|
93
|
+
* Streamed as well: the snapshot is opened for a header peek, for a gzip
|
|
94
|
+
* pass that only counts unless the caller measured the length already
|
|
95
|
+
* (`hints`; the homeserver needs the exact ciphertext length up front and
|
|
96
|
+
* gzip output is deterministic for identical input) and for the gzip pass
|
|
97
|
+
* that is uploaded — and never held whole.
|
|
97
98
|
*/
|
|
98
|
-
async save(
|
|
99
|
+
async save(
|
|
100
|
+
snapshot: FileSnapshot,
|
|
101
|
+
hints: SaveHints = {},
|
|
102
|
+
): Promise<SaveResult> {
|
|
99
103
|
const checked = await assertSqliteStream(
|
|
100
104
|
snapshot.open(),
|
|
101
105
|
`MatrixMediaOwnerStore: refusing to save — the working copy (${snapshot.size} bytes)`,
|
|
102
106
|
);
|
|
103
107
|
await checked.cancel().catch(() => undefined);
|
|
104
|
-
const gzLength =
|
|
108
|
+
const gzLength =
|
|
109
|
+
hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
|
|
105
110
|
const { eventId } = await this.gateway.uploadUserSnapshotStream(
|
|
106
111
|
this.userDid,
|
|
107
112
|
this.storageKey,
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One streamed pass over a snapshot that yields BOTH what a flush needs
|
|
3
|
+
* before it uploads: the SHA-256 of the raw SQLite bytes (the "did anything
|
|
4
|
+
* change" gate against the last upload) and the exact gzipped length (the
|
|
5
|
+
* VFS needs `Upload-Length` when the tus session is created). Before this
|
|
6
|
+
* helper the object hashed in one pass and the store counted the gzip in a
|
|
7
|
+
* second one; with the R2 page tier every pass over a cold file is one R2
|
|
8
|
+
* GET per segment, so the two are folded together and the upload itself is
|
|
9
|
+
* the only other read.
|
|
10
|
+
*
|
|
11
|
+
* The length is exact for the store's own gzip: `gzipStream` is the same
|
|
12
|
+
* `CompressionStream('gzip')` the stores pipe the upload through, and its
|
|
13
|
+
* output is deterministic for identical input.
|
|
14
|
+
*/
|
|
15
|
+
import { createHash } from 'node:crypto';
|
|
16
|
+
import { countStream, gzipStream } from './types';
|
|
17
|
+
|
|
18
|
+
export interface SnapshotMeasure {
|
|
19
|
+
/** Hex SHA-256 of the raw (uncompressed) bytes. */
|
|
20
|
+
sha256Hex: string;
|
|
21
|
+
/** Byte length of `gzipStream(<the same bytes>)`. */
|
|
22
|
+
gzippedLength: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function measureForSave(
|
|
26
|
+
stream: ReadableStream<Uint8Array>,
|
|
27
|
+
): Promise<SnapshotMeasure> {
|
|
28
|
+
const hash = createHash('sha256');
|
|
29
|
+
const tapped = stream.pipeThrough(
|
|
30
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
31
|
+
transform(chunk, controller) {
|
|
32
|
+
hash.update(chunk);
|
|
33
|
+
controller.enqueue(chunk);
|
|
34
|
+
},
|
|
35
|
+
}),
|
|
36
|
+
);
|
|
37
|
+
const gzippedLength = await countStream(gzipStream(tapped));
|
|
38
|
+
return { sha256Hex: hash.digest('hex'), gzippedLength };
|
|
39
|
+
}
|
|
@@ -20,7 +20,13 @@
|
|
|
20
20
|
* Matrix copy so no second copy of the user's history lingers.
|
|
21
21
|
*/
|
|
22
22
|
import { VfsNoDelegationError } from './ixo-vfs-store';
|
|
23
|
-
import type {
|
|
23
|
+
import type {
|
|
24
|
+
FileSnapshot,
|
|
25
|
+
OwnerCopy,
|
|
26
|
+
OwnerStore,
|
|
27
|
+
SaveHints,
|
|
28
|
+
SaveResult,
|
|
29
|
+
} from './types';
|
|
24
30
|
|
|
25
31
|
export interface MigratingOwnerStoreOptions {
|
|
26
32
|
/** System of record (IXO VFS). */
|
|
@@ -104,8 +110,8 @@ export class MigratingOwnerStore implements OwnerStore {
|
|
|
104
110
|
* "not on VFS yet" (no delegation) — so the object keeps the copy dirty
|
|
105
111
|
* and retries; Matrix media is never written.
|
|
106
112
|
*/
|
|
107
|
-
async save(snapshot: FileSnapshot): Promise<SaveResult> {
|
|
108
|
-
return this.primary.save(snapshot);
|
|
113
|
+
async save(snapshot: FileSnapshot, hints?: SaveHints): Promise<SaveResult> {
|
|
114
|
+
return this.primary.save(snapshot, hints);
|
|
109
115
|
}
|
|
110
116
|
|
|
111
117
|
/**
|
package/src/owner-store/types.ts
CHANGED
|
@@ -15,9 +15,13 @@
|
|
|
15
15
|
* Both directions STREAM: `load()` hands back the raw (already gunzipped)
|
|
16
16
|
* SQLite bytes as a `ReadableStream` the object writes into its VFS chunk by
|
|
17
17
|
* chunk, and `save()` takes a `FileSnapshot` — a re-openable, consistent
|
|
18
|
-
* view of the working copy — that the store reads as many times as it needs
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* view of the working copy — that the store reads as many times as it needs.
|
|
19
|
+
* The object measures the hash and the gzipped length in ONE pass
|
|
20
|
+
* (`measureForSave`) and hands the length in as a `SaveHints`, so a store
|
|
21
|
+
* only reads the snapshot once more, for the upload itself; a store called
|
|
22
|
+
* without the hint counts the gzip in a pass of its own. That matters with
|
|
23
|
+
* the R2 page tier, where every pass over a cold file is one R2 GET per
|
|
24
|
+
* segment. Nothing on this path ever holds the whole file in memory — the
|
|
21
25
|
* legacy Matrix media path included: the gateway hands the media over as a
|
|
22
26
|
* stream and decryption, gunzip and the SQLite header check run chunk by
|
|
23
27
|
* chunk in the object. Compression is the store's concern. `etag` lets the object skip a reload
|
|
@@ -26,7 +30,7 @@
|
|
|
26
30
|
export interface OwnerStore {
|
|
27
31
|
readonly kind: 'matrix' | 'vfs';
|
|
28
32
|
load(): Promise<OwnerCopy | null>;
|
|
29
|
-
save(snapshot: FileSnapshot): Promise<SaveResult>;
|
|
33
|
+
save(snapshot: FileSnapshot, hints?: SaveHints): Promise<SaveResult>;
|
|
30
34
|
/** Cheap upstream freshness probe — null when unsupported/unknown. */
|
|
31
35
|
head(): Promise<{ etag: string } | null>;
|
|
32
36
|
/** Remove the durable copy (user asked to be forgotten). */
|
|
@@ -60,6 +64,12 @@ export interface OwnerCopy {
|
|
|
60
64
|
fromLegacy?: true;
|
|
61
65
|
}
|
|
62
66
|
|
|
67
|
+
/** What the caller already measured about the snapshot (see `measureForSave`). */
|
|
68
|
+
export interface SaveHints {
|
|
69
|
+
/** Byte length of `gzipStream(snapshot.open())`; spares the store its counting pass. */
|
|
70
|
+
gzippedLength?: number;
|
|
71
|
+
}
|
|
72
|
+
|
|
63
73
|
export interface SaveResult {
|
|
64
74
|
etag: string;
|
|
65
75
|
/** Bytes actually sent upstream (compressed size on the VFS path). */
|
package/src/shell/app.ts
CHANGED
|
@@ -471,6 +471,18 @@ export function createShell(
|
|
|
471
471
|
app.post('/debug/storage/flush', async (c) =>
|
|
472
472
|
c.json(await userStub(c.env, c.get('auth').userDid).flushToOwnerStore()),
|
|
473
473
|
);
|
|
474
|
+
app.post('/debug/storage/tier-flush', async (c) => {
|
|
475
|
+
const body: unknown = await c.req.json().catch(() => ({}));
|
|
476
|
+
const rec = typeof body === 'object' && body !== null ? body : {};
|
|
477
|
+
const force = Reflect.get(rec, 'force') === true;
|
|
478
|
+
const max = Number(Reflect.get(rec, 'maxSegments'));
|
|
479
|
+
return c.json(
|
|
480
|
+
await userStub(c.env, c.get('auth').userDid).tierFlush({
|
|
481
|
+
force,
|
|
482
|
+
...(Number.isInteger(max) && max > 0 && { maxSegments: max }),
|
|
483
|
+
}),
|
|
484
|
+
);
|
|
485
|
+
});
|
|
474
486
|
app.post('/debug/storage/reset', async (c) =>
|
|
475
487
|
c.json(await userStub(c.env, c.get('auth').userDid).resetWorkingCopy()),
|
|
476
488
|
);
|
|
@@ -19,22 +19,43 @@ export function parseChunkCacheBytes(
|
|
|
19
19
|
raw: string | undefined,
|
|
20
20
|
warn: (message: string) => void = () => undefined,
|
|
21
21
|
): number {
|
|
22
|
-
|
|
22
|
+
return parseByteSize(raw, {
|
|
23
|
+
name: 'CHUNK_CACHE_BYTES',
|
|
24
|
+
fallback: DEFAULT_CHUNK_CACHE_BYTES,
|
|
25
|
+
min: MIN_CHUNK_CACHE_BYTES,
|
|
26
|
+
max: MAX_CHUNK_CACHE_BYTES,
|
|
27
|
+
warn,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Same grammar for any byte-sized env knob (`TIER_HOT_BUDGET_BYTES`, ...). */
|
|
32
|
+
export function parseByteSize(
|
|
33
|
+
raw: string | undefined,
|
|
34
|
+
options: {
|
|
35
|
+
name: string;
|
|
36
|
+
fallback: number;
|
|
37
|
+
min: number;
|
|
38
|
+
max: number;
|
|
39
|
+
warn?: (message: string) => void;
|
|
40
|
+
},
|
|
41
|
+
): number {
|
|
42
|
+
const warn = options.warn ?? (() => undefined);
|
|
43
|
+
if (raw === undefined || raw.trim() === '') return options.fallback;
|
|
23
44
|
const match = /^\s*(\d+)\s*([kKmM]?)\s*$/.exec(raw);
|
|
24
45
|
if (!match) {
|
|
25
46
|
warn(
|
|
26
|
-
|
|
47
|
+
`${options.name}=${JSON.stringify(raw)} is not a byte count — using the default ${options.fallback}`,
|
|
27
48
|
);
|
|
28
|
-
return
|
|
49
|
+
return options.fallback;
|
|
29
50
|
}
|
|
30
51
|
const unit = match[2]?.toLowerCase();
|
|
31
52
|
const multiplier = unit === 'k' ? 1024 : unit === 'm' ? 1024 * 1024 : 1;
|
|
32
53
|
const bytes = Number(match[1]) * multiplier;
|
|
33
|
-
if (bytes <
|
|
54
|
+
if (bytes < options.min || bytes > options.max) {
|
|
34
55
|
warn(
|
|
35
|
-
|
|
56
|
+
`${options.name}=${raw} is outside ${options.min}..${options.max} — using the default ${options.fallback}`,
|
|
36
57
|
);
|
|
37
|
-
return
|
|
58
|
+
return options.fallback;
|
|
38
59
|
}
|
|
39
60
|
return bytes;
|
|
40
61
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The storage grain shared by the DO VFS and the R2 page tier. Kept in its
|
|
3
|
+
* own module so `page-tier.ts` can import it without a cycle through
|
|
4
|
+
* `do-vfs.ts`.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Bytes per SQLite page. Matches `PRAGMA page_size=4096`. */
|
|
8
|
+
export const VFS_PAGE_SIZE = 4096;
|
|
9
|
+
|
|
10
|
+
/** SQLite pages packed into one stored row (billing grain). */
|
|
11
|
+
export const PAGES_PER_CHUNK = 16;
|
|
12
|
+
|
|
13
|
+
/** Bytes per stored chunk row (64 KiB — well under the 2 MB DO value cap). */
|
|
14
|
+
export const CHUNK_SIZE = VFS_PAGE_SIZE * PAGES_PER_CHUNK;
|