@aigne/afs-did-space 1.12.0-beta.5

package/dist/inline-content-migration.cjs

@@ -0,0 +1,212 @@
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+const require_cid = require('./cid.cjs');
+require('./mime-utils.cjs');
+const require_inline_content = require('./inline-content.cjs');
+const require_object_key = require('./object-key.cjs');
+let _aigne_afs_provider_utils = require("@aigne/afs-provider-utils");
+
+//#region src/inline-content-migration.ts
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/**
+* Page over content-NULL rows and backfill the inline copy from R2. Returns
+* stats; resumable across invocations (a fresh call re-scans from the start and
+* re-selects whatever is still NULL).
+*/
+async function backfillInlineContent(backend, objectStore, opts = {}) {
+	const threshold = opts.threshold ?? require_inline_content.DEFAULT_INLINE_THRESHOLD;
+	const pageSize = opts.pageSize ?? 40;
+	const maxPages = opts.maxPages ?? Number.POSITIVE_INFINITY;
+	const retries = opts.retries ?? 2;
+	const throttleMs = opts.throttleMs ?? 0;
+	const sleep = opts.sleep ?? defaultSleep;
+	const log = opts.log ?? (() => {});
+	const stats = {
+		scanned: 0,
+		backfilled: 0,
+		skippedNonText: 0,
+		skippedMissing: 0,
+		skippedMismatch: 0,
+		deferredError: 0,
+		pages: 0,
+		lastCursor: "",
+		done: false
+	};
+	let cursor = "";
+	while (stats.pages < maxPages) {
+		const rows = await backend.selectCandidates(cursor, pageSize, threshold);
+		if (rows.length === 0) {
+			stats.done = true;
+			break;
+		}
+		stats.pages++;
+		for (const row of rows) {
+			stats.scanned++;
+			cursor = row.path;
+			stats.lastCursor = cursor;
+			const mime = (0, _aigne_afs_provider_utils.detectMimeType)(row.path);
+			if (!(0, _aigne_afs_provider_utils.isTextMime)(mime)) {
+				stats.skippedNonText++;
+				continue;
+			}
+			let bytes = null;
+			let lastErr;
+			let absent = false;
+			for (let attempt = 0; attempt <= retries; attempt++) try {
+				const obj = await objectStore.get(require_object_key.objectKeyForCid(row.cid));
+				if (!obj) absent = true;
+				else bytes = new Uint8Array(await obj.arrayBuffer());
+				lastErr = void 0;
+				break;
+			} catch (e) {
+				lastErr = e;
+				if (attempt < retries) await sleep(throttleMs || 50);
+			}
+			if (lastErr !== void 0) {
+				stats.deferredError++;
+				log(`defer ${row.path}: R2 read error ${String(lastErr)}`);
+				continue;
+			}
+			if (absent) {
+				stats.skippedMissing++;
+				log(`skip ${row.path}: object ${row.cid} missing (404)`);
+				continue;
+			}
+			const content = require_inline_content.decodeInlineText(bytes, mime ?? void 0);
+			if (content === null) {
+				stats.skippedNonText++;
+				continue;
+			}
+			if (await require_cid.computeCID(content) !== row.cid) {
+				stats.skippedMismatch++;
+				log(`skip ${row.path}: cid mismatch (content hash != ${row.cid})`);
+				continue;
+			}
+			await backend.backfillRow(row.path, content, mime ?? null);
+			stats.backfilled++;
+			if (throttleMs > 0) await sleep(throttleMs);
+		}
+	}
+	return stats;
+}
+const SELECT_SQL = "SELECT path, cid, size FROM ds_entries WHERE scope = ? AND kind = 'file' AND content IS NULL AND size <= ? AND path > ? ORDER BY path ASC LIMIT ?";
+const UPDATE_SQL = "UPDATE ds_entries SET content = ?, mime_type = ? WHERE scope = ? AND path = ? AND content IS NULL";
+/** D1 (Cloudflare) backend adapter — the same SQL the index uses. */
+function makeD1InlineBackfillBackend(db, scope) {
+	return {
+		async selectCandidates(cursor, limit, threshold) {
+			return ((await db.prepare(SELECT_SQL).bind(scope, threshold, cursor, limit).all()).results ?? []).map((r) => ({
+				path: String(r.path),
+				cid: String(r.cid),
+				size: Number(r.size)
+			}));
+		},
+		async backfillRow(path, content, mimeType) {
+			await db.prepare(UPDATE_SQL).bind(content, mimeType, scope, path).run();
+		}
+	};
+}
+/** Node SQLite (libsql) backend adapter. */
+function makeSqliteInlineBackfillBackend(db, scope) {
+	return {
+		async selectCandidates(cursor, limit, threshold) {
+			return (await db.execute({
+				sql: SELECT_SQL,
+				args: [
+					scope,
+					threshold,
+					cursor,
+					limit
+				]
+			})).rows.map((r) => ({
+				path: String(r.path),
+				cid: String(r.cid),
+				size: Number(r.size)
+			}));
+		},
+		async backfillRow(path, content, mimeType) {
+			await db.execute({
+				sql: UPDATE_SQL,
+				args: [
+					content,
+					mimeType,
+					scope,
+					path
+				]
+			});
+		}
+	};
+}
+const DISTINCT_NULL_SCOPES_SQL = "SELECT DISTINCT scope FROM ds_entries WHERE kind = 'file' AND content IS NULL AND size <= ?";
+/** Run one D1 HTTP query (with transient-error retry); returns the rows or throws. */
+async function d1HttpQuery(cfg, sql, params) {
+	const f = cfg.fetchImpl ?? fetch;
+	const sleep = cfg.sleep ?? defaultSleep;
+	const maxRetries = cfg.maxRetries ?? 4;
+	const url = `https://api.cloudflare.com/client/v4/accounts/${cfg.accountId}/d1/database/${cfg.databaseId}/query`;
+	for (let attempt = 0;; attempt++) {
+		let status = 0;
+		let json;
+		let networkErr;
+		try {
+			const res = await f(url, {
+				method: "POST",
+				headers: {
+					authorization: `Bearer ${cfg.apiToken}`,
+					"content-type": "application/json"
+				},
+				body: JSON.stringify({
+					sql,
+					params
+				})
+			});
+			status = res.status;
+			json = await res.json();
+		} catch (e) {
+			networkErr = e;
+		}
+		if (!networkErr && status >= 200 && status < 300 && json?.success !== false) return json?.result?.[0]?.results ?? [];
+		const transient = networkErr != null || status >= 500 || status === 429;
+		const msg = networkErr ? `D1 HTTP network error: ${String(networkErr).slice(0, 160)}` : `D1 HTTP ${status}: ${JSON.stringify(json?.errors ?? json).slice(0, 200)}`;
+		if (!transient || attempt >= maxRetries) throw new Error(msg);
+		await sleep(Math.min(2e3, 200 * 2 ** attempt));
+	}
+}
+/** CF D1 HTTP-API backend adapter (offline mechanism A for staging/prod). */
+function makeD1HttpInlineBackfillBackend(cfg) {
+	return {
+		async selectCandidates(cursor, limit, threshold) {
+			return (await d1HttpQuery(cfg, SELECT_SQL, [
+				cfg.scope,
+				threshold,
+				cursor,
+				limit
+			])).map((r) => ({
+				path: String(r.path),
+				cid: String(r.cid),
+				size: Number(r.size)
+			}));
+		},
+		async backfillRow(path, content, mimeType) {
+			await d1HttpQuery(cfg, UPDATE_SQL, [
+				content,
+				mimeType,
+				cfg.scope,
+				path
+			]);
+		}
+	};
+}
+/**
+* List the distinct scopes that still have content-NULL file rows (≤ threshold)
+* on a CF D1, so a bulk sweep can migrate each scope-aware. One HTTP query.
+*/
+async function listD1NullContentScopes(cfg, threshold = require_inline_content.DEFAULT_INLINE_THRESHOLD) {
+	return (await d1HttpQuery(cfg, DISTINCT_NULL_SCOPES_SQL, [threshold])).map((r) => String(r.scope));
+}
+
+//#endregion
+exports.backfillInlineContent = backfillInlineContent;
+exports.listD1NullContentScopes = listD1NullContentScopes;
+exports.makeD1HttpInlineBackfillBackend = makeD1HttpInlineBackfillBackend;
+exports.makeD1InlineBackfillBackend = makeD1InlineBackfillBackend;
+exports.makeSqliteInlineBackfillBackend = makeSqliteInlineBackfillBackend;

package/dist/inline-content-migration.d.cts

@@ -0,0 +1,110 @@
+import { ObjectStore } from "./platform/r2/dist/types.cjs";
+
+//#region src/inline-content-migration.d.ts
+/** A content-NULL file row eligible for inline backfill. */
+interface InlineBackfillCandidate {
+  path: string;
+  cid: string;
+  size: number;
+}
+/** Backend-specific SQL for the sweep (D1 vs Node SQLite). */
+interface InlineBackfillBackend {
+  /**
+   * Select up to `limit` content-NULL FILE rows with `path > cursor` and
+   * `size <= threshold`, ordered by `path ASC` (the keyset cursor). Empty array
+   * ⇒ no more candidates after `cursor`.
+   */
+  selectCandidates(cursor: string, limit: number, threshold: number): Promise<InlineBackfillCandidate[]>;
+  /**
+   * Set `content` + `mime_type` for `path` ONLY IF the row is still
+   * `content IS NULL` (idempotent — a concurrent/earlier fill wins). `mimeType`
+   * is `null` for extensionless / unknown-extension paths — stored as SQL NULL
+   * to match the write path (so `isTextMime(null)` stays true and the read path
+   * still serves the inline copy; an empty string would flip `isTextMime` false).
+   */
+  backfillRow(path: string, content: string, mimeType: string | null): Promise<void>;
+}
+interface InlineBackfillOptions {
+  /** `size <= threshold` filter (default {@link DEFAULT_INLINE_THRESHOLD} = 2 KB). */
+  threshold?: number;
+  /** Rows per page = R2 GETs per page. Default 40 (free-tier 50-subrequest safe). */
+  pageSize?: number;
+  /** Cap pages this invocation (cross-invocation paging). Default unbounded. */
+  maxPages?: number;
+  /** Sleep (ms) after each R2 read to pace subrequests. Default 0. */
+  throttleMs?: number;
+  /** Bounded retries on a transient R2 error before deferring the row. Default 2. */
+  retries?: number;
+  /** Injectable sleep (tests pass a no-op). */
+  sleep?: (ms: number) => Promise<void>;
+  /** Progress / skip logger. */
+  log?: (msg: string) => void;
+}
+interface InlineBackfillStats {
+  scanned: number;
+  backfilled: number;
+  skippedNonText: number;
+  skippedMissing: number;
+  skippedMismatch: number;
+  deferredError: number;
+  pages: number;
+  lastCursor: string;
+  /** True when a page came back empty (the sweep reached the end). */
+  done: boolean;
+}
+/**
+ * Page over content-NULL rows and backfill the inline copy from R2. Returns
+ * stats; resumable across invocations (a fresh call re-scans from the start and
+ * re-selects whatever is still NULL).
+ */
+declare function backfillInlineContent(backend: InlineBackfillBackend, objectStore: ObjectStore, opts?: InlineBackfillOptions): Promise<InlineBackfillStats>;
+/** D1 (Cloudflare) backend adapter — the same SQL the index uses. */
+declare function makeD1InlineBackfillBackend(db: D1Database, scope: string): InlineBackfillBackend;
+/**
+ * Minimal libsql client shape the SQLite adapter needs (Node `_metadata.db`).
+ * `args` is narrowed to `(string | number)[]` — exactly what the sweep binds —
+ * so a real `@libsql/client` `Client` is structurally assignable (a wider
+ * `unknown[]` is not assignable to libsql's `InValue[]`).
+ */
+interface LibsqlLike {
+  execute(stmt: {
+    sql: string;
+    args: (string | number | null)[];
+  }): Promise<{
+    rows: Array<Record<string, unknown>>;
+  }>;
+}
+/** Node SQLite (libsql) backend adapter. */
+declare function makeSqliteInlineBackfillBackend(db: LibsqlLike, scope: string): InlineBackfillBackend;
+/**
+ * CF D1 over the HTTP API (`POST /accounts/{acct}/d1/database/{db}/query`) — the
+ * OFFLINE realization of mechanism A for staging/prod CF: a Node process holds a
+ * Cloudflare API token with D1-edit and runs the SAME paged sweep SQL the worker
+ * would, while reading R2 objects out-of-band (the script supplies the
+ * ObjectStore — e.g. R2 presigned GET). `fetchImpl` is injectable for tests.
+ */
+interface D1HttpConfig {
+  accountId: string;
+  databaseId: string;
+  apiToken: string;
+  scope: string;
+  fetchImpl?: typeof fetch;
+  /**
+   * Bounded retries for TRANSIENT D1 HTTP failures (5xx — incl. the occasional
+   * `code 7500 internal error` — / 429 / network) with exponential backoff. A
+   * one-off CF hiccup must NOT abort a multi-thousand-row sweep. Permanent 4xx
+   * (auth/SQL) throw immediately. Default 4. `sleep` injectable for tests.
+   */
+  maxRetries?: number;
+  sleep?: (ms: number) => Promise<void>;
+}
+/** CF D1 HTTP-API backend adapter (offline mechanism A for staging/prod). */
+declare function makeD1HttpInlineBackfillBackend(cfg: D1HttpConfig): InlineBackfillBackend;
+/**
+ * List the distinct scopes that still have content-NULL file rows (≤ threshold)
+ * on a CF D1, so a bulk sweep can migrate each scope-aware. One HTTP query.
+ */
+declare function listD1NullContentScopes(cfg: Omit<D1HttpConfig, "scope">, threshold?: number): Promise<string[]>;
+//#endregion
+export { D1HttpConfig, InlineBackfillBackend, InlineBackfillCandidate, InlineBackfillOptions, InlineBackfillStats, LibsqlLike, backfillInlineContent, listD1NullContentScopes, makeD1HttpInlineBackfillBackend, makeD1InlineBackfillBackend, makeSqliteInlineBackfillBackend };
+//# sourceMappingURL=inline-content-migration.d.cts.map

package/dist/inline-content-migration.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inline-content-migration.d.cts","names":[],"sources":["../src/inline-content-migration.ts"],"mappings":";;;AAoEA;AAAA,UA5BiB,uBAAA;EACf,IAAA;EACA,GAAA;EACA,IAAA;AAAA;;UAIe,qBAAA;EA+Bf;;;;;EAzBA,gBAAA,CACE,MAAA,UACA,KAAA,UACA,SAAA,WACC,OAAA,CAAQ,uBAAA;EAyBO;;AAGpB;;;;;EApBE,WAAA,CAAY,IAAA,UAAc,OAAA,UAAiB,QAAA,kBAA0B,OAAA;AAAA;AAAA,UAGtD,qBAAA;EAsBf;EApBA,SAAA;EAsBA;EApBA,QAAA;EAuBA;EArBA,QAAA;EAqBI;EAnBJ,UAAA;EA6ByC;EA3BzC,OAAA;EA4BS;EA1BT,KAAA,IAAS,EAAA,aAAe,OAAA;EA4BlB;EA1BN,GAAA,IAAO,GAAA;AAAA;AAAA,UAGQ,mBAAA;EACf,OAAA;EACA,UAAA;EACA,cAAA;EACA,cAAA;EACA,eAAA;EACA,aAAA;EACA,KAAA;EACA,UAAA;EAgBS;EAdT,IAAA;AAAA;AAkHF;;;;;AAAA,iBAxGsB,qBAAA,CACpB,OAAA,EAAS,qBAAA,EACT,WAAA,EAAa,WAAA,EACb,IAAA,GAAM,qBAAA,GACL,OAAA,CAAQ,mBAAA;;iBAoGK,2BAAA,CAA4B,EAAA,EAAI,UAAA,EAAY,KAAA,WAAgB,qBAAA;;;AAsB5E;;;;UAAiB,UAAA;EACf,OAAA,CAAQ,IAAA;IAAQ,GAAA;IAAa,IAAA;EAAA,IAAqC,OAAA;IAChE,IAAA,EAAM,KAAA,CAAM,MAAA;EAAA;AAAA;;iBAKA,+BAAA,CACd,EAAA,EAAI,UAAA,EACJ,KAAA,WACC,qBAAA;;;;;AAHH;;;UA6BiB,YAAA;EACf,SAAA;EACA,UAAA;EACA,QAAA;EACA,KAAA;EACA,SAAA,UAAmB,KAAA;EA/BG;AA0BxB;;;;;EAYE,UAAA;EACA,KAAA,IAAS,EAAA,aAAe,OAAA;AAAA;;iBAgDV,+BAAA,CAAgC,GAAA,EAAK,YAAA,GAAe,qBAAA;;;;;iBAgB9C,uBAAA,CACpB,GAAA,EAAK,IAAA,CAAK,YAAA,YACV,SAAA,YACC,OAAA"}

package/dist/inline-content-migration.d.mts

@@ -0,0 +1,111 @@
+import { ObjectStore } from "./providers/platform/r2/dist/types.mjs";
+import "./providers/platform/r2/dist/index.mjs";
+
+//#region src/inline-content-migration.d.ts
+/** A content-NULL file row eligible for inline backfill. */
+interface InlineBackfillCandidate {
+  path: string;
+  cid: string;
+  size: number;
+}
+/** Backend-specific SQL for the sweep (D1 vs Node SQLite). */
+interface InlineBackfillBackend {
+  /**
+   * Select up to `limit` content-NULL FILE rows with `path > cursor` and
+   * `size <= threshold`, ordered by `path ASC` (the keyset cursor). Empty array
+   * ⇒ no more candidates after `cursor`.
+   */
+  selectCandidates(cursor: string, limit: number, threshold: number): Promise<InlineBackfillCandidate[]>;
+  /**
+   * Set `content` + `mime_type` for `path` ONLY IF the row is still
+   * `content IS NULL` (idempotent — a concurrent/earlier fill wins). `mimeType`
+   * is `null` for extensionless / unknown-extension paths — stored as SQL NULL
+   * to match the write path (so `isTextMime(null)` stays true and the read path
+   * still serves the inline copy; an empty string would flip `isTextMime` false).
+   */
+  backfillRow(path: string, content: string, mimeType: string | null): Promise<void>;
+}
+interface InlineBackfillOptions {
+  /** `size <= threshold` filter (default {@link DEFAULT_INLINE_THRESHOLD} = 2 KB). */
+  threshold?: number;
+  /** Rows per page = R2 GETs per page. Default 40 (free-tier 50-subrequest safe). */
+  pageSize?: number;
+  /** Cap pages this invocation (cross-invocation paging). Default unbounded. */
+  maxPages?: number;
+  /** Sleep (ms) after each R2 read to pace subrequests. Default 0. */
+  throttleMs?: number;
+  /** Bounded retries on a transient R2 error before deferring the row. Default 2. */
+  retries?: number;
+  /** Injectable sleep (tests pass a no-op). */
+  sleep?: (ms: number) => Promise<void>;
+  /** Progress / skip logger. */
+  log?: (msg: string) => void;
+}
+interface InlineBackfillStats {
+  scanned: number;
+  backfilled: number;
+  skippedNonText: number;
+  skippedMissing: number;
+  skippedMismatch: number;
+  deferredError: number;
+  pages: number;
+  lastCursor: string;
+  /** True when a page came back empty (the sweep reached the end). */
+  done: boolean;
+}
+/**
+ * Page over content-NULL rows and backfill the inline copy from R2. Returns
+ * stats; resumable across invocations (a fresh call re-scans from the start and
+ * re-selects whatever is still NULL).
+ */
+declare function backfillInlineContent(backend: InlineBackfillBackend, objectStore: ObjectStore, opts?: InlineBackfillOptions): Promise<InlineBackfillStats>;
+/** D1 (Cloudflare) backend adapter — the same SQL the index uses. */
+declare function makeD1InlineBackfillBackend(db: D1Database, scope: string): InlineBackfillBackend;
+/**
+ * Minimal libsql client shape the SQLite adapter needs (Node `_metadata.db`).
+ * `args` is narrowed to `(string | number)[]` — exactly what the sweep binds —
+ * so a real `@libsql/client` `Client` is structurally assignable (a wider
+ * `unknown[]` is not assignable to libsql's `InValue[]`).
+ */
+interface LibsqlLike {
+  execute(stmt: {
+    sql: string;
+    args: (string | number | null)[];
+  }): Promise<{
+    rows: Array<Record<string, unknown>>;
+  }>;
+}
+/** Node SQLite (libsql) backend adapter. */
+declare function makeSqliteInlineBackfillBackend(db: LibsqlLike, scope: string): InlineBackfillBackend;
+/**
+ * CF D1 over the HTTP API (`POST /accounts/{acct}/d1/database/{db}/query`) — the
+ * OFFLINE realization of mechanism A for staging/prod CF: a Node process holds a
+ * Cloudflare API token with D1-edit and runs the SAME paged sweep SQL the worker
+ * would, while reading R2 objects out-of-band (the script supplies the
+ * ObjectStore — e.g. R2 presigned GET). `fetchImpl` is injectable for tests.
+ */
+interface D1HttpConfig {
+  accountId: string;
+  databaseId: string;
+  apiToken: string;
+  scope: string;
+  fetchImpl?: typeof fetch;
+  /**
+   * Bounded retries for TRANSIENT D1 HTTP failures (5xx — incl. the occasional
+   * `code 7500 internal error` — / 429 / network) with exponential backoff. A
+   * one-off CF hiccup must NOT abort a multi-thousand-row sweep. Permanent 4xx
+   * (auth/SQL) throw immediately. Default 4. `sleep` injectable for tests.
+   */
+  maxRetries?: number;
+  sleep?: (ms: number) => Promise<void>;
+}
+/** CF D1 HTTP-API backend adapter (offline mechanism A for staging/prod). */
+declare function makeD1HttpInlineBackfillBackend(cfg: D1HttpConfig): InlineBackfillBackend;
+/**
+ * List the distinct scopes that still have content-NULL file rows (≤ threshold)
+ * on a CF D1, so a bulk sweep can migrate each scope-aware. One HTTP query.
+ */
+declare function listD1NullContentScopes(cfg: Omit<D1HttpConfig, "scope">, threshold?: number): Promise<string[]>;
+//#endregion
+export { D1HttpConfig, InlineBackfillBackend, InlineBackfillCandidate, InlineBackfillOptions, InlineBackfillStats, LibsqlLike, backfillInlineContent, listD1NullContentScopes, makeD1HttpInlineBackfillBackend, makeD1InlineBackfillBackend, makeSqliteInlineBackfillBackend };
+//# sourceMappingURL=inline-content-migration.d.mts.map

package/dist/inline-content-migration.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inline-content-migration.d.mts","names":[],"sources":["../src/inline-content-migration.ts"],"mappings":";;;;;UAwCiB,uBAAA;EACf,IAAA;EACA,GAAA;EACA,IAAA;AAAA;;UAIe,qBAAA;EA6Bf;;;;;EAvBA,gBAAA,CACE,MAAA,UACA,KAAA,UACA,SAAA,WACC,OAAA,CAAQ,uBAAA;EAyBJ;;;AAGT;;;;EApBE,WAAA,CAAY,IAAA,UAAc,OAAA,UAAiB,QAAA,kBAA0B,OAAA;AAAA;AAAA,UAGtD,qBAAA;EAqBf;EAnBA,SAAA;EAqBA;EAnBA,QAAA;EAqBA;EAnBA,QAAA;EAqBI;EAnBJ,UAAA;EA6BoB;EA3BpB,OAAA;;EAEA,KAAA,IAAS,EAAA,aAAe,OAAA;EA2BX;EAzBb,GAAA,IAAO,GAAA;AAAA;AAAA,UAGQ,mBAAA;EACf,OAAA;EACA,UAAA;EACA,cAAA;EACA,cAAA;EACA,eAAA;EACA,aAAA;EACA,KAAA;EACA,UAAA;EAgBC;EAdD,IAAA;AAAA;;AAkHF;;;;iBAxGsB,qBAAA,CACpB,OAAA,EAAS,qBAAA,EACT,WAAA,EAAa,WAAA,EACb,IAAA,GAAM,qBAAA,GACL,OAAA,CAAQ,mBAAA;;iBAoGK,2BAAA,CAA4B,EAAA,EAAI,UAAA,EAAY,KAAA,WAAgB,qBAAA;;;;AAsB5E;;;UAAiB,UAAA;EACf,OAAA,CAAQ,IAAA;IAAQ,GAAA;IAAa,IAAA;EAAA,IAAqC,OAAA;IAChE,IAAA,EAAM,KAAA,CAAM,MAAA;EAAA;AAAA;;iBAKA,+BAAA,CACd,EAAA,EAAI,UAAA,EACJ,KAAA,WACC,qBAAA;;;;;;AAHH;;UA6BiB,YAAA;EACf,SAAA;EACA,UAAA;EACA,QAAA;EACA,KAAA;EACA,SAAA,UAAmB,KAAA;EA/BG;;AA0BxB;;;;EAYE,UAAA;EACA,KAAA,IAAS,EAAA,aAAe,OAAA;AAAA;;iBAgDV,+BAAA,CAAgC,GAAA,EAAK,YAAA,GAAe,qBAAA;;;;;iBAgB9C,uBAAA,CACpB,GAAA,EAAK,IAAA,CAAK,YAAA,YACV,SAAA,YACC,OAAA"}

package/dist/inline-content-migration.mjs

@@ -0,0 +1,207 @@
+import { computeCID } from "./cid.mjs";
+import { detectMimeType, isTextMime } from "./mime-utils.mjs";
+import { DEFAULT_INLINE_THRESHOLD, decodeInlineText } from "./inline-content.mjs";
+import { objectKeyForCid } from "./object-key.mjs";
+
+//#region src/inline-content-migration.ts
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/**
+* Page over content-NULL rows and backfill the inline copy from R2. Returns
+* stats; resumable across invocations (a fresh call re-scans from the start and
+* re-selects whatever is still NULL).
+*/
+async function backfillInlineContent(backend, objectStore, opts = {}) {
+	const threshold = opts.threshold ?? DEFAULT_INLINE_THRESHOLD;
+	const pageSize = opts.pageSize ?? 40;
+	const maxPages = opts.maxPages ?? Number.POSITIVE_INFINITY;
+	const retries = opts.retries ?? 2;
+	const throttleMs = opts.throttleMs ?? 0;
+	const sleep = opts.sleep ?? defaultSleep;
+	const log = opts.log ?? (() => {});
+	const stats = {
+		scanned: 0,
+		backfilled: 0,
+		skippedNonText: 0,
+		skippedMissing: 0,
+		skippedMismatch: 0,
+		deferredError: 0,
+		pages: 0,
+		lastCursor: "",
+		done: false
+	};
+	let cursor = "";
+	while (stats.pages < maxPages) {
+		const rows = await backend.selectCandidates(cursor, pageSize, threshold);
+		if (rows.length === 0) {
+			stats.done = true;
+			break;
+		}
+		stats.pages++;
+		for (const row of rows) {
+			stats.scanned++;
+			cursor = row.path;
+			stats.lastCursor = cursor;
+			const mime = detectMimeType(row.path);
+			if (!isTextMime(mime)) {
+				stats.skippedNonText++;
+				continue;
+			}
+			let bytes = null;
+			let lastErr;
+			let absent = false;
+			for (let attempt = 0; attempt <= retries; attempt++) try {
+				const obj = await objectStore.get(objectKeyForCid(row.cid));
+				if (!obj) absent = true;
+				else bytes = new Uint8Array(await obj.arrayBuffer());
+				lastErr = void 0;
+				break;
+			} catch (e) {
+				lastErr = e;
+				if (attempt < retries) await sleep(throttleMs || 50);
+			}
+			if (lastErr !== void 0) {
+				stats.deferredError++;
+				log(`defer ${row.path}: R2 read error ${String(lastErr)}`);
+				continue;
+			}
+			if (absent) {
+				stats.skippedMissing++;
+				log(`skip ${row.path}: object ${row.cid} missing (404)`);
+				continue;
+			}
+			const content = decodeInlineText(bytes, mime ?? void 0);
+			if (content === null) {
+				stats.skippedNonText++;
+				continue;
+			}
+			if (await computeCID(content) !== row.cid) {
+				stats.skippedMismatch++;
+				log(`skip ${row.path}: cid mismatch (content hash != ${row.cid})`);
+				continue;
+			}
+			await backend.backfillRow(row.path, content, mime ?? null);
+			stats.backfilled++;
+			if (throttleMs > 0) await sleep(throttleMs);
+		}
+	}
+	return stats;
+}
+const SELECT_SQL = "SELECT path, cid, size FROM ds_entries WHERE scope = ? AND kind = 'file' AND content IS NULL AND size <= ? AND path > ? ORDER BY path ASC LIMIT ?";
+const UPDATE_SQL = "UPDATE ds_entries SET content = ?, mime_type = ? WHERE scope = ? AND path = ? AND content IS NULL";
+/** D1 (Cloudflare) backend adapter — the same SQL the index uses. */
+function makeD1InlineBackfillBackend(db, scope) {
+	return {
+		async selectCandidates(cursor, limit, threshold) {
+			return ((await db.prepare(SELECT_SQL).bind(scope, threshold, cursor, limit).all()).results ?? []).map((r) => ({
+				path: String(r.path),
+				cid: String(r.cid),
+				size: Number(r.size)
+			}));
+		},
+		async backfillRow(path, content, mimeType) {
+			await db.prepare(UPDATE_SQL).bind(content, mimeType, scope, path).run();
+		}
+	};
+}
+/** Node SQLite (libsql) backend adapter. */
+function makeSqliteInlineBackfillBackend(db, scope) {
+	return {
+		async selectCandidates(cursor, limit, threshold) {
+			return (await db.execute({
+				sql: SELECT_SQL,
+				args: [
+					scope,
+					threshold,
+					cursor,
+					limit
+				]
+			})).rows.map((r) => ({
+				path: String(r.path),
+				cid: String(r.cid),
+				size: Number(r.size)
+			}));
+		},
+		async backfillRow(path, content, mimeType) {
+			await db.execute({
+				sql: UPDATE_SQL,
+				args: [
+					content,
+					mimeType,
+					scope,
+					path
+				]
+			});
+		}
+	};
+}
+const DISTINCT_NULL_SCOPES_SQL = "SELECT DISTINCT scope FROM ds_entries WHERE kind = 'file' AND content IS NULL AND size <= ?";
+/** Run one D1 HTTP query (with transient-error retry); returns the rows or throws. */
+async function d1HttpQuery(cfg, sql, params) {
+	const f = cfg.fetchImpl ?? fetch;
+	const sleep = cfg.sleep ?? defaultSleep;
+	const maxRetries = cfg.maxRetries ?? 4;
+	const url = `https://api.cloudflare.com/client/v4/accounts/${cfg.accountId}/d1/database/${cfg.databaseId}/query`;
+	for (let attempt = 0;; attempt++) {
+		let status = 0;
+		let json;
+		let networkErr;
+		try {
+			const res = await f(url, {
+				method: "POST",
+				headers: {
+					authorization: `Bearer ${cfg.apiToken}`,
+					"content-type": "application/json"
+				},
+				body: JSON.stringify({
+					sql,
+					params
+				})
+			});
+			status = res.status;
+			json = await res.json();
+		} catch (e) {
+			networkErr = e;
+		}
+		if (!networkErr && status >= 200 && status < 300 && json?.success !== false) return json?.result?.[0]?.results ?? [];
+		const transient = networkErr != null || status >= 500 || status === 429;
+		const msg = networkErr ? `D1 HTTP network error: ${String(networkErr).slice(0, 160)}` : `D1 HTTP ${status}: ${JSON.stringify(json?.errors ?? json).slice(0, 200)}`;
+		if (!transient || attempt >= maxRetries) throw new Error(msg);
+		await sleep(Math.min(2e3, 200 * 2 ** attempt));
+	}
+}
+/** CF D1 HTTP-API backend adapter (offline mechanism A for staging/prod). */
+function makeD1HttpInlineBackfillBackend(cfg) {
+	return {
+		async selectCandidates(cursor, limit, threshold) {
+			return (await d1HttpQuery(cfg, SELECT_SQL, [
+				cfg.scope,
+				threshold,
+				cursor,
+				limit
+			])).map((r) => ({
+				path: String(r.path),
+				cid: String(r.cid),
+				size: Number(r.size)
+			}));
+		},
+		async backfillRow(path, content, mimeType) {
+			await d1HttpQuery(cfg, UPDATE_SQL, [
+				content,
+				mimeType,
+				cfg.scope,
+				path
+			]);
+		}
+	};
+}
+/**
+* List the distinct scopes that still have content-NULL file rows (≤ threshold)
+* on a CF D1, so a bulk sweep can migrate each scope-aware. One HTTP query.
+*/
+async function listD1NullContentScopes(cfg, threshold = DEFAULT_INLINE_THRESHOLD) {
+	return (await d1HttpQuery(cfg, DISTINCT_NULL_SCOPES_SQL, [threshold])).map((r) => String(r.scope));
+}
+
+//#endregion
+export { backfillInlineContent, listD1NullContentScopes, makeD1HttpInlineBackfillBackend, makeD1InlineBackfillBackend, makeSqliteInlineBackfillBackend };
+//# sourceMappingURL=inline-content-migration.mjs.map