@alteran/astro 0.7.7 → 0.8.2

package/migrations/meta/_journal.json

@@ -71,6 +71,27 @@
       "when": 1778624701795,
       "tag": "0009_oauth_session_state",
       "breakpoints": true
+    },
+    {
+      "idx": 10,
+      "version": "6",
+      "when": 1778710862250,
+      "tag": "0010_eminent_klaw",
+      "breakpoints": true
+    },
+    {
+      "idx": 11,
+      "version": "6",
+      "when": 1778754994653,
+      "tag": "0011_chief_darwin",
+      "breakpoints": true
+    },
+    {
+      "idx": 12,
+      "version": "6",
+      "when": 1778790000000,
+      "tag": "0012_backfill_blob_usage",
+      "breakpoints": true
     }
   ]
-}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alteran/astro",
-  "version": "0.7.7",
+  "version": "0.8.2",
   "description": "Astro integration for running a Cloudflare-hosted Bluesky PDS with Alteran.",
   "module": "index.js",
   "types": "index.d.ts",
@@ -25,54 +25,37 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "dev": "astro dev",
-    "build": "astro build",
-    "preview": "astro preview",
-    "test:oauth": "bun test tests/oauth-*.test.ts tests/resource-auth.test.ts",
-    "deploy": "astro build && bunx wrangler deploy --env production",
-    "iac:deploy": "bunx alchemy deploy iac/alchemy.run.ts",
-    "iac:destroy": "bunx alchemy destroy iac/alchemy.run.ts",
-    "iac:plan": "bunx alchemy plan iac/alchemy.run.ts",
-    "db:generate": "bunx drizzle-kit generate",
-    "db:apply": "bunx wrangler d1 migrations apply pds",
-    "db:apply:local": "bunx wrangler d1 migrations apply pds --local",
-    "db:apply:local:direct": "wrangler d1 migrations apply pds --local",
-    "db:reset:local": "rm -rf .wrangler/state && rm -rf migrations && bun run db:generate && bun run db:apply:local",
-    "secrets:setup": "bun run scripts/setup-secrets.ts",
-    "relay:request-crawl": "bun run scripts/request-crawl.ts",
-    "pds:test-create-session": "bun run scripts/test-create-session.ts"
-  },
+  "scripts": {},
   "devDependencies": {
-    "@astrojs/cloudflare": "^12.6.9",
-    "@atproto/api": "^0.17.0",
-    "@cloudflare/vite-plugin": "^1.13.8",
-    "@types/bun": "latest",
-    "drizzle-kit": "^0.31.5",
-    "miniflare": "3",
-    "vite": "^6.3.6",
-    "wrangler": "^4.40.3"
+    "@astrojs/cloudflare": "^12.6.13",
+    "@cloudflare/vite-plugin": "^1.37.0",
+    "drizzle-kit": "^0.31.10",
+    "miniflare": "^3.20250718.3",
+    "vite": "^6.4.2",
+    "wrangler": "^4.91.0"
   },
   "peerDependencies": {
     "typescript": "^5"
   },
   "dependencies": {
-    "@atproto/crypto": "^0.4.4",
-    "@cloudflare/workers-types": "^4.20251001.0",
+    "@atproto/api": "^0.17.7",
+    "@atproto/crypto": "^0.4.5",
+    "@atproto/syntax": "^0.4.3",
+    "@cloudflare/workers-types": "^4.20260511.1",
     "@did-plc/lib": "^0.0.4",
-    "@iconify-json/simple-icons": "^1.2.53",
-    "@ipld/car": "^5.4.2",
-    "@ipld/dag-cbor": "^9.2.5",
-    "@noble/hashes": "^2.0.1",
-    "astro": "^6.1.5",
+    "@iconify-json/simple-icons": "^1.2.82",
+    "@ipld/car": "^5.4.6",
+    "@ipld/dag-cbor": "^9.2.7",
+    "@noble/hashes": "^2.2.0",
+    "astro": "^6.3.3",
     "astro-icon": "^1.1.5",
-    "dotenv": "^17.2.3",
-    "drizzle-orm": "^0.44.6",
-    "get-port": "^7.1.0",
-    "hono": "^4.9.9",
-    "multiformats": "^13.4.1",
-    "uint8arrays": "^5.1.0",
-    "jose": "^5.9.6"
+    "dotenv": "^17.4.2",
+    "drizzle-orm": "^0.44.7",
+    "get-port": "^7.2.0",
+    "hono": "^4.12.18",
+    "multiformats": "^13.4.2",
+    "uint8arrays": "^5.1.1",
+    "jose": "^5.10.0"
   },
   "repository": {
     "type": "git",

package/src/db/blob.ts

@@ -0,0 +1,323 @@
+import { eq, sql } from 'drizzle-orm';
+import { getDb } from './client';
+import { blob_ref, blob_quota } from './schema';
+import type { Env } from '../env';
+
+export type BlobKeyRef = {
+  did: string;
+  key: string;
+};
+
+export type BlobRefMetadata = {
+  did: string;
+  cid: string;
+  key: string;
+  mime: string;
+  size: number;
+  uploadedAt: number;
+};
+
+export type BlobRegistrationResult =
+  | { tag: 'registered'; blob: BlobRefMetadata }
+  | { tag: 'alreadyExists'; blob: BlobRefMetadata }
+  | { tag: 'quotaExceeded' };
+
+export async function putBlobRef(env: Env, did: string, cid: string, key: string, mime: string, size: number) {
+  const db = getDb(env);
+  const uploadedAt = Date.now();
+  await db
+    .insert(blob_ref)
+    .values({ did, cid, key, mime, size, uploadedAt })
+    .onConflictDoUpdate({
+      target: [blob_ref.did, blob_ref.cid],
+      set: {
+        key: sql.raw(`excluded.${blob_ref.key.name}`),
+        mime: sql.raw(`excluded.${blob_ref.mime.name}`),
+        size: sql.raw(`excluded.${blob_ref.size.name}`),
+        uploadedAt: sql.raw(`excluded.${blob_ref.uploadedAt.name}`),
+      },
+    });
+}
+
+export async function registerBlobRefWithQuota(
+  env: Env,
+  did: string,
+  cid: string,
+  key: string,
+  mime: string,
+  size: number,
+): Promise<BlobRegistrationResult> {
+  const existing = await getBlobRef(env, did, cid);
+  if (existing) {
+    const uploadedAt = Date.now();
+    await env.ALTERAN_DB.batch([
+      env.ALTERAN_DB.prepare(
+        `UPDATE blob
+         SET uploaded_at = ?
+         WHERE did = ? AND cid = ?`,
+      ).bind(uploadedAt, did, cid),
+      env.ALTERAN_DB.prepare(
+        `UPDATE blob_quota
+         SET updated_at = ?
+         WHERE did = ?`,
+      ).bind(uploadedAt, did),
+    ]);
+    return { tag: 'alreadyExists', blob: { ...existing, uploadedAt } };
+  }
+
+  const quotaBytes = parseInt(env.PDS_BLOB_QUOTA_BYTES || '10737418240', 10);
+  const maxBytes = Number.isFinite(quotaBytes) && quotaBytes > 0 ? quotaBytes : 10737418240;
+  const uploadedAt = Date.now();
+  const results = await env.ALTERAN_DB.batch([
+    env.ALTERAN_DB.prepare(
+      `INSERT OR IGNORE INTO blob_quota (did, total_bytes, blob_count, updated_at)
+       VALUES (?, 0, 0, ?)`,
+    ).bind(did, uploadedAt),
+    env.ALTERAN_DB.prepare(
+      `UPDATE blob_quota
+       SET updated_at = ?
+       WHERE did = ?`,
+    ).bind(uploadedAt, did),
+    env.ALTERAN_DB.prepare(
+      `INSERT OR IGNORE INTO blob (did, cid, key, mime, size, uploaded_at)
+       SELECT ?, ?, ?, ?, ?, ?
+       FROM blob_quota
+       WHERE did = ?
+         AND total_bytes + ? <= ?`,
+    ).bind(did, cid, key, mime, size, uploadedAt, did, size, maxBytes),
+    env.ALTERAN_DB.prepare(
+      `UPDATE blob_quota
+       SET total_bytes = (
+             SELECT COALESCE(SUM(size), 0) FROM blob WHERE did = ?
+           ),
+           blob_count = (
+             SELECT COUNT(*) FROM blob WHERE did = ?
+           ),
+           updated_at = ?
+       WHERE did = ?`,
+    ).bind(did, did, Date.now(), did),
+  ]);
+
+  const blob = await getBlobRef(env, did, cid);
+  if (blob && changedRows(results[2]) === 1) return { tag: 'registered', blob };
+  if (blob) return { tag: 'alreadyExists', blob };
+  return { tag: 'quotaExceeded' };
+}
+
+export async function getRecordBlobKeys(env: Env, did: string, uri: string): Promise<string[]> {
+  const result = await env.ALTERAN_DB.prepare(
+    'SELECT key FROM blob_usage WHERE did = ? AND record_uri = ?',
+  )
+    .bind(did, uri)
+    .all<{ key: string }>();
+
+  return result.results?.map((row: { key: string }) => row.key) ?? [];
+}
+
+export async function blobKeyHasUsage(env: Env, did: string, key: string): Promise<boolean> {
+  const row = await env.ALTERAN_DB.prepare(
+    'SELECT 1 FROM blob_usage WHERE did = ? AND key = ? LIMIT 1',
+  )
+    .bind(did, key)
+    .first();
+
+  return row !== null;
+}
+
+export async function deleteUnreferencedBlobKeys(
+  env: Env,
+  refs: readonly BlobKeyRef[],
+): Promise<number> {
+  let deleted = 0;
+  for (const { did, key } of uniqueBlobKeyRefs(refs)) {
+    const blob = await env.ALTERAN_DB.prepare(
+      'SELECT 1 FROM blob WHERE did = ? AND key = ? LIMIT 1',
+    )
+      .bind(did, key)
+      .first();
+    if (!blob) continue;
+
+    const cutoff = blobDeletionCutoff();
+    const object = await env.ALTERAN_BLOBS.head(key);
+    if (object && !uploadedBeforeCutoff(object, cutoff)) continue;
+
+    const [deletion] = await env.ALTERAN_DB.batch([
+      env.ALTERAN_DB.prepare(
+        `DELETE FROM blob
+         WHERE did = ?
+           AND key = ?
+           AND uploaded_at <= ?
+           AND NOT EXISTS (
+             SELECT 1 FROM blob_usage WHERE did = ? AND key = ?
+           )`,
+      ).bind(did, key, cutoff, did, key),
+      recomputeQuotaStatement(env, did),
+    ]);
+    if (changedRows(deletion) !== 1) continue;
+
+    // R2 deletes cannot share D1's commit guard. Metadata is the visibility
+    // gate; object cleanup needs a separate lease/tombstone flow.
+    deleted++;
+  }
+
+  return deleted;
+}
+
+export async function sweepEligibleUnreferencedBlobKeys(
+  env: Env,
+  options: { did?: string; limit?: number } = {},
+): Promise<number> {
+  return deleteUnreferencedBlobKeys(env, await listOrphanBlobRefs(env, {
+    did: options.did,
+    limit: options.limit ?? 100,
+  }));
+}
+
+export async function listOrphanBlobKeys(env: Env): Promise<string[]> {
+  return (await listOrphanBlobRefs(env)).map((ref) => ref.key);
+}
+
+export async function listOrphanBlobRefs(
+  env: Env,
+  options: { did?: string; limit?: number } = {},
+): Promise<BlobKeyRef[]> {
+  const limit = Math.max(1, Math.min(options.limit ?? 100, 1000));
+  const cutoff = blobDeletionCutoff();
+  const result = options.did
+    ? await env.ALTERAN_DB.prepare(
+      `SELECT did, key
+       FROM blob b
+       WHERE b.did = ?
+         AND b.uploaded_at <= ?
+         AND NOT EXISTS (
+           SELECT 1 FROM blob_usage u WHERE u.did = b.did AND u.key = b.key
+         )
+       ORDER BY b.uploaded_at, b.did, b.key
+       LIMIT ?`,
+    ).bind(options.did, cutoff, limit).all<BlobKeyRef>()
+    : await env.ALTERAN_DB.prepare(
+      `SELECT did, key
+       FROM blob b
+       WHERE b.uploaded_at <= ?
+         AND NOT EXISTS (
+           SELECT 1 FROM blob_usage u WHERE u.did = b.did AND u.key = b.key
+         )
+       ORDER BY b.uploaded_at, b.did, b.key
+       LIMIT ?`,
+    ).bind(cutoff, limit).all<BlobKeyRef>();
+  return result.results ?? [];
+}
+
+export async function deleteBlobByKey(env: Env, key: string) {
+  const db = getDb(env);
+  await db.delete(blob_ref).where(eq(blob_ref.key, key)).run();
+}
+
+export async function getBlobQuota(env: Env, did: string) {
+  const db = getDb(env);
+  const quota = await db.select().from(blob_quota).where(eq(blob_quota.did, did)).get();
+  return quota ?? { did, total_bytes: 0, blob_count: 0, updated_at: Date.now() };
+}
+
+export async function updateBlobQuota(env: Env, did: string, bytesAdded: number, countAdded: number) {
+  const db = getDb(env);
+  const current = await getBlobQuota(env, did);
+
+  const newTotalBytes = Math.max(0, current.total_bytes + bytesAdded);
+  const newBlobCount = Math.max(0, current.blob_count + countAdded);
+  const now = Date.now();
+
+  await db
+    .insert(blob_quota)
+    .values({
+      did,
+      total_bytes: newTotalBytes,
+      blob_count: newBlobCount,
+      updated_at: now,
+    })
+    .onConflictDoUpdate({
+      target: blob_quota.did,
+      set: {
+        total_bytes: sql.raw(`excluded.${blob_quota.total_bytes.name}`),
+        blob_count: sql.raw(`excluded.${blob_quota.blob_count.name}`),
+        updated_at: sql.raw(`excluded.${blob_quota.updated_at.name}`),
+      },
+    });
+}
+
+export async function checkBlobQuota(env: Env, did: string, additionalBytes: number): Promise<boolean> {
+  const quota = await getBlobQuota(env, did);
+  const maxBytes = parseInt(env.PDS_BLOB_QUOTA_BYTES || '10737418240', 10);
+
+  return quota.total_bytes + additionalBytes <= maxBytes;
+}
+
+export async function deleteUnreferencedBlobRef(env: Env, did: string, cid: string): Promise<void> {
+  await env.ALTERAN_DB.batch([
+    env.ALTERAN_DB.prepare(
+      `DELETE FROM blob
+       WHERE did = ?
+         AND cid = ?
+         AND NOT EXISTS (
+           SELECT 1 FROM blob_usage u WHERE u.did = blob.did AND u.key = blob.key
+         )`,
+    ).bind(did, cid),
+    recomputeQuotaStatement(env, did),
+  ]);
+}
+
+function uniqueBlobKeyRefs(refs: readonly BlobKeyRef[]): BlobKeyRef[] {
+  const seen = new Set<string>();
+  const unique: BlobKeyRef[] = [];
+  for (const ref of refs) {
+    const id = `${ref.did}\0${ref.key}`;
+    if (seen.has(id)) continue;
+    seen.add(id);
+    unique.push(ref);
+  }
+  return unique;
+}
+
+function blobDeletionCutoff(): number {
+  return Date.now() - 60 * 60 * 1000;
+}
+
+function changedRows(result: unknown): number {
+  const meta = (result as { meta?: Record<string, unknown> } | undefined)?.meta;
+  const changes = meta?.changes ?? meta?.rows_written ?? meta?.rowsWritten;
+  return typeof changes === 'number' ? changes : 0;
+}
+
+function recomputeQuotaStatement(env: Env, did: string) {
+  return env.ALTERAN_DB.prepare(
+    `UPDATE blob_quota
+     SET total_bytes = (
+           SELECT COALESCE(SUM(size), 0) FROM blob WHERE did = ?
+         ),
+         blob_count = (
+           SELECT COUNT(*) FROM blob WHERE did = ?
+         ),
+         updated_at = ?
+     WHERE did = ?`,
+  ).bind(did, did, Date.now(), did);
+}
+
+async function getBlobRef(env: Env, did: string, cid: string): Promise<BlobRefMetadata | null> {
+  const row = await env.ALTERAN_DB.prepare(
+    `SELECT did, cid, key, mime, size, uploaded_at as uploadedAt
+     FROM blob
+     WHERE did = ? AND cid = ?
+     LIMIT 1`,
+  ).bind(did, cid).first<BlobRefMetadata>();
+  return row ?? null;
+}
+
+function uploadedBeforeCutoff(object: { uploaded?: Date | string | number }, cutoff: number): boolean {
+  const uploaded = object.uploaded;
+  if (uploaded instanceof Date) return uploaded.getTime() <= cutoff;
+  if (typeof uploaded === 'string' || typeof uploaded === 'number') {
+    const timestamp = new Date(uploaded).getTime();
+    return Number.isFinite(timestamp) && timestamp <= cutoff;
+  }
+  return false;
+}