@alteran/astro 0.7.6 → 0.8.1

package/src/pages/xrpc/com.atproto.sync.getBlob.ts

@@ -1,11 +1,11 @@
 import type { APIContext } from 'astro';
+import { ensureValidDid } from '@atproto/syntax';
+import { CID } from 'multiformats/cid';
 import { getDb } from '../../db/client';
 import { blob_ref } from '../../db/schema';
-import { eq } from 'drizzle-orm';
-import { CID } from 'multiformats/cid';
-import { sha256 } from 'multiformats/hashes/sha2';
-import { putBlobRef } from '../../db/dal';
-import { isAccountActive } from '../../db/dal';
+import { and, eq } from 'drizzle-orm';
+import { blobKeyHasUsage, isAccountActive } from '../../db/dal';
+import { resolveSecret } from '../../lib/secrets';
 
 export const prerender = false;
 
@@ -23,98 +23,50 @@ export async function GET({ locals, url }: APIContext) {
   const { env } = locals.runtime;
 
   try {
-    const configuredDid = typeof env.PDS_DID === 'string' ? env.PDS_DID : undefined;
-    const did = url.searchParams.get('did') ?? configuredDid;
+    const configuredDid = await resolveSecret(env.PDS_DID);
+    const did = url.searchParams.get('did');
     const cid = url.searchParams.get('cid');
     if (!did || !cid) {
-      return new Response(
-        JSON.stringify({
-          error: 'InvalidRequest',
-          message: 'did and cid parameters are required'
-        }),
-        { status: 400, headers: { 'Content-Type': 'application/json' } }
-      );
+      return jsonError('InvalidRequest', 'did and cid parameters are required');
+    }
+    if (!isValidDid(did) || !isValidCid(cid)) {
+      return jsonError('InvalidRequest', 'did and cid parameters must be valid');
+    }
+    if (!configuredDid || did !== configuredDid) {
+      return jsonError('RepoNotFound', 'Repo not found');
     }
 
     const active = await isAccountActive(env, did);
     if (!active) {
-      return new Response(
-        JSON.stringify({ error: 'AccountInactive', message: 'Account is not active' }),
-        { status: 403, headers: { 'Content-Type': 'application/json' } }
-      );
+      return jsonError('RepoDeactivated', 'Repo is deactivated');
     }
 
     const db = getDb(env);
 
-    // Look up blob metadata by CID
-    let blobMeta = await db
+    const blobMeta = await db
       .select()
       .from(blob_ref)
-      .where(eq(blob_ref.cid, cid))
+      .where(and(eq(blob_ref.did, did), eq(blob_ref.cid, cid)))
       .get();
 
-    let key: string | null = blobMeta?.key ?? null;
-    let mime: string = blobMeta?.mime ?? 'application/octet-stream';
-    let size: number | null = blobMeta?.size ?? null;
-
-    // Fallback for older uploads: derive R2 key from CID (raw/sha256) if DB row missing
-    if (!key) {
-      try {
-        const link = CID.parse(cid);
-        // blob CIDs are raw (0x55) with sha256 multihash
-        if (link.multihash.code !== sha256.code) {
-          return new Response(
-            JSON.stringify({ error: 'InvalidRequest', message: 'Unsupported multihash' }),
-            { status: 400, headers: { 'Content-Type': 'application/json' } }
-          );
-        }
-        // Recreate legacy R2 key scheme used by store.put()
-        const digest = link.multihash.digest; // Uint8Array
-        // base64url encode
-        let s = '';
-        for (const b of digest) s += String.fromCharCode(b);
-        const b64url = btoa(s).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '');
-        key = `blobs/by-cid/${b64url}`;
-      } catch {
-        key = null;
-      }
-    }
-
-    if (!key) {
-      return new Response(
-        JSON.stringify({ error: 'InvalidRequest', message: 'Blob not found' }),
-        { status: 400, headers: { 'Content-Type': 'application/json' } }
-      );
+    if (!blobMeta || !(await blobKeyHasUsage(env, did, blobMeta.key))) {
+      return blobNotFound();
     }
 
     // Fetch blob from R2
     const r2 = env.ALTERAN_BLOBS;
-    const object = await r2.get(key);
+    const object = await r2.get(blobMeta.key);
 
-    if (!object) {
-      return new Response(
-        JSON.stringify({ error: 'InvalidRequest', message: 'Blob not found' }),
-        { status: 400, headers: { 'Content-Type': 'application/json' } }
-      );
+    if (!object || object.size !== Number(blobMeta.size)) {
+      return blobNotFound();
     }
 
-    if (!blobMeta) {
-      try {
-        size = object.size ?? size ?? 0;
-        await putBlobRef(env, did, cid, key, mime, Number(size ?? 0));
-      } catch (backfillError) {
-        // Backfill is opportunistic; serving the blob is the priority.
-        console.warn('getBlob backfill failed:', backfillError);
-      }
-    }
-
-    // workers-types' ReadableStream lacks the DOM-types readMany member; the
-    // shapes are wire-compatible at runtime, so widen through unknown.
-    return new Response(object.body as unknown as ReadableStream<Uint8Array>, {
+    const body = await responseBody(object);
+    return new Response(body, {
       status: 200,
       headers: {
-        'Content-Type': mime,
-        ...(size != null ? { 'Content-Length': String(size) } : {}),
+        'Content-Type': blobMeta.mime,
+        'Content-Length': String(blobMeta.size),
         'Cache-Control': 'public, max-age=31536000, immutable',
         'x-content-type-options': 'nosniff',
         'content-security-policy': "default-src 'none'; sandbox",
@@ -131,3 +83,69 @@ export async function GET({ locals, url }: APIContext) {
     );
   }
 }
+
+function blobNotFound(): Response {
+  return jsonError('BlobNotFound', 'Blob not found');
+}
+
+function jsonError(error: string, message: string, status = 400): Response {
+  return new Response(
+    JSON.stringify({ error, message }),
+    { status, headers: { 'Content-Type': 'application/json' } },
+  );
+}
+
+function isValidDid(did: string): boolean {
+  try {
+    ensureValidDid(did);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function isValidCid(cid: string): boolean {
+  try {
+    CID.parse(cid);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function responseBody(object: {
+  body: unknown;
+  arrayBuffer(): Promise<ArrayBuffer>;
+}): Promise<BodyInit> {
+  try {
+    return localReadableStream(object.body as ReadableStream<Uint8Array>);
+  } catch (error) {
+    // Miniflare exposes R2 bodies via a workerd object that can't be cloned
+    // across the worker boundary; fall back to a buffered read when that happens.
+    if (error instanceof Error && error.message.includes('can not be cloned')) {
+      return object.arrayBuffer();
+    }
+    throw error;
+  }
+}
+
+function localReadableStream(source: ReadableStream<Uint8Array>): ReadableStream<Uint8Array> {
+  const reader = source.getReader();
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const chunk = await reader.read();
+        if (chunk.done) {
+          controller.close();
+          return;
+        }
+        controller.enqueue(new Uint8Array(chunk.value));
+      } catch (error) {
+        controller.error(error);
+      }
+    },
+    cancel(reason) {
+      return reader.cancel(reason);
+    },
+  });
+}

package/src/pages/xrpc/com.atproto.sync.listBlobs.ts

@@ -1,7 +1,8 @@
 import type { APIContext } from 'astro';
-import { drizzle } from 'drizzle-orm/d1';
-import { blob_ref } from '../../db/schema';
-import { eq, gt, and } from 'drizzle-orm';
+import { ensureValidDid, isValidTid } from '@atproto/syntax';
+import { isAccountActive } from '../../db/dal';
+import { listBlobCids } from '../../services/repo/list-blobs';
+import { resolveSecret } from '../../lib/secrets';
 
 export const prerender = false;
 
@@ -12,31 +13,29 @@ export const prerender = false;
 export async function GET({ locals, url }: APIContext) {
   const { env } = locals.runtime;
 
-  const did = url.searchParams.get('did') || (env.PDS_DID as string);
-  const since = url.searchParams.get('since') || '';
-  const limit = parseInt(url.searchParams.get('limit') || '500', 10);
+  const configuredDid = await resolveSecret(env.PDS_DID);
+  const did = url.searchParams.get('did') ?? undefined;
+  const since = url.searchParams.get('since') ?? undefined;
+  const cursor = url.searchParams.get('cursor') ?? undefined;
+  const limit = parseLimit(url.searchParams.get('limit'));
+  if (!did) return jsonError('InvalidRequest', 'did is required');
+  if (!isValidDid(did)) return jsonError('InvalidRequest', 'did must be valid');
+  if (!limit) return jsonError('InvalidRequest', 'limit must be an integer between 1 and 1000');
+  if (since && !isValidTid(since)) return jsonError('InvalidRequest', 'since must be a valid TID');
+  if (!configuredDid || did !== configuredDid) return jsonError('RepoNotFound', 'Repo not found');
 
   try {
-    const db = drizzle(env.ALTERAN_DB);
-
-    const blobs = since
-      ? await db
-          .select()
-          .from(blob_ref)
-          .where(and(eq(blob_ref.did, did), gt(blob_ref.cid, since)))
-          .limit(limit)
-          .all()
-      : await db
-          .select()
-          .from(blob_ref)
-          .where(eq(blob_ref.did, did))
-          .limit(limit)
-          .all();
+    const active = await isAccountActive(env, did);
+    if (!active) {
+      return jsonError('RepoDeactivated', 'Repo is deactivated');
+    }
+
+    const blobs = await listBlobCids(env, { did, since, cursor, limit });
 
     return new Response(
       JSON.stringify({
-        cids: blobs.map(b => b.cid),
-        cursor: blobs.length > 0 ? blobs[blobs.length - 1].cid : undefined,
+        cids: blobs.cids,
+        ...(blobs.cursor ? { cursor: blobs.cursor } : {}),
       }),
       {
         status: 200,
@@ -51,3 +50,26 @@ export async function GET({ locals, url }: APIContext) {
     );
   }
 }
+
+function parseLimit(value: string | null): number | null {
+  if (value === null || value === '') return 500;
+  if (!/^\d+$/.test(value)) return null;
+  const limit = Number(value);
+  return Number.isInteger(limit) && limit >= 1 && limit <= 1000 ? limit : null;
+}
+
+function jsonError(error: string, message: string): Response {
+  return new Response(
+    JSON.stringify({ error, message }),
+    { status: 400, headers: { 'Content-Type': 'application/json' } },
+  );
+}
+
+function isValidDid(did: string): boolean {
+  try {
+    ensureValidDid(did);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/services/car.ts

@@ -174,10 +174,15 @@ export async function encodeBlocksForCommit(
   mstRoot: CID,
   ops: Array<{ path: string; cid: CID | null }>,
   newMstBlocks?: Array<[CID, Uint8Array]>,
+  commitBlock?: Uint8Array,
+  newRecordBlocks?: Array<readonly [CID, Uint8Array]>,
 ): Promise<Uint8Array> {
   const blockstore = new D1Blockstore(env);
   const blocks: { cid: CID; bytes: Uint8Array }[] = [];
   const seen = new Set<string>();
+  const stagedRecordBlocks = new Map(
+    (newRecordBlocks ?? []).map(([cid, bytes]) => [cid.toString(), bytes]),
+  );
 
   // Helper to add block if not already seen
   const addBlock = async (cid: CID) => {
@@ -185,6 +190,14 @@ export async function encodeBlocksForCommit(
     if (seen.has(cidStr)) return;
     seen.add(cidStr);
     let bytes = await blockstore.get(cid);
+    if (!bytes) {
+      bytes = stagedRecordBlocks.get(cidStr) ?? null;
+    }
+    if (!bytes) {
+      if (cidStr === commitCid.toString() && commitBlock) {
+        bytes = commitBlock;
+      }
+    }
     if (!bytes) {
       // Attempt to reconstruct commit block from commit_log if this is the commit cid
       if (cidStr === commitCid.toString()) {

package/src/services/repo/apply-prepared-writes.ts

@@ -0,0 +1,185 @@
+import type { CID } from 'multiformats/cid';
+import type { Env } from '../../env';
+import { bumpRoot } from '../../db/repo';
+import {
+  deleteRecordStatements,
+  getRecordBlobKeys,
+  putRecordStatements,
+  setRecordBlobUsageStatements,
+  type BlobKeyRef,
+} from '../../db/dal';
+import type { RepoOp } from '../../lib/firehose/frames';
+import { RepoWriteError } from '../../lib/repo-write-error';
+import type { MST } from '../../lib/mst';
+import type { PreparedWrite, ValidationStatus } from '../../lib/repo-write-validation';
+import {
+  collectUnstoredMstBlocks,
+  encodeRecordBlock,
+  type EncodedBlock,
+} from './blockstore-ops';
+import { assertBlobKeysAvailable } from './blob-refs';
+
+export interface BatchCommitResult {
+  commit: {
+    cid: string;
+    rev: string;
+  } | null;
+  commitCid?: string;
+  rev?: string;
+  ops: RepoOp[];
+  commitData?: string;
+  sig?: string;
+  blocks?: string;
+  results: Array<{
+    $type: string;
+    uri?: string;
+    cid?: string;
+    validationStatus?: ValidationStatus;
+  }>;
+  dereferencedBlobKeys: BlobKeyRef[];
+}
+
+type SideEffect =
+  | { action: 'put'; uri: string; cid: string; record: unknown; blobKeys: string[] }
+  | { action: 'delete'; uri: string };
+
+export async function applyPreparedWritesToRepo(
+  env: Env,
+  did: string,
+  root: MST,
+  writes: PreparedWrite[],
+  expectedCommitCid?: string | null,
+): Promise<BatchCommitResult> {
+  let mst = root;
+  const prevMstRoot = await mst.getPointer();
+  const ops: RepoOp[] = [];
+  const sideEffects: SideEffect[] = [];
+  const results: BatchCommitResult['results'] = [];
+  const recordBlocks: EncodedBlock[] = [];
+
+  for (const write of writes) {
+    const path = `${write.collection}/${write.rkey}`;
+    const uri = `at://${did}/${path}`;
+    if (write.action === 'delete') {
+      const prev = await mst.get(path);
+      if (!prev) throw new RepoWriteError('InvalidRequest', 'record does not exist');
+      mst = await mst.delete(path);
+      ops.push({ action: 'delete', path, cid: null, prev });
+      sideEffects.push({ action: 'delete', uri });
+      results.push({ $type: 'com.atproto.repo.applyWrites#deleteResult' });
+      continue;
+    }
+
+    const prev = await mst.get(path);
+    if (write.action === 'update') {
+      if (!prev) throw new RepoWriteError('InvalidRequest', 'record does not exist');
+      const recordBlock = await encodeRecordBlock(write.record);
+      const [recordCid] = recordBlock;
+      recordBlocks.push(recordBlock);
+      mst = await mst.update(path, recordCid);
+      ops.push({ action: 'update', path, cid: recordCid, prev });
+      addPutEffect(sideEffects, results, uri, recordCid, write);
+      continue;
+    }
+
+    if (prev) throw new RepoWriteError('InvalidRequest', 'record already exists');
+    const recordBlock = await encodeRecordBlock(write.record);
+    const [recordCid] = recordBlock;
+    recordBlocks.push(recordBlock);
+    mst = await mst.add(path, recordCid);
+    ops.push({ action: 'create', path, cid: recordCid });
+    addPutEffect(sideEffects, results, uri, recordCid, write);
+  }
+
+  const dereferencedBlobKeys = await findDereferencedBlobKeys(env, did, sideEffects);
+  const currentRoot = await mst.getPointer();
+  const newMstBlocks = await collectUnstoredMstBlocks(mst);
+  const requiredBlobKeys = sideEffects.flatMap((effect) =>
+    effect.action === 'put' ? effect.blobKeys : [],
+  );
+  await assertBlobKeysAvailable(env, did, requiredBlobKeys);
+
+  const { commitCid, rev, commitData, sig, blocks } = await bumpRoot(
+    env,
+    prevMstRoot ?? undefined,
+    currentRoot,
+    {
+      ops,
+      newMstBlocks: Array.from(newMstBlocks),
+      newRecordBlocks: recordBlocks,
+      expectedCommitCid,
+      requiredBlobKeys,
+      sideEffectStatements: (guard) => sideEffects.flatMap((effect) => {
+        if (effect.action === 'delete') {
+          return [
+            ...deleteRecordStatements(env, effect.uri, guard),
+            ...setRecordBlobUsageStatements(env, did, effect.uri, [], guard),
+          ];
+        }
+        return [
+          ...putRecordStatements(env, {
+            uri: effect.uri,
+            did,
+            cid: effect.cid,
+            json: JSON.stringify(effect.record),
+          }, guard),
+          ...setRecordBlobUsageStatements(env, did, effect.uri, effect.blobKeys, guard),
+        ];
+      }),
+    },
+  );
+
+  return {
+    commit: { cid: commitCid, rev },
+    commitCid,
+    rev,
+    ops,
+    commitData,
+    sig,
+    blocks,
+    results,
+    dereferencedBlobKeys: dereferencedBlobKeys.map((key) => ({ did, key })),
+  };
+}
+
+function addPutEffect(
+  sideEffects: SideEffect[],
+  results: BatchCommitResult['results'],
+  uri: string,
+  recordCid: CID,
+  write: Exclude<PreparedWrite, { action: 'delete' }>,
+): void {
+  const cid = recordCid.toString();
+  sideEffects.push({
+    action: 'put',
+    uri,
+    cid,
+    record: write.record,
+    blobKeys: write.blobKeys,
+  });
+  results.push({
+    $type: `com.atproto.repo.applyWrites#${write.action}Result`,
+    uri,
+    cid,
+    validationStatus: write.validationStatus,
+  });
+}
+
+async function findDereferencedBlobKeys(
+  env: Env,
+  did: string,
+  sideEffects: SideEffect[],
+): Promise<string[]> {
+  const previousBlobKeysByUri = new Map<string, string[]>();
+  const finalBlobKeysByUri = new Map<string, string[]>();
+  for (const effect of sideEffects) {
+    if (!previousBlobKeysByUri.has(effect.uri)) {
+      previousBlobKeysByUri.set(effect.uri, await getRecordBlobKeys(env, did, effect.uri));
+    }
+    finalBlobKeysByUri.set(effect.uri, effect.action === 'put' ? effect.blobKeys : []);
+  }
+  return Array.from(previousBlobKeysByUri).flatMap(([uri, previousKeys]) => {
+    const finalKeys = finalBlobKeysByUri.get(uri) ?? [];
+    return previousKeys.filter((key) => !finalKeys.includes(key));
+  });
+}

package/src/services/repo/blob-refs.ts

@@ -0,0 +1,48 @@
+import type { Env } from '../../env';
+import { RepoWriteError } from '../../lib/repo-write-error';
+import { collectBlobRefs } from '../../lib/repo-write-data';
+
+export async function assertBlobKeysAvailable(env: Env, did: string, keys: string[]): Promise<void> {
+  for (const key of new Set(keys)) {
+    const row = await env.ALTERAN_DB.prepare(
+      'SELECT cid, size FROM blob WHERE did = ? AND key = ? LIMIT 1',
+    )
+      .bind(did, key)
+      .first<{ cid: string; size: number }>();
+    if (!row) {
+      throw new RepoWriteError('BlobNotFound', 'blob not found');
+    }
+    const object = await env.ALTERAN_BLOBS.head(key);
+    if (!object || object.size !== Number(row.size)) {
+      throw new RepoWriteError('BlobNotFound', `blob not found: ${row.cid}`);
+    }
+  }
+}
+
+export async function resolveRecordBlobKeys(env: Env, did: string, record: unknown): Promise<string[]> {
+  const refs: Array<{ cid: string; mimeType: string; size: number }> = [];
+  collectBlobRefs(record, refs);
+  const keys = new Set<string>();
+  for (const ref of refs) {
+    const row = await env.ALTERAN_DB.prepare(
+      'SELECT key, mime, size FROM blob WHERE did = ? AND cid = ? LIMIT 1',
+    )
+      .bind(did, ref.cid)
+      .first<{ key: string; mime: string; size: number }>();
+    if (!row) {
+      throw new RepoWriteError('BlobNotFound', `blob not found: ${ref.cid}`);
+    }
+    if (row.mime !== ref.mimeType) {
+      throw new RepoWriteError('InvalidMimeType', `blob mime type mismatch: ${ref.cid}`);
+    }
+    if (Number(row.size) !== ref.size) {
+      throw new RepoWriteError('InvalidSize', `blob size mismatch: ${ref.cid}`);
+    }
+    const object = await env.ALTERAN_BLOBS.head(row.key);
+    if (!object || object.size !== Number(row.size)) {
+      throw new RepoWriteError('BlobNotFound', `blob not found: ${ref.cid}`);
+    }
+    keys.add(row.key);
+  }
+  return Array.from(keys);
+}

package/src/services/repo/blockstore-ops.ts

@@ -1,29 +1,71 @@
 import { CID } from 'multiformats/cid';
 import * as dagCbor from '@ipld/dag-cbor';
+import { fromString as bytesFromString } from 'uint8arrays/from-string';
 import { cidForCbor } from '../../lib/mst/util';
-import type { D1Blockstore, MST, BlockMap } from '../../lib/mst';
+import type { MST, BlockMap } from '../../lib/mst';
+import { RepoWriteError } from '../../lib/repo-write-error';
 
-export async function storeRecord(
-  blockstore: D1Blockstore,
+export type EncodedBlock = readonly [CID, Uint8Array];
+
+export function recordToIpld(record: unknown, depth = 0): unknown {
+  if (depth > 100) {
+    throw new RepoWriteError('InvalidRequest', 'record is too deeply nested');
+  }
+  if (Array.isArray(record)) {
+    return record.map((item) => recordToIpld(item, depth + 1));
+  }
+  if (!record || typeof record !== 'object') {
+    return record;
+  }
+  if (record instanceof Uint8Array || CID.asCID(record)) {
+    return record;
+  }
+
+  const obj = record as Record<string, unknown>;
+  const keys = Object.keys(obj);
+  if (keys.length === 1 && typeof obj.$link === 'string') {
+    return CID.parse(obj.$link);
+  }
+  if (keys.length === 1 && typeof obj.$bytes === 'string') {
+    return bytesFromString(obj.$bytes, 'base64');
+  }
+
+  const converted: Record<string, unknown> = Object.create(null);
+  for (const [key, value] of Object.entries(obj)) {
+    if (key === '__proto__') {
+      throw new RepoWriteError('InvalidRequest', 'record contains a forbidden object key');
+    }
+    converted[key] = recordToIpld(value, depth + 1);
+  }
+  return converted;
+}
+
+export async function cidForRecord(record: unknown): Promise<CID> {
+  try {
+    return cidForCbor(recordToIpld(record));
+  } catch (error) {
+    if (error instanceof RepoWriteError) throw error;
+    throw new RepoWriteError('InvalidRequest', 'record is not dag-cbor encodable');
+  }
+}
+
+export async function encodeRecordBlock(
   record: unknown,
-): Promise<CID> {
-  const bytes = dagCbor.encode(record);
-  const cid = await cidForCbor(record);
-  await blockstore.put(cid, bytes);
-  return cid;
+): Promise<EncodedBlock> {
+  try {
+    const ipldRecord = recordToIpld(record);
+    const bytes = dagCbor.encode(ipldRecord);
+    const cid = await cidForCbor(ipldRecord);
+    return [cid, bytes] as const;
+  } catch (error) {
+    if (error instanceof RepoWriteError) throw error;
+    throw new RepoWriteError('InvalidRequest', 'record is not dag-cbor encodable');
+  }
 }
 
-export async function storeMstBlocks(
-  blockstore: D1Blockstore,
+export async function collectUnstoredMstBlocks(
   mst: MST,
 ): Promise<BlockMap> {
   const diff = await mst.getUnstoredBlocks();
-  for (const [cid, bytes] of diff.blocks) {
-    console.log(
-      `[RepoManager] Storing new MST block: ${cid.toString()}, size: ${bytes.length}`,
-    );
-    await blockstore.put(cid, bytes);
-  }
-  console.log(`[RepoManager] Stored ${diff.blocks.size} new MST blocks`);
   return diff.blocks;
 }