@alteran/astro 0.7.6 → 0.8.1

package/src/services/repo/list-blobs.ts

@@ -0,0 +1,43 @@
+import type { Env } from '../../env';
+
+type ListBlobCidsParams = {
+  did: string;
+  since?: string;
+  cursor?: string;
+  limit: number;
+};
+
+export async function listBlobCids(
+  env: Env,
+  { did, since, cursor, limit }: ListBlobCidsParams,
+): Promise<{ cids: string[]; cursor?: string }> {
+  const pageSize = limit + 1;
+  const clauses = ['did = ?'];
+  const values: Array<string | number> = [did];
+  if (since) {
+    clauses.push('repo_rev > ?');
+    values.push(since);
+  }
+  if (cursor) {
+    clauses.push('cid > ?');
+    values.push(cursor);
+  }
+  values.push(pageSize);
+
+  const result = await env.ALTERAN_DB.prepare(
+    `SELECT DISTINCT cid
+     FROM blob_usage
+     WHERE ${clauses.join(' AND ')}
+     ORDER BY cid
+     LIMIT ?`,
+  )
+    .bind(...values)
+    .all<{ cid: string }>();
+
+  const rows = result.results ?? [];
+  const page = rows.slice(0, limit).map((row) => row.cid);
+  return {
+    cids: page,
+    ...(page.length > 0 ? { cursor: page[page.length - 1] } : {}),
+  };
+}

package/src/services/repo-manager.ts

@@ -5,18 +5,38 @@ import { drizzle } from 'drizzle-orm/d1';
 import { repo_root } from '../db/schema';
 import { eq, sql } from 'drizzle-orm';
 import type { RepoOp } from '../lib/firehose/frames';
-import { putRecord as dalPutRecord, deleteRecord as dalDeleteRecord } from '../db/dal';
-import { bumpRoot } from '../db/repo';
+import {
+  putRecordStatements,
+  repairRecordBlobUsageForCurrentRecord,
+  setRecordBlobUsageStatements,
+  getRecordBlobKeys,
+  type BlobKeyRef,
+} from '../db/dal';
+import { assertRepoHead, bumpRoot, RepoBlobNotFoundError, RepoCommitConflictError } from '../db/repo';
 import { generateTid } from '../lib/commit';
 import { resolveSecret } from '../lib/secrets';
-import { storeRecord, storeMstBlocks } from './repo/blockstore-ops';
+import {
+  collectUnstoredMstBlocks,
+  encodeRecordBlock,
+  cidForRecord,
+  type EncodedBlock,
+} from './repo/blockstore-ops';
 import { extractOps as extractOpsImpl } from './repo/operations';
 import { ServerMisconfigured } from '../lib/errors';
+import { RepoWriteError } from '../lib/repo-write-error';
+import type { PreparedWrite } from '../lib/repo-write-validation';
+import { assertBlobKeysAvailable, resolveRecordBlobKeys } from './repo/blob-refs';
+import {
+  applyPreparedWritesToRepo,
+  type BatchCommitResult,
+} from './repo/apply-prepared-writes';
 
 interface RecordMutation {
   mst: MST;
   recordCid: CID;
+  recordBlock: EncodedBlock;
   prevMstRoot: CID | null;
+  expectedCommitCid: string | null;
   newMstBlocks: BlockMap;
 }
 
@@ -29,8 +49,23 @@ interface CommitResult {
   commitData: string;
   sig: string;
   blocks: string;
+  dereferencedBlobKeys: BlobKeyRef[];
 }
 
+interface NoopRecordResult {
+  uri: string;
+  cid: string;
+  ops: RepoOp[];
+  dereferencedBlobKeys?: undefined;
+  commitCid?: undefined;
+  rev?: undefined;
+  commitData?: undefined;
+  sig?: undefined;
+  blocks?: undefined;
+}
+
+type RecordWriteResult = CommitResult | NoopRecordResult;
+
 export class RepoManager {
   private blockstore: D1Blockstore;
 
@@ -44,59 +79,61 @@ export class RepoManager {
     return did;
   }
 
-  async getRoot(): Promise<MST | null> {
-    try {
-      const db = drizzle(this.env.ALTERAN_DB);
-      const did = await this.getDid();
-
-      const rows = await db
-        .select()
-        .from(repo_root)
-        .where(eq(repo_root.did, did))
-        .limit(1);
-
-      const row = rows[0];
-      if (!row) return null;
-
-      const commit = await this.env.ALTERAN_DB.prepare(
-        `SELECT data FROM commit_log WHERE cid = ? LIMIT 1`,
-      )
-        .bind(row.commitCid)
-        .first();
-
-      if (!commit) {
-        console.error(`[RepoManager] No commit found for CID: ${row.commitCid}`);
-        return null;
-      }
+  private async getRootSnapshot(): Promise<{ mst: MST; commitCid: string } | null> {
+    const db = drizzle(this.env.ALTERAN_DB);
+    const did = await this.getDid();
 
-      const parsed = JSON.parse(String(commit.data));
-      const mstRoot = CID.parse(String(parsed.data));
+    const rows = await db
+      .select()
+      .from(repo_root)
+      .where(eq(repo_root.did, did))
+      .limit(1);
 
-      console.log(
-        `[RepoManager] Loading MST root: ${mstRoot.toString()} from commit: ${row.commitCid}`,
-      );
+    const row = rows[0];
+    if (!row) return null;
 
-      return MST.load(this.blockstore, mstRoot);
-    } catch (error) {
-      console.error('[RepoManager] Error in getRoot:', error);
-      return null;
+    const commit = await this.env.ALTERAN_DB.prepare(
+      `SELECT data FROM commit_log WHERE cid = ? LIMIT 1`,
+    )
+      .bind(row.commitCid)
+      .first();
+
+    if (!commit) {
+      throw new Error(`repo root points at missing commit ${row.commitCid}`);
     }
+
+    const parsed = JSON.parse(String(commit.data));
+    const mstRoot = CID.parse(String(parsed.data));
+
+    console.log(
+      `[RepoManager] Loading MST root: ${mstRoot.toString()} from commit: ${row.commitCid}`,
+    );
+
+    return { mst: await MST.load(this.blockstore, mstRoot), commitCid: row.commitCid };
   }
 
-  async getOrCreateRoot(): Promise<MST> {
-    const existing = await this.getRoot();
+  async getRoot(): Promise<MST | null> {
+    return (await this.getRootSnapshot())?.mst ?? null;
+  }
+
+  private async getOrCreateRootSnapshot(): Promise<{ mst: MST; commitCid: string | null }> {
+    const existing = await this.getRootSnapshot();
     if (existing) {
-      const pointer = await existing.getPointer();
+      const pointer = await existing.mst.getPointer();
       console.log(`[RepoManager] Loaded existing MST root: ${pointer.toString()}`);
       return existing;
     }
 
     console.log('[RepoManager] Creating new empty MST');
     const mst = await MST.create(this.blockstore, []);
-    await storeMstBlocks(this.blockstore, mst);
     const pointer = await mst.getPointer();
     console.log(`[RepoManager] Created new MST root: ${pointer.toString()}`);
-    return mst;
+    return { mst, commitCid: null };
+  }
+
+  async getOrCreateRoot(): Promise<MST> {
+    const snapshot = await this.getOrCreateRootSnapshot();
+    return snapshot.mst;
   }
 
   async addRecord(
@@ -105,44 +142,70 @@ export class RepoManager {
     record: unknown,
   ): Promise<RecordMutation> {
     const key = `${collection}/${rkey}`;
-    const currentMst = await this.getOrCreateRoot();
-    const prevMstRoot = await currentMst.getPointer();
-    const recordCid = await storeRecord(this.blockstore, record);
+    const { mst: currentMst, commitCid: expectedCommitCid } = await this.getOrCreateRootSnapshot();
+    const prevMstRoot = expectedCommitCid === null ? null : await currentMst.getPointer();
+    const recordBlock = await encodeRecordBlock(record);
+    const [recordCid] = recordBlock;
     const newMst = await currentMst.add(key, recordCid);
-    const newMstBlocks = await storeMstBlocks(this.blockstore, newMst);
-    return { mst: newMst, recordCid, prevMstRoot, newMstBlocks };
+    const newMstBlocks = await collectUnstoredMstBlocks(newMst);
+    return { mst: newMst, recordCid, recordBlock, prevMstRoot, expectedCommitCid, newMstBlocks };
   }
 
   async createRecord(
     collection: string,
     record: unknown,
     rkey?: string,
+    blobKeys?: string[],
+    expectedCommitCid?: string | null,
   ): Promise<CommitResult> {
     const key = rkey ?? generateTid();
-    const { mst, recordCid, prevMstRoot, newMstBlocks } = await this.addRecord(
+    const { mst, recordCid, recordBlock, prevMstRoot, expectedCommitCid: rootCommitCid, newMstBlocks } = await this.addRecord(
       collection,
       key,
       record,
     );
 
     const did = await this.getDid();
+    const effectiveBlobKeys = blobKeys ?? await resolveRecordBlobKeys(this.env, did, record);
     const uri = `at://${did}/${collection}/${key}`;
-    await dalPutRecord(this.env, {
-      uri,
-      did,
-      cid: recordCid.toString(),
-      json: JSON.stringify(record),
-    });
 
     const currentRoot = await mst.getPointer();
+    const commitGuard = expectedCommitCid === undefined ? rootCommitCid : expectedCommitCid;
+    const opsForCommit: RepoOp[] = [{ action: 'create', path: `${collection}/${key}`, cid: recordCid }];
+    await assertBlobKeysAvailable(this.env, did, effectiveBlobKeys);
     const { commitCid, rev, ops, commitData, sig, blocks } = await bumpRoot(
       this.env,
       prevMstRoot ?? undefined,
       currentRoot,
-      { newMstBlocks: Array.from(newMstBlocks) },
+      {
+        ops: opsForCommit,
+        newMstBlocks: Array.from(newMstBlocks),
+        newRecordBlocks: [recordBlock],
+        expectedCommitCid: commitGuard,
+        requiredBlobKeys: effectiveBlobKeys,
+        sideEffectStatements: (guard) => [
+          ...putRecordStatements(this.env, {
+            uri,
+            did,
+            cid: recordCid.toString(),
+            json: JSON.stringify(record),
+          }, guard),
+          ...setRecordBlobUsageStatements(this.env, did, uri, effectiveBlobKeys, guard),
+        ],
+      },
     );
 
-    return { uri, cid: recordCid.toString(), commitCid, rev, ops, commitData, sig, blocks };
+    return {
+      uri,
+      cid: recordCid.toString(),
+      commitCid,
+      rev,
+      ops,
+      commitData,
+      sig,
+      blocks,
+      dereferencedBlobKeys: [],
+    };
   }
 
   async updateRecord(
@@ -151,53 +214,133 @@ export class RepoManager {
     record: unknown,
   ): Promise<RecordMutation> {
     const key = `${collection}/${rkey}`;
-    const currentMst = await this.getOrCreateRoot();
-    const prevMstRoot = await currentMst.getPointer();
-    const recordCid = await storeRecord(this.blockstore, record);
+    const { mst: currentMst, commitCid: expectedCommitCid } = await this.getOrCreateRootSnapshot();
+    const prevMstRoot = expectedCommitCid === null ? null : await currentMst.getPointer();
+    const recordBlock = await encodeRecordBlock(record);
+    const [recordCid] = recordBlock;
     const newMst = await currentMst.update(key, recordCid);
-    const newMstBlocks = await storeMstBlocks(this.blockstore, newMst);
-    return { mst: newMst, recordCid, prevMstRoot, newMstBlocks };
+    const newMstBlocks = await collectUnstoredMstBlocks(newMst);
+    return { mst: newMst, recordCid, recordBlock, prevMstRoot, expectedCommitCid, newMstBlocks };
   }
 
-  async putRecord(collection: string, rkey: string, record: unknown): Promise<CommitResult> {
-    const { mst, recordCid, prevMstRoot, newMstBlocks } = await this.updateRecord(
-      collection,
-      rkey,
-      record,
-    );
+  async putRecord(
+    collection: string,
+    rkey: string,
+    record: unknown,
+    blobKeys?: string[],
+    expectedCommitCid?: string | null,
+  ): Promise<RecordWriteResult> {
+    const key = `${collection}/${rkey}`;
+    const { mst: currentMst, commitCid: rootCommitCid } = await this.getOrCreateRootSnapshot();
+    const prevMstRoot = rootCommitCid === null ? null : await currentMst.getPointer();
+    const existingCid = await currentMst.get(key);
+    const recordCid = await cidForRecord(record);
     const did = await this.getDid();
+    const effectiveBlobKeys = blobKeys ?? await resolveRecordBlobKeys(this.env, did, record);
     const uri = `at://${did}/${collection}/${rkey}`;
-    await dalPutRecord(this.env, {
-      uri,
-      did,
-      cid: recordCid.toString(),
-      json: JSON.stringify(record),
-    });
+    const commitGuard = expectedCommitCid === undefined ? rootCommitCid : expectedCommitCid;
+    if (existingCid?.toString() === recordCid.toString()) {
+      if (typeof commitGuard !== 'string') {
+        await assertRepoHead(this.env, did, commitGuard);
+      } else {
+        const repairResult = await repairRecordBlobUsageForCurrentRecord(
+          this.env,
+          did,
+          uri,
+          recordCid.toString(),
+          effectiveBlobKeys,
+          commitGuard,
+        );
+        if (repairResult.tag === 'blobNotFound') throw new RepoBlobNotFoundError();
+        if (repairResult.tag === 'conflict') throw new RepoCommitConflictError();
+      }
+      return { uri, cid: recordCid.toString(), ops: [] };
+    }
+
+    const previousBlobKeys = await getRecordBlobKeys(this.env, did, uri);
+    const recordBlock = await encodeRecordBlock(record);
+    const mst = existingCid
+      ? await currentMst.update(key, recordCid)
+      : await currentMst.add(key, recordCid);
+    const newMstBlocks = await collectUnstoredMstBlocks(mst);
     const currentRoot = await mst.getPointer();
+    const opsForCommit: RepoOp[] = existingCid
+      ? [{ action: 'update', path: key, cid: recordCid, prev: existingCid }]
+      : [{ action: 'create', path: key, cid: recordCid }];
+    await assertBlobKeysAvailable(this.env, did, effectiveBlobKeys);
     const { commitCid, rev, ops, commitData, sig, blocks } = await bumpRoot(
       this.env,
       prevMstRoot ?? undefined,
       currentRoot,
-      { newMstBlocks: Array.from(newMstBlocks) },
+      {
+        ops: opsForCommit,
+        newMstBlocks: Array.from(newMstBlocks),
+        newRecordBlocks: [recordBlock],
+        expectedCommitCid: commitGuard,
+        requiredBlobKeys: effectiveBlobKeys,
+        sideEffectStatements: (guard) => [
+          ...putRecordStatements(this.env, {
+            uri,
+            did,
+            cid: recordCid.toString(),
+            json: JSON.stringify(record),
+          }, guard),
+          ...setRecordBlobUsageStatements(this.env, did, uri, effectiveBlobKeys, guard),
+        ],
+      },
+    );
+    return {
+      uri,
+      cid: recordCid.toString(),
+      commitCid,
+      rev,
+      ops,
+      commitData,
+      sig,
+      blocks,
+      dereferencedBlobKeys: previousBlobKeys
+        .filter((key) => !effectiveBlobKeys.includes(key))
+        .map((key) => ({ did, key })),
+    };
+  }
+
+  async getRecordCid(collection: string, rkey: string): Promise<CID | null> {
+    const currentMst = await this.getRoot();
+    if (!currentMst) return null;
+    return currentMst.get(`${collection}/${rkey}`);
+  }
+
+  async applyPreparedWrites(
+    writes: PreparedWrite[],
+    expectedCommitCid?: string | null,
+  ): Promise<BatchCommitResult> {
+    const did = await this.getDid();
+    const snapshot = await this.getOrCreateRootSnapshot();
+    return applyPreparedWritesToRepo(
+      this.env,
+      did,
+      snapshot.mst,
+      writes,
+      expectedCommitCid === undefined ? snapshot.commitCid : expectedCommitCid,
     );
-    return { uri, cid: recordCid.toString(), commitCid, rev, ops, commitData, sig, blocks };
   }
 
   async deleteRecord(
     collection: string,
     rkey: string,
-  ): Promise<{ mst: MST; prevMstRoot: CID | null; uri: string; newMstBlocks: BlockMap }> {
+  ): Promise<{ mst: MST; prevMstRoot: CID | null; uri: string; newMstBlocks: BlockMap; currentCid: CID }> {
     const key = `${collection}/${rkey}`;
     const currentMst = await this.getOrCreateRoot();
     const prevMstRoot = await currentMst.getPointer();
+    const currentCid = await currentMst.get(key);
+    if (!currentCid) throw new RepoWriteError('InvalidRequest', 'record does not exist');
     const newMst = await currentMst.delete(key);
-    const newMstBlocks = await storeMstBlocks(this.blockstore, newMst);
+    const newMstBlocks = await collectUnstoredMstBlocks(newMst);
 
     const did = await this.getDid();
     const uri = `at://${did}/${collection}/${rkey}`;
-    await dalDeleteRecord(this.env, uri);
 
-    return { mst: newMst, prevMstRoot, uri, newMstBlocks };
+    return { mst: newMst, prevMstRoot, uri, newMstBlocks, currentCid };
   }
 
   async getRecord(collection: string, rkey: string): Promise<unknown | null> {

package/src/worker/runtime.ts

@@ -166,7 +166,7 @@ export function normalizePdsRequestForAstro(request: WorkersRequest): WorkersReq
   }
 
   const headerRecord: Record<string, string> = {};
-  request.headers.forEach((value, key) => {
+  request.headers.forEach((value: string, key: string) => {
     headerRecord[key] = value;
   });
   headerRecord.origin = url.origin;

package/src/worker/sequencer/upgrade.ts

@@ -30,6 +30,8 @@ export type UpgradeContext = {
   readonly onClient: (id: string, cursor: number, server: HibernatableSocket) => void;
 };
 
+type WebSocketResponseInit = ResponseInit & { readonly webSocket: WebSocket };
+
 // Reject NaN / negative / non-integer cursors at the boundary. parseInt('abc')
 // yields NaN, which would silently bypass `cursor > nextSeq - 1` (all NaN
 // comparisons are false) and get persisted into the attachment.
@@ -137,5 +139,6 @@ export function handleUpgrade(
 function buildUpgradeResponse(client: WebSocket, requestedProtocol: string | undefined): Response {
   const headers = new Headers();
   if (requestedProtocol) headers.set('Sec-WebSocket-Protocol', requestedProtocol);
-  return new Response(null, { status: 101, webSocket: client, headers });
+  const init: WebSocketResponseInit = { status: 101, webSocket: client, headers };
+  return new Response(null, init);
 }