@alteran/astro 0.7.7 → 0.8.2

package/src/services/repo/apply-prepared-writes.ts

@@ -0,0 +1,185 @@
+import type { CID } from 'multiformats/cid';
+import type { Env } from '../../env';
+import { bumpRoot } from '../../db/repo';
+import {
+  deleteRecordStatements,
+  getRecordBlobKeys,
+  putRecordStatements,
+  setRecordBlobUsageStatements,
+  type BlobKeyRef,
+} from '../../db/dal';
+import type { RepoOp } from '../../lib/firehose/frames';
+import { RepoWriteError } from '../../lib/repo-write-error';
+import type { MST } from '../../lib/mst';
+import type { PreparedWrite, ValidationStatus } from '../../lib/repo-write-validation';
+import {
+  collectUnstoredMstBlocks,
+  encodeRecordBlock,
+  type EncodedBlock,
+} from './blockstore-ops';
+import { assertBlobKeysAvailable } from './blob-refs';
+
+export interface BatchCommitResult {
+  commit: {
+    cid: string;
+    rev: string;
+  } | null;
+  commitCid?: string;
+  rev?: string;
+  ops: RepoOp[];
+  commitData?: string;
+  sig?: string;
+  blocks?: string;
+  results: Array<{
+    $type: string;
+    uri?: string;
+    cid?: string;
+    validationStatus?: ValidationStatus;
+  }>;
+  dereferencedBlobKeys: BlobKeyRef[];
+}
+
+type SideEffect =
+  | { action: 'put'; uri: string; cid: string; record: unknown; blobKeys: string[] }
+  | { action: 'delete'; uri: string };
+
+export async function applyPreparedWritesToRepo(
+  env: Env,
+  did: string,
+  root: MST,
+  writes: PreparedWrite[],
+  expectedCommitCid?: string | null,
+): Promise<BatchCommitResult> {
+  let mst = root;
+  const prevMstRoot = await mst.getPointer();
+  const ops: RepoOp[] = [];
+  const sideEffects: SideEffect[] = [];
+  const results: BatchCommitResult['results'] = [];
+  const recordBlocks: EncodedBlock[] = [];
+
+  for (const write of writes) {
+    const path = `${write.collection}/${write.rkey}`;
+    const uri = `at://${did}/${path}`;
+    if (write.action === 'delete') {
+      const prev = await mst.get(path);
+      if (!prev) throw new RepoWriteError('InvalidRequest', 'record does not exist');
+      mst = await mst.delete(path);
+      ops.push({ action: 'delete', path, cid: null, prev });
+      sideEffects.push({ action: 'delete', uri });
+      results.push({ $type: 'com.atproto.repo.applyWrites#deleteResult' });
+      continue;
+    }
+
+    const prev = await mst.get(path);
+    if (write.action === 'update') {
+      if (!prev) throw new RepoWriteError('InvalidRequest', 'record does not exist');
+      const recordBlock = await encodeRecordBlock(write.record);
+      const [recordCid] = recordBlock;
+      recordBlocks.push(recordBlock);
+      mst = await mst.update(path, recordCid);
+      ops.push({ action: 'update', path, cid: recordCid, prev });
+      addPutEffect(sideEffects, results, uri, recordCid, write);
+      continue;
+    }
+
+    if (prev) throw new RepoWriteError('InvalidRequest', 'record already exists');
+    const recordBlock = await encodeRecordBlock(write.record);
+    const [recordCid] = recordBlock;
+    recordBlocks.push(recordBlock);
+    mst = await mst.add(path, recordCid);
+    ops.push({ action: 'create', path, cid: recordCid });
+    addPutEffect(sideEffects, results, uri, recordCid, write);
+  }
+
+  const dereferencedBlobKeys = await findDereferencedBlobKeys(env, did, sideEffects);
+  const currentRoot = await mst.getPointer();
+  const newMstBlocks = await collectUnstoredMstBlocks(mst);
+  const requiredBlobKeys = sideEffects.flatMap((effect) =>
+    effect.action === 'put' ? effect.blobKeys : [],
+  );
+  await assertBlobKeysAvailable(env, did, requiredBlobKeys);
+
+  const { commitCid, rev, commitData, sig, blocks } = await bumpRoot(
+    env,
+    prevMstRoot ?? undefined,
+    currentRoot,
+    {
+      ops,
+      newMstBlocks: Array.from(newMstBlocks),
+      newRecordBlocks: recordBlocks,
+      expectedCommitCid,
+      requiredBlobKeys,
+      sideEffectStatements: (guard) => sideEffects.flatMap((effect) => {
+        if (effect.action === 'delete') {
+          return [
+            ...deleteRecordStatements(env, effect.uri, guard),
+            ...setRecordBlobUsageStatements(env, did, effect.uri, [], guard),
+          ];
+        }
+        return [
+          ...putRecordStatements(env, {
+            uri: effect.uri,
+            did,
+            cid: effect.cid,
+            json: JSON.stringify(effect.record),
+          }, guard),
+          ...setRecordBlobUsageStatements(env, did, effect.uri, effect.blobKeys, guard),
+        ];
+      }),
+    },
+  );
+
+  return {
+    commit: { cid: commitCid, rev },
+    commitCid,
+    rev,
+    ops,
+    commitData,
+    sig,
+    blocks,
+    results,
+    dereferencedBlobKeys: dereferencedBlobKeys.map((key) => ({ did, key })),
+  };
+}
+
+function addPutEffect(
+  sideEffects: SideEffect[],
+  results: BatchCommitResult['results'],
+  uri: string,
+  recordCid: CID,
+  write: Exclude<PreparedWrite, { action: 'delete' }>,
+): void {
+  const cid = recordCid.toString();
+  sideEffects.push({
+    action: 'put',
+    uri,
+    cid,
+    record: write.record,
+    blobKeys: write.blobKeys,
+  });
+  results.push({
+    $type: `com.atproto.repo.applyWrites#${write.action}Result`,
+    uri,
+    cid,
+    validationStatus: write.validationStatus,
+  });
+}
+
+async function findDereferencedBlobKeys(
+  env: Env,
+  did: string,
+  sideEffects: SideEffect[],
+): Promise<string[]> {
+  const previousBlobKeysByUri = new Map<string, string[]>();
+  const finalBlobKeysByUri = new Map<string, string[]>();
+  for (const effect of sideEffects) {
+    if (!previousBlobKeysByUri.has(effect.uri)) {
+      previousBlobKeysByUri.set(effect.uri, await getRecordBlobKeys(env, did, effect.uri));
+    }
+    finalBlobKeysByUri.set(effect.uri, effect.action === 'put' ? effect.blobKeys : []);
+  }
+  return Array.from(previousBlobKeysByUri).flatMap(([uri, previousKeys]) => {
+    const finalKeys = finalBlobKeysByUri.get(uri) ?? [];
+    return previousKeys.filter((key) => !finalKeys.includes(key));
+  });
+}

package/src/services/repo/blob-refs.ts

@@ -0,0 +1,48 @@
+import type { Env } from '../../env';
+import { RepoWriteError } from '../../lib/repo-write-error';
+import { collectBlobRefs } from '../../lib/repo-write-data';
+
+export async function assertBlobKeysAvailable(env: Env, did: string, keys: string[]): Promise<void> {
+  for (const key of new Set(keys)) {
+    const row = await env.ALTERAN_DB.prepare(
+      'SELECT cid, size FROM blob WHERE did = ? AND key = ? LIMIT 1',
+    )
+      .bind(did, key)
+      .first<{ cid: string; size: number }>();
+    if (!row) {
+      throw new RepoWriteError('BlobNotFound', 'blob not found');
+    }
+    const object = await env.ALTERAN_BLOBS.head(key);
+    if (!object || object.size !== Number(row.size)) {
+      throw new RepoWriteError('BlobNotFound', `blob not found: ${row.cid}`);
+    }
+  }
+}
+
+export async function resolveRecordBlobKeys(env: Env, did: string, record: unknown): Promise<string[]> {
+  const refs: Array<{ cid: string; mimeType: string; size: number }> = [];
+  collectBlobRefs(record, refs);
+  const keys = new Set<string>();
+  for (const ref of refs) {
+    const row = await env.ALTERAN_DB.prepare(
+      'SELECT key, mime, size FROM blob WHERE did = ? AND cid = ? LIMIT 1',
+    )
+      .bind(did, ref.cid)
+      .first<{ key: string; mime: string; size: number }>();
+    if (!row) {
+      throw new RepoWriteError('BlobNotFound', `blob not found: ${ref.cid}`);
+    }
+    if (row.mime !== ref.mimeType) {
+      throw new RepoWriteError('InvalidMimeType', `blob mime type mismatch: ${ref.cid}`);
+    }
+    if (Number(row.size) !== ref.size) {
+      throw new RepoWriteError('InvalidSize', `blob size mismatch: ${ref.cid}`);
+    }
+    const object = await env.ALTERAN_BLOBS.head(row.key);
+    if (!object || object.size !== Number(row.size)) {
+      throw new RepoWriteError('BlobNotFound', `blob not found: ${ref.cid}`);
+    }
+    keys.add(row.key);
+  }
+  return Array.from(keys);
+}

package/src/services/repo/blockstore-ops.ts

@@ -1,29 +1,71 @@
 import { CID } from 'multiformats/cid';
 import * as dagCbor from '@ipld/dag-cbor';
+import { fromString as bytesFromString } from 'uint8arrays/from-string';
 import { cidForCbor } from '../../lib/mst/util';
-import type { D1Blockstore, MST, BlockMap } from '../../lib/mst';
+import type { MST, BlockMap } from '../../lib/mst';
+import { RepoWriteError } from '../../lib/repo-write-error';
 
-export async function storeRecord(
-  blockstore: D1Blockstore,
+export type EncodedBlock = readonly [CID, Uint8Array];
+
+export function recordToIpld(record: unknown, depth = 0): unknown {
+  if (depth > 100) {
+    throw new RepoWriteError('InvalidRequest', 'record is too deeply nested');
+  }
+  if (Array.isArray(record)) {
+    return record.map((item) => recordToIpld(item, depth + 1));
+  }
+  if (!record || typeof record !== 'object') {
+    return record;
+  }
+  if (record instanceof Uint8Array || CID.asCID(record)) {
+    return record;
+  }
+
+  const obj = record as Record<string, unknown>;
+  const keys = Object.keys(obj);
+  if (keys.length === 1 && typeof obj.$link === 'string') {
+    return CID.parse(obj.$link);
+  }
+  if (keys.length === 1 && typeof obj.$bytes === 'string') {
+    return bytesFromString(obj.$bytes, 'base64');
+  }
+
+  const converted: Record<string, unknown> = Object.create(null);
+  for (const [key, value] of Object.entries(obj)) {
+    if (key === '__proto__') {
+      throw new RepoWriteError('InvalidRequest', 'record contains a forbidden object key');
+    }
+    converted[key] = recordToIpld(value, depth + 1);
+  }
+  return converted;
+}
+
+export async function cidForRecord(record: unknown): Promise<CID> {
+  try {
+    return cidForCbor(recordToIpld(record));
+  } catch (error) {
+    if (error instanceof RepoWriteError) throw error;
+    throw new RepoWriteError('InvalidRequest', 'record is not dag-cbor encodable');
+  }
+}
+
+export async function encodeRecordBlock(
   record: unknown,
-): Promise<CID> {
-  const bytes = dagCbor.encode(record);
-  const cid = await cidForCbor(record);
-  await blockstore.put(cid, bytes);
-  return cid;
+): Promise<EncodedBlock> {
+  try {
+    const ipldRecord = recordToIpld(record);
+    const bytes = dagCbor.encode(ipldRecord);
+    const cid = await cidForCbor(ipldRecord);
+    return [cid, bytes] as const;
+  } catch (error) {
+    if (error instanceof RepoWriteError) throw error;
+    throw new RepoWriteError('InvalidRequest', 'record is not dag-cbor encodable');
+  }
 }
 
-export async function storeMstBlocks(
-  blockstore: D1Blockstore,
+export async function collectUnstoredMstBlocks(
   mst: MST,
 ): Promise<BlockMap> {
   const diff = await mst.getUnstoredBlocks();
-  for (const [cid, bytes] of diff.blocks) {
-    console.log(
-      `[RepoManager] Storing new MST block: ${cid.toString()}, size: ${bytes.length}`,
-    );
-    await blockstore.put(cid, bytes);
-  }
-  console.log(`[RepoManager] Stored ${diff.blocks.size} new MST blocks`);
   return diff.blocks;
 }

package/src/services/repo/list-blobs.ts

@@ -0,0 +1,43 @@
+import type { Env } from '../../env';
+
+type ListBlobCidsParams = {
+  did: string;
+  since?: string;
+  cursor?: string;
+  limit: number;
+};
+
+export async function listBlobCids(
+  env: Env,
+  { did, since, cursor, limit }: ListBlobCidsParams,
+): Promise<{ cids: string[]; cursor?: string }> {
+  const pageSize = limit + 1;
+  const clauses = ['did = ?'];
+  const values: Array<string | number> = [did];
+  if (since) {
+    clauses.push('repo_rev > ?');
+    values.push(since);
+  }
+  if (cursor) {
+    clauses.push('cid > ?');
+    values.push(cursor);
+  }
+  values.push(pageSize);
+
+  const result = await env.ALTERAN_DB.prepare(
+    `SELECT DISTINCT cid
+     FROM blob_usage
+     WHERE ${clauses.join(' AND ')}
+     ORDER BY cid
+     LIMIT ?`,
+  )
+    .bind(...values)
+    .all<{ cid: string }>();
+
+  const rows = result.results ?? [];
+  const page = rows.slice(0, limit).map((row) => row.cid);
+  return {
+    cids: page,
+    ...(page.length > 0 ? { cursor: page[page.length - 1] } : {}),
+  };
+}