@alteran/astro 0.1.0

package/src/services/car.ts

@@ -0,0 +1,249 @@
+import type { Env } from '../env';
+import { listRecords } from '../db/dal';
+import { drizzle } from 'drizzle-orm/d1';
+import { desc, and, gte, lte } from 'drizzle-orm';
+import { commit_log } from '../db/schema';
+import { CID } from 'multiformats/cid';
+import * as dagCbor from '@ipld/dag-cbor';
+import { sha256 } from 'multiformats/hashes/sha2';
+import { MST, Leaf, D1Blockstore } from '../lib/mst';
+
+export type CarSnapshot = {
+  bytes: Uint8Array;
+  root: CID;
+  blocks: { cid: CID; bytes: Uint8Array }[];
+};
+
+export async function encodeRecordBlock(value: unknown) {
+  const bytes = dagCbor.encode(value);
+  const hash = await sha256.digest(bytes);
+  const cid = CID.createV1(dagCbor.code, hash);
+  return { cid, bytes } as const;
+}
+
+export async function buildRepoCar(env: Env, did: string): Promise<CarSnapshot> {
+  // Prefer the latest signed commit from commit_log (authoritative root)
+  const db = drizzle(env.DB);
+  const tip = await db.select().from(commit_log).orderBy(desc(commit_log.seq)).limit(1).get();
+
+  if (tip) {
+    try {
+      // Reconstruct the exact signed commit object that produced tip.cid
+      const parsed = JSON.parse(tip.data);
+      const prevStr = parsed.prev ?? null;
+      const signedCommit = {
+        did: parsed.did as string,
+        version: parsed.version as number,
+        data: CID.parse(String(parsed.data)),
+        rev: String(parsed.rev),
+        prev: prevStr ? CID.parse(String(prevStr)) : null,
+        sig: (() => {
+          const bin = atob(String(tip.sig));
+          const u8 = new Uint8Array(bin.length);
+          for (let i = 0; i < bin.length; i++) u8[i] = bin.charCodeAt(i);
+          return u8;
+        })(),
+      } as const;
+
+      // Encode to CBOR and verify CID matches tip
+      const commitBytes = dagCbor.encode(signedCommit);
+      const hash = await sha256.digest(commitBytes);
+      const commitCid = CID.createV1(dagCbor.code, hash);
+
+      if (commitCid.toString() === (tip as any).cid) {
+        // Build a full snapshot CAR: commit block + all MST nodes + all record blocks
+        const blockstore = new D1Blockstore(env);
+        const blocks: { cid: CID; bytes: Uint8Array }[] = [{ cid: commitCid, bytes: commitBytes }];
+        const seen = new Set<string>([commitCid.toString()]);
+
+        const addBlock = async (cid: CID) => {
+          const key = cid.toString();
+          if (seen.has(key)) return;
+          const bytes = await blockstore.get(cid);
+          if (bytes) {
+            seen.add(key);
+            blocks.push({ cid, bytes });
+          }
+        };
+
+        const mstRoot = CID.parse(String(parsed.data));
+        // 1) Add all MST node blocks
+        await addMstBlocks(blockstore, mstRoot, seen, blocks);
+
+        // 2) Add all record leaf blocks by walking the MST
+        try {
+          const mst = MST.load(blockstore, mstRoot);
+          for await (const leaf of mst.walkLeavesFrom('')) {
+            await addBlock(leaf.value);
+          }
+        } catch (e) {
+          console.warn('Snapshot: failed traversing MST leaves:', e);
+        }
+
+        const bytes = encodeCar([commitCid], blocks);
+        return { bytes, root: commitCid, blocks };
+      }
+    } catch (e) {
+      // Fall through to deterministic snapshot
+      console.warn('Failed to reconstruct signed commit from tip; falling back to snapshot:', e);
+    }
+  }
+
+  // Fallback: deterministic snapshot built from current records
+  const rows = await listRecords(env);
+  const blocks: { cid: CID; bytes: Uint8Array }[] = [];
+  for (const r of rows) {
+    if (!r.uri.startsWith(`at://${did}/`)) continue;
+    const value = JSON.parse(r.json);
+    const block = await encodeRecordBlock(value);
+    blocks.push(block);
+  }
+  const commitObj = { type: 'commit', did, records: blocks.map((b) => b.cid.toString()).sort() };
+  const commit = await encodeRecordBlock(commitObj);
+  const bytes = encodeCar([commit.cid], [...blocks, commit]);
+  return { bytes, root: commit.cid, blocks: [...blocks, commit] };
+}
+
+export async function buildRepoCarRange(env: Env, fromSeq: number, toSeq: number): Promise<CarSnapshot> {
+  const db = drizzle(env.DB);
+  const rows = await db.select().from(commit_log).where(and(gte(commit_log.seq, fromSeq), lte(commit_log.seq, toSeq))).all();
+  const blocks: { cid: CID; bytes: Uint8Array }[] = [];
+  for (const r of rows) {
+    const b = await encodeRecordBlock({ type: 'commit', rev: r.rev, head: r.cid, ts: r.ts });
+    blocks.push(b);
+  }
+  const root = blocks[blocks.length - 1]?.cid ?? (await encodeRecordBlock({})).cid;
+  const bytes = encodeCar([root], blocks);
+  return { bytes, root, blocks };
+}
+
+export async function buildBlocksCar(values: unknown[]): Promise<CarSnapshot> {
+  const blocks: { cid: CID; bytes: Uint8Array }[] = [];
+  for (const v of values) {
+    const block = await encodeRecordBlock(v);
+    blocks.push(block);
+  }
+  const root = blocks[0]?.cid ?? (await encodeRecordBlock({})).cid;
+  const bytes = encodeCar([root], blocks);
+  return { bytes, root, blocks };
+}
+
+/**
+ * Encode a list of already-encoded blocks into a CAR v1 file.
+ */
+export function encodeBlocksToCAR(root: CID, blocks: { cid: CID; bytes: Uint8Array }[]): Uint8Array {
+  return encodeCar([root], blocks);
+}
+
+export function encodeExistingBlocksToCAR(roots: CID[], blocks: { cid: CID; bytes: Uint8Array }[]): Uint8Array {
+  return encodeCar(roots, blocks);
+}
+
+function concat(parts: Uint8Array[]): Uint8Array {
+  const size = parts.reduce((n, p) => n + p.byteLength, 0);
+  const buf = new Uint8Array(size);
+  let off = 0;
+  for (const p of parts) { buf.set(p, off); off += p.byteLength; }
+  return buf;
+}
+
+function varint(n: number): Uint8Array {
+  const bytes: number[] = [];
+  while (n >= 0x80) {
+    bytes.push((n & 0x7f) | 0x80);
+    n >>>= 7;
+  }
+  bytes.push(n);
+  return new Uint8Array(bytes);
+}
+
+function encodeCar(roots: CID[], blocks: { cid: CID; bytes: Uint8Array }[]): Uint8Array {
+  const header = dagCbor.encode({ version: 1, roots });
+  const chunks: Uint8Array[] = [];
+  chunks.push(varint(header.byteLength));
+  chunks.push(header);
+  for (const { cid, bytes } of blocks) {
+    const block = concat([cid.bytes, bytes]);
+    chunks.push(varint(block.byteLength));
+    chunks.push(block);
+  }
+  return concat(chunks);
+}
+
+/**
+ * Encode blocks for firehose commit frame
+ * Includes commit block, MST nodes, and record blocks
+ */
+export async function encodeBlocksForCommit(
+  env: Env,
+  commitCid: CID,
+  mstRoot: CID,
+  ops: Array<{ path: string; cid: CID | null }>,
+): Promise<Uint8Array> {
+  const blockstore = new D1Blockstore(env);
+  const blocks: { cid: CID; bytes: Uint8Array }[] = [];
+  const seen = new Set<string>();
+
+  // Helper to add block if not already seen
+  const addBlock = async (cid: CID) => {
+    const cidStr = cid.toString();
+    if (seen.has(cidStr)) return;
+    seen.add(cidStr);
+
+    const bytes = await blockstore.get(cid);
+    if (bytes) {
+      blocks.push({ cid, bytes });
+    }
+  };
+
+  // 1. Add commit block
+  await addBlock(commitCid);
+
+  // 2. Add MST nodes by traversing the tree
+  await addMstBlocks(blockstore, mstRoot, seen, blocks);
+
+  // 3. Add record blocks for all operations
+  for (const op of ops) {
+    if (op.cid) {
+      await addBlock(op.cid);
+    }
+  }
+
+  // Encode as CAR with commit as root
+  return encodeCar([commitCid], blocks);
+}
+
+/**
+ * Recursively add all MST node blocks
+ */
+async function addMstBlocks(
+  blockstore: D1Blockstore,
+  rootCid: CID,
+  seen: Set<string>,
+  blocks: { cid: CID; bytes: Uint8Array }[],
+): Promise<void> {
+  const cidStr = rootCid.toString();
+  if (seen.has(cidStr)) return;
+  seen.add(cidStr);
+
+  // Add the MST node block itself
+  const bytes = await blockstore.get(rootCid);
+  if (!bytes) return;
+  blocks.push({ cid: rootCid, bytes });
+
+  // Load MST and traverse children
+  try {
+    const mst = MST.load(blockstore, rootCid);
+    const entries = await mst.getEntries();
+
+    for (const entry of entries) {
+      if (entry.isTree()) {
+        // Recursively add child MST blocks
+        const childCid = await entry.getPointer();
+        await addMstBlocks(blockstore, childCid, seen, blocks);
+      }
+    }
+  } catch (error) {
+    console.error('Error traversing MST:', error);
+  }
+}

package/src/services/r2-blob-store.ts

@@ -0,0 +1,87 @@
+// Types via tsconfig.app.json
+import type { R2ObjectBody } from '@cloudflare/workers-types';
+import type { Env } from '../env';
+
+export type PutOptions = {
+  contentType?: string;
+  maxBytes?: number; // default from env or 5 MiB
+};
+
+export type PutResult = {
+  key: string;
+  size: number;
+  sha256: string; // base64url
+};
+
+type BlobBody = ArrayBuffer | ArrayBufferView;
+
+export class R2BlobStore {
+  constructor(private env: Env) {}
+
+  private maxBytes(defaultMax = 5 * 1024 * 1024): number {
+    const raw = (this.env as any).PDS_MAX_BLOB_SIZE as string | undefined;
+    const n = raw ? Number(raw) : defaultMax;
+    return Number.isFinite(n) && n > 0 ? n : defaultMax;
+  }
+
+  private static asUint8Array(data: BlobBody): Uint8Array {
+    if (data instanceof ArrayBuffer) return new Uint8Array(data);
+    if (ArrayBuffer.isView(data)) {
+      const view = data as ArrayBufferView;
+      return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
+    }
+    // Fallback should never happen because BlobBody restricts input, but keep typing satisfied
+    return new Uint8Array(data as ArrayBuffer);
+  }
+
+  private static toArrayBuffer(data: Uint8Array): ArrayBuffer {
+    const bufferLike = data.buffer;
+    if (bufferLike instanceof ArrayBuffer) {
+      if (data.byteOffset === 0 && data.byteLength === bufferLike.byteLength) {
+        return bufferLike;
+      }
+      return bufferLike.slice(data.byteOffset, data.byteOffset + data.byteLength);
+    }
+    const buffer = new ArrayBuffer(data.byteLength);
+    new Uint8Array(buffer).set(data);
+    return buffer;
+  }
+
+  private static b64url(bytes: ArrayBuffer | Uint8Array): string {
+    const view = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+    let s = '';
+    for (const v of view) s += String.fromCharCode(v);
+    return btoa(s).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '');
+  }
+
+  private static hex(bytes: Uint8Array): string {
+    return Array.from(bytes).map((v) => v.toString(16).padStart(2, '0')).join('');
+  }
+
+  private static cidKey(shaB64url: string, prefix = 'blobs/by-cid/'): string {
+    return `${prefix}${shaB64url}`;
+  }
+
+  async put(body: BlobBody, opts: PutOptions = {}): Promise<PutResult> {
+    const view = R2BlobStore.asUint8Array(body);
+    const size = view.byteLength;
+    const limit = opts.maxBytes ?? this.maxBytes();
+    if (size > limit) throw new Error(`BlobTooLarge:${size}>${limit}`);
+
+    const contentType = opts.contentType ?? 'application/octet-stream';
+    const sha = await crypto.subtle.digest('SHA-256', view);
+    const shaB64 = R2BlobStore.b64url(sha);
+    const key = R2BlobStore.cidKey(shaB64);
+    const buffer = R2BlobStore.toArrayBuffer(view);
+    await this.env.BLOBS.put(key, buffer, { httpMetadata: { contentType } });
+    return { key, size, sha256: shaB64 };
+  }
+
+  async get(key: string): Promise<R2ObjectBody | null> {
+    return this.env.BLOBS.get(key);
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.env.BLOBS.delete(key);
+  }
+}

package/src/services/repo-manager.ts

@@ -0,0 +1,339 @@
+import { CID } from 'multiformats/cid';
+import type { Env } from '../env';
+import { MST, D1Blockstore, Leaf } from '../lib/mst';
+import { drizzle } from 'drizzle-orm/d1';
+import { repo_root } from '../db/schema';
+import { eq } from 'drizzle-orm';
+import type { RepoOp } from '../lib/firehose/frames';
+import * as dagCbor from '@ipld/dag-cbor';
+import { cidForCbor } from '../lib/mst/util';
+import { putRecord as dalPutRecord, deleteRecord as dalDeleteRecord } from '../db/dal';
+import { bumpRoot } from '../db/repo';
+import { generateTid } from '../lib/commit';
+
+/**
+ * Repository Manager
+ * Manages MST-based repository operations
+ */
+export class RepoManager {
+  private blockstore: D1Blockstore;
+  private did: string;
+
+  constructor(private env: Env) {
+    this.blockstore = new D1Blockstore(env);
+    this.did = env.PDS_DID ?? 'did:example:single-user';
+  }
+
+  /**
+   * Get the current MST root
+   */
+  async getRoot(): Promise<MST | null> {
+    const db = drizzle(this.env.DB);
+    const row = await db
+      .select()
+      .from(repo_root)
+      .where(eq(repo_root.did, this.did))
+      .get();
+
+    if (!row || !row.commitCid) return null;
+
+    try {
+      const rootCid = CID.parse(row.commitCid);
+      return MST.load(this.blockstore, rootCid);
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Get or create the MST root
+   */
+  async getOrCreateRoot(): Promise<MST> {
+    const existing = await this.getRoot();
+    if (existing) return existing;
+
+    // Create new empty MST
+    const mst = await MST.create(this.blockstore, []);
+    return mst;
+  }
+
+  /**
+   * Add a record to the repository
+   */
+  async addRecord(collection: string, rkey: string, record: unknown): Promise<{
+    mst: MST;
+    recordCid: CID;
+    prevMstRoot: CID | null;
+  }> {
+    const key = `${collection}/${rkey}`;
+
+    // Get previous MST root for op extraction
+    const currentMst = await this.getOrCreateRoot();
+    const prevMstRoot = await currentMst.getPointer();
+
+    // Encode record and store in blockstore
+    const recordCid = await this.storeRecord(record);
+
+    // Add the new record
+    const newMst = await currentMst.add(key, recordCid);
+
+    // Store all new MST blocks
+    await this.storeMstBlocks(newMst);
+
+    return { mst: newMst, recordCid, prevMstRoot };
+  }
+
+  /**
+   * High-level helper: create record, persist JSON, bump root, return commit info
+   */
+  async createRecord(collection: string, record: unknown, rkey?: string): Promise<{
+    uri: string;
+    cid: string;
+    commitCid: string;
+    rev: string;
+    ops: RepoOp[];
+  }> {
+    const key = rkey ?? generateTid();
+    const { mst, recordCid, prevMstRoot } = await this.addRecord(collection, key, record);
+
+    // Persist JSON to table for easy reads
+    const uri = `at://${this.did}/${collection}/${key}`;
+    await dalPutRecord(this.env, { uri, did: this.did, cid: recordCid.toString(), json: JSON.stringify(record) } as any);
+
+    // Update repo root with signed commit and extract ops
+    const { commitCid, rev, ops } = await bumpRoot(this.env, prevMstRoot ?? undefined);
+
+    return { uri, cid: recordCid.toString(), commitCid, rev, ops };
+  }
+
+  /**
+   * Update a record in the repository
+   */
+  async updateRecord(collection: string, rkey: string, record: unknown): Promise<{
+    mst: MST;
+    recordCid: CID;
+    prevMstRoot: CID | null;
+  }> {
+    const key = `${collection}/${rkey}`;
+
+    // Get previous MST root for op extraction
+    const currentMst = await this.getOrCreateRoot();
+    const prevMstRoot = await currentMst.getPointer();
+
+    // Encode record and store in blockstore
+    const recordCid = await this.storeRecord(record);
+
+    // Update the record
+    const newMst = await currentMst.update(key, recordCid);
+
+    // Store all new MST blocks
+    await this.storeMstBlocks(newMst);
+
+    return { mst: newMst, recordCid, prevMstRoot };
+  }
+
+  /**
+   * High-level helper: put record (update), persist JSON, bump root
+   */
+  async putRecord(collection: string, rkey: string, record: unknown): Promise<{
+    uri: string;
+    cid: string;
+    commitCid: string;
+    rev: string;
+    ops: RepoOp[];
+  }> {
+    const { mst, recordCid, prevMstRoot } = await this.updateRecord(collection, rkey, record);
+    const uri = `at://${this.did}/${collection}/${rkey}`;
+    await dalPutRecord(this.env, { uri, did: this.did, cid: recordCid.toString(), json: JSON.stringify(record) } as any);
+    const { commitCid, rev, ops } = await bumpRoot(this.env, prevMstRoot ?? undefined);
+    return { uri, cid: recordCid.toString(), commitCid, rev, ops };
+  }
+
+  /**
+   * Delete a record from the repository
+   */
+  async deleteRecord(collection: string, rkey: string): Promise<{
+    uri: string;
+    commitCid: string;
+    rev: string;
+    ops: RepoOp[];
+  }> {
+    const key = `${collection}/${rkey}`;
+
+    // Get previous MST root for op extraction
+    const currentMst = await this.getOrCreateRoot();
+    const prevMstRoot = await currentMst.getPointer();
+
+    // Delete the record
+    const newMst = await currentMst.delete(key);
+
+    // Store all new MST blocks
+    await this.storeMstBlocks(newMst);
+
+    // Delete from records table
+    const uri = `at://${this.did}/${collection}/${rkey}`;
+    await dalDeleteRecord(this.env, uri);
+
+    const { commitCid, rev, ops } = await bumpRoot(this.env, prevMstRoot ?? undefined);
+    return { uri, commitCid, rev, ops };
+  }
+
+  /**
+   * Get a record from the repository
+   */
+  async getRecord(collection: string, rkey: string): Promise<unknown | null> {
+    const key = `${collection}/${rkey}`;
+
+    const currentMst = await this.getRoot();
+    if (!currentMst) return null;
+
+    const recordCid = await currentMst.get(key);
+    if (!recordCid) return null;
+
+    return this.blockstore.readObj(recordCid);
+  }
+
+  /**
+   * List records in a collection
+   */
+  async listRecords(collection: string, limit = 50, cursor?: string): Promise<{ key: string; cid: CID }[]> {
+    const currentMst = await this.getRoot();
+    if (!currentMst) return [];
+
+    const prefix = `${collection}/`;
+    const leaves = await currentMst.listWithPrefix(prefix, limit);
+
+    return leaves
+      .filter(leaf => !cursor || leaf.key > `${collection}/${cursor}`)
+      .map(leaf => ({
+        key: leaf.key.replace(prefix, ''),
+        cid: leaf.value,
+      }));
+  }
+
+  /**
+   * Update the repo root to point to the new MST
+   */
+  async updateRoot(mst: MST, rev: number): Promise<void> {
+    const db = drizzle(this.env.DB);
+    const rootCid = await mst.getPointer();
+
+    await db
+      .insert(repo_root)
+      .values({
+        did: this.did,
+        commitCid: rootCid.toString(),
+        rev,
+      })
+      .onConflictDoUpdate({
+        target: repo_root.did,
+        set: {
+          commitCid: rootCid.toString(),
+          rev,
+        },
+      })
+      .run();
+  }
+
+  /**
+   * Extract operations from MST diff between two commits
+   * Compares old MST root with new MST root to identify create/update/delete operations
+   */
+  async extractOps(prevRoot: CID | null, newRoot: CID): Promise<RepoOp[]> {
+    const ops: RepoOp[] = [];
+
+    // Load both trees
+    const newMst = await MST.load(this.blockstore, newRoot).getEntries();
+    const prevMst = prevRoot ? await MST.load(this.blockstore, prevRoot).getEntries() : [];
+
+    // Build maps for efficient lookup
+    const prevMap = new Map<string, CID>();
+    const newMap = new Map<string, CID>();
+
+    // Collect all leaves from previous tree
+    await this.collectLeaves(prevMst, prevMap);
+
+    // Collect all leaves from new tree
+    await this.collectLeaves(newMst, newMap);
+
+    // Find creates and updates
+    for (const [path, cid] of Array.from(newMap.entries())) {
+      const prevCid = prevMap.get(path);
+      if (!prevCid) {
+        // New key - create operation
+        ops.push({
+          action: 'create',
+          path,
+          cid,
+        });
+      } else if (!prevCid.equals(cid)) {
+        // Key exists but CID changed - update operation
+        ops.push({
+          action: 'update',
+          path,
+          cid,
+          prev: prevCid,
+        });
+      }
+    }
+
+    // Find deletes
+    for (const [path, prevCid] of Array.from(prevMap.entries())) {
+      if (!newMap.has(path)) {
+        // Key no longer exists - delete operation
+        ops.push({
+          action: 'delete',
+          path,
+          cid: null,
+          prev: prevCid,
+        });
+      }
+    }
+
+    // Sort ops by path for deterministic ordering
+    ops.sort((a, b) => a.path.localeCompare(b.path));
+
+    return ops;
+  }
+
+  /**
+   * Recursively collect all leaves from MST entries into a map
+   */
+  private async collectLeaves(entries: (MST | Leaf)[], map: Map<string, CID>): Promise<void> {
+    for (const entry of entries) {
+      if (entry.isLeaf()) {
+        map.set(entry.key, entry.value);
+      } else {
+        // Recursively collect from subtree
+        const subEntries = await entry.getEntries();
+        await this.collectLeaves(subEntries, map);
+      }
+    }
+  }
+
+  /**
+   * Store a record in the blockstore and return its CID
+   */
+  private async storeRecord(record: unknown): Promise<CID> {
+    const bytes = dagCbor.encode(record);
+    const cid = await cidForCbor(record);
+    await this.blockstore.put(cid, bytes);
+    return cid;
+  }
+
+  /**
+   * Store all blocks from an MST to the blockstore
+   */
+  private async storeMstBlocks(mst: MST): Promise<void> {
+    const { cid, bytes } = await mst.serialize();
+    await this.blockstore.put(cid, bytes);
+
+    // Recursively store child blocks
+    const entries = await mst.getEntries();
+    for (const entry of entries) {
+      if (entry.isTree()) {
+        await this.storeMstBlocks(entry);
+      }
+    }
+  }
+}

package/src/shims/astro-internal-handler.d.ts

@@ -0,0 +1,4 @@
+declare module 'astro/internal/handler' {
+  export function handle(...args: any[]): Promise<Response>;
+}
+