cojson 0.8.12 → 0.8.17

package/src/storage/FileSystem.ts

@@ -1,6 +1,6 @@
-import { CoValueChunk } from "./index.js";
-import { RawCoID } from "../ids.js";
 import { CryptoProvider, StreamingHash } from "../crypto/crypto.js";
+import { RawCoID } from "../ids.js";
+import { CoValueChunk } from "./index.js";
 
 export type BlockFilename = `L${number}-${string}-${string}-H${number}.jsonl`;
 
@@ -11,123 +11,115 @@ export type WalEntry = { id: RawCoID } & CoValueChunk;
 export type WalFilename = `wal-${number}.jsonl`;
 
 export interface FileSystem<WriteHandle, ReadHandle> {
-    crypto: CryptoProvider;
-    createFile(filename: string): Promise<WriteHandle>;
-    append(handle: WriteHandle, data: Uint8Array): Promise<void>;
-    close(handle: ReadHandle | WriteHandle): Promise<void>;
-    closeAndRename(handle: WriteHandle, filename: BlockFilename): Promise<void>;
-    openToRead(filename: string): Promise<{ handle: ReadHandle; size: number }>;
-    read(
-        handle: ReadHandle,
-        offset: number,
-        length: number,
-    ): Promise<Uint8Array>;
-    listFiles(): Promise<string[]>;
-    removeFile(filename: BlockFilename | WalFilename): Promise<void>;
+  crypto: CryptoProvider;
+  createFile(filename: string): Promise<WriteHandle>;
+  append(handle: WriteHandle, data: Uint8Array): Promise<void>;
+  close(handle: ReadHandle | WriteHandle): Promise<void>;
+  closeAndRename(handle: WriteHandle, filename: BlockFilename): Promise<void>;
+  openToRead(filename: string): Promise<{ handle: ReadHandle; size: number }>;
+  read(handle: ReadHandle, offset: number, length: number): Promise<Uint8Array>;
+  listFiles(): Promise<string[]>;
+  removeFile(filename: BlockFilename | WalFilename): Promise<void>;
 }
 
 export const textEncoder = new TextEncoder();
 export const textDecoder = new TextDecoder();
 
 export async function readChunk<RH, FS extends FileSystem<unknown, RH>>(
-    handle: RH,
-    header: { start: number; length: number },
-    fs: FS,
+  handle: RH,
+  header: { start: number; length: number },
+  fs: FS,
 ): Promise<CoValueChunk> {
-    const chunkBytes = await fs.read(handle, header.start, header.length);
+  const chunkBytes = await fs.read(handle, header.start, header.length);
 
-    const chunk = JSON.parse(textDecoder.decode(chunkBytes));
-    return chunk;
+  const chunk = JSON.parse(textDecoder.decode(chunkBytes));
+  return chunk;
 }
 
 export async function readHeader<RH, FS extends FileSystem<unknown, RH>>(
-    filename: string,
-    handle: RH,
-    size: number,
-    fs: FS,
+  filename: string,
+  handle: RH,
+  size: number,
+  fs: FS,
 ): Promise<BlockHeader> {
-    const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
+  const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
 
-    const headerBytes = await fs.read(
-        handle,
-        size - headerLength,
-        headerLength,
-    );
+  const headerBytes = await fs.read(handle, size - headerLength, headerLength);
 
-    const header = JSON.parse(textDecoder.decode(headerBytes));
-    return header;
+  const header = JSON.parse(textDecoder.decode(headerBytes));
+  return header;
 }
 
 export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
-    chunks: Map<RawCoID, CoValueChunk>,
-    level: number,
-    blockNumber: number,
-    fs: FS,
+  chunks: Map<RawCoID, CoValueChunk>,
+  level: number,
+  blockNumber: number,
+  fs: FS,
 ): Promise<BlockFilename> {
-    if (chunks.size === 0) {
-        throw new Error("No chunks to write");
-    }
-
-    const blockHeader: BlockHeader = [];
-
-    let offset = 0;
-
-    const file = await fs.createFile(
-        "wipBlock" + Math.random().toString(36).substring(7) + ".tmp.jsonl",
-    );
-    const hash = new StreamingHash(fs.crypto);
-
-    const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) =>
-        id1.localeCompare(id2),
-    );
-
-    for (const [id, chunk] of chunksSortedById) {
-        const encodedBytes = hash.update(chunk);
-        const encodedBytesWithNewline = new Uint8Array(encodedBytes.length + 1);
-        encodedBytesWithNewline.set(encodedBytes);
-        encodedBytesWithNewline[encodedBytes.length] = 10;
-        await fs.append(file, encodedBytesWithNewline);
-        const length = encodedBytesWithNewline.length;
-        blockHeader.push({ id, start: offset, length });
-        offset += length;
-    }
-
-    const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
-    await fs.append(file, headerBytes);
-
-    // console.log(
-    //     "full file",
-    //     yield* $(
-    //         fs.read(file as unknown as RH, 0, offset + headerBytes.length),
-    //     ),
-    // );
-
-    const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
-        3,
-        "0",
-    )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
-        headerBytes.length
-    }.jsonl`;
-    // console.log("renaming to" + filename);
-    await fs.closeAndRename(file, filename);
-
-    return filename;
-
-    // console.log("Wrote block", filename, blockHeader);
-    // console.log("IDs in block", blockHeader.map(e => e.id));
+  if (chunks.size === 0) {
+    throw new Error("No chunks to write");
+  }
+
+  const blockHeader: BlockHeader = [];
+
+  let offset = 0;
+
+  const file = await fs.createFile(
+    "wipBlock" + Math.random().toString(36).substring(7) + ".tmp.jsonl",
+  );
+  const hash = new StreamingHash(fs.crypto);
+
+  const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) =>
+    id1.localeCompare(id2),
+  );
+
+  for (const [id, chunk] of chunksSortedById) {
+    const encodedBytes = hash.update(chunk);
+    const encodedBytesWithNewline = new Uint8Array(encodedBytes.length + 1);
+    encodedBytesWithNewline.set(encodedBytes);
+    encodedBytesWithNewline[encodedBytes.length] = 10;
+    await fs.append(file, encodedBytesWithNewline);
+    const length = encodedBytesWithNewline.length;
+    blockHeader.push({ id, start: offset, length });
+    offset += length;
+  }
+
+  const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
+  await fs.append(file, headerBytes);
+
+  // console.log(
+  //     "full file",
+  //     yield* $(
+  //         fs.read(file as unknown as RH, 0, offset + headerBytes.length),
+  //     ),
+  // );
+
+  const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
+    3,
+    "0",
+  )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
+    headerBytes.length
+  }.jsonl`;
+  // console.log("renaming to" + filename);
+  await fs.closeAndRename(file, filename);
+
+  return filename;
+
+  // console.log("Wrote block", filename, blockHeader);
+  // console.log("IDs in block", blockHeader.map(e => e.id));
 }
 
 export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
-    handle: WH,
-    fs: FS,
-    id: RawCoID,
-    chunk: CoValueChunk,
+  handle: WH,
+  fs: FS,
+  id: RawCoID,
+  chunk: CoValueChunk,
 ) {
-    const walEntry: WalEntry = {
-        id,
-        ...chunk,
-    };
-    const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-    console.log("writing to WAL", handle, id, bytes.length);
-    return fs.append(handle, bytes);
+  const walEntry: WalEntry = {
+    id,
+    ...chunk,
+  };
+  const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
+  console.log("writing to WAL", handle, id, bytes.length);
+  return fs.append(handle, bytes);
 }

package/src/storage/chunksAndKnownStates.ts

@@ -1,142 +1,137 @@
-import { RawCoID, SessionID } from "../ids.js";
 import { MAX_RECOMMENDED_TX_SIZE } from "../coValueCore.js";
+import { RawCoID, SessionID } from "../ids.js";
 import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { CoValueChunk } from "./index.js";
 
 export function contentSinceChunk(
-    id: RawCoID,
-    chunk: CoValueChunk,
-    known?: CoValueKnownState,
+  id: RawCoID,
+  chunk: CoValueChunk,
+  known?: CoValueKnownState,
 ): NewContentMessage[] {
-    const newContentPieces: NewContentMessage[] = [];
-
-    newContentPieces.push({
-        id: id,
-        action: "content",
-        header: known?.header ? undefined : chunk.header,
-        new: {},
-        priority: getPriorityFromHeader(chunk.header),
-    });
-
-    for (const [sessionID, sessionsEntry] of Object.entries(
-        chunk.sessionEntries,
-    )) {
-        for (const entry of sessionsEntry) {
-            const knownStart = known?.sessions[sessionID as SessionID] || 0;
-
-            if (entry.after + entry.transactions.length <= knownStart) {
-                continue;
-            }
-
-            const actuallyNewTransactions = entry.transactions.slice(
-                Math.max(0, knownStart - entry.after),
-            );
-
-            const newAfter =
-                entry.after +
-                (actuallyNewTransactions.length - entry.transactions.length);
-
-            let newContentEntry =
-                newContentPieces[0]?.new[sessionID as SessionID];
-
-            if (!newContentEntry) {
-                newContentEntry = {
-                    after: newAfter,
-                    lastSignature: entry.lastSignature,
-                    newTransactions: actuallyNewTransactions,
-                };
-                newContentPieces[0]!.new[sessionID as SessionID] =
-                    newContentEntry;
-            } else {
-                newContentEntry.newTransactions.push(
-                    ...actuallyNewTransactions,
-                );
-                newContentEntry.lastSignature = entry.lastSignature;
-            }
-        }
+  const newContentPieces: NewContentMessage[] = [];
+
+  newContentPieces.push({
+    id: id,
+    action: "content",
+    header: known?.header ? undefined : chunk.header,
+    new: {},
+    priority: getPriorityFromHeader(chunk.header),
+  });
+
+  for (const [sessionID, sessionsEntry] of Object.entries(
+    chunk.sessionEntries,
+  )) {
+    for (const entry of sessionsEntry) {
+      const knownStart = known?.sessions[sessionID as SessionID] || 0;
+
+      if (entry.after + entry.transactions.length <= knownStart) {
+        continue;
+      }
+
+      const actuallyNewTransactions = entry.transactions.slice(
+        Math.max(0, knownStart - entry.after),
+      );
+
+      const newAfter =
+        entry.after +
+        (actuallyNewTransactions.length - entry.transactions.length);
+
+      let newContentEntry = newContentPieces[0]?.new[sessionID as SessionID];
+
+      if (!newContentEntry) {
+        newContentEntry = {
+          after: newAfter,
+          lastSignature: entry.lastSignature,
+          newTransactions: actuallyNewTransactions,
+        };
+        newContentPieces[0]!.new[sessionID as SessionID] = newContentEntry;
+      } else {
+        newContentEntry.newTransactions.push(...actuallyNewTransactions);
+        newContentEntry.lastSignature = entry.lastSignature;
+      }
     }
+  }
 
-    return newContentPieces;
+  return newContentPieces;
 }
 
 export function chunkToKnownState(id: RawCoID, chunk: CoValueChunk) {
-    const ourKnown: CoValueKnownState = {
-        id,
-        header: !!chunk.header,
-        sessions: {},
-    };
-
-    for (const [sessionID, sessionEntries] of Object.entries(
-        chunk.sessionEntries,
-    )) {
-        for (const entry of sessionEntries) {
-            ourKnown.sessions[sessionID as SessionID] =
-                entry.after + entry.transactions.length;
-        }
+  const ourKnown: CoValueKnownState = {
+    id,
+    header: !!chunk.header,
+    sessions: {},
+  };
+
+  for (const [sessionID, sessionEntries] of Object.entries(
+    chunk.sessionEntries,
+  )) {
+    for (const entry of sessionEntries) {
+      ourKnown.sessions[sessionID as SessionID] =
+        entry.after + entry.transactions.length;
     }
-    return ourKnown;
+  }
+  return ourKnown;
 }
 
 export function mergeChunks(
-    chunkA: CoValueChunk,
-    chunkB: CoValueChunk,
+  chunkA: CoValueChunk,
+  chunkB: CoValueChunk,
 ): "nonContigous" | CoValueChunk {
-    const header = chunkA.header || chunkB.header;
+  const header = chunkA.header || chunkB.header;
 
-    const newSessions = { ...chunkA.sessionEntries };
-    for (const sessionID in chunkB.sessionEntries) {
-        // figure out if we can merge the chunks
-        const sessionEntriesA = chunkA.sessionEntries[sessionID];
-        const sessionEntriesB = chunkB.sessionEntries[sessionID]!;
+  const newSessions = { ...chunkA.sessionEntries };
+  for (const sessionID in chunkB.sessionEntries) {
+    // figure out if we can merge the chunks
+    const sessionEntriesA = chunkA.sessionEntries[sessionID];
+    const sessionEntriesB = chunkB.sessionEntries[sessionID]!;
 
-        if (!sessionEntriesA) {
-            newSessions[sessionID] = sessionEntriesB;
-            continue;
-        }
-
-        const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1]!;
-        const firstEntryOfB = sessionEntriesB[0]!;
+    if (!sessionEntriesA) {
+      newSessions[sessionID] = sessionEntriesB;
+      continue;
+    }
 
+    const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1]!;
+    const firstEntryOfB = sessionEntriesB[0]!;
+
+    if (
+      lastEntryOfA.after + lastEntryOfA.transactions.length ===
+      firstEntryOfB.after
+    ) {
+      const newEntries = [];
+      let bytesSinceLastSignature = 0;
+      for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
+        const entryByteLength = entry.transactions.reduce(
+          (sum, tx) =>
+            sum +
+            (tx.privacy === "private"
+              ? tx.encryptedChanges.length
+              : tx.changes.length),
+          0,
+        );
         if (
-            lastEntryOfA.after + lastEntryOfA.transactions.length ===
-            firstEntryOfB.after
+          newEntries.length === 0 ||
+          bytesSinceLastSignature + entryByteLength > MAX_RECOMMENDED_TX_SIZE
         ) {
-            const newEntries = [];
-            let bytesSinceLastSignature = 0;
-            for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
-                const entryByteLength = entry.transactions.reduce(
-                    (sum, tx) =>
-                        sum +
-                        (tx.privacy === "private"
-                            ? tx.encryptedChanges.length
-                            : tx.changes.length),
-                    0,
-                );
-                if (
-                    newEntries.length === 0 ||
-                    bytesSinceLastSignature + entryByteLength >
-                        MAX_RECOMMENDED_TX_SIZE
-                ) {
-                    newEntries.push({
-                        after: entry.after,
-                        lastSignature: entry.lastSignature,
-                        transactions: entry.transactions,
-                    });
-                    bytesSinceLastSignature = 0;
-                } else {
-                    const lastNewEntry = newEntries[newEntries.length - 1]!;
-                    lastNewEntry.transactions.push(...entry.transactions);
-                    lastNewEntry.lastSignature = entry.lastSignature;
-
-                    bytesSinceLastSignature += entry.transactions.length;
-                }
-            }
-            newSessions[sessionID] = newEntries;
+          newEntries.push({
+            after: entry.after,
+            lastSignature: entry.lastSignature,
+            transactions: entry.transactions,
+          });
+          bytesSinceLastSignature = 0;
         } else {
-            return "nonContigous" as const;
+          const lastNewEntry = newEntries[newEntries.length - 1]!;
+          lastNewEntry.transactions.push(...entry.transactions);
+          lastNewEntry.lastSignature = entry.lastSignature;
+
+          bytesSinceLastSignature += entry.transactions.length;
         }
+      }
+      newSessions[sessionID] = newEntries;
+    } else {
+      return "nonContigous" as const;
     }
+  }
 
-    return { header, sessionEntries: newSessions };
+  return { header, sessionEntries: newSessions };
 }