cojson 0.13.10 → 0.13.12

package/src/storage/FileSystem.ts

@@ -1,113 +0,0 @@
-import { CryptoProvider, StreamingHash } from "../crypto/crypto.js";
-import { RawCoID } from "../ids.js";
-import { CoValueChunk } from "./index.js";
-
-export type BlockFilename = `L${number}-${string}-${string}-H${number}.jsonl`;
-
-export type BlockHeader = { id: RawCoID; start: number; length: number }[];
-
-export type WalEntry = { id: RawCoID } & CoValueChunk;
-
-export type WalFilename = `wal-${number}.jsonl`;
-
-export interface FileSystem<WriteHandle, ReadHandle> {
-  crypto: CryptoProvider;
-  createFile(filename: string): Promise<WriteHandle>;
-  append(handle: WriteHandle, data: Uint8Array): Promise<void>;
-  close(handle: ReadHandle | WriteHandle): Promise<void>;
-  closeAndRename(handle: WriteHandle, filename: BlockFilename): Promise<void>;
-  openToRead(filename: string): Promise<{ handle: ReadHandle; size: number }>;
-  read(handle: ReadHandle, offset: number, length: number): Promise<Uint8Array>;
-  listFiles(): Promise<string[]>;
-  removeFile(filename: BlockFilename | WalFilename): Promise<void>;
-}
-
-export const textEncoder = new TextEncoder();
-export const textDecoder = new TextDecoder();
-
-export async function readChunk<RH, FS extends FileSystem<unknown, RH>>(
-  handle: RH,
-  header: { start: number; length: number },
-  fs: FS,
-): Promise<CoValueChunk> {
-  const chunkBytes = await fs.read(handle, header.start, header.length);
-
-  const chunk = JSON.parse(textDecoder.decode(chunkBytes));
-  return chunk;
-}
-
-export async function readHeader<RH, FS extends FileSystem<unknown, RH>>(
-  filename: string,
-  handle: RH,
-  size: number,
-  fs: FS,
-): Promise<BlockHeader> {
-  const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
-
-  const headerBytes = await fs.read(handle, size - headerLength, headerLength);
-
-  const header = JSON.parse(textDecoder.decode(headerBytes));
-  return header;
-}
-
-export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
-  chunks: Map<RawCoID, CoValueChunk>,
-  level: number,
-  blockNumber: number,
-  fs: FS,
-): Promise<BlockFilename> {
-  if (chunks.size === 0) {
-    throw new Error("No chunks to write");
-  }
-
-  const blockHeader: BlockHeader = [];
-
-  let offset = 0;
-
-  const file = await fs.createFile(
-    "wipBlock" + Math.random().toString(36).substring(7) + ".tmp.jsonl",
-  );
-  const hash = new StreamingHash(fs.crypto);
-
-  const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) =>
-    id1.localeCompare(id2),
-  );
-
-  for (const [id, chunk] of chunksSortedById) {
-    const encodedBytes = hash.update(chunk);
-    const encodedBytesWithNewline = new Uint8Array(encodedBytes.length + 1);
-    encodedBytesWithNewline.set(encodedBytes);
-    encodedBytesWithNewline[encodedBytes.length] = 10;
-    await fs.append(file, encodedBytesWithNewline);
-    const length = encodedBytesWithNewline.length;
-    blockHeader.push({ id, start: offset, length });
-    offset += length;
-  }
-
-  const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
-  await fs.append(file, headerBytes);
-
-  const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
-    3,
-    "0",
-  )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
-    headerBytes.length
-  }.jsonl`;
-  await fs.closeAndRename(file, filename);
-
-  return filename;
-}
-
-export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
-  handle: WH,
-  fs: FS,
-  id: RawCoID,
-  chunk: CoValueChunk,
-) {
-  const walEntry: WalEntry = {
-    id,
-    ...chunk,
-  };
-  const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-  return fs.append(handle, bytes);
-}

package/src/storage/chunksAndKnownStates.ts

@@ -1,137 +0,0 @@
-import { MAX_RECOMMENDED_TX_SIZE } from "../coValueCore.js";
-import { RawCoID, SessionID } from "../ids.js";
-import { getPriorityFromHeader } from "../priority.js";
-import { CoValueKnownState, NewContentMessage } from "../sync.js";
-import { CoValueChunk } from "./index.js";
-
-export function contentSinceChunk(
-  id: RawCoID,
-  chunk: CoValueChunk,
-  known?: CoValueKnownState,
-): NewContentMessage[] {
-  const newContentPieces: NewContentMessage[] = [];
-
-  newContentPieces.push({
-    id: id,
-    action: "content",
-    header: known?.header ? undefined : chunk.header,
-    new: {},
-    priority: getPriorityFromHeader(chunk.header),
-  });
-
-  for (const [sessionID, sessionsEntry] of Object.entries(
-    chunk.sessionEntries,
-  )) {
-    for (const entry of sessionsEntry) {
-      const knownStart = known?.sessions[sessionID as SessionID] || 0;
-
-      if (entry.after + entry.transactions.length <= knownStart) {
-        continue;
-      }
-
-      const actuallyNewTransactions = entry.transactions.slice(
-        Math.max(0, knownStart - entry.after),
-      );
-
-      const newAfter =
-        entry.after +
-        (actuallyNewTransactions.length - entry.transactions.length);
-
-      let newContentEntry = newContentPieces[0]?.new[sessionID as SessionID];
-
-      if (!newContentEntry) {
-        newContentEntry = {
-          after: newAfter,
-          lastSignature: entry.lastSignature,
-          newTransactions: actuallyNewTransactions,
-        };
-        newContentPieces[0]!.new[sessionID as SessionID] = newContentEntry;
-      } else {
-        newContentEntry.newTransactions.push(...actuallyNewTransactions);
-        newContentEntry.lastSignature = entry.lastSignature;
-      }
-    }
-  }
-
-  return newContentPieces;
-}
-
-export function chunkToKnownState(id: RawCoID, chunk: CoValueChunk) {
-  const ourKnown: CoValueKnownState = {
-    id,
-    header: !!chunk.header,
-    sessions: {},
-  };
-
-  for (const [sessionID, sessionEntries] of Object.entries(
-    chunk.sessionEntries,
-  )) {
-    for (const entry of sessionEntries) {
-      ourKnown.sessions[sessionID as SessionID] =
-        entry.after + entry.transactions.length;
-    }
-  }
-  return ourKnown;
-}
-
-export function mergeChunks(
-  chunkA: CoValueChunk,
-  chunkB: CoValueChunk,
-): "nonContigous" | CoValueChunk {
-  const header = chunkA.header || chunkB.header;
-
-  const newSessions = { ...chunkA.sessionEntries };
-  for (const sessionID in chunkB.sessionEntries) {
-    // figure out if we can merge the chunks
-    const sessionEntriesA = chunkA.sessionEntries[sessionID];
-    const sessionEntriesB = chunkB.sessionEntries[sessionID]!;
-
-    if (!sessionEntriesA) {
-      newSessions[sessionID] = sessionEntriesB;
-      continue;
-    }
-
-    const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1]!;
-    const firstEntryOfB = sessionEntriesB[0]!;
-
-    if (
-      lastEntryOfA.after + lastEntryOfA.transactions.length ===
-      firstEntryOfB.after
-    ) {
-      const newEntries = [];
-      let bytesSinceLastSignature = 0;
-      for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
-        const entryByteLength = entry.transactions.reduce(
-          (sum, tx) =>
-            sum +
-            (tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length),
-          0,
-        );
-        if (
-          newEntries.length === 0 ||
-          bytesSinceLastSignature + entryByteLength > MAX_RECOMMENDED_TX_SIZE
-        ) {
-          newEntries.push({
-            after: entry.after,
-            lastSignature: entry.lastSignature,
-            transactions: entry.transactions,
-          });
-          bytesSinceLastSignature = 0;
-        } else {
-          const lastNewEntry = newEntries[newEntries.length - 1]!;
-          lastNewEntry.transactions.push(...entry.transactions);
-          lastNewEntry.lastSignature = entry.lastSignature;
-
-          bytesSinceLastSignature += entry.transactions.length;
-        }
-      }
-      newSessions[sessionID] = newEntries;
-    } else {
-      return "nonContigous" as const;
-    }
-  }
-
-  return { header, sessionEntries: newSessions };
-}