cojson 0.7.0-alpha.37 → 0.7.0-alpha.38

package/src/storage/FileSystem.ts

@@ -1,12 +1,11 @@
 import { Effect } from "effect";
 import { CoValueChunk } from "./index.js";
 import { RawCoID } from "../ids.js";
-import { StreamingHash } from "../crypto.js";
+import { CryptoProvider, StreamingHash } from "../crypto/crypto.js";
 
-export type BlockFilename =
-    `${string}-L${number}-H${number}.jsonl`;
+export type BlockFilename = `${string}-L${number}-H${number}.jsonl`;
 
-    export type BlockHeader = { id: RawCoID; start: number; length: number }[];
+export type BlockHeader = { id: RawCoID; start: number; length: number }[];
 
 export type WalEntry = { id: RawCoID } & CoValueChunk;
 
@@ -18,25 +17,26 @@ export type FSErr = {
 };
 
 export interface FileSystem<WriteHandle, ReadHandle> {
+    crypto: CryptoProvider;
     createFile(filename: string): Effect.Effect<WriteHandle, FSErr>;
     append(handle: WriteHandle, data: Uint8Array): Effect.Effect<void, FSErr>;
-    close(
-        handle: ReadHandle | WriteHandle,
-    ): Effect.Effect<void, FSErr>;
+    close(handle: ReadHandle | WriteHandle): Effect.Effect<void, FSErr>;
     closeAndRename(
         handle: WriteHandle,
-        filename: BlockFilename
+        filename: BlockFilename,
     ): Effect.Effect<void, FSErr>;
     openToRead(
-        filename: string
-    ): Effect.Effect<{ handle: ReadHandle; size: number; }, FSErr>;
+        filename: string,
+    ): Effect.Effect<{ handle: ReadHandle; size: number }, FSErr>;
     read(
         handle: ReadHandle,
         offset: number,
-        length: number
+        length: number,
     ): Effect.Effect<Uint8Array, FSErr>;
     listFiles(): Effect.Effect<string[], FSErr>;
-    removeFile(filename: BlockFilename | WalFilename): Effect.Effect<void, FSErr>;
+    removeFile(
+        filename: BlockFilename | WalFilename,
+    ): Effect.Effect<void, FSErr>;
 }
 
 export const textEncoder = new TextEncoder();
@@ -44,12 +44,12 @@ export const textDecoder = new TextDecoder();
 
 export function readChunk<RH, FS extends FileSystem<unknown, RH>>(
     handle: RH,
-    header: { start: number; length: number; },
-    fs: FS
+    header: { start: number; length: number },
+    fs: FS,
 ): Effect.Effect<CoValueChunk, FSErr> {
     return Effect.gen(function* ($) {
         const chunkBytes = yield* $(
-            fs.read(handle, header.start, header.length)
+            fs.read(handle, header.start, header.length),
         );
 
         const chunk = JSON.parse(textDecoder.decode(chunkBytes));
@@ -61,15 +61,13 @@ export function readHeader<RH, FS extends FileSystem<unknown, RH>>(
     filename: string,
     handle: RH,
     size: number,
-    fs: FS
+    fs: FS,
 ): Effect.Effect<BlockHeader, FSErr> {
     return Effect.gen(function* ($) {
-
-
         const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
 
         const headerBytes = yield* $(
-            fs.read(handle, size - headerLength, headerLength)
+            fs.read(handle, size - headerLength, headerLength),
         );
 
         const header = JSON.parse(textDecoder.decode(headerBytes));
@@ -80,7 +78,7 @@ export function readHeader<RH, FS extends FileSystem<unknown, RH>>(
 export function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
     chunks: Map<RawCoID, CoValueChunk>,
     level: number,
-    fs: FS
+    fs: FS,
 ): Effect.Effect<void, FSErr> {
     if (chunks.size === 0) {
         return Effect.die(new Error("No chunks to write"));
@@ -94,19 +92,20 @@ export function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
         const file = yield* $(
             fs.createFile(
                 "wipBlock" +
-                Math.random().toString(36).substring(7) +
-                ".tmp.jsonl"
-            )
+                    Math.random().toString(36).substring(7) +
+                    ".tmp.jsonl",
+            ),
         );
-        const hash = new StreamingHash();
+        const hash = new StreamingHash(fs.crypto);
 
-        const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) => id1.localeCompare(id2)
+        const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) =>
+            id1.localeCompare(id2),
         );
 
         for (const [id, chunk] of chunksSortedById) {
             const encodedBytes = hash.update(chunk);
             const encodedBytesWithNewline = new Uint8Array(
-                encodedBytes.length + 1
+                encodedBytes.length + 1,
             );
             encodedBytesWithNewline.set(encodedBytes);
             encodedBytesWithNewline[encodedBytes.length] = 10;
@@ -122,11 +121,13 @@ export function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
         console.log(
             "full file",
             yield* $(
-                fs.read(file as unknown as RH, 0, offset + headerBytes.length)
-            )
+                fs.read(file as unknown as RH, 0, offset + headerBytes.length),
+            ),
         );
 
-        const filename: BlockFilename = `${hash.digest()}-L${level}-H${headerBytes.length}.jsonl`;
+        const filename: BlockFilename = `${hash.digest()}-L${level}-H${
+            headerBytes.length
+        }.jsonl`;
         console.log("renaming to" + filename);
         yield* $(fs.closeAndRename(file, filename));
 
@@ -138,7 +139,7 @@ export function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
     handle: WH,
     fs: FS,
     id: RawCoID,
-    chunk: CoValueChunk
+    chunk: CoValueChunk,
 ): Effect.Effect<void, FSErr> {
     return Effect.gen(function* ($) {
         const walEntry: WalEntry = {

package/src/storage/chunksAndKnownStates.ts

@@ -1,13 +1,13 @@
-import { Either } from 'effect';
-import { RawCoID, SessionID } from '../ids.js';
-import { MAX_RECOMMENDED_TX_SIZE } from '../index.js';
+import { Either } from "effect";
+import { RawCoID, SessionID } from "../ids.js";
+import { MAX_RECOMMENDED_TX_SIZE } from "../index.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { CoValueChunk } from "./index.js";
 
 export function contentSinceChunk(
     id: RawCoID,
     chunk: CoValueChunk,
-    known?: CoValueKnownState
+    known?: CoValueKnownState,
 ): NewContentMessage[] {
     const newContentPieces: NewContentMessage[] = [];
 
@@ -19,7 +19,7 @@ export function contentSinceChunk(
     });
 
     for (const [sessionID, sessionsEntry] of Object.entries(
-        chunk.sessionEntries
+        chunk.sessionEntries,
     )) {
         for (const entry of sessionsEntry) {
             const knownStart = known?.sessions[sessionID as SessionID] || 0;
@@ -29,13 +29,15 @@ export function contentSinceChunk(
             }
 
             const actuallyNewTransactions = entry.transactions.slice(
-                Math.max(0, knownStart - entry.after)
+                Math.max(0, knownStart - entry.after),
             );
 
-            const newAfter = entry.after +
+            const newAfter =
+                entry.after +
                 (actuallyNewTransactions.length - entry.transactions.length);
 
-            let newContentEntry = newContentPieces[0]?.new[sessionID as SessionID];
+            let newContentEntry =
+                newContentPieces[0]?.new[sessionID as SessionID];
 
             if (!newContentEntry) {
                 newContentEntry = {
@@ -47,7 +49,7 @@ export function contentSinceChunk(
                     newContentEntry;
             } else {
                 newContentEntry.newTransactions.push(
-                    ...actuallyNewTransactions
+                    ...actuallyNewTransactions,
                 );
                 newContentEntry.lastSignature = entry.lastSignature;
             }
@@ -65,7 +67,7 @@ export function chunkToKnownState(id: RawCoID, chunk: CoValueChunk) {
     };
 
     for (const [sessionID, sessionEntries] of Object.entries(
-        chunk.sessionEntries
+        chunk.sessionEntries,
     )) {
         for (const entry of sessionEntries) {
             ourKnown.sessions[sessionID as SessionID] =
@@ -77,7 +79,7 @@ export function chunkToKnownState(id: RawCoID, chunk: CoValueChunk) {
 
 export function mergeChunks(
     chunkA: CoValueChunk,
-    chunkB: CoValueChunk
+    chunkB: CoValueChunk,
 ): Either.Either<"nonContigous", CoValueChunk> {
     const header = chunkA.header || chunkB.header;
 
@@ -95,21 +97,26 @@ export function mergeChunks(
         const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1]!;
         const firstEntryOfB = sessionEntriesB[0]!;
 
-        if (lastEntryOfA.after + lastEntryOfA.transactions.length ===
-            firstEntryOfB.after) {
+        if (
+            lastEntryOfA.after + lastEntryOfA.transactions.length ===
+            firstEntryOfB.after
+        ) {
             const newEntries = [];
             let bytesSinceLastSignature = 0;
             for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
                 const entryByteLength = entry.transactions.reduce(
-                    (sum, tx) => sum +
+                    (sum, tx) =>
+                        sum +
                         (tx.privacy === "private"
                             ? tx.encryptedChanges.length
                             : tx.changes.length),
-                    0
+                    0,
                 );
-                if (newEntries.length === 0 ||
+                if (
+                    newEntries.length === 0 ||
                     bytesSinceLastSignature + entryByteLength >
-                    MAX_RECOMMENDED_TX_SIZE) {
+                        MAX_RECOMMENDED_TX_SIZE
+                ) {
                     newEntries.push({
                         after: entry.after,
                         lastSignature: entry.lastSignature,

package/src/storage/index.ts

@@ -7,7 +7,7 @@ import {
 import { Effect, Either, SynchronizedRef } from "effect";
 import { RawCoID } from "../ids.js";
 import { CoValueHeader, Transaction } from "../coValueCore.js";
-import { Signature } from "../crypto.js";
+import { Signature } from "../crypto/crypto.js";
 import {
     CoValueKnownState,
     NewContentMessage,
@@ -63,7 +63,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
     constructor(
         fs: FS,
         fromLocalNode: ReadableStream<SyncMessage>,
-        toLocalNode: WritableStream<SyncMessage>
+        toLocalNode: WritableStream<SyncMessage>,
     ) {
         this.fs = fs;
         this.fromLocalNode = fromLocalNode.getReader();
@@ -76,7 +76,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                 let done = false;
                 while (!done) {
                     const result = yield* Effect.promise(() =>
-                        this.fromLocalNode.read()
+                        this.fromLocalNode.read(),
                     );
                     done = result.done;
 
@@ -91,14 +91,14 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                             yield* this.sendNewContent(
                                 result.value.id,
                                 result.value,
-                                undefined
+                                undefined,
                             );
                         }
                     }
                 }
 
                 return;
-            })
+            }),
         );
 
         setTimeout(() => this.compact(), 20000);
@@ -107,10 +107,10 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
     sendNewContent(
         id: RawCoID,
         known: CoValueKnownState | undefined,
-        asDependencyOf: RawCoID | undefined
+        asDependencyOf: RawCoID | undefined,
     ): Effect.Effect<void, FSErr> {
         return SynchronizedRef.updateEffect(this.coValues, (coValues) =>
-            this.sendNewContentInner(coValues, id, known, asDependencyOf)
+            this.sendNewContentInner(coValues, id, known, asDependencyOf),
         );
     }
 
@@ -118,7 +118,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         coValues: { [id: `co_z${string}`]: CoValueChunk | undefined },
         id: RawCoID,
         known: CoValueKnownState | undefined,
-        asDependencyOf: RawCoID | undefined
+        asDependencyOf: RawCoID | undefined,
     ): Effect.Effect<
         { [id: `co_z${string}`]: CoValueChunk | undefined },
         FSErr,
@@ -139,7 +139,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                         header: false,
                         sessions: {},
                         asDependencyOf,
-                    })
+                    }),
                 );
 
                 return coValues;
@@ -153,7 +153,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                     coValues,
                     coValue.header.ruleset.group,
                     undefined,
-                    asDependencyOf || id
+                    asDependencyOf || id,
                 );
             } else if (
                 !known?.header &&
@@ -182,7 +182,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                         coValues,
                         account as CoID<RawCoValue>,
                         undefined,
-                        asDependencyOf || id
+                        asDependencyOf || id,
                     );
                 }
             }
@@ -190,7 +190,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
             const newContentMessages = contentSinceChunk(
                 id,
                 coValue,
-                known
+                known,
             ).map((message) => ({ ...message, asDependencyOf }));
 
             const ourKnown: CoValueKnownState = chunkToKnownState(id, coValue);
@@ -200,7 +200,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                     action: "known",
                     ...ourKnown,
                     asDependencyOf,
-                })
+                }),
             );
 
             for (const message of newContentMessages) {
@@ -213,7 +213,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
     }
 
     withWAL(
-        handler: (wal: WH) => Effect.Effect<void, FSErr>
+        handler: (wal: WH) => Effect.Effect<void, FSErr>,
     ): Effect.Effect<void, FSErr> {
         return SynchronizedRef.updateEffect(this.currentWal, (wal) =>
             Effect.gen(this, function* () {
@@ -222,17 +222,17 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                     newWal = yield* this.fs.createFile(
                         `wal-${new Date().toISOString()}-${Math.random()
                             .toString(36)
-                            .slice(2)}.jsonl`
+                            .slice(2)}.jsonl`,
                     );
                 }
                 yield* handler(newWal);
                 return newWal;
-            })
+            }),
         );
     }
 
     handleNewContent(
-        newContent: NewContentMessage
+        newContent: NewContentMessage,
     ): Effect.Effect<void, FSErr> {
         return SynchronizedRef.updateEffect(this.coValues, (coValues) =>
             Effect.gen(this, function* () {
@@ -253,8 +253,8 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                                             newInSession.newTransactions,
                                     },
                                 ],
-                            ]
-                        )
+                            ],
+                        ),
                     ),
                 };
 
@@ -266,8 +266,8 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                                 wal,
                                 this.fs,
                                 newContent.id,
-                                newContentAsChunk
-                            )
+                                newContentAsChunk,
+                            ),
                         );
 
                         return {
@@ -287,7 +287,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                         //     )
                         // );
                         console.warn(
-                            "Incontiguous incoming update for " + newContent.id
+                            "Incontiguous incoming update for " + newContent.id,
                         );
                         return coValues;
                     }
@@ -295,7 +295,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                     const merged = mergeChunks(coValue, newContentAsChunk);
                     if (Either.isRight(merged)) {
                         yield* Effect.logWarning(
-                            "Non-contigous new content for " + newContent.id
+                            "Non-contigous new content for " + newContent.id,
                         );
 
                         // yield* Effect.promise(() =>
@@ -314,27 +314,27 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                                 wal,
                                 this.fs,
                                 newContent.id,
-                                newContentAsChunk
-                            )
+                                newContentAsChunk,
+                            ),
                         );
 
                         return { ...coValues, [newContent.id]: merged.left };
                     }
                 }
-            })
+            }),
         );
     }
 
     loadCoValue<WH, RH, FS extends FileSystem<WH, RH>>(
         id: RawCoID,
-        fs: FS
+        fs: FS,
     ): Effect.Effect<CoValueChunk | undefined, FSErr> {
         // return _loadChunkFromWal(id, fs);
         return Effect.gen(this, function* () {
             const files = this.fileCache || (yield* fs.listFiles());
             this.fileCache = files;
             const blockFiles = files.filter((name) =>
-                name.startsWith("hash_")
+                name.startsWith("hash_"),
             ) as BlockFilename[];
 
             for (const blockFile of blockFiles) {
@@ -346,7 +346,12 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
                 if (!cachedHeader) {
                     cachedHeader = {};
-                    const header = yield* readHeader(blockFile, handle, size, fs);
+                    const header = yield* readHeader(
+                        blockFile,
+                        handle,
+                        size,
+                        fs,
+                    );
                     for (const entry of header) {
                         cachedHeader[entry.id] = {
                             start: entry.start,
@@ -361,12 +366,11 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                 let result;
                 if (headerEntry) {
                     result = yield* readChunk(handle, headerEntry, fs);
-
                 }
 
                 yield* fs.close(handle);
 
-                return result
+                return result;
             }
 
             return undefined;
@@ -379,7 +383,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                 const fileNames = yield* this.fs.listFiles();
 
                 const walFiles = fileNames.filter((name) =>
-                    name.startsWith("wal-")
+                    name.startsWith("wal-"),
                 ) as WalFilename[];
                 walFiles.sort();
 
@@ -394,7 +398,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                             yield* this.fs.close(wal);
                         }
                         return undefined;
-                    })
+                    }),
                 );
 
                 for (const fileName of walFiles) {
@@ -402,7 +406,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                         yield* this.fs.openToRead(fileName);
                     if (size === 0) {
                         yield* this.fs.close(handle);
-                        continue
+                        continue;
                     }
                     const bytes = yield* this.fs.read(handle, 0, size);
 
@@ -424,7 +428,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                                         ", " +
                                         fileName,
                                     existingChunk,
-                                    chunk
+                                    chunk,
                                 );
                             } else {
                                 coValues.set(chunk.id, merged.left);
@@ -442,7 +446,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                     yield* this.fs.removeFile(walFile);
                 }
                 this.fileCache = undefined;
-            })
+            }),
         );
 
         setTimeout(() => this.compact(), 5000);
@@ -464,7 +468,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                 peer1role: "client",
                 peer2role: "server",
                 trace,
-            }
+            },
         );
 
         new LSMStorage(fs, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);

package/src/streamUtils.ts

@@ -16,7 +16,7 @@ export function connectedPeers(
         trace?: boolean;
         peer1role?: Peer["role"];
         peer2role?: Peer["role"];
-    } = {}
+    } = {},
 ): [Peer, Peer] {
     const [inRx1, inTx1] = newStreamPair<SyncMessage>(peer1id + "_in");
     const [outRx1, outTx1] = newStreamPair<SyncMessage>(peer1id + "_out");
@@ -29,7 +29,7 @@ export function connectedPeers(
             new TransformStream({
                 transform(
                     chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void }
+                    controller: { enqueue: (msg: SyncMessage) => void },
                 ) {
                     trace &&
                         console.debug(
@@ -40,12 +40,12 @@ export function connectedPeers(
                                     k === "changes" || k === "encryptedChanges"
                                         ? v.slice(0, 20) + "..."
                                         : v,
-                                2
-                            )
+                                2,
+                            ),
                         );
                     controller.enqueue(chunk);
                 },
-            })
+            }),
         )
         .pipeTo(inTx1);
 
@@ -54,7 +54,7 @@ export function connectedPeers(
             new TransformStream({
                 transform(
                     chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void }
+                    controller: { enqueue: (msg: SyncMessage) => void },
                 ) {
                     trace &&
                         console.debug(
@@ -65,12 +65,12 @@ export function connectedPeers(
                                     k === "changes" || k === "encryptedChanges"
                                         ? v.slice(0, 20) + "..."
                                         : v,
-                                2
-                            )
+                                2,
+                            ),
                         );
                     controller.enqueue(chunk);
                 },
-            })
+            }),
         )
         .pipeTo(inTx2);
 
@@ -92,7 +92,7 @@ export function connectedPeers(
 }
 
 export function newStreamPair<T>(
-    pairName?: string
+    pairName?: string,
 ): [ReadableStream<T>, WritableStream<T>] {
     let queueLength = 0;
     let readerClosed = false;
@@ -138,13 +138,13 @@ export function newStreamPair<T>(
         new TransformStream<any, any>({
             transform(
                 chunk: SyncMessage,
-                controller: { enqueue: (msg: SyncMessage) => void }
+                controller: { enqueue: (msg: SyncMessage) => void },
             ) {
                 queueLength -= 1;
                 maybeReportQueueLength();
                 controller.enqueue(chunk);
             },
-        })
+        }),
     ) as ReadableStream<T>;
 
     let lastWritePromise = Promise.resolve();