@automerge/automerge-repo 2.0.0-alpha.6 → 2.0.0-collectionsync-alpha.1

package/src/ferigan.ts

@@ -0,0 +1,184 @@
+import { EventEmitter } from "eventemitter3"
+import { MessageContents, RepoMessage } from "./network/messages.js"
+import { AutomergeUrl } from "./types.js"
+import { Repo } from "./Repo.js"
+
+const URL_RE = /automerge:([\w\d+/=]+)(?:\?([\w\d+/=&]*))*/
+
+export interface Ferigan extends EventEmitter<FeriganEvents> {
+  receiveMessage(message: RepoMessage): Promise<void>
+  load(
+    doc: ChangeLogId,
+    since: ChangeHash[]
+  ): AsyncIterableIterator<Progress<ChangeLog | undefined>>
+  loadCollection(
+    doc: ChangeLogId
+  ): AsyncIterableIterator<Progress<Index | undefined>>
+  append(
+    doc: ChangeLogId,
+    parents: ChangeHash[],
+    changes: Uint8Array
+  ): Promise<void>
+  replace(
+    doc: ChangeLogId,
+    start: ChangeHash,
+    end: ChangeHash,
+    changes: Uint8Array
+  ): Promise<void>
+}
+
+export function makeFerigan(repo: Repo): Ferigan {
+  function fakeProgress<T>(): AsyncIterableIterator<Progress<T>> {
+    return (async function*() {
+      yield { type: "synchronizing_index" }
+    })()
+  }
+  class FakeFerigan extends EventEmitter<FeriganEvents> {
+    #repo: Repo
+
+    constructor(repo: Repo) {
+      super()
+      this.#repo = repo
+    }
+
+    async receiveMessage(message: RepoMessage): Promise<void> { }
+    load(
+      doc: ChangeLogId
+    ): AsyncIterableIterator<Progress<ChangeLog | undefined>> {
+      return fakeProgress()
+    }
+    async *loadCollection(
+      doc: ChangeLogId
+    ): AsyncIterableIterator<Progress<Index | undefined>> {
+      yield { type: "synchronizing_index" }
+
+      const index: Index = { rootUrl: doc as AutomergeUrl, entries: [] }
+
+      function findLinks(
+        obj: unknown
+      ): { url: string }[] {
+        const links: { url: string; }[] = []
+
+        const traverse = (value: unknown): void => {
+          if (typeof value === "string") {
+            const url = parseUrl(value)
+            if (url != null) {
+              links.push(url)
+            }
+          } else if (Array.isArray(value)) {
+            value.forEach(traverse)
+          } else if (typeof value === "object" && value !== null) {
+            Object.values(value).forEach(traverse)
+          }
+        }
+
+        traverse(obj)
+        return links
+      }
+
+      const indexDoc = this.#repo.find(doc as AutomergeUrl)
+      const handlesToProcess = [indexDoc]
+
+      while (handlesToProcess.length > 0) {
+        const handle = handlesToProcess.pop()
+        if (!handle) {
+          continue
+        }
+
+        const doc = await handle.doc()
+        if (!doc) {
+          continue
+        }
+
+        handle.on("change", change => {
+          for (const patch of change.patches) {
+            if (patch.action === "splice") {
+              const possibleUrl = parseUrl(patch.value)
+              if (possibleUrl != null) {
+                this.emit("indexChanged", {
+                  indexUrl: indexDoc.url,
+                  change: {
+                    type: "add",
+                    url: possibleUrl.url,
+                  }
+                })
+              }
+            }
+          }
+        })
+
+        const links = findLinks(doc)
+        for (const { url: urlStr } of links) {
+          const url = urlStr as AutomergeUrl
+          if (index.entries.some(entry => entry === url)) {
+            continue
+          }
+          index.entries.push(url)
+
+          const childHandle = this.#repo.find(url as AutomergeUrl)
+          handlesToProcess.push(childHandle)
+        }
+      }
+      yield { type: "done", value: index }
+    }
+    append(
+      doc: ChangeLogId,
+      parents: ChangeHash[],
+      changes: Uint8Array
+    ): Promise<void> {
+      return Promise.resolve()
+    }
+    replace(
+      doc: ChangeLogId,
+      start: ChangeHash,
+      end: ChangeHash,
+      changes: Uint8Array
+    ): Promise<void> {
+      return Promise.resolve()
+    }
+  }
+  return new FakeFerigan(repo)
+}
+
+interface FeriganEvents {
+  message: (event: {message: MessageContents}) => void
+  changed: (event: {changedLog: ChangeLogId}) => void
+  indexChanged: (event: {indexUrl: ChangeLogId, change: IndexChange}) => void
+}
+
+type IndexChange = {
+  type: "add"
+  url: AutomergeUrl
+}
+
+type ChangeLogId = string
+
+type ChangeHash = string
+
+type ChangeLog = { start: ChangeHash; end: ChangeHash; changes: Uint8Array }
+
+export type Progress<T> =
+  | { type: "synchronizing_index" }
+  | {
+    type: "synchronizing_docs"
+    progress: number
+    total: number
+  }
+  | { type: "done"; value: T }
+
+export type Index = {
+  rootUrl: AutomergeUrl,
+  entries: AutomergeUrl[],
+}
+
+function parseUrl(
+  urlStr: string
+): { url: AutomergeUrl } | undefined {
+  const match = urlStr.match(URL_RE)
+  if (!match) {
+    return undefined
+  }
+  const url = new URL(match[0])
+  const normalisedUrl = `automerge:${url.pathname}`
+  return { url: normalisedUrl as AutomergeUrl }
+}

package/src/helpers/tests/storage-adapter-tests.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest"
+import { describe, expect, beforeEach, it as _it } from "vitest"
 
 import type { StorageAdapterInterface } from "../../storage/StorageAdapterInterface.js"
 
@@ -8,72 +8,61 @@ const PAYLOAD_C = () => new Uint8Array([2, 111, 74, 131, 236, 96, 142, 193])
 
 const LARGE_PAYLOAD = new Uint8Array(100000).map(() => Math.random() * 256)
 
-export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
-  const setup = async () => {
-    const { adapter, teardown = NO_OP } = await _setup()
-    return { adapter, teardown }
-  }
+type AdapterTestContext = {
+  adapter: StorageAdapterInterface
+}
+
+const it = _it<AdapterTestContext>
+
+export function runStorageAdapterTests(setup: SetupFn, title?: string): void {
+  beforeEach<AdapterTestContext>(async ctx => {
+    const { adapter, teardown = NO_OP } = await setup()
+    ctx.adapter = adapter
+    return teardown
+  })
 
   describe(`Storage adapter acceptance tests ${
     title ? `(${title})` : ""
   }`, () => {
     describe("load", () => {
-      it("should return undefined if there is no data", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return undefined if there is no data", async ({ adapter }) => {
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toBeUndefined()
-
-        teardown()
       })
     })
 
     describe("save and load", () => {
-      it("should return data that was saved", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return data that was saved", async ({ adapter }) => {
         await adapter.save(["storage-adapter-id"], PAYLOAD_A())
         const actual = await adapter.load(["storage-adapter-id"])
         expect(actual).toStrictEqual(PAYLOAD_A())
-
-        teardown()
       })
 
-      it("should work with composite keys", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should work with composite keys", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toStrictEqual(PAYLOAD_A())
-
-        teardown()
       })
 
-      it("should work with a large payload", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should work with a large payload", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], LARGE_PAYLOAD)
         const actual = await adapter.load(["AAAAA", "sync-state", "xxxxx"])
         expect(actual).toStrictEqual(LARGE_PAYLOAD)
-
-        teardown()
       })
     })
 
     describe("loadRange", () => {
-      it("should return an empty array if there is no data", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return an empty array if there is no data", async ({
+        adapter,
+      }) => {
         expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([])
-
-        teardown()
       })
     })
 
     describe("save and loadRange", () => {
-      it("should return all the data that matches the key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should return all the data that matches the key", async ({
+        adapter,
+      }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "snapshot", "yyyyy"], PAYLOAD_B())
         await adapter.save(["AAAAA", "sync-state", "zzzzz"], PAYLOAD_C())
@@ -92,13 +81,9 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
             { key: ["AAAAA", "sync-state", "zzzzz"], data: PAYLOAD_C() },
           ])
         )
-
-        teardown()
       })
 
-      it("should only load values that match they key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should only load values that match they key", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["BBBBB", "sync-state", "zzzzz"], PAYLOAD_C())
 
@@ -113,15 +98,11 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
             { key: ["BBBBB", "sync-state", "zzzzz"], data: PAYLOAD_C() },
           ])
         )
-
-        teardown()
       })
     })
 
     describe("save and remove", () => {
-      it("after removing, should be empty", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("after removing, should be empty", async ({ adapter }) => {
         await adapter.save(["AAAAA", "snapshot", "xxxxx"], PAYLOAD_A())
         await adapter.remove(["AAAAA", "snapshot", "xxxxx"])
 
@@ -129,30 +110,24 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(
           await adapter.load(["AAAAA", "snapshot", "xxxxx"])
         ).toBeUndefined()
-
-        teardown()
       })
     })
 
     describe("save and save", () => {
-      it("should overwrite data saved with the same key", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should overwrite data saved with the same key", async ({
+        adapter,
+      }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_B())
 
         expect(await adapter.loadRange(["AAAAA", "sync-state"])).toStrictEqual([
           { key: ["AAAAA", "sync-state", "xxxxx"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
     })
 
     describe("removeRange", () => {
-      it("should remove a range of records", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should remove a range of records", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["AAAAA", "snapshot", "yyyyy"], PAYLOAD_B())
         await adapter.save(["AAAAA", "sync-state", "zzzzz"], PAYLOAD_C())
@@ -162,13 +137,9 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(await adapter.loadRange(["AAAAA"])).toStrictEqual([
           { key: ["AAAAA", "snapshot", "yyyyy"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
 
-      it("should not remove records that don't match", async () => {
-        const { adapter, teardown } = await setup()
-
+      it("should not remove records that don't match", async ({ adapter }) => {
         await adapter.save(["AAAAA", "sync-state", "xxxxx"], PAYLOAD_A())
         await adapter.save(["BBBBB", "sync-state", "zzzzz"], PAYLOAD_B())
 
@@ -178,8 +149,6 @@ export function runStorageAdapterTests(_setup: SetupFn, title?: string): void {
         expect(actual).toStrictEqual([
           { key: ["BBBBB", "sync-state", "zzzzz"], data: PAYLOAD_B() },
         ])
-
-        teardown()
       })
     })
   })
@@ -189,5 +158,5 @@ const NO_OP = () => {}
 
 export type SetupFn = () => Promise<{
   adapter: StorageAdapterInterface
-  teardown?: () => void
+  teardown?: () => void | Promise<void>
 }>

package/src/index.ts

@@ -41,6 +41,8 @@ export type { NetworkAdapterInterface } from "./network/NetworkAdapterInterface.
 export { isRepoMessage } from "./network/messages.js"
 export { StorageAdapter } from "./storage/StorageAdapter.js"
 export type { StorageAdapterInterface } from "./storage/StorageAdapterInterface.js"
+export type { Index, Progress } from "./ferigan.js"
+export { CollectionHandle } from "./CollectionHandle.js"
 
 /** @hidden **/
 export * as cbor from "./helpers/cbor.js"

package/src/network/NetworkSubsystem.ts

@@ -38,6 +38,10 @@ export class NetworkSubsystem extends EventEmitter<NetworkSubsystemEvents> {
     adapters.forEach(a => this.addNetworkAdapter(a))
   }
 
+  get peers(): PeerId[] {
+    return Array.from(Object.keys(this.#adaptersByPeer)) as PeerId[]
+  }
+
   disconnect() {
     this.adapters.forEach(a => a.disconnect())
   }

package/src/network/messages.ts

@@ -130,6 +130,14 @@ export type RepoMessage =
   | DocumentUnavailableMessage
   | RemoteSubscriptionControlMessage
   | RemoteHeadsChanged
+  | BeelayMessage
+
+export type BeelayMessage = {
+  type: "beelay"
+  senderId: PeerId
+  targetId: PeerId
+  message: Uint8Array
+}
 
 /** These are message types that are handled by the {@link CollectionSynchronizer}.*/
 export type DocMessage =
@@ -167,10 +175,11 @@ export const isRepoMessage = (message: Message): message is RepoMessage =>
   isRequestMessage(message) ||
   isDocumentUnavailableMessage(message) ||
   isRemoteSubscriptionControlMessage(message) ||
-  isRemoteHeadsChanged(message)
+  isRemoteHeadsChanged(message) ||
+  true
 
 // prettier-ignore
-export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage => 
+export const isDocumentUnavailableMessage = (msg: Message): msg is DocumentUnavailableMessage =>
   msg.type === "doc-unavailable"
 
 export const isRequestMessage = (msg: Message): msg is RequestMessage =>

package/src/storage/StorageAdapter.ts

@@ -34,3 +34,45 @@ export abstract class StorageAdapter implements StorageAdapterInterface {
   /** Remove all values with keys that start with `keyPrefix` */
   abstract removeRange(keyPrefix: StorageKey): Promise<void>
 }
+
+export class InMemoryStorageAdapter extends StorageAdapter {
+  #data: Record<string, Uint8Array> = {}
+
+  #keyToString(key: string[]): string {
+    return key.join(".")
+  }
+
+  #stringToKey(key: string): string[] {
+    return key.split(".")
+  }
+
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+    const range = Object.entries(this.#data)
+      .filter(([key, _]) => key.startsWith(this.#keyToString(keyPrefix)))
+      .map(([key, data]) => ({ key: this.#stringToKey(key), data }))
+    return Promise.resolve(range)
+  }
+
+  async removeRange(keyPrefix: string[]): Promise<void> {
+    Object.entries(this.#data)
+      .filter(([key, _]) => key.startsWith(this.#keyToString(keyPrefix)))
+      .forEach(([key, _]) => delete this.#data[key])
+  }
+
+  async load(key: string[]): Promise<Uint8Array | undefined> {
+    return new Promise(resolve => resolve(this.#data[this.#keyToString(key)]))
+  }
+
+  async save(key: string[], binary: Uint8Array) {
+    this.#data[this.#keyToString(key)] = binary
+    return Promise.resolve()
+  }
+
+  async remove(key: string[]) {
+    delete this.#data[this.#keyToString(key)]
+  }
+
+  keys() {
+    return Object.keys(this.#data)
+  }
+}