effect 4.0.5 → 4.0.6

package/src/eventlog/EventLogServer.ts

@@ -14,16 +14,19 @@ import * as Arr from "../Array.ts"
 import * as Cache from "../Cache.ts"
 import * as Context from "../Context.ts"
 import * as Data from "../Data.ts"
+import * as Duration from "../Duration.ts"
 import * as Effect from "../Effect.ts"
 import * as Equal from "../Equal.ts"
+import * as Exit from "../Exit.ts"
 import * as Hash from "../Hash.ts"
+import * as Uuid from "../internal/uuid.ts"
 import * as Layer from "../Layer.ts"
 import * as Option from "../Option.ts"
 import * as Redacted from "../Redacted.ts"
 import type * as Rpc from "../rpc/Rpc.ts"
 import type * as RpcGroup from "../rpc/RpcGroup.ts"
 import * as Stream from "../Stream.ts"
-import type { RemoteId } from "./EventJournal.ts"
+import type { EntryId, RemoteId } from "./EventJournal.ts"
 import * as EventLog from "./EventLog.ts"
 import {
   ChunkedMessage,
@@ -32,7 +35,8 @@ import {
   EventLogRemoteRpcs,
   HelloResponse,
   SingleMessage,
-  type StoreId
+  type StoreId,
+  type WriteAck
 } from "./EventLogMessage.ts"
 import * as EventLogSessionAuth from "./EventLogSessionAuth.ts"
 
@@ -72,12 +76,16 @@ export const layerAuthMiddleware: Layer.Layer<
  * chunked writes, delegates write and change handling to the supplied callbacks,
  * and frames large change payloads into chunks.
  *
+ * Use the connection's `ConnectionWrites.track` in `onWrite` and
+ * `ConnectionWrites.filter` in `changes` to suppress its own write echoes.
+ *
  * @stability unstable
  * @category layers
  * @since 4.0.0
  */
 export const layerRpcHandlers = (options: {
   readonly remoteId: RemoteId
+  readonly firstSequence?: number | undefined
   readonly authorizeIdentity?: (publicKey: string) => Effect.Effect<void, EventLogProtocolError>
   readonly getOrCreateSessionAuthBinding: (
     publicKey: string,
@@ -85,12 +93,14 @@ export const layerRpcHandlers = (options: {
   ) => Effect.Effect<Uint8Array<ArrayBuffer>>
   readonly onWrite: (
     data: Uint8Array<ArrayBuffer>,
-    authenticatedPublicKeys: ReadonlySet<string>
-  ) => Effect.Effect<void, EventLogProtocolError>
+    authenticatedPublicKeys: ReadonlySet<string>,
+    writes: ConnectionWrites
+  ) => Effect.Effect<ReadonlyArray<WriteAck>, EventLogProtocolError>
   readonly changes: (options: {
     readonly publicKey: string
     readonly storeId: StoreId
     readonly startSequence: number
+    readonly writes: ConnectionWrites
   }) => Stream.Stream<Uint8Array<ArrayBuffer>, unknown>
 }): Layer.Layer<
   Rpc.ToHandler<RpcGroup.Rpcs<typeof EventLogRemoteRpcs>> | EventLogAuthentication
@@ -103,8 +113,8 @@ export const layerRpcHandlers = (options: {
     })
     let chunkedIdCounter = 0
 
-    const persistedSigningPublicKeys = yield* Cache.make({
-      lookup: (key: SessionAuthCacheKey) =>
+    const persistedSigningPublicKeys = yield* Cache.makeWith(
+      (key: SessionAuthCacheKey) =>
         options.getOrCreateSessionAuthBinding(key.publicKey, key.signingPublicKey).pipe(
           Effect.catchCause((_) =>
             Effect.fail(
@@ -117,15 +127,19 @@ export const layerRpcHandlers = (options: {
             )
           )
         ),
-      capacity: 4096
-    })
+      {
+        capacity: 4096,
+        timeToLive: (exit) => Exit.isSuccess(exit) ? Duration.infinity : Duration.zero
+      }
+    )
 
     return EventLogRemoteRpcs.of({
       "EventLog.Hello": Effect.fnUntraced(function*(_, { client }) {
         const challenge = yield* Cache.get(clientChallenges, client.id)
         return new HelloResponse({
           remoteId: options.remoteId,
-          challenge
+          challenge,
+          firstSequence: options.firstSequence
         })
       }),
       "EventLog.Authenticate": Effect.fnUntraced(function*(request, { client }) {
@@ -181,21 +195,22 @@ export const layerRpcHandlers = (options: {
           .annotate(AuthenticatedIdentities, authenticatedIdentities)
           .annotate(ChunkedMessageState, new Map())
       }),
-      "EventLog.WriteSingle": Effect.fnUntraced(function*(request, { client }) {
-        yield* options.onWrite(
+      "EventLog.WriteSingle": (request, { client }) =>
+        options.onWrite(
           request.data,
-          Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set()
-        )
-      }),
-      "EventLog.WriteChunked": Effect.fnUntraced(function*(request, { client }) {
+          Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set(),
+          getConnectionWrites(client)
+        ),
+      "EventLog.WriteChunked": (request, { client }) => {
         const state = Context.get(client.annotations, ChunkedMessageState)
         const data = ChunkedMessage.join(state, request)
-        if (!data) return
-        yield* options.onWrite(
+        if (!data) return Effect.succeed([])
+        return options.onWrite(
           data,
-          Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set()
+          Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set(),
+          getConnectionWrites(client)
         )
-      }),
+      },
       "EventLog.Changes": (request, { client }) => {
         const authenticatedIdentities = Context.getOrUndefined(client.annotations, AuthenticatedIdentities)
         if (!authenticatedIdentities?.has(request.publicKey)) {
@@ -211,7 +226,8 @@ export const layerRpcHandlers = (options: {
         return options.changes({
           publicKey: request.publicKey,
           storeId: request.storeId,
-          startSequence: request.startSequence
+          startSequence: request.startSequence,
+          writes: getConnectionWrites(client)
         }).pipe(
           Stream.mapArray(Arr.flatMap((data): NonEmptyReadonlyArray<SingleMessage | ChunkedMessage> => {
             if (data.byteLength <= ChunkedMessage.chunkSize) {
@@ -259,6 +275,132 @@ export class ChunkedMessageState extends Context.Reference<
   defaultValue: () => new Map()
 }) {}
 
+/**
+ * Tracks writes to suppress their echoes on the same RPC connection.
+ *
+ * **Details**
+ *
+ * Only tracks entries for stores with an open change stream.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.0.6
+ */
+export class ConnectionWrites {
+  private readonly streams = new Map<string, Set<ConnectionStream>>()
+
+  /**
+   * Runs a storage write while suppressing its entries on this connection's streams.
+   *
+   * @stability unstable
+   * @since 4.0.6
+   */
+  track<E, R>(
+    options: {
+      readonly publicKey: string
+      readonly storeId: StoreId
+      readonly entryIds: ReadonlyArray<EntryId>
+    },
+    write: Effect.Effect<ReadonlyArray<WriteAck>, E, R>
+  ): Effect.Effect<ReadonlyArray<WriteAck>, E, R> {
+    const open = this.streams.get(makeStreamKey(options.publicKey, options.storeId))
+    if (open === undefined || options.entryIds.length === 0) return write
+    const streams = Array.from(open)
+    const started = streams.map((stream) => stream.position)
+    const ids = options.entryIds.map(Uuid.stringify)
+    // Track before writing: storage can publish an entry before the write
+    // returns. Each occurrence is counted, as an id can be stored more than once.
+    for (const stream of streams) {
+      for (const id of ids) stream.ids.set(id, (stream.ids.get(id) ?? 0) + 1)
+    }
+    return Effect.onExit(write, (exit) =>
+      Effect.sync(() => {
+        for (let i = 0; i < streams.length; i++) {
+          const stream = streams[i]
+          if (exit._tag === "Failure") {
+            for (const id of ids) release(stream.ids, id)
+            continue
+          }
+          // Release occurrences the stream had already passed when the write began.
+          for (const ack of exit.value) {
+            if (ack.sequence <= started[i]) release(stream.ids, Uuid.stringify(ack.entryId))
+          }
+        }
+      }))
+  }
+
+  /**
+   * Filters this connection's writes from a store's change stream. An item can
+   * hold several entries; `sequence` returns its highest one.
+   *
+   * @stability unstable
+   * @since 4.0.6
+   */
+  filter<A, E, R>(
+    options: {
+      readonly publicKey: string
+      readonly storeId: StoreId
+      readonly entryIds: (item: A) => ReadonlyArray<EntryId>
+      readonly sequence: (item: A) => number
+    },
+    stream: Stream.Stream<A, E, R>
+  ): Stream.Stream<A, E, R> {
+    return Stream.suspend(() => {
+      const key = makeStreamKey(options.publicKey, options.storeId)
+      const state: ConnectionStream = { ids: new Map(), position: -1 }
+      let open = this.streams.get(key)
+      if (open === undefined) {
+        open = new Set()
+        this.streams.set(key, open)
+      }
+      open.add(state)
+      return stream.pipe(
+        Stream.filter((item) => {
+          const sequence = options.sequence(item)
+          if (sequence > state.position) state.position = sequence
+          if (state.ids.size === 0) return true
+          // An item is written by one request, so it is either all or none of ours.
+          const ids = options.entryIds(item).map(Uuid.stringify)
+          if (!ids.every((id) => state.ids.has(id))) return true
+          for (const id of ids) release(state.ids, id)
+          return false
+        }),
+        Stream.ensuring(Effect.sync(() => {
+          open.delete(state)
+          if (open.size === 0) this.streams.delete(key)
+        }))
+      )
+    })
+  }
+}
+
+interface ConnectionStream {
+  // Outstanding occurrences of this connection's writes, by entry id.
+  readonly ids: Map<string, number>
+  position: number
+}
+
+const release = (ids: Map<string, number>, id: string): void => {
+  const count = ids.get(id)
+  if (count === undefined) return
+  if (count <= 1) ids.delete(id)
+  else ids.set(id, count - 1)
+}
+
+const makeStreamKey = (publicKey: string, storeId: StoreId): string => `${publicKey}/${storeId}`
+
+class ConnectionWritesState extends Context.Service<ConnectionWritesState, ConnectionWrites>()(
+  "effect/eventlog/EventLogServer/ConnectionWrites"
+) {}
+
+const getConnectionWrites = (client: Rpc.ServerClient): ConnectionWrites => {
+  const existing = Context.getOrUndefined(client.annotations, ConnectionWritesState)
+  if (existing !== undefined) return existing
+  const writes = new ConnectionWrites()
+  client.annotate(ConnectionWritesState, writes)
+  return writes
+}
+
 /**
  * Annotation containing the public keys authenticated on an RPC connection.
  *

package/src/eventlog/EventLogServerEncrypted.ts

@@ -14,18 +14,24 @@
 import * as Arr from "../Array.ts"
 import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
+import * as Equal from "../Equal.ts"
 import * as Uuid from "../internal/uuid.ts"
 import * as Layer from "../Layer.ts"
 import * as PubSub from "../PubSub.ts"
 import * as RcMap from "../RcMap.ts"
 import * as RpcServer from "../rpc/RpcServer.ts"
-import * as Schema from "../Schema.ts"
 import type * as Scope from "../Scope.ts"
 import * as Stream from "../Stream.ts"
-import * as Transferable from "../workers/Transferable.ts"
-import { EntryId, makeRemoteIdUnsafe, type RemoteId } from "./EventJournal.ts"
-import type { EncryptedRemoteEntry } from "./EventLogEncryption.ts"
-import { ChangesRpc, EventLogProtocolError, EventLogRemoteRpcs, type StoreId, WriteEntries } from "./EventLogMessage.ts"
+import { type EntryId, makeRemoteIdUnsafe, type RemoteId } from "./EventJournal.ts"
+import type { EncryptedBatch, EncryptedRemoteBatch } from "./EventLogEncryption.ts"
+import {
+  ChangesRpc,
+  EventLogProtocolError,
+  EventLogRemoteRpcs,
+  type StoreId,
+  type WriteAck,
+  WriteEntries
+} from "./EventLogMessage.ts"
 import * as EventLogServer from "./EventLogServer.ts"
 
 /**
@@ -33,9 +39,9 @@ import * as EventLogServer from "./EventLogServer.ts"
  *
  * **Details**
  *
- * Incoming encrypted write payloads are decoded and persisted through `Storage`;
- * change streams read encrypted entries from storage and encode them for the
- * remote protocol.
+ * Decodes and persists writes through `Storage`, acknowledging their stored
+ * sequences. Change streams encode stored batches, excluding this connection's
+ * writes.
  *
  * @stability unstable
  * @category layers
@@ -47,9 +53,10 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() {
 
   return EventLogServer.layerRpcHandlers({
     remoteId,
+    firstSequence: storage.firstSequence,
     getOrCreateSessionAuthBinding: (publicKey, signingPublicKey) =>
       storage.getOrCreateSessionAuthBinding(publicKey, signingPublicKey),
-    onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys) {
+    onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys, writes) {
       const request = yield* WriteEntries.decode(data).pipe(
         Effect.mapError((_) =>
           new EventLogProtocolError({
@@ -68,15 +75,11 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() {
           message: "Identity is not authenticated"
         })
       }
-      if (request.encryptedEntries.length === 0) return
-      const entries = request.encryptedEntries.map(({ encryptedEntry, entryId, iv }) =>
-        new PersistedEntry({
-          entryId,
-          iv,
-          encryptedEntry
-        })
-      )
-      return yield* storage.write(request.publicKey, request.storeId, entries).pipe(
+      if (request.batches.length === 0) return []
+      const entryIds = request.batches.flatMap((batch) => batch.entryIds)
+      return yield* storage.write(request.publicKey, request.storeId, request.batches).pipe(
+        Effect.flatMap((stored) => toWriteAcks(storage, request.publicKey, request.storeId, request.batches, stored)),
+        (write) => writes.track({ publicKey: request.publicKey, storeId: request.storeId, entryIds }, write),
         Effect.catchCause((_) =>
           Effect.fail(
             new EventLogProtocolError({
@@ -89,13 +92,53 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() {
         )
       )
     }),
-    changes: ({ publicKey, storeId, startSequence }) =>
-      storage.changes(publicKey, storeId, startSequence).pipe(
-        Stream.mapArrayEffect((entries) => Effect.map(ChangesRpc.encodeEncrypted(entries), Arr.of))
+    changes: ({ publicKey, startSequence, storeId, writes }) =>
+      writes.filter(
+        {
+          publicKey,
+          storeId,
+          entryIds: (batch) => batch.entryIds,
+          sequence: (batch) => batch.sequence + batch.entryIds.length - 1
+        },
+        storage.changes(publicKey, storeId, startSequence)
+      ).pipe(
+        Stream.mapArrayEffect((batches) => Effect.map(ChangesRpc.encodeEncrypted(batches), Arr.of))
       )
   })
 }))
 
+// Acknowledges each entry occurrence in request order: at its position in the
+// batch this write stored, or, for a skipped batch, where it was first stored.
+const toWriteAcks = Effect.fnUntraced(function*(
+  storage: Storage["Service"],
+  publicKey: string,
+  storeId: StoreId,
+  batches: ReadonlyArray<EncryptedBatch>,
+  stored: ReadonlyArray<EncryptedRemoteBatch>
+) {
+  // Stored batches keep request order, and each batch has its own random iv.
+  let next = 0
+  const storedAt = batches.map((batch) => {
+    if (stored[next] === undefined || !Equal.equals(stored[next].iv, batch.iv)) return undefined
+    return stored[next++].sequence
+  })
+  const skipped = batches.flatMap((batch, i) => storedAt[i] === undefined ? batch.entryIds : [])
+  const firstStored = new Map<string, number>()
+  if (skipped.length > 0) {
+    for (const ack of yield* storage.entrySequences(publicKey, storeId, skipped)) {
+      firstStored.set(Uuid.stringify(ack.entryId), ack.sequence)
+    }
+  }
+  const acks: Array<WriteAck> = []
+  batches.forEach((batch, b) =>
+    batch.entryIds.forEach((entryId, i) => {
+      const sequence = storedAt[b] === undefined ? firstStored.get(Uuid.stringify(entryId)) : storedAt[b] + i
+      if (sequence !== undefined) acks.push({ entryId, sequence })
+    })
+  )
+  return acks
+})
+
 /**
  * Provides an encrypted event-log RPC server using `EventLogRemoteRpcs` and the
  * encrypted server RPC handlers.
@@ -108,7 +151,7 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() {
  * **Details**
  *
  * This layer installs `EventLogRemoteRpcs` on the provided RPC server protocol
- * and wires those RPCs to `layerRpcHandlers`. Encrypted entries, session
+ * and wires those RPCs to `layerRpcHandlers`. Encrypted batches, session
  * authentication bindings, remote ids, and change streams are delegated to
  * `Storage`.
  *
@@ -124,31 +167,6 @@ export const layer: Layer.Layer<never, never, RpcServer.Protocol | Storage> = Rp
   Layer.provide(layerRpcHandlers)
 )
 
-/**
- * Schema for encrypted entries persisted by the encrypted event-log server.
- *
- * @stability unstable
- * @category models
- * @since 4.0.0
- */
-export class PersistedEntry extends Schema.Class<PersistedEntry>(
-  "effect/eventlog/EventLogServerEncrypted/PersistedEntry"
-)({
-  entryId: EntryId,
-  iv: Transferable.Uint8Array,
-  encryptedEntry: Transferable.Uint8Array
-}) {
-  /**
-   * String representation of the encrypted entry id.
-   *
-   * @stability unstable
-   * @since 4.0.0
-   */
-  get entryIdString(): string {
-    return Uuid.stringify(this.entryId)
-  }
-}
-
 /**
  * Defines the backing store service used by the encrypted event-log server.
  *
@@ -160,15 +178,27 @@ export class PersistedEntry extends Schema.Class<PersistedEntry>(
  * **Details**
  *
  * It provides the server remote id, stores session authentication bindings,
- * persists encrypted entries, and streams encrypted changes for a public key and
+ * persists encrypted batches, and streams encrypted changes for a public key and
  * store id.
  *
+ * A batch owns one remote sequence per entry, starting at its `sequence`.
+ * `write` stores a batch unless every one of its entry ids is already stored,
+ * which makes retried writes idempotent, and returns the batches it stored.
+ * `entrySequences` returns the sequence each entry was first stored at, so
+ * skipped batches can still be acknowledged. `changes` streams every batch
+ * whose last sequence is at or after `startSequence`, followed by newly stored
+ * batches.
+ *
  * @stability unstable
  * @category services
  * @since 4.0.0
  */
 export class Storage extends Context.Service<Storage, {
   readonly getId: Effect.Effect<RemoteId>
+  /**
+   * Lowest assigned sequence (default 0). Overestimating skips entries.
+   */
+  readonly firstSequence?: number | undefined
   readonly getOrCreateSessionAuthBinding: (
     publicKey: string,
     signingPublicKey: Uint8Array<ArrayBuffer>
@@ -176,13 +206,18 @@ export class Storage extends Context.Service<Storage, {
   readonly write: (
     publicKey: string,
     storeId: StoreId,
-    entries: ReadonlyArray<PersistedEntry>
-  ) => Effect.Effect<ReadonlyArray<EncryptedRemoteEntry>>
+    batches: ReadonlyArray<EncryptedBatch>
+  ) => Effect.Effect<ReadonlyArray<EncryptedRemoteBatch>>
+  readonly entrySequences: (
+    publicKey: string,
+    storeId: StoreId,
+    entryIds: ReadonlyArray<EntryId>
+  ) => Effect.Effect<ReadonlyArray<WriteAck>>
   readonly changes: (
     publicKey: string,
     storeId: StoreId,
     startSequence: number
-  ) => Stream.Stream<EncryptedRemoteEntry>
+  ) => Stream.Stream<EncryptedRemoteBatch>
 }>()("effect/eventlog/EventLogServer/Storage") {}
 
 /**
@@ -198,28 +233,25 @@ export class Storage extends Context.Service<Storage, {
  * @since 4.0.0
  */
 export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.Scope> = Effect.gen(function*() {
-  const knownIds = new Map<string, Map<string, number>>()
-  const journals = new Map<string, Array<EncryptedRemoteEntry>>()
+  const journals = new Map<string, {
+    // Entry id to the sequence it was first stored at.
+    readonly knownIds: Map<string, number>
+    readonly batches: Array<EncryptedRemoteBatch>
+    nextSequence: number
+  }>()
   const sessionAuthBindings = new Map<string, Uint8Array<ArrayBuffer>>()
   const remoteId = makeRemoteIdUnsafe()
-  const ensureKnownIds = (scopeKey: string): Map<string, number> => {
-    let storeKnownIds = knownIds.get(scopeKey)
-    if (storeKnownIds) return storeKnownIds
-    storeKnownIds = new Map<string, number>()
-    knownIds.set(scopeKey, storeKnownIds)
-    return storeKnownIds
-  }
   const ensureJournal = (scopeKey: string) => {
     let journal = journals.get(scopeKey)
     if (journal) return journal
-    journal = []
+    journal = { knownIds: new Map(), batches: [], nextSequence: 0 }
     journals.set(scopeKey, journal)
     return journal
   }
   const pubsubs = yield* RcMap.make({
     lookup: (_scopeKey: string) =>
       Effect.acquireRelease(
-        PubSub.unbounded<EncryptedRemoteEntry>(),
+        PubSub.unbounded<EncryptedRemoteBatch>(),
         PubSub.shutdown
       ),
     idleTimeToLive: 60000
@@ -234,33 +266,42 @@ export const makeStorageMemory: Effect.Effect<Storage["Service"], never, Scope.S
         sessionAuthBindings.set(publicKey, signingPublicKey)
         return signingPublicKey
       }),
-    write: Effect.fnUntraced(function*(publicKey, storeId, entries) {
+    write: Effect.fnUntraced(function*(publicKey, storeId, batches) {
       const scopeKey = makeEncryptedScopeKey({ publicKey, storeId })
       const pubsub = yield* RcMap.get(pubsubs, scopeKey)
-      const storeKnownIds = ensureKnownIds(scopeKey)
       const journal = ensureJournal(scopeKey)
-      const encryptedEntries: Array<EncryptedRemoteEntry> = []
-      for (const entry of entries) {
-        const idString = entry.entryIdString
-        if (storeKnownIds.has(idString)) continue
-        const encrypted: EncryptedRemoteEntry = {
-          sequence: journal.length,
-          entryId: entry.entryId,
-          iv: entry.iv,
-          encryptedEntry: entry.encryptedEntry
-        }
-        encryptedEntries.push(encrypted)
-        storeKnownIds.set(idString, encrypted.sequence)
-        journal.push(encrypted)
-        PubSub.publishUnsafe(pubsub, encrypted)
+      const stored: Array<EncryptedRemoteBatch> = []
+      for (const batch of batches) {
+        const ids = batch.entryIds.map(Uuid.stringify)
+        if (ids.every((id) => journal.knownIds.has(id))) continue
+        ids.forEach((id, i) => {
+          if (!journal.knownIds.has(id)) journal.knownIds.set(id, journal.nextSequence + i)
+        })
+        const remoteBatch: EncryptedRemoteBatch = { sequence: journal.nextSequence, ...batch }
+        journal.nextSequence += ids.length
+        journal.batches.push(remoteBatch)
+        stored.push(remoteBatch)
+        PubSub.publishUnsafe(pubsub, remoteBatch)
       }
-      return encryptedEntries
+      return stored
     }, Effect.scoped),
+    entrySequences: (publicKey, storeId, entryIds) =>
+      Effect.sync(() => {
+        const knownIds = ensureJournal(makeEncryptedScopeKey({ publicKey, storeId })).knownIds
+        const acks: Array<WriteAck> = []
+        for (const entryId of entryIds) {
+          const sequence = knownIds.get(Uuid.stringify(entryId))
+          if (sequence !== undefined) acks.push({ entryId, sequence })
+        }
+        return acks
+      }),
     changes: Effect.fnUntraced(function*(publicKey, storeId, startSequence) {
       const scopeKey = makeEncryptedScopeKey({ publicKey, storeId })
       const pubsub = yield* RcMap.get(pubsubs, scopeKey)
       const subscription = yield* PubSub.subscribe(pubsub)
-      return Stream.fromArray(ensureJournal(scopeKey).slice(startSequence)).pipe(
+      const batches = ensureJournal(scopeKey).batches
+      const first = batches.findIndex((batch) => batch.sequence + batch.entryIds.length > startSequence)
+      return Stream.fromArray(first === -1 ? [] : batches.slice(first)).pipe(
         Stream.concat(Stream.fromSubscription(subscription))
       )
     }, Stream.unwrap)