@livestore/sync-cf 0.5.0-dev.0 → 0.5.0-dev.1

package/src/cf-worker/do/observability.test.ts

@@ -0,0 +1,337 @@
+import { ROOT_CONTEXT, type SpanStatus, SpanStatusCode, trace } from '@opentelemetry/api'
+
+import { BackendIdMismatchError, UnknownError } from '@livestore/common'
+import { type CfTypes, layerProtocolDurableObject, WsContext } from '@livestore/common-cf'
+import { makeNoopSpan, makeNoopTracer } from '@livestore/utils'
+import { Vitest } from '@livestore/utils-dev/node-vitest'
+import {
+  Deferred,
+  Effect,
+  FetchHttpClient,
+  Fiber,
+  Layer,
+  Option,
+  OtelTracer,
+  RpcClient,
+  RpcTest,
+  Schema,
+  Stream,
+  TestClock,
+  Tracer,
+} from '@livestore/utils/effect'
+
+import { SyncDoRpc } from '../../common/do-rpc-schema.ts'
+import { SyncWsRpc } from '../../common/ws-rpc-schema.ts'
+import { WebSocketAttachmentSchema } from '../shared.ts'
+import type * as DoCtx from './layer.ts'
+import { makeObservability, rpcSpanOptions, withRpcSpan, withRpcStreamSpan } from './observability.ts'
+import { createDoRpcHandler } from './transport/do-rpc-server.ts'
+import { makeRpcHandlers } from './transport/ws-rpc-server.ts'
+
+Vitest.describe('sync-cf telemetry ownership', () => {
+  Vitest.afterEach(() => Vitest.vi.restoreAllMocks())
+
+  Vitest.effect('runs sync effects and streams without exporting when telemetry is omitted', () =>
+    Effect.gen(function* () {
+      const fetch: typeof globalThis.fetch = Vitest.vi.fn(async () => new Response('{}'))
+      const observability = makeObservability(undefined)
+      const result = yield* Effect.succeed('ack').pipe(
+        Effect.withSpan('push'),
+        observability.effect,
+        Effect.provideService(FetchHttpClient.Fetch, fetch),
+      )
+      const history = yield* Stream.make('event').pipe(
+        Stream.withSpan('pull'),
+        observability.stream,
+        Stream.runCollect,
+        Effect.provideService(FetchHttpClient.Fetch, fetch),
+      )
+
+      yield* TestClock.adjust('4 seconds')
+      Vitest.expect(result).toBe('ack')
+      Vitest.expect(history).toEqual(['event'])
+      Vitest.expect(fetch).not.toHaveBeenCalled()
+    }),
+  )
+
+  Vitest.effect('builds and finalizes a supplied layer independently for overlapping operations', () =>
+    Effect.gen(function* () {
+      const firstStarted = yield* Deferred.make<void>()
+      const secondStarted = yield* Deferred.make<void>()
+      const firstGate = yield* Deferred.make<void>()
+      const secondGate = yield* Deferred.make<void>()
+      const firstReleased = yield* Deferred.make<void>()
+      const secondReleased = yield* Deferred.make<void>()
+      const builds: string[] = []
+      const releases: string[] = []
+      const layer = Layer.effectDiscard(
+        Effect.acquireRelease(
+          Effect.currentSpan.pipe(
+            Effect.orDie,
+            Effect.tap((span) => Effect.sync(() => builds.push(span.name))),
+          ),
+          (span) =>
+            Effect.sync(() => releases.push(span.name)).pipe(
+              Effect.andThen(Deferred.succeed(span.name === 'first' ? firstReleased : secondReleased, undefined)),
+            ),
+        ),
+      )
+      const observability = makeObservability(layer)
+      const first = yield* Deferred.succeed(firstStarted, undefined).pipe(
+        Effect.andThen(Deferred.await(firstGate)),
+        observability.effect,
+        Effect.withSpan('first'),
+        Effect.forkChild,
+      )
+      yield* Deferred.await(firstStarted)
+      const second = yield* Deferred.succeed(secondStarted, undefined).pipe(
+        Effect.andThen(Deferred.await(secondGate)),
+        observability.effect,
+        Effect.withSpan('second'),
+        Effect.forkChild,
+      )
+      yield* Deferred.await(secondStarted)
+      Vitest.expect(builds).toEqual(['first', 'second'])
+      Vitest.expect(releases).toEqual([])
+
+      yield* Deferred.succeed(firstGate, undefined)
+      yield* Fiber.join(first)
+      yield* Deferred.await(firstReleased)
+      Vitest.expect(releases).toEqual(['first'])
+      Vitest.expect(second.pollUnsafe()).toBeUndefined()
+
+      yield* Deferred.succeed(secondGate, undefined)
+      yield* Fiber.join(second)
+      yield* Deferred.await(secondReleased)
+      Vitest.expect(releases).toEqual(['first', 'second'])
+    }),
+  )
+
+  Vitest.effect('finalizes finite history while the live subscription is still open', () =>
+    Effect.gen(function* () {
+      const { layer, ended } = makeRecordingTracer()
+      const finalized = yield* Deferred.make<ReadonlyArray<string>>()
+      const live = yield* Deferred.make<void>()
+      const observability = makeObservability(
+        Layer.mergeAll(layer, Layer.effectDiscard(Effect.addFinalizer(() => Deferred.succeed(finalized, [...ended])))),
+      )
+      const fiber = yield* Stream.make('history').pipe(
+        Stream.withSpan('pull-history'),
+        observability.stream,
+        Stream.concat(Stream.fromEffect(Deferred.succeed(live, undefined)).pipe(Stream.concat(Stream.never))),
+        Stream.runDrain,
+        Effect.forkChild,
+      )
+      yield* Effect.addFinalizer(() => Fiber.interrupt(fiber))
+      yield* Deferred.await(live)
+      Vitest.expect(yield* Deferred.await(finalized)).toContain('pull-history')
+      Vitest.expect(fiber.pollUnsafe()).toBeUndefined()
+    }),
+  )
+
+  Vitest.effect('ends WebSocket telemetry before the production handler enters its live tail', () =>
+    Effect.gen(function* () {
+      const { layer, ended } = makeRecordingTracer()
+      const finalized = yield* Deferred.make<void>()
+      const observability = makeObservability(
+        Layer.mergeAll(layer, Layer.effectDiscard(Effect.addFinalizer(() => Deferred.succeed(finalized, undefined)))),
+      )
+      const input = makeDoInput()
+      const handlers = makeRpcHandlers({ ...input, observability }).pipe(
+        Layer.provide(Layer.succeed(WsContext, WsContext.of({ ws: makeWebSocket() }))),
+      )
+      const client = yield* RpcTest.makeClient(SyncWsRpc).pipe(Effect.provide(handlers))
+      const pullFiber = yield* client['SyncWsRpc.Pull']({
+        storeId: 'store',
+        cursor: Option.none(),
+        live: true,
+      }).pipe(Stream.runDrain, Effect.forkChild)
+      yield* Effect.addFinalizer(() => Fiber.interrupt(pullFiber))
+
+      yield* Deferred.await(finalized).pipe(Effect.timeout('1 second'))
+      Vitest.expect(ended).toContain('RpcServer.SyncWsRpc.Pull')
+      Vitest.expect(pullFiber.pollUnsafe()).toBeUndefined()
+    }),
+  )
+
+  Vitest.effect('installs telemetry inside the Durable Object RPC stream runtime', () =>
+    Effect.gen(function* () {
+      const { layer, ended } = makeRecordingTracer()
+      const observability = makeObservability(layer)
+      const input = makeDoInput()
+      const protocol = layerProtocolDurableObject({
+        callerContext: { bindingName: 'TEST', durableObjectId: 'client' },
+        callRpc: (payload) =>
+          createDoRpcHandler({
+            // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- Effect RPC owns this ArrayBuffer-backed payload.
+            payload: payload as Uint8Array<ArrayBuffer>,
+            input,
+            observability,
+          }).pipe(Effect.runPromise),
+      })
+      const responses = yield* Effect.gen(function* () {
+        const client = yield* RpcClient.make(SyncDoRpc)
+        return yield* client['SyncDoRpc.Pull']({ storeId: 'store', cursor: Option.none() }).pipe(Stream.runCollect)
+      }).pipe(Effect.provide(protocol))
+
+      Vitest.expect(responses).toHaveLength(1)
+      Vitest.expect(ended).toContain('RpcServer.SyncDoRpc.Pull')
+    }),
+  )
+
+  Vitest.effect('attaches a finite span to a sampled caller across an unexported RPC envelope', () =>
+    Effect.gen(function* () {
+      const { layer, spans, ended } = makeRecordingTracer()
+      const observability = makeObservability(layer)
+      const caller = Tracer.externalSpan({ traceId: '1'.repeat(32), spanId: '2222222222222222', sampled: true })
+      yield* Effect.flatMap(rpcSpanOptions, (options) =>
+        withRpcSpan(Effect.void.pipe(Effect.withSpan('push')), 'finite-rpc', options),
+      ).pipe(observability.effect, Effect.withSpan('unexported-rpc-envelope'), Effect.withParentSpan(caller))
+
+      Vitest.expect(spans.get('finite-rpc')?.parentId).toBe(caller.spanId)
+      Vitest.expect(spans.get('push')?.parentId).toBe(spans.get('finite-rpc')?.spanId)
+      Vitest.expect(ended).toEqual(['push', 'finite-rpc'])
+    }),
+  )
+
+  Vitest.effect('starts a backend root when the RPC caller is unsampled', () =>
+    Effect.gen(function* () {
+      const { layer, spans, ended } = makeRecordingTracer()
+      const observability = makeObservability(layer)
+      const caller = Tracer.externalSpan({ traceId: '1'.repeat(32), spanId: '2222222222222222', sampled: false })
+      yield* Effect.flatMap(rpcSpanOptions, (options) => withRpcSpan(Effect.void, 'finite-rpc', options)).pipe(
+        observability.effect,
+        Effect.withSpan('unexported-rpc-envelope'),
+        Effect.withParentSpan(caller),
+      )
+
+      Vitest.expect(spans.get('finite-rpc')?.parentId).toBeUndefined()
+      Vitest.expect(ended).toEqual(['finite-rpc'])
+    }),
+  )
+
+  Vitest.effect('keeps anticipated RPC recovery failures out of boundary error telemetry', () =>
+    Effect.gen(function* () {
+      const { layer, statuses } = makeRecordingTracer()
+      const observability = makeObservability(layer)
+      const expected = new BackendIdMismatchError({ expected: 'backend-a', received: 'backend-b' })
+      const unexpected = new UnknownError({ cause: new Error('boom') })
+
+      const effectError = yield* withRpcSpan(Effect.fail(expected), 'expected-effect').pipe(
+        observability.effect,
+        Effect.flip,
+      )
+      const streamError = yield* Stream.fail(expected).pipe(
+        (_) => withRpcStreamSpan(_, 'expected-stream'),
+        observability.stream,
+        Stream.runDrain,
+        Effect.flip,
+      )
+      const unexpectedError = yield* withRpcSpan(Effect.fail(unexpected), 'unexpected-effect').pipe(
+        observability.effect,
+        Effect.flip,
+      )
+
+      Vitest.expect(effectError).toBe(expected)
+      Vitest.expect(streamError).toBe(expected)
+      Vitest.expect(unexpectedError).toBe(unexpected)
+      Vitest.expect(statuses.get('expected-effect')).toBe(SpanStatusCode.OK)
+      Vitest.expect(statuses.get('expected-stream')).toBe(SpanStatusCode.OK)
+      Vitest.expect(statuses.get('unexpected-effect')).toBe(SpanStatusCode.ERROR)
+    }),
+  )
+
+  Vitest.effect('forwards the operation exit to supplied layer finalizers', () =>
+    Effect.gen(function* () {
+      const released = yield* Deferred.make<string>()
+      const expected = new BackendIdMismatchError({ expected: 'backend-a', received: 'backend-b' })
+      const observability = makeObservability(
+        Layer.effectDiscard(
+          Effect.acquireRelease(Effect.void, (_, exit) => Deferred.succeed(released, exit._tag).pipe(Effect.asVoid)),
+        ),
+      )
+
+      const error = yield* Effect.fail(expected).pipe(observability.effect, Effect.flip)
+
+      Vitest.expect(error).toBe(expected)
+      Vitest.expect(yield* Deferred.await(released)).toBe('Failure')
+    }),
+  )
+
+  Vitest.live('acknowledges before the owned OTLP collector responds', () =>
+    Effect.gen(function* () {
+      const started = Promise.withResolvers<void>()
+      const gate = Promise.withResolvers<void>()
+      const acknowledged = yield* Deferred.make<void>()
+      const fetch: typeof globalThis.fetch = Vitest.vi.fn(() => {
+        started.resolve()
+        return gate.promise.then(() => new Response('{}', { headers: { 'content-type': 'application/json' } }))
+      })
+      yield* Effect.addFinalizer(() => Effect.sync(() => gate.resolve()))
+      const observability = makeObservability({ baseUrl: 'http://collector.invalid' })
+      yield* Effect.void.pipe(
+        Effect.withSpan('push'),
+        observability.effect,
+        Effect.andThen(Deferred.succeed(acknowledged, undefined)),
+        Effect.provideService(FetchHttpClient.Fetch, fetch),
+        Effect.forkChild,
+      )
+      yield* Effect.promise(() => started.promise)
+      yield* Effect.yieldNow
+      Vitest.expect(yield* Deferred.isDone(acknowledged)).toBe(true)
+    }),
+  )
+})
+
+/** Record the public OTel boundary; span IDs are compared relationally, not fixed to creation order. */
+const makeRecordingTracer = () => {
+  const tracer = makeNoopTracer()
+  const ended: string[] = []
+  const statuses = new Map<string, SpanStatusCode>()
+  const spans = new Map<string, { spanId: string; parentId: string | undefined }>()
+  let nextId = 0
+  Vitest.vi.spyOn(tracer, 'startSpan').mockImplementation((name, _options, context) => {
+    const spanId = (++nextId).toString(16).padStart(16, '0')
+    spans.set(name, { spanId, parentId: trace.getSpanContext(context ?? ROOT_CONTEXT)?.spanId })
+    const noopSpan = makeNoopSpan()
+    return {
+      ...noopSpan,
+      spanContext: () => ({ traceId: '1'.repeat(32), spanId, traceFlags: 1 }),
+      setStatus: (status: SpanStatus) => {
+        statuses.set(name, status.code)
+        return noopSpan
+      },
+      end: () => {
+        ended.push(name)
+      },
+    }
+  })
+  const layer = OtelTracer.layerWithoutOtelTracer.pipe(Layer.provide(Layer.succeed(OtelTracer.OtelTracer, tracer)))
+  return { layer, ended, spans, statuses }
+}
+
+/** Minimal empty SQLite-backed Durable Object boundary for transport composition tests. */
+const makeDoInput = (): Omit<DoCtx.DoCtxInput, 'from'> => {
+  const emptyCursor = Object.assign([], { toArray: () => [] })
+  const doSelf = {
+    env: {},
+    ctx: {
+      storage: {
+        sql: { exec: () => emptyCursor },
+        kv: { get: () => undefined, put: () => undefined, delete: () => undefined },
+        deleteAll: async () => undefined,
+      },
+    },
+  }
+
+  // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- The test intentionally stubs the narrow Worker APIs reached by an empty pull.
+  return { doSelf: doSelf as unknown as DoCtx.DoCtxInput['doSelf'], doOptions: undefined }
+}
+
+const makeWebSocket = (): CfTypes.WebSocket => {
+  const attachment = Schema.encodeSync(WebSocketAttachmentSchema)({ storeId: 'store', pullRequestIds: [] })
+  const ws = { deserializeAttachment: () => attachment, close: Vitest.vi.fn() }
+
+  // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- The handler only reads the attachment and closes invalid sockets.
+  return ws as unknown as CfTypes.WebSocket
+}

package/src/cf-worker/do/observability.ts

@@ -0,0 +1,121 @@
+import type { BackendIdMismatchError, ServerAheadError, UnknownError } from '@livestore/common'
+import {
+  Effect,
+  FetchHttpClient,
+  Layer,
+  Option,
+  Otlp,
+  Ref,
+  Result,
+  Scope,
+  Stream,
+  type Tracer,
+} from '@livestore/utils/effect'
+
+import type { SyncBackendOtelOptions } from '../shared.ts'
+
+/** Keep exporter ownership separate from sync scopes: their finalizers must not delay acknowledgments. */
+export const makeObservability = (options: SyncBackendOtelOptions | undefined) => {
+  const layer = makeLayer(options)
+
+  const backgroundLayer = Layer.effectContext(
+    Effect.acquireRelease(Scope.make(), (scope, exit) => Scope.close(scope, exit).pipe(runInBackground)).pipe(
+      Effect.flatMap((scope) => Layer.buildWithScope(layer, scope)),
+    ),
+  )
+
+  return {
+    effect: <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+      options === undefined ? effect : effect.pipe(Effect.provide(backgroundLayer, { local: true })),
+    stream: <A, E, R>(stream: Stream.Stream<A, E, R>) =>
+      options === undefined ? stream : stream.pipe(Stream.provide(backgroundLayer, { local: true })),
+  }
+}
+
+/**
+ * Effect RPC's envelope lives until a live subscription closes and may never be exported.
+ * Give finite sync work its own exported boundary, attached to a sampled caller or a new backend root.
+ */
+export const rpcSpanOptions = Effect.currentSpan.pipe(
+  Effect.map((span) => makeRpcSpanOptions(span.parent)),
+  Effect.catchTag('NoSuchElementError', () => Effect.succeed({ root: true })),
+)
+
+export const makeRpcSpanOptions = (parent: Option.Option<Tracer.AnySpan>): Tracer.SpanOptions => ({
+  // Client runtimes use an unsampled no-op span by default. Parenting to it would suppress backend-only telemetry.
+  parent: parent.pipe(
+    Option.filter((span) => span.sampled),
+    Option.getOrUndefined,
+  ),
+  root: true,
+})
+
+/** Keep expected sync-recovery results out of error telemetry while preserving the typed failure for RPC callers. */
+export const withRpcSpan = <A, E extends SyncRpcError, R>(
+  effect: Effect.Effect<A, E, R>,
+  name: string,
+  options?: Tracer.SpanOptions,
+): Effect.Effect<A, E, R> =>
+  effect.pipe(
+    Effect.map(Result.succeed),
+    Effect.catchIf(isExpectedSyncError, (error) => Effect.succeed(Result.fail(error))),
+    Effect.withSpan(name, options),
+    Effect.flatMap(resultToEffect),
+  )
+
+/** Stream equivalent of `withRpcSpan`; expected failures are restored after the boundary span ends successfully. */
+export const withRpcStreamSpan = <A, E extends SyncRpcError, R>(
+  stream: Stream.Stream<A, E, R>,
+  name: string,
+  options?: Tracer.SpanOptions,
+): Stream.Stream<A, E, R> =>
+  Stream.unwrap(
+    Effect.gen(function* () {
+      const expectedError = yield* Ref.make(Option.none<E>())
+      const instrumented = stream.pipe(
+        Stream.catchIf(isExpectedSyncError, (error) =>
+          Stream.fromEffect(Ref.set(expectedError, Option.some(error))).pipe(Stream.drain),
+        ),
+        Stream.withSpan(name, options),
+      )
+      const restoreExpectedError = Stream.fromEffect(Ref.get(expectedError)).pipe(
+        Stream.flatMap(
+          Option.match({
+            onNone: () => Stream.empty,
+            onSome: Stream.fail,
+          }),
+        ),
+      )
+
+      return instrumented.pipe(Stream.concat(restoreExpectedError))
+    }),
+  )
+
+type SyncRpcError = UnknownError | ServerAheadError | BackendIdMismatchError
+
+const makeLayer = (options: SyncBackendOtelOptions | undefined): Layer.Layer<never> => {
+  if (isTracerLayer(options) === true) return options
+  if (options?.baseUrl !== undefined) {
+    return Otlp.layerJson({
+      baseUrl: options.baseUrl,
+      tracerExportInterval: 50,
+      shutdownTimeout: 3000,
+      resource: { serviceName: options.serviceName ?? 'sync-cf-do' },
+    }).pipe(Layer.provide(FetchHttpClient.layer))
+  }
+  return Layer.empty
+}
+
+/** DOs stay active for pending work; no shutdown hook or connection-wide timer is required. */
+const runInBackground = (effect: Effect.Effect<void>) =>
+  effect.pipe(Effect.ignoreCause, Effect.withTracerEnabled(false), Effect.forkDetach, Effect.asVoid)
+
+/** Preserve the known Layer requirements when narrowing the configuration union. */
+const isTracerLayer = (options: SyncBackendOtelOptions | undefined): options is Layer.Layer<never> =>
+  Layer.isLayer(options)
+
+const isExpectedSyncError = (error: SyncRpcError): boolean =>
+  error._tag === 'ServerAheadError' || error._tag === 'BackendIdMismatchError'
+
+const resultToEffect = <A, E>(result: Result.Result<A, E>): Effect.Effect<A, E> =>
+  Result.isSuccess(result) === true ? Effect.succeed(result.success) : Effect.fail(result.failure)

package/src/cf-worker/do/push.ts

@@ -173,13 +173,14 @@ export const makePush =
           )
           if (rpcSubscriptions.length > 0) {
             for (const [, subscription] of rpcSubscriptions) {
-              for (const { encoded } of responses) {
+              for (const { response } of responses) {
                 yield* emitStreamResponse({
                   callerContext: subscription.callerContext,
                   env,
                   requestId: subscription.requestId,
                   storeId: subscription.storeId,
-                  values: [encoded],
+                  schema: SyncMessage.PullResponse,
+                  values: [response],
                 }).pipe(Effect.tapCauseLogPretty, Effect.exit)
               }
             }

package/src/cf-worker/do/transport/do-rpc-server.test.ts

@@ -0,0 +1,32 @@
+import { Vitest } from '@livestore/utils-dev/node-vitest'
+
+import { type RpcSubscription, rpcSubscriptionKeyPrefix } from '../../shared.ts'
+import { dropRpcSubscription } from './do-rpc-server.ts'
+
+const durableObjectId = 'client-do'
+const key = `${rpcSubscriptionKeyPrefix}${durableObjectId}`
+
+const subscribedWith = (requestId: string) => {
+  const rows = new Map<string, RpcSubscription>([
+    [
+      key,
+      { storeId: 'store', subscribedAt: 0, requestId, callerContext: { bindingName: 'CLIENT_DO', durableObjectId } },
+    ],
+  ])
+  return { rows, kv: { get: (k: string) => rows.get(k), delete: (k: string) => rows.delete(k) } }
+}
+
+Vitest.describe('sync-cf DO-RPC unsubscribe', () => {
+  // A replacement pull overwrote the row with its own request id; the earlier pull's late unsubscribe must be a no-op.
+  Vitest.test('keeps the row when the request id was superseded', () => {
+    const { rows, kv } = subscribedWith('pull-2')
+    dropRpcSubscription(kv, { durableObjectId, requestId: 'pull-1' })
+    Vitest.expect(rows.has(key)).toBe(true)
+  })
+
+  Vitest.test('drops the row for the pull that registered it', () => {
+    const { rows, kv } = subscribedWith('pull-1')
+    dropRpcSubscription(kv, { durableObjectId, requestId: 'pull-1' })
+    Vitest.expect(rows.has(key)).toBe(false)
+  })
+})

package/src/cf-worker/do/transport/do-rpc-server.ts

@@ -5,23 +5,42 @@ import { Effect, Headers, Option, Stream } from '@livestore/utils/effect'
 import { SyncDoRpc } from '../../../common/do-rpc-schema.ts'
 import { rpcSubscriptionKeyPrefix, type RpcSubscription } from '../../shared.ts'
 import * as DoCtx from '../layer.ts'
+import { makeRpcSpanOptions, type makeObservability, withRpcStreamSpan } from '../observability.ts'
 import { makeEndingPullStream } from '../pull.ts'
 import { makePush } from '../push.ts'
 
 export interface DoRpcHandlerOptions {
   payload: Uint8Array<ArrayBuffer>
   input: Omit<DoCtx.DoCtxInput, 'from'>
+  observability: ReturnType<typeof makeObservability>
+}
+
+/**
+ * Drops a live-pull subscription row only if it still belongs to the pull that asked. A replacement store booted on
+ * the same client DO while a slow shutdown is still draining overwrites the row with its own request id, and the
+ * earlier pull's late unsubscribe must not take that subscription down.
+ */
+export const dropRpcSubscription = (
+  kv: { get: (key: string) => RpcSubscription | undefined; delete: (key: string) => unknown },
+  { durableObjectId, requestId }: { durableObjectId: string; requestId: string },
+): void => {
+  const key = `${rpcSubscriptionKeyPrefix}${durableObjectId}`
+  if (kv.get(key)?.requestId === requestId) {
+    kv.delete(key)
+  }
 }
 
 export const createDoRpcHandler = (
   options: DoRpcHandlerOptions,
 ): Effect.Effect<Uint8Array<ArrayBuffer> | CfTypes.ReadableStream> =>
   Effect.gen({ self: this }, function* () {
-    const { payload, input } = options
+    const { payload, input, observability } = options
+    const parent = yield* Effect.currentParentSpan.pipe(Effect.option)
 
     // TODO add admin RPCs
     const RpcLive = SyncDoRpc.toLayer({
       'SyncDoRpc.Ping': () => Effect.void,
+      'SyncDoRpc.Unsubscribe': (req) => Effect.sync(() => dropRpcSubscription(input.doSelf.ctx.storage.kv, req)),
       'SyncDoRpc.Pull': (req, { headers }) =>
         Effect.gen({ self: this }, function* () {
           const { ctx } = yield* DoCtx.DoCtx
@@ -55,6 +74,9 @@ export const createDoRpcHandler = (
               : new UnknownError({ cause }),
           ),
           Stream.tapCause(Effect.log),
+          // The bridge runs this stream in a separate runtime. Install telemetry there too.
+          (_) => withRpcStreamSpan(_, 'RpcServer.SyncDoRpc.Pull', makeRpcSpanOptions(parent)),
+          observability.stream,
         ),
       'SyncDoRpc.Push': (req) =>
         Effect.gen({ self: this }, function* () {

package/src/cf-worker/do/transport/ws-rpc-server.ts

@@ -5,47 +5,71 @@ import { Effect, identity, Layer, Result, RpcServer, Schema, Stream } from '@liv
 import { SyncWsRpc } from '../../../common/ws-rpc-schema.ts'
 import { headersRecordToMap, WebSocketAttachmentSchema } from '../../shared.ts'
 import * as DoCtx from '../layer.ts'
+import { type makeObservability, rpcSpanOptions, withRpcSpan, withRpcStreamSpan } from '../observability.ts'
 import { makeEndingPullStream } from '../pull.ts'
 import { makePush } from '../push.ts'
 
-export const makeRpcServer = ({ doSelf, doOptions }: Omit<DoCtx.DoCtxInput, 'from'>) => {
-  const handlersLayer = SyncWsRpc.toLayer({
+export const makeRpcServer = ({
+  doSelf,
+  doOptions,
+  observability,
+}: Omit<DoCtx.DoCtxInput, 'from'> & { observability: ReturnType<typeof makeObservability> }) => {
+  const handlersLayer = makeRpcHandlers({ doSelf, doOptions, observability })
+
+  return RpcServer.layer(SyncWsRpc).pipe(Layer.provide(handlersLayer))
+}
+
+/** Production handler layer, exported so transport tests can exercise the exact RPC composition in memory. */
+export const makeRpcHandlers = ({
+  doSelf,
+  doOptions,
+  observability,
+}: Omit<DoCtx.DoCtxInput, 'from'> & { observability: ReturnType<typeof makeObservability> }) =>
+  SyncWsRpc.toLayer({
     'SyncWsRpc.Pull': (req) =>
       Effect.gen(function* () {
+        const spanOptions = yield* rpcSpanOptions
         const headers = yield* getForwardedHeaders
         return makeEndingPullStream({ req, payload: req.payload, headers }).pipe(
-          // Needed to keep the stream alive on the client side for phase 2 (i.e. not send the `Exit` stream RPC message)
-          req.live === true ? Stream.concat(Stream.never) : identity,
           Stream.provide(DoCtx.layer({ doSelf, doOptions, from: { storeId: req.storeId } })),
           Stream.mapError((cause) =>
             cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
               ? cause
               : new UnknownError({ cause }),
           ),
+          (_) => withRpcStreamSpan(_, 'RpcServer.SyncWsRpc.Pull', spanOptions),
         )
-      }).pipe(Stream.unwrap),
+      }).pipe(
+        Stream.unwrap,
+        // Drain finite history before entering the live phase, which survives DO hibernation.
+        observability.stream,
+        // Keep the client subscription open without retaining an exporter scope or timer.
+        req.live === true ? Stream.concat(Stream.never) : identity,
+      ),
     'SyncWsRpc.Push': (req) =>
-      Effect.gen(function* () {
-        const { doOptions, storeId, ctx, env } = yield* DoCtx.DoCtx
-        const headers = yield* getForwardedHeaders
+      Effect.flatMap(rpcSpanOptions, (spanOptions) =>
+        Effect.gen(function* () {
+          const { doOptions, storeId, ctx, env } = yield* DoCtx.DoCtx
+          const headers = yield* getForwardedHeaders
 
-        const push = makePush({ options: doOptions, storeId, payload: req.payload, headers, ctx, env })
+          const push = makePush({ options: doOptions, storeId, payload: req.payload, headers, ctx, env })
 
-        return yield* push(req)
-      }).pipe(
-        Effect.provide(DoCtx.layer({ doSelf, doOptions, from: { storeId: req.storeId } })),
-        Effect.mapError((cause) =>
-          cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
-            ? cause
-            : new UnknownError({ cause }),
+          return yield* push(req)
+        }).pipe(
+          Effect.provide(DoCtx.layer({ doSelf, doOptions, from: { storeId: req.storeId } })),
+          Effect.mapError((cause) =>
+            cause._tag === 'UnknownError' ||
+            cause._tag === 'ServerAheadError' ||
+            cause._tag === 'BackendIdMismatchError'
+              ? cause
+              : new UnknownError({ cause }),
+          ),
+          Effect.tapCauseLogPretty,
+          (_) => withRpcSpan(_, 'RpcServer.SyncWsRpc.Push', spanOptions),
         ),
-        Effect.tapCauseLogPretty,
-      ),
+      ).pipe(observability.effect),
   })
 
-  return RpcServer.layer(SyncWsRpc).pipe(Layer.provide(handlersLayer))
-}
-
 /** Extracts forwarded headers from the WebSocket attachment */
 const getForwardedHeaders = Effect.gen(function* () {
   const { ws } = yield* WsContext