@livestore/utils 0.3.0-dev.11 → 0.3.0-dev.13

package/src/effect/WebChannel/broadcastChannelWithAck.ts

@@ -1,8 +1,9 @@
-import type { Scope } from 'effect'
-import { Deferred, Effect, Predicate, Queue, Schema, Stream } from 'effect'
+import { Deferred, Exit, Predicate, Queue, Schema, Scope, Stream } from 'effect'
 
-import type { WebChannel } from './common.js'
+import * as Effect from '../Effect.js'
+import type { InputSchema, WebChannel } from './common.js'
 import { WebChannelSymbol } from './common.js'
+import { listenToDebugPing, mapSchema } from './WebChannel.js'
 
 const ConnectMessage = Schema.TaggedStruct('ConnectMessage', {
   from: Schema.String,
@@ -31,96 +32,99 @@ const Message = Schema.Union(ConnectMessage, ConnectAckMessage, DisconnectMessag
  */
 export const broadcastChannelWithAck = <MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded>({
   channelName,
-  listenSchema,
-  sendSchema,
+  schema: inputSchema,
 }: {
   channelName: string
-  listenSchema: Schema.Schema<MsgListen, MsgListenEncoded>
-  sendSchema: Schema.Schema<MsgSend, MsgSendEncoded>
+  schema: InputSchema<MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded>
 }): Effect.Effect<WebChannel<MsgListen, MsgSend>, never, Scope.Scope> =>
-  Effect.gen(function* () {
-    const channel = new BroadcastChannel(channelName)
-    const messageQueue = yield* Queue.unbounded<MsgSend>()
-    const connectionId = crypto.randomUUID()
+  Effect.scopeWithCloseable((scope) =>
+    Effect.gen(function* () {
+      const channel = new BroadcastChannel(channelName)
+      const messageQueue = yield* Queue.unbounded<MsgSend>()
+      const connectionId = crypto.randomUUID()
+      const schema = mapSchema(inputSchema)
 
-    const peerIdRef = { current: undefined as undefined | string }
-    const connectedLatch = yield* Effect.makeLatch(false)
-    const supportsTransferables = false
+      const peerIdRef = { current: undefined as undefined | string }
+      const connectedLatch = yield* Effect.makeLatch(false)
+      const supportsTransferables = false
 
-    const postMessage = (msg: typeof Message.Type) => channel.postMessage(Schema.encodeSync(Message)(msg))
+      const postMessage = (msg: typeof Message.Type) => channel.postMessage(Schema.encodeSync(Message)(msg))
 
-    const send = (message: MsgSend) =>
-      Effect.gen(function* () {
-        yield* connectedLatch.await
-
-        const payload = yield* Schema.encode(sendSchema)(message)
-        postMessage(PayloadMessage.make({ from: connectionId, to: peerIdRef.current!, payload }))
-      })
-
-    const listen = Stream.fromEventListener<MessageEvent>(channel, 'message').pipe(
-      Stream.map(({ data }) => data),
-      Stream.map(Schema.decodeOption(Message)),
-      Stream.filterMap((_) => _),
-      Stream.mapEffect((data) =>
+      const send = (message: MsgSend) =>
         Effect.gen(function* () {
-          switch (data._tag) {
-            // Case: other side sends connect message (because otherside wasn't yet online when this side send their connect message)
-            case 'ConnectMessage': {
-              peerIdRef.current = data.from
-              postMessage(ConnectAckMessage.make({ from: connectionId, to: data.from }))
-              yield* connectedLatch.open
-              break
-            }
-            // Case: other side sends connect-ack message (because otherside was already online when this side connected)
-            case 'ConnectAckMessage': {
-              if (data.to === connectionId) {
+          yield* connectedLatch.await
+
+          const payload = yield* Schema.encode(schema.send)(message)
+          postMessage(PayloadMessage.make({ from: connectionId, to: peerIdRef.current!, payload }))
+        })
+
+      const listen = Stream.fromEventListener<MessageEvent>(channel, 'message').pipe(
+        Stream.map(({ data }) => data),
+        Stream.map(Schema.decodeOption(Message)),
+        Stream.filterMap((_) => _),
+        Stream.mapEffect((data) =>
+          Effect.gen(function* () {
+            switch (data._tag) {
+              // Case: other side sends connect message (because otherside wasn't yet online when this side send their connect message)
+              case 'ConnectMessage': {
                 peerIdRef.current = data.from
+                postMessage(ConnectAckMessage.make({ from: connectionId, to: data.from }))
                 yield* connectedLatch.open
+                break
               }
-              break
-            }
-            case 'DisconnectMessage': {
-              if (data.from === peerIdRef.current) {
-                peerIdRef.current = undefined
-                yield* connectedLatch.close
-                yield* establishConnection
+              // Case: other side sends connect-ack message (because otherside was already online when this side connected)
+              case 'ConnectAckMessage': {
+                if (data.to === connectionId) {
+                  peerIdRef.current = data.from
+                  yield* connectedLatch.open
+                }
+                break
               }
-              break
-            }
-            case 'PayloadMessage': {
-              if (data.to === connectionId) {
-                return Schema.decodeEither(listenSchema)(data.payload)
+              case 'DisconnectMessage': {
+                if (data.from === peerIdRef.current) {
+                  peerIdRef.current = undefined
+                  yield* connectedLatch.close
+                  yield* establishConnection
+                }
+                break
+              }
+              case 'PayloadMessage': {
+                if (data.to === connectionId) {
+                  return Schema.decodeEither(schema.listen)(data.payload)
+                }
+                break
               }
-              break
             }
-          }
+          }),
+        ),
+        Stream.filter(Predicate.isNotUndefined),
+        listenToDebugPing(channelName),
+      )
+
+      const establishConnection = Effect.gen(function* () {
+        postMessage(ConnectMessage.make({ from: connectionId }))
+      })
+
+      yield* establishConnection
+
+      yield* Effect.addFinalizer(() =>
+        Effect.gen(function* () {
+          postMessage(DisconnectMessage.make({ from: connectionId }))
+          channel.close()
+          yield* Queue.shutdown(messageQueue)
         }),
-      ),
-      Stream.filter(Predicate.isNotUndefined),
-    )
-
-    const establishConnection = Effect.gen(function* () {
-      postMessage(ConnectMessage.make({ from: connectionId }))
-    })
-
-    yield* establishConnection
-
-    yield* Effect.addFinalizer(() =>
-      Effect.gen(function* () {
-        postMessage(DisconnectMessage.make({ from: connectionId }))
-        channel.close()
-        yield* Queue.shutdown(messageQueue)
-      }),
-    )
-
-    const closedDeferred = yield* Deferred.make<void>()
-
-    return {
-      [WebChannelSymbol]: WebChannelSymbol,
-      send,
-      listen,
-      closedDeferred,
-      schema: { listen: listenSchema, send: sendSchema },
-      supportsTransferables,
-    }
-  }).pipe(Effect.withSpan(`WebChannel:broadcastChannelWithAck(${channelName})`))
+      )
+
+      const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+
+      return {
+        [WebChannelSymbol]: WebChannelSymbol,
+        send,
+        listen,
+        closedDeferred,
+        shutdown: Scope.close(scope, Exit.void),
+        schema,
+        supportsTransferables,
+      }
+    }).pipe(Effect.withSpan(`WebChannel:broadcastChannelWithAck(${channelName})`)),
+  )

package/src/effect/WebChannel/common.ts

@@ -1,5 +1,5 @@
-import type { Deferred, Effect, Either, ParseResult, Schema, Stream } from 'effect'
-import { Predicate } from 'effect'
+import type { Deferred, Effect, Either, ParseResult } from 'effect'
+import { Predicate, Schema, Stream } from 'effect'
 
 export const WebChannelSymbol = Symbol('WebChannel')
 export type WebChannelSymbol = typeof WebChannelSymbol
@@ -13,5 +13,59 @@ export interface WebChannel<MsgListen, MsgSend, E = never> {
   listen: Stream.Stream<Either.Either<MsgListen, ParseResult.ParseError>, E>
   supportsTransferables: boolean
   closedDeferred: Deferred.Deferred<void>
+  shutdown: Effect.Effect<void>
   schema: { listen: Schema.Schema<MsgListen, any>; send: Schema.Schema<MsgSend, any> }
+  debugInfo?: Record<string, any>
+}
+
+export const DebugPingMessage = Schema.TaggedStruct('WebChannel.DebugPing', {
+  message: Schema.String,
+  payload: Schema.optional(Schema.String),
+})
+
+export const schemaWithDebugPing = <MsgListen, MsgSend>(
+  schema: OutputSchema<MsgListen, MsgSend, any, any>,
+): OutputSchema<MsgListen | typeof DebugPingMessage.Type, MsgSend | typeof DebugPingMessage.Type, any, any> => ({
+  send: Schema.Union(schema.send, DebugPingMessage),
+  listen: Schema.Union(schema.listen, DebugPingMessage),
+})
+
+export type InputSchema<MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded> =
+  | Schema.Schema<MsgListen | MsgSend, MsgListenEncoded | MsgSendEncoded>
+  | OutputSchema<MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded>
+
+export type OutputSchema<MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded> = {
+  listen: Schema.Schema<MsgListen, MsgListenEncoded>
+  send: Schema.Schema<MsgSend, MsgSendEncoded>
+}
+
+export const mapSchema = <MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded>(
+  schema: InputSchema<MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded>,
+): OutputSchema<MsgListen, MsgSend, MsgListenEncoded, MsgSendEncoded> =>
+  Predicate.hasProperty(schema, 'send') && Predicate.hasProperty(schema, 'listen')
+    ? schemaWithDebugPing(schema)
+    : (schemaWithDebugPing({ send: schema, listen: schema }) as any)
+
+export const listenToDebugPing = (channelName: string) => {
+  const threadName = (() => {
+    if (typeof globalThis !== 'undefined' && Predicate.hasProperty(globalThis, 'name') && self.name !== '') {
+      return self.name
+    } else if (typeof globalThis !== 'undefined') {
+      return 'window'
+    }
+    return 'unknown thread'
+  })()
+
+  return <MsgListen>(
+    stream: Stream.Stream<Either.Either<MsgListen, ParseResult.ParseError>, never>,
+  ): Stream.Stream<Either.Either<MsgListen, ParseResult.ParseError>, never> =>
+    stream.pipe(
+      Stream.filter((msg) => {
+        if (msg._tag === 'Right' && Schema.is(DebugPingMessage)(msg.right)) {
+          console.log(`[${threadName}] WebChannel:ping [${channelName}]`, msg.right.message, msg.right.payload)
+          return false
+        }
+        return true
+      }),
+    )
 }

package/src/effect/index.ts

@@ -56,6 +56,9 @@ export {
   Match,
   TestServices,
   Mailbox,
+  ExecutionStrategy,
+  PrimaryKey,
+  Types,
 } from 'effect'
 
 export { dual } from 'effect/Function'
@@ -69,7 +72,7 @@ export * as Subscribable from './Subscribable.js'
 
 export * as Logger from './Logger.js'
 
-export * as WebChannel from './WebChannel.js'
+export * as WebChannel from './WebChannel/WebChannel.js'
 export * as WebSocket from './WebSocket.js'
 
 export * as SchemaAST from 'effect/SchemaAST'

package/src/env.ts

@@ -29,3 +29,7 @@ export const isDevEnv = () => {
 export const TRACE_VERBOSE = env('LS_TRACE_VERBOSE') !== undefined || env('VITE_LS_TRACE_VERBOSE') !== undefined
 
 export const LS_DEV = env('LS_DEV') !== undefined || env('VITE_LS_DEV') !== undefined
+
+const envTruish = (env: string | undefined) => env !== undefined && env !== 'false' && env !== '0'
+
+export const IS_CI = envTruish(env('CI'))

package/src/index.ts

@@ -14,12 +14,12 @@ export * as base64 from './base64.js'
 export { default as prettyBytes } from 'pretty-bytes'
 
 import type * as otel from '@opentelemetry/api'
+import type { Types } from 'effect'
 
 import { isDevEnv } from './env.js'
 import { objectToString } from './misc.js'
 
 export type Prettify<T> = T extends infer U ? { [K in keyof U]: Prettify<U[K]> } : never
-export type PrettifyFlat<T> = T extends infer U ? { [K in keyof U]: U[K] } : never
 
 export type TypeEq<A, B> = (<T>() => T extends A ? 1 : 2) extends <T>() => T extends B ? 1 : 2 ? true : false
 
@@ -30,6 +30,8 @@ export type AssertTrue<T extends true> = T
 export type Writeable<T> = { -readonly [P in keyof T]: T[P] }
 export type DeepWriteable<T> = { -readonly [P in keyof T]: DeepWriteable<T[P]> }
 
+export type Nullable<T> = { [K in keyof T]: T[K] | null }
+
 export type Primitive = null | undefined | string | number | boolean | symbol | bigint
 
 export type LiteralUnion<LiteralType, BaseType extends Primitive> = LiteralType | (BaseType & Record<never, never>)
@@ -134,7 +136,7 @@ export const unwrapThunk = <T>(_: T | (() => T)): T => {
   }
 }
 
-export type NullableFieldsToOptional<T> = PrettifyFlat<
+export type NullableFieldsToOptional<T> = Types.Simplify<
   Partial<T> & {
     [K in keyof T as null extends T[K] ? K : never]?: Exclude<T[K], null>
   } & {

package/src/node/ChildProcessRunner/ChildProcessRunner.ts

@@ -3,7 +3,7 @@ import process from 'node:process'
 
 import { WorkerError } from '@effect/platform/WorkerError'
 import * as Runner from '@effect/platform/WorkerRunner'
-import { FiberId } from 'effect'
+import { Cause } from 'effect'
 import * as Context from 'effect/Context'
 import * as Deferred from 'effect/Deferred'
 import * as Effect from 'effect/Effect'
@@ -15,7 +15,7 @@ import * as Scope from 'effect/Scope'
 
 const platformRunnerImpl = Runner.PlatformRunner.of({
   [Runner.PlatformRunnerTypeId]: Runner.PlatformRunnerTypeId,
-  start<I, O>() {
+  start<I, O>(closeLatch: Deferred.Deferred<void, WorkerError>) {
     return Effect.gen(function* () {
       if (!process.send) {
         return yield* new WorkerError({ reason: 'spawn', cause: new Error('not in a child process') })
@@ -27,32 +27,43 @@ const platformRunnerImpl = Runner.PlatformRunner.of({
       }
       const send = (_portId: number, message: O, _transfers?: ReadonlyArray<unknown>) =>
         Effect.sync(() => port.postMessage([1, message] /*, transfers as any*/))
-      const run = <A, E, R>(handler: (portId: number, message: I) => Effect.Effect<A, E, R>) =>
-        Effect.uninterruptibleMask((restore) =>
-          Effect.gen(function* () {
-            const runtime = (yield* Effect.runtime<R | Scope.Scope>()).pipe(
-              Runtime.updateContext(Context.omit(Scope.Scope)),
-            ) as Runtime.Runtime<R>
-            const fiberSet = yield* FiberSet.make<any, WorkerError | E>()
-            const runFork = Runtime.runFork(runtime)
-            port.on('message', (message: Runner.BackingRunner.Message<I>) => {
-              if (message[0] === 0) {
-                FiberSet.unsafeAdd(fiberSet, runFork(restore(handler(0, message[1]))))
-              } else {
-                Deferred.unsafeDone(fiberSet.deferred, Exit.interrupt(FiberId.none))
-                port.close()
-              }
-            })
-            port.on('messageerror', (cause) => {
-              Deferred.unsafeDone(fiberSet.deferred, new WorkerError({ reason: 'decode', cause }))
-            })
-            port.on('error', (cause) => {
-              Deferred.unsafeDone(fiberSet.deferred, new WorkerError({ reason: 'unknown', cause }))
-            })
-            port.postMessage([0])
-            return (yield* restore(FiberSet.join(fiberSet))) as never
-          }).pipe(Effect.scoped),
-        )
+
+      const run = Effect.fnUntraced(function* <A, E, R>(
+        handler: (portId: number, message: I) => Effect.Effect<A, E, R>,
+      ) {
+        const runtime = (yield* Effect.interruptible(Effect.runtime<R | Scope.Scope>())).pipe(
+          Runtime.updateContext(Context.omit(Scope.Scope)),
+        ) as Runtime.Runtime<R>
+        const fiberSet = yield* FiberSet.make<any, WorkerError | E>()
+        const runFork = Runtime.runFork(runtime)
+        const onExit = (exit: Exit.Exit<any, E>) => {
+          if (exit._tag === 'Failure') {
+            Deferred.unsafeDone(closeLatch, Exit.die(Cause.squash(exit.cause)))
+          }
+        }
+        port.on('message', (message: Runner.BackingRunner.Message<I>) => {
+          if (message[0] === 0) {
+            const fiber = runFork(handler(0, message[1]))
+            fiber.addObserver(onExit)
+            FiberSet.unsafeAdd(fiberSet, fiber)
+          } else {
+            Deferred.unsafeDone(closeLatch, Exit.void)
+            port.close()
+            // TODO get rid of this timeout (needs help from Tim Smart)
+            setTimeout(() => {
+              // eslint-disable-next-line unicorn/no-process-exit
+              process.exit(0)
+            }, 1000)
+          }
+        })
+        port.on('messageerror', (cause) => {
+          Deferred.unsafeDone(closeLatch, new WorkerError({ reason: 'decode', cause }))
+        })
+        port.on('error', (cause) => {
+          Deferred.unsafeDone(closeLatch, new WorkerError({ reason: 'unknown', cause }))
+        })
+        port.postMessage([0])
+      })
 
       return { run, send }
     })

package/src/node/mod.ts

@@ -14,7 +14,6 @@ import { makeNoopTracer } from '../NoopTracer.js'
 // import { tapCauseLogPretty } from '../effect/Effect.js'
 
 export * as Cli from '@effect/cli'
-export * as PlatformBun from '@effect/platform-bun'
 export * as PlatformNode from '@effect/platform-node'
 export * as SocketServer from '@effect/experimental/SocketServer'
 export * as OtelResource from '@effect/opentelemetry/Resource'
@@ -52,8 +51,7 @@ export const OtelLiveHttp = ({
 > =>
   Effect.gen(function* () {
     const config = yield* Config.all({
-      exporterUrlTracing: Config.string('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'),
-      exporterUrlMracing: Config.string('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT'),
+      exporterUrl: Config.string('OTEL_EXPORTER_OTLP_ENDPOINT'),
       serviceName: serviceName
         ? Config.succeed(serviceName)
         : Config.string('OTEL_SERVICE_NAME').pipe(Config.withDefault('overtone-node-utils-default-service')),
@@ -65,7 +63,7 @@ export const OtelLiveHttp = ({
     const resource = { serviceName: config.serviceName }
 
     // METRICS
-    const metricExporter = new OTLPMetricExporter({ url: config.exporterUrlMracing })
+    const metricExporter = new OTLPMetricExporter({ url: `${config.exporterUrl}/v1/metrics` })
 
     const metricReader = new PeriodicExportingMetricReader({
       exporter: metricExporter,
@@ -76,7 +74,9 @@ export const OtelLiveHttp = ({
     const OtelLive = OtelNodeSdk.layer(() => ({
       resource,
       metricReader,
-      spanProcessor: new BatchSpanProcessor(new OTLPTraceExporter({ url: config.exporterUrlTracing, headers: {} })),
+      spanProcessor: new BatchSpanProcessor(
+        new OTLPTraceExporter({ url: `${config.exporterUrl}/v1/traces`, headers: {} }),
+      ),
     }))
 
     const RootSpanLive = Layer.span(config.rootSpanName, {
@@ -104,7 +104,7 @@ export const logTraceUiUrlForSpan = (printMsg?: (url: string) => string) => (spa
 
 export const getTracingBackendUrl = (span: otel.Span) =>
   Effect.gen(function* () {
-    const endpoint = yield* Config.string('TRACING_UI_ENDPOINT').pipe(Config.option, Effect.orDie)
+    const endpoint = yield* Config.string('GRAFANA_ENDPOINT').pipe(Config.option, Effect.orDie)
     if (endpoint._tag === 'None') return
 
     const traceId = span.spanContext().traceId

package/tsconfig.json

@@ -6,5 +6,5 @@
     "tsBuildInfoFile": "./dist/.tsbuildinfo.json"
   },
   "include": ["./src"],
-  "references": [{ "path": "../db-schema"}]
+  "references": []
 }