@livestore/utils 0.2.0-dev.2 → 0.3.0-dev.0

package/src/effect/WebSocket.ts

@@ -0,0 +1,72 @@
+import type { Schedule, Scope } from 'effect'
+import { Effect, Exit, identity, Schema } from 'effect'
+
+export class WebSocketError extends Schema.TaggedError<WebSocketError>()('WebSocketError', {
+  cause: Schema.Defect,
+}) {}
+
+// TODO refactor using Effect socket implementation
+// https://github.com/Effect-TS/effect/blob/main/packages%2Fexperimental%2Fsrc%2FDevTools%2FClient.ts#L113
+// "In a Stream pipeline everything above the pipeThrough is the outgoing (send) messages. Everything below is the incoming (message event) messages."
+// https://github.com/Effect-TS/effect/blob/main/packages%2Fplatform%2Fsrc%2FSocket.ts#L451
+
+/**
+ * Creates a WebSocket connection and waits for the connection to be established.
+ * Automatically closes the connection when the scope is closed.
+ */
+export const makeWebSocket = ({
+  url,
+  reconnect,
+}: {
+  url: string
+  reconnect?: Schedule.Schedule<unknown> | false
+}): Effect.Effect<globalThis.WebSocket, WebSocketError, Scope.Scope> =>
+  Effect.gen(function* () {
+    const socket = yield* Effect.tryPromise({
+      try: async () => {
+        // console.debug('[WebSocket] connecting to', url)
+        const socket = new globalThis.WebSocket(url)
+
+        if (socket.readyState === globalThis.WebSocket.OPEN) {
+          return socket
+        }
+
+        return await new Promise<globalThis.WebSocket>((resolve, reject) => {
+          socket.addEventListener('open', () => resolve(socket), { once: true })
+          // eslint-disable-next-line unicorn/prefer-add-event-listener
+          socket.onerror = (event) => reject(event)
+        })
+      },
+      catch: (errorEvent: any) => {
+        if (errorEvent.currentTarget != null && errorEvent.currentTarget instanceof globalThis.WebSocket) {
+          errorEvent.currentTarget.close(3000, `closing websocket connection due to error: ${errorEvent.toString()}`)
+        }
+
+        return new WebSocketError({ cause: errorEvent })
+      },
+    }).pipe(
+      /**
+       * Common WebSocket close codes: https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/close
+       *   1000: Normal closure
+       *   1001: Endpoint is going away, a server is terminating the connection because it has received a request that indicates the client is ending the connection.
+       *   1002: Protocol error, a server is terminating the connection because it has received data on the connection that was not consistent with the type of the connection.
+       *   1011: Internal server error, a server is terminating the connection because it encountered an unexpected condition that prevented it from fulfilling the request.
+       *
+       * For reference, here are the valid WebSocket close code ranges:
+       *   1000-1999: Reserved for protocol usage
+       *   2000-2999: Reserved for WebSocket extensions
+       *   3000-3999: Available for libraries and frameworks
+       *   4000-4999: Available for applications
+       */
+      Effect.acquireRelease((socket, exit) =>
+        Effect.sync(() =>
+          Exit.isFailure(exit)
+            ? socket.close(3000, `closing webmesh websocket connection due to error: ${exit.cause.toString()}`)
+            : socket.close(1000, 'closing webmesh websocket connection gracefully'),
+        ),
+      ),
+      reconnect ? Effect.retry(reconnect) : identity,
+    )
+
+    return socket
+  })

package/src/effect/index.ts

@@ -39,9 +39,11 @@ export {
   HashSet,
   MutableHashSet,
   MutableHashMap,
+  TQueue,
   Option,
   LogLevel,
-  Logger,
+  // Logger,
+  Config,
   Layer,
   STM,
   TRef,
@@ -51,15 +53,21 @@ export {
   identity,
   GlobalValue,
   Match,
+  TestServices,
 } from 'effect'
 
 export { dual } from 'effect/Function'
 
 export * as Stream from './Stream.js'
 
+export * as BucketQueue from './BucketQueue.js'
+
 export * as SubscriptionRef from './SubscriptionRef.js'
 
+export * as Logger from './Logger.js'
+
 export * as WebChannel from './WebChannel.js'
+export * as WebSocket from './WebSocket.js'
 
 export * as SchemaAST from 'effect/SchemaAST'
 export { TreeFormatter } from 'effect/ParseResult'
@@ -67,6 +75,8 @@ export { ParseResult, Pretty } from 'effect'
 export type { Serializable, SerializableWithResult } from 'effect/Schema'
 export * as Schema from './Schema/index.js'
 export * as OtelTracer from '@effect/opentelemetry/Tracer'
+// export * as OtelResource from '@effect/opentelemetry/Resource'
+export * as TaskTracing from './TaskTracing.js'
 
 export {
   Transferable,
@@ -81,9 +91,12 @@ export {
   HttpClientRequest,
   HttpClientResponse,
   FetchHttpClient,
+  Socket,
 } from '@effect/platform'
 export { BrowserWorker, BrowserWorkerRunner } from '@effect/platform-browser'
 
+// export { DevTools as EffectDevtools } from '@effect/experimental'
+
 export * as Effect from './Effect.js'
 export * as Schedule from './Schedule.js'
 export * as Scheduler from './Scheduler.js'

package/src/env.ts

@@ -0,0 +1,31 @@
+export const env = (name: string): string | undefined => {
+  if (typeof process !== 'undefined' && process.env !== undefined) {
+    return process.env[name]
+  }
+
+  // TODO re-enable the full guard code once `import.meta` is supported in Expo
+  // if (import.meta !== undefined && import.meta.env !== undefined) {
+  if (import.meta.env !== undefined) {
+    return import.meta.env[name]
+  }
+
+  return undefined
+}
+
+export const isDevEnv = () => {
+  if (typeof process !== 'undefined' && process.env !== undefined) {
+    return process.env.NODE_ENV !== 'production'
+  }
+
+  // TODO re-enable the full guard code once `import.meta` is supported in Expo
+  // if (import.meta !== undefined && import.meta.env !== undefined) {
+  if (import.meta.env !== undefined) {
+    return import.meta.env.DEV
+  }
+
+  return false
+}
+
+export const TRACE_VERBOSE = env('LS_TRACE_VERBOSE') !== undefined || env('VITE_LS_TRACE_VERBOSE') !== undefined
+
+export const LS_DEV = env('LS_DEV') !== undefined || env('VITE_LS_DEV') !== undefined

package/src/index.ts

@@ -8,12 +8,14 @@ export * from './set.js'
 export * from './browser.js'
 export * from './Deferred.js'
 export * from './misc.js'
+export * from './env.js'
 export * from './fast-deep-equal.js'
 export * as base64 from './base64.js'
 export { default as prettyBytes } from 'pretty-bytes'
 
 import type * as otel from '@opentelemetry/api'
 
+import { isDevEnv } from './env.js'
 import { objectToString } from './misc.js'
 
 export type Prettify<T> = T extends infer U ? { [K in keyof U]: Prettify<U[K]> } : never
@@ -88,7 +90,7 @@ export function casesHandled(unexpectedCase: never): never {
 
 export const shouldNeverHappen = (msg?: string, ...args: any[]): never => {
   console.error(msg, ...args)
-  if (isDev()) {
+  if (isDevEnv()) {
     debugger
   }
 
@@ -228,13 +230,3 @@ export const isPromise = (value: any): value is Promise<unknown> => typeof value
 export const isIterable = <T>(value: any): value is Iterable<T> => typeof value?.[Symbol.iterator] === 'function'
 
 export { objectToString as errorToString } from './misc.js'
-
-const isDev = memoizeByRef(() => {
-  if (import.meta.env !== undefined) {
-    return import.meta.env.DEV || import.meta.env.VITE_DEV
-  } else if (typeof process !== 'undefined' && process.env !== undefined) {
-    return process.env.DEV
-  } else {
-    return false
-  }
-})

package/src/nanoid/index.browser.ts

@@ -0,0 +1,2 @@
+// @ts-expect-error TODO
+export { nanoid } from 'nanoid/index.browser.js'

package/src/node/ChildProcessRunner/ChildProcessRunner.ts

@@ -0,0 +1,63 @@
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+import process from 'node:process'
+
+import { WorkerError } from '@effect/platform/WorkerError'
+import * as Runner from '@effect/platform/WorkerRunner'
+import { FiberId } from 'effect'
+import * as Context from 'effect/Context'
+import * as Deferred from 'effect/Deferred'
+import * as Effect from 'effect/Effect'
+import * as Exit from 'effect/Exit'
+import * as FiberSet from 'effect/FiberSet'
+import * as Layer from 'effect/Layer'
+import * as Runtime from 'effect/Runtime'
+import * as Scope from 'effect/Scope'
+
+const platformRunnerImpl = Runner.PlatformRunner.of({
+  [Runner.PlatformRunnerTypeId]: Runner.PlatformRunnerTypeId,
+  start<I, O>() {
+    return Effect.gen(function* () {
+      if (!process.send) {
+        return yield* new WorkerError({ reason: 'spawn', cause: new Error('not in a child process') })
+      }
+      const port = {
+        postMessage: (message: any) => process.send!(message),
+        on: (event: string, handler: (message: any) => void) => process.on(event, handler),
+        close: () => process.disconnect(),
+      }
+      const send = (_portId: number, message: O, _transfers?: ReadonlyArray<unknown>) =>
+        Effect.sync(() => port.postMessage([1, message] /*, transfers as any*/))
+      const run = <A, E, R>(handler: (portId: number, message: I) => Effect.Effect<A, E, R>) =>
+        Effect.uninterruptibleMask((restore) =>
+          Effect.gen(function* () {
+            const runtime = (yield* Effect.runtime<R | Scope.Scope>()).pipe(
+              Runtime.updateContext(Context.omit(Scope.Scope)),
+            ) as Runtime.Runtime<R>
+            const fiberSet = yield* FiberSet.make<any, WorkerError | E>()
+            const runFork = Runtime.runFork(runtime)
+            port.on('message', (message: Runner.BackingRunner.Message<I>) => {
+              if (message[0] === 0) {
+                FiberSet.unsafeAdd(fiberSet, runFork(restore(handler(0, message[1]))))
+              } else {
+                Deferred.unsafeDone(fiberSet.deferred, Exit.interrupt(FiberId.none))
+                port.close()
+              }
+            })
+            port.on('messageerror', (cause) => {
+              Deferred.unsafeDone(fiberSet.deferred, new WorkerError({ reason: 'decode', cause }))
+            })
+            port.on('error', (cause) => {
+              Deferred.unsafeDone(fiberSet.deferred, new WorkerError({ reason: 'unknown', cause }))
+            })
+            port.postMessage([0])
+            return (yield* restore(FiberSet.join(fiberSet))) as never
+          }).pipe(Effect.scoped),
+        )
+
+      return { run, send }
+    })
+  },
+})
+
+/** @internal */
+export const layer = Layer.succeed(Runner.PlatformRunner, platformRunnerImpl)

package/src/node/ChildProcessRunner/ChildProcessWorker.ts

@@ -0,0 +1,66 @@
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+
+import type * as ChildProcess from 'node:child_process'
+
+import * as Worker from '@effect/platform/Worker'
+import { WorkerError } from '@effect/platform/WorkerError'
+import * as Deferred from 'effect/Deferred'
+import * as Effect from 'effect/Effect'
+import * as Exit from 'effect/Exit'
+import * as Layer from 'effect/Layer'
+import * as Scope from 'effect/Scope'
+
+const platformWorkerImpl = Worker.makePlatform<ChildProcess.ChildProcess>()({
+  setup({ scope, worker: childProcess }) {
+    return Effect.flatMap(Deferred.make<void, WorkerError>(), (exitDeferred) => {
+      childProcess.on('exit', () => {
+        Deferred.unsafeDone(exitDeferred, Exit.void)
+      })
+      return Effect.as(
+        Scope.addFinalizer(
+          scope,
+          Effect.suspend(() => {
+            childProcess.send([1])
+            return Deferred.await(exitDeferred)
+          }).pipe(
+            Effect.interruptible,
+            Effect.timeout(5000),
+            Effect.catchAllCause(() => Effect.sync(() => childProcess.kill())),
+          ),
+        ),
+        {
+          postMessage: (message: any) => childProcess.send(message),
+          on: (event: string, handler: (message: any) => void) => childProcess.on(event, handler),
+        },
+      )
+    })
+  },
+  listen({ deferred, emit, port }) {
+    port.on('message', (message) => {
+      emit(message)
+    })
+    port.on('messageerror', (cause) => {
+      Deferred.unsafeDone(deferred, new WorkerError({ reason: 'decode', cause }))
+    })
+    port.on('error', (cause) => {
+      Deferred.unsafeDone(deferred, new WorkerError({ reason: 'unknown', cause }))
+    })
+    port.on('exit', (code) => {
+      Deferred.unsafeDone(
+        deferred,
+        new WorkerError({ reason: 'unknown', cause: new Error(`exited with code ${code}`) }),
+      )
+    })
+    return Effect.void
+  },
+})
+
+/** @internal */
+export const layerWorker = Layer.succeed(Worker.PlatformWorker, platformWorkerImpl)
+
+/** @internal */
+export const layerManager = Layer.provide(Worker.layerManager, layerWorker)
+
+/** @internal */
+export const layer = (spawn: (id: number) => ChildProcess.ChildProcess) =>
+  Layer.merge(layerManager, Worker.layerSpawner(spawn))

package/src/node/mod.ts

@@ -0,0 +1,115 @@
+import * as OtelNodeSdk from '@effect/opentelemetry/NodeSdk'
+import type * as otel from '@opentelemetry/api'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { Config, Effect, Layer } from 'effect'
+import type { ParentSpan } from 'effect/Tracer'
+
+import { tapCauseLogPretty } from '../effect/Effect.js'
+import type { OtelTracer } from '../effect/index.js'
+
+// import { tapCauseLogPretty } from '../effect/Effect.js'
+
+export * as Cli from '@effect/cli'
+export * as PlatformBun from '@effect/platform-bun'
+export * as PlatformNode from '@effect/platform-node'
+export * as SocketServer from '@effect/experimental/SocketServer'
+export * as OtelResource from '@effect/opentelemetry/Resource'
+
+export { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
+export { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
+export * as OtelNodeSdk from '@effect/opentelemetry/NodeSdk'
+
+export * as ChildProcessRunner from './ChildProcessRunner/ChildProcessRunner.js'
+export * as ChildProcessWorker from './ChildProcessRunner/ChildProcessWorker.js'
+
+// Enable debug logging for OpenTelemetry
+// otel.diag.setLogger(new otel.DiagConsoleLogger(), otel.DiagLogLevel.ERROR)
+
+// export const OtelLiveHttp = (args: any): Layer.Layer<never> => Layer.empty
+
+export const OtelLiveHttp = ({
+  serviceName,
+  rootSpanName,
+  skipLogUrl,
+}: { serviceName?: string; rootSpanName?: string; skipLogUrl?: boolean } = {}): Layer.Layer<
+  OtelTracer.OtelTracer | ParentSpan,
+  never,
+  never
+> =>
+  Effect.gen(function* () {
+    const config = yield* Config.all({
+      exporterUrlTracing: Config.string('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'),
+      exporterUrlMracing: Config.string('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT'),
+      serviceName: serviceName
+        ? Config.succeed(serviceName)
+        : Config.string('OTEL_SERVICE_NAME').pipe(Config.withDefault('overtone-node-utils-default-service')),
+      rootSpanName: rootSpanName
+        ? Config.succeed(rootSpanName)
+        : Config.string('OTEL_ROOT_SPAN_NAME').pipe(Config.withDefault('RootSpan')),
+    }).pipe(tapCauseLogPretty, Effect.orDie)
+
+    const resource = { serviceName: config.serviceName }
+
+    // METRICS
+    const metricExporter = new OTLPMetricExporter({ url: config.exporterUrlMracing })
+
+    const metricReader = new PeriodicExportingMetricReader({
+      exporter: metricExporter,
+      exportIntervalMillis: 1000,
+    })
+
+    // TRACING
+    const OtelLive = OtelNodeSdk.layer(() => ({
+      resource,
+      metricReader,
+      spanProcessor: new BatchSpanProcessor(new OTLPTraceExporter({ url: config.exporterUrlTracing, headers: {} })),
+    }))
+
+    const RootSpanLive = Layer.span(config.rootSpanName, {
+      attributes: { config },
+      onEnd: skipLogUrl ? undefined : (span: any) => logTraceUiUrlForSpan()(span.span),
+    })
+
+    return RootSpanLive.pipe(Layer.provideMerge(OtelLive))
+  }).pipe(Layer.unwrapEffect) as any
+
+export const logTraceUiUrlForSpan = (printMsg?: (url: string) => string) => (span: otel.Span) =>
+  getTracingBackendUrl(span).pipe(
+    Effect.tap((url) => {
+      if (url === undefined) {
+        console.warn('No tracing backend url found')
+      } else {
+        if (printMsg) {
+          console.log(printMsg(url))
+        } else {
+          console.log(`Trace URL: ${url}`)
+        }
+      }
+    }),
+  )
+
+export const getTracingBackendUrl = (span: otel.Span) =>
+  Effect.gen(function* () {
+    const endpoint = yield* Config.string('TRACING_UI_ENDPOINT').pipe(Config.option, Effect.orDie)
+    if (endpoint._tag === 'None') return
+
+    const traceId = span.spanContext().traceId
+
+    // Grafana + Tempo
+
+    const grafanaEndpoint = endpoint.value
+    const searchParams = new URLSearchParams({
+      orgId: '1',
+      left: JSON.stringify({
+        datasource: 'tempo',
+        queries: [{ query: traceId, queryType: 'traceql', refId: 'A' }],
+        range: { from: 'now-1h', to: 'now' },
+      }),
+    })
+
+    // TODO make dynamic via env var
+    return `${grafanaEndpoint}/explore?${searchParams.toString()}`
+  })

package/src/node-vitest/mod.ts

@@ -0,0 +1 @@
+export * as Vitest from '@effect/vitest'

package/src/node-vitest/polyfill.ts

@@ -0,0 +1 @@
+process.stdout.isTTY = true

package/tmp/effect-deferred-repro.ts

@@ -0,0 +1,29 @@
+/// <reference lib="es2024" />
+
+import { Effect, Logger } from 'effect'
+
+const main = Effect.gen(function* () {
+  yield* Effect.addFinalizer((_) => Effect.log('finalizer', _))
+
+  // const finished = yield* Deferred.make<void>()
+
+  const def = Promise.withResolvers()
+
+  setTimeout(() => {
+    def.resolve(undefined)
+  }, 1000)
+
+  console.log('awaiting')
+
+  // const fiber = yield* Deferred.await(finished).pipe(Effect.fork)
+  // yield* Fiber.join(fiber)
+  // yield* finished
+  yield* Effect.promise(() => def.promise)
+  console.log('finished')
+})
+
+main
+  .pipe(Effect.scoped, Effect.provide(Logger.pretty), Effect.tapErrorCause(Effect.logError), Effect.runPromise)
+  .finally(() => {
+    console.log('finally')
+  })

package/tmp/effect-semaphore-repro.ts

@@ -0,0 +1,93 @@
+/// <reference lib="es2024" />
+import * as NodePlatform from '@effect/platform-node'
+import { Deferred, Effect, FiberSet, Logger, Ref, Scheduler } from 'effect'
+
+const main = Effect.gen(function* () {
+  const semaphore = yield* Effect.makeSemaphore(1)
+
+  const fiberset = yield* FiberSet.make()
+
+  // setInterval(() => {
+  //   // semaphore.release(1)
+  // }, 1_000_000)
+
+  const stateRef = yield* Ref.make<'open' | 'closed'>('open')
+
+  // let counter = 0
+  // const nextExpectedCounter = 0
+  // const def = Promise.withResolvers()
+
+  const finished = yield* Deferred.make<void>()
+
+  const MAX = 100
+
+  const doWorkWithSemaphore = (counter: number) =>
+    Effect.gen(function* () {
+      // yield* Effect.log(`doWorkWithSemaphore: ${counter} - waiting for semaphore`)
+      // yield* semaphore.take(1)
+      // if (counter !== nextExpectedCounter) {
+      //   throw new Error(`Expected counter ${nextExpectedCounter} but got ${counter}`)
+      // }
+      // nextExpectedCounter = counter + 1
+      yield* Effect.log(`doWorkWithSemaphore: ${counter} - got semaphore`)
+      // setTimeout(() => {}, 120)
+      yield* Effect.sleep(40)
+      // yield* Effect.yieldNow()
+      yield* Effect.log(`doWorkWithSemaphore: ${counter} - releasing semaphore`)
+      yield* Ref.set(stateRef, 'open')
+      yield* semaphore.release(1)
+
+      if (counter === MAX - 1) {
+        yield* Deferred.succeed(finished, undefined)
+        // def.resolve(undefined)
+      }
+    }).pipe(Effect.scoped)
+
+  for (let i = 0; i < MAX; i++) {
+    yield* doWorkWithSemaphore(i).pipe(FiberSet.run(fiberset))
+    // yield* Effect.sleep(20)
+    yield* Effect.gen(function* () {
+      const state = yield* Ref.get(stateRef)
+      if (state === 'open') {
+        yield* Effect.logWarning(`loop: ${i} - semaphore is open, taking`)
+      } else {
+        yield* Effect.log(`loop: ${i} - semaphore is closed, waiting`)
+      }
+
+      yield* semaphore.take(1)
+      yield* Ref.set(stateRef, 'closed')
+
+      yield* doWorkWithSemaphore(i).pipe(Effect.fork)
+
+      const sleepTime = Math.random() * 40
+      yield* Effect.log(`loop: ${i} - sleeping for ${sleepTime}ms`)
+      yield* Effect.sleep(sleepTime)
+
+      // yield* Effect.yieldNow()
+    }).pipe(FiberSet.run(fiberset))
+  }
+
+  // yield* Effect.never
+
+  yield* finished
+  // yield* Effect.promise(() => def.promise)
+  // console.log('finished')
+})
+
+export const messageChannel = (shouldYield: Scheduler.Scheduler['shouldYield'] = Scheduler.defaultShouldYield) =>
+  Scheduler.makeBatched((task) => {
+    const messageChannel = new MessageChannel()
+
+    messageChannel.port1.postMessage(undefined)
+
+    // eslint-disable-next-line unicorn/prefer-add-event-listener
+    messageChannel.port2.onmessage = task
+  }, shouldYield)
+
+main.pipe(
+  Effect.scoped,
+  Effect.provide(Logger.pretty),
+  Effect.tapErrorCause(Effect.logError),
+  Effect.withScheduler(messageChannel()),
+  NodePlatform.NodeRuntime.runMain,
+)