@livestore/adapter-web 0.4.0-dev.1 → 0.4.0-dev.11

package/src/web-worker/client-session/sqlite-loader.ts

@@ -0,0 +1,19 @@
+import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
+
+/**
+ * Browser sessions benefit from downloading and compiling the wasm binary as soon as
+ * possible to hide network and IO latency behind the rest of the boot process. We kick
+ * that work off eagerly on the client while still returning the shared promise.
+ *
+ * The Cloudflare / Workerd runtime has stricter rules: async fetches during module
+ * evaluation are blocked, so we defer loading until the worker asks for it.
+ */
+const isServerRuntime = String(import.meta.env.SSR) === 'true'
+
+let sqlite3Promise: ReturnType<typeof loadSqlite3Wasm> | undefined
+
+if (isServerRuntime === false) {
+  sqlite3Promise = loadSqlite3Wasm()
+}
+
+export const loadSqlite3 = () => (isServerRuntime ? loadSqlite3Wasm() : (sqlite3Promise ?? loadSqlite3Wasm()))

package/src/web-worker/common/persisted-sqlite.ts

@@ -1,12 +1,14 @@
 import { liveStoreStorageFormatVersion, UnexpectedError } from '@livestore/common'
 import type { LiveStoreSchema } from '@livestore/common/schema'
-import { decodeSAHPoolFilename, HEADER_OFFSET_DATA } from '@livestore/sqlite-wasm/browser'
+import { decodeSAHPoolFilename, HEADER_OFFSET_DATA, type WebDatabaseMetadataOpfs } from '@livestore/sqlite-wasm/browser'
+import { isDevEnv } from '@livestore/utils'
 import { Effect, Schedule, Schema } from '@livestore/utils/effect'
 
 import * as OpfsUtils from '../../opfs-utils.ts'
 import type * as WorkerSchema from './worker-schema.ts'
 
 export class PersistedSqliteError extends Schema.TaggedError<PersistedSqliteError>()('PersistedSqliteError', {
+  message: Schema.String,
   cause: Schema.Defect,
 }) {}
 
@@ -141,3 +143,220 @@ export const getStateDbFileName = (schema: LiveStoreSchema) => {
     schema.state.sqlite.migrations.strategy === 'manual' ? 'fixed' : schema.state.sqlite.hash.toString()
   return `state${schemaHashSuffix}.db`
 }
+
+export const MAX_ARCHIVED_STATE_DBS_IN_DEV = 3
+
+/**
+ * Cleanup old state database files after successful migration.
+ * This prevents OPFS file pool capacity from being exhausted by accumulated schema files.
+ *
+ * @param vfs - The AccessHandlePoolVFS instance for safe file operations
+ * @param currentSchema - Current schema (to avoid deleting the active database)
+ */
+export const cleanupOldStateDbFiles = Effect.fn('@livestore/adapter-web:cleanupOldStateDbFiles')(
+  function* ({
+    vfs,
+    currentSchema,
+    opfsDirectory,
+  }: {
+    vfs: WebDatabaseMetadataOpfs['vfs']
+    currentSchema: LiveStoreSchema
+    opfsDirectory: string
+  }) {
+    // Only cleanup for auto migration strategy because:
+    // - Auto strategy: Creates new database files per schema change (e.g., state123.db, state456.db)
+    //   which accumulate over time and can exhaust OPFS file pool capacity
+    // - Manual strategy: Always reuses the same database file (statefixed.db) across schema changes,
+    //   so there are never multiple old files to clean up
+    if (currentSchema.state.sqlite.migrations.strategy === 'manual') {
+      yield* Effect.logDebug('Skipping state db cleanup - manual migration strategy uses fixed filename')
+      return
+    }
+
+    const isDev = isDevEnv()
+    const currentDbFileName = getStateDbFileName(currentSchema)
+    const currentPath = `/${currentDbFileName}`
+
+    const allPaths = yield* Effect.sync(() => vfs.getTrackedFilePaths())
+    const oldStateDbPaths = allPaths.filter(
+      (path) => path.startsWith('/state') && path.endsWith('.db') && path !== currentPath,
+    )
+
+    if (oldStateDbPaths.length === 0) {
+      yield* Effect.logDebug('State db cleanup completed: no old database files found')
+      return
+    }
+
+    yield* Effect.logDebug(`Found ${oldStateDbPaths.length} old state database file(s) to clean up`)
+
+    let deletedCount = 0
+    const archivedFileNames: string[] = []
+    let archiveDirHandle: FileSystemDirectoryHandle | undefined
+
+    for (const path of oldStateDbPaths) {
+      const fileName = path.startsWith('/') ? path.slice(1) : path
+
+      if (isDev) {
+        archiveDirHandle = yield* Effect.tryPromise({
+          try: () => OpfsUtils.getDirHandle(`${opfsDirectory}/archive`, { create: true }),
+          catch: (cause) => new ArchiveStateDbError({ message: 'Failed to ensure archive directory', cause }),
+        })
+
+        const archivedFileName = yield* archiveStateDbFile({
+          vfs,
+          fileName,
+          archiveDirHandle,
+        })
+
+        archivedFileNames.push(archivedFileName)
+      }
+
+      const vfsResultCode = yield* Effect.try({
+        try: () => vfs.jDelete(fileName, 0),
+        catch: (cause) => new SqliteVfsError({ operation: 'jDelete', fileName, cause }),
+      })
+
+      // 0 indicates a successful result in SQLite.
+      // See https://www.sqlite.org/c3ref/c_abort.html
+      if (vfsResultCode !== 0) {
+        return yield* new SqliteVfsError({
+          operation: 'jDelete',
+          fileName,
+          vfsResultCode,
+        })
+      }
+
+      deletedCount++
+      yield* Effect.logDebug(`Successfully deleted old state database file: ${fileName}`)
+    }
+
+    if (isDev && archiveDirHandle !== undefined) {
+      const pruneResult = yield* pruneArchiveDir({
+        archiveDirHandle,
+        keep: MAX_ARCHIVED_STATE_DBS_IN_DEV,
+      })
+
+      yield* Effect.logDebug(
+        `State db cleanup completed: archived ${archivedFileNames.length} file(s); removed ${deletedCount} old database file(s) from active pool; archive retained ${pruneResult.retained.length} file(s)`,
+      )
+    } else {
+      yield* Effect.logDebug(`State db cleanup completed: removed ${deletedCount} old database file(s)`)
+    }
+  },
+  Effect.mapError(
+    (error) =>
+      new PersistedSqliteError({
+        message: 'Failed to clean up old state database file(s)',
+        cause: error,
+      }),
+  ),
+)
+
+const archiveStateDbFile = Effect.fn('@livestore/adapter-web:archiveStateDbFile')(function* ({
+  vfs,
+  fileName,
+  archiveDirHandle,
+}: {
+  vfs: WebDatabaseMetadataOpfs['vfs']
+  fileName: string
+  archiveDirHandle: FileSystemDirectoryHandle
+}) {
+  const stateDbBuffer = vfs.readFilePayload(fileName)
+
+  const archiveFileName = `${Date.now()}-${fileName}`
+
+  const archiveFileHandle = yield* Effect.tryPromise({
+    try: () => archiveDirHandle.getFileHandle(archiveFileName, { create: true }),
+    catch: (cause) =>
+      new ArchiveStateDbError({
+        message: 'Failed to open archive file handle',
+        fileName: archiveFileName,
+        cause,
+      }),
+  })
+
+  const archiveFileAccessHandle = yield* Effect.acquireRelease(
+    Effect.tryPromise({
+      try: () => archiveFileHandle.createSyncAccessHandle(),
+      catch: (cause) =>
+        new ArchiveStateDbError({
+          message: 'Failed to create sync access handle for archived file',
+          fileName: archiveFileName,
+          cause,
+        }),
+    }),
+    (handle) => Effect.sync(() => handle.close()).pipe(Effect.ignoreLogged),
+  )
+
+  yield* Effect.try({
+    try: () => {
+      archiveFileAccessHandle.write(stateDbBuffer)
+      archiveFileAccessHandle.flush()
+    },
+    catch: (cause) =>
+      new ArchiveStateDbError({
+        message: 'Failed to write archived state database',
+        fileName: archiveFileName,
+        cause,
+      }),
+  })
+
+  return archiveFileName
+}, Effect.scoped)
+
+const pruneArchiveDir = Effect.fn('@livestore/adapter-web:pruneArchiveDir')(function* ({
+  archiveDirHandle,
+  keep,
+}: {
+  archiveDirHandle: FileSystemDirectoryHandle
+  keep: number
+}) {
+  const files = yield* Effect.tryPromise({
+    try: async () => {
+      const result: { name: string; lastModified: number }[] = []
+
+      for await (const entry of archiveDirHandle.values()) {
+        if (entry.kind !== 'file') continue
+        const fileHandle = await archiveDirHandle.getFileHandle(entry.name)
+        const file = await fileHandle.getFile()
+        result.push({ name: entry.name, lastModified: file.lastModified })
+      }
+
+      return result.sort((a, b) => b.lastModified - a.lastModified)
+    },
+    catch: (cause) => new ArchiveStateDbError({ message: 'Failed to enumerate archived state databases', cause }),
+  })
+
+  const retained = files.slice(0, keep)
+  const toDelete = files.slice(keep)
+
+  yield* Effect.forEach(toDelete, ({ name }) =>
+    Effect.tryPromise({
+      try: () => archiveDirHandle.removeEntry(name),
+      catch: (cause) =>
+        new ArchiveStateDbError({
+          message: 'Failed to delete archived state database',
+          fileName: name,
+          cause,
+        }),
+    }),
+  )
+
+  return {
+    retained,
+    deleted: toDelete,
+  }
+})
+
+export class ArchiveStateDbError extends Schema.TaggedError<ArchiveStateDbError>()('ArchiveStateDbError', {
+  message: Schema.String,
+  fileName: Schema.optional(Schema.String),
+  cause: Schema.Defect,
+}) {}
+
+export class SqliteVfsError extends Schema.TaggedError<SqliteVfsError>()('SqliteVfsError', {
+  operation: Schema.String,
+  fileName: Schema.String,
+  vfsResultCode: Schema.optional(Schema.Number),
+  cause: Schema.optional(Schema.Defect),
+}) {}

package/src/web-worker/common/worker-schema.ts

@@ -4,6 +4,7 @@ import {
   LeaderAheadError,
   liveStoreVersion,
   MigrationsReport,
+  SyncBackend,
   SyncState,
   UnexpectedError,
 } from '@livestore/common'
@@ -146,6 +147,33 @@ export class LeaderWorkerInnerGetLeaderSyncState extends Schema.TaggedRequest<Le
   },
 ) {}
 
+export class LeaderWorkerInnerSyncStateStream extends Schema.TaggedRequest<LeaderWorkerInnerSyncStateStream>()(
+  'SyncStateStream',
+  {
+    payload: {},
+    success: SyncState.SyncState,
+    failure: UnexpectedError,
+  },
+) {}
+
+export class LeaderWorkerInnerGetNetworkStatus extends Schema.TaggedRequest<LeaderWorkerInnerGetNetworkStatus>()(
+  'GetNetworkStatus',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: UnexpectedError,
+  },
+) {}
+
+export class LeaderWorkerInnerNetworkStatusStream extends Schema.TaggedRequest<LeaderWorkerInnerNetworkStatusStream>()(
+  'NetworkStatusStream',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: UnexpectedError,
+  },
+) {}
+
 export class LeaderWorkerInnerShutdown extends Schema.TaggedRequest<LeaderWorkerInnerShutdown>()('Shutdown', {
   payload: {},
   success: Schema.Void,
@@ -173,6 +201,9 @@ export const LeaderWorkerInnerRequest = Schema.Union(
   LeaderWorkerInnerGetRecreateSnapshot,
   LeaderWorkerInnerGetLeaderHead,
   LeaderWorkerInnerGetLeaderSyncState,
+  LeaderWorkerInnerSyncStateStream,
+  LeaderWorkerInnerGetNetworkStatus,
+  LeaderWorkerInnerNetworkStatusStream,
   LeaderWorkerInnerShutdown,
   LeaderWorkerInnerExtraDevtoolsMessage,
   WebmeshWorker.Schema.CreateConnection,
@@ -218,6 +249,9 @@ export class SharedWorkerRequest extends Schema.Union(
   LeaderWorkerInnerExportEventlog,
   LeaderWorkerInnerGetLeaderHead,
   LeaderWorkerInnerGetLeaderSyncState,
+  LeaderWorkerInnerSyncStateStream,
+  LeaderWorkerInnerGetNetworkStatus,
+  LeaderWorkerInnerNetworkStatusStream,
   LeaderWorkerInnerShutdown,
   LeaderWorkerInnerExtraDevtoolsMessage,
 

package/src/web-worker/leader-worker/make-leader-worker.ts

@@ -26,7 +26,7 @@ import {
 import type * as otel from '@opentelemetry/api'
 
 import * as OpfsUtils from '../../opfs-utils.ts'
-import { getStateDbFileName, sanitizeOpfsDir } from '../common/persisted-sqlite.ts'
+import { cleanupOldStateDbFiles, getStateDbFileName, sanitizeOpfsDir } from '../common/persisted-sqlite.ts'
 import { makeShutdownChannel } from '../common/shutdown-channel.ts'
 import * as WorkerSchema from '../common/worker-schema.ts'
 
@@ -59,16 +59,16 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       )
     : undefined
 
+  const layer = Layer.mergeAll(Logger.prettyWithThread(self.name), FetchHttpClient.layer, TracingLive ?? Layer.empty)
+
   return makeWorkerRunnerOuter(options).pipe(
     Layer.provide(BrowserWorkerRunner.layer),
     WorkerRunner.launch,
     Effect.scoped,
     Effect.tapCauseLogPretty,
     Effect.annotateLogs({ thread: self.name }),
-    Effect.provide(Logger.prettyWithThread(self.name)),
-    Effect.provide(FetchHttpClient.layer),
+    Effect.provide(layer),
     LS_DEV ? TaskTracing.withAsyncTaggingTracing((name) => (console as any).createTask(name)) : identity,
-    TracingLive ? Effect.provide(TracingLive) : identity,
     // We're using this custom scheduler to improve op batching behaviour and reduce the overhead
     // of the Effect fiber runtime given we have different tradeoffs on a worker thread.
     // Despite the "message channel" name, is has nothing to do with the `incomingRequestsPort` above.
@@ -131,6 +131,16 @@ const makeWorkerRunnerInner = ({ schema, sync: syncOptions }: WorkerOptions) =>
           concurrency: 2,
         })
 
+        // Clean up old state database files after successful database creation
+        // This prevents OPFS file pool capacity exhaustion from accumulated state db files after schema changes/migrations
+        if (dbState.metadata._tag === 'opfs') {
+          yield* cleanupOldStateDbFiles({
+            vfs: dbState.metadata.vfs,
+            currentSchema: schema,
+            opfsDirectory: dbState.metadata.persistenceInfo.opfsDirectory,
+          })
+        }
+
         const devtoolsOptions = yield* makeDevtoolsOptions({ devtoolsEnabled, dbState, dbEventlog })
         const shutdownChannel = yield* makeShutdownChannel(storeId)
 
@@ -212,6 +222,21 @@ const makeWorkerRunnerInner = ({ schema, sync: syncOptions }: WorkerOptions) =>
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-web:worker:GetLeaderSyncState'),
       ),
+    SyncStateStream: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return workerCtx.syncProcessor.syncState.changes
+      }).pipe(Stream.unwrapScoped),
+    GetNetworkStatus: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return yield* workerCtx.networkStatus
+      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-web:worker:GetNetworkStatus')),
+    NetworkStatusStream: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return workerCtx.networkStatus.changes
+      }).pipe(Stream.unwrapScoped),
     Shutdown: () =>
       Effect.gen(function* () {
         yield* Effect.logDebug('[@livestore/adapter-web:worker] Shutdown')

package/src/web-worker/shared-worker/make-shared-worker.ts

@@ -28,6 +28,20 @@ import {
 import { makeShutdownChannel } from '../common/shutdown-channel.ts'
 import * as WorkerSchema from '../common/worker-schema.ts'
 
+// Extract from `livestore-shared-worker-${storeId}`
+const storeId = self.name.replace('livestore-shared-worker-', '')
+
+// We acquire a lock that is held as long as this shared worker is alive.
+// This way, when the shared worker is terminated (e.g. by the browser when the page is closed),
+// the lock is released and any thread waiting for the lock can be notified.
+const LIVESTORE_SHARED_WORKER_TERMINATION_LOCK = `livestore-shared-worker-termination-lock-${storeId}`
+navigator.locks.request(
+  LIVESTORE_SHARED_WORKER_TERMINATION_LOCK,
+  { steal: true },
+  // We use a never-resolving promise to hold the lock
+  async () => new Promise(() => {}),
+)
+
 if (isDevEnv()) {
   globalThis.__debugLiveStoreUtils = {
     blobUrl: (buffer: Uint8Array<ArrayBuffer>) =>
@@ -56,12 +70,15 @@ const makeWorkerRunner = Effect.gen(function* () {
 
   const forwardRequest = <TReq extends WorkerSchema.LeaderWorkerInnerRequest>(
     req: TReq,
-  ): TReq extends Schema.WithResult<infer A, infer _I, infer _E, infer _EI, infer _R>
-    ? Effect.Effect<A, UnexpectedError, never>
-    : never =>
+  ): Effect.Effect<
+    Schema.WithResult.Success<TReq>,
+    UnexpectedError | Schema.WithResult.Failure<TReq>,
+    Schema.WithResult.Context<TReq>
+  > =>
+    // Forward the request to the active worker and normalize platform errors into UnexpectedError.
     waitForWorker.pipe(
       // Effect.logBefore(`forwardRequest: ${req._tag}`),
-      Effect.andThen((worker) => worker.executeEffect(req) as Effect.Effect<unknown, unknown, never>),
+      Effect.andThen((worker) => worker.executeEffect(req) as Effect.Effect<unknown, unknown, unknown>),
       // Effect.tap((_) => Effect.log(`forwardRequest: ${req._tag}`, _)),
       // Effect.tapError((cause) => Effect.logError(`forwardRequest err: ${req._tag}`, cause)),
       Effect.interruptible,
@@ -78,17 +95,23 @@ const makeWorkerRunner = Effect.gen(function* () {
       ),
       Effect.catchAllDefect((cause) => new UnexpectedError({ cause })),
       Effect.tapCauseLogPretty,
-    ) as any
+    ) as Effect.Effect<
+      Schema.WithResult.Success<TReq>,
+      UnexpectedError | Schema.WithResult.Failure<TReq>,
+      Schema.WithResult.Context<TReq>
+    >
 
   const forwardRequestStream = <TReq extends WorkerSchema.LeaderWorkerInnerRequest>(
     req: TReq,
-  ): TReq extends Schema.WithResult<infer A, infer _I, infer _E, infer _EI, infer _R>
-    ? Stream.Stream<A, UnexpectedError, never>
-    : never =>
+  ): Stream.Stream<
+    Schema.WithResult.Success<TReq>,
+    UnexpectedError | Schema.WithResult.Failure<TReq>,
+    Schema.WithResult.Context<TReq>
+  > =>
     Effect.gen(function* () {
       yield* Effect.logDebug(`forwardRequestStream: ${req._tag}`)
       const { worker, scope } = yield* SubscriptionRef.waitUntil(leaderWorkerContextSubRef, isNotUndefined)
-      const stream = worker.execute(req) as Stream.Stream<unknown, unknown, never>
+      const stream = worker.execute(req) as Stream.Stream<unknown, unknown, unknown>
 
       // It seems the request stream is not automatically interrupted when the scope shuts down
       // so we need to manually interrupt it when the scope shuts down
@@ -109,7 +132,11 @@ const makeWorkerRunner = Effect.gen(function* () {
       Stream.unwrap,
       Stream.ensuring(Effect.logDebug(`shutting down stream for ${req._tag}`)),
       UnexpectedError.mapToUnexpectedErrorStream,
-    ) as any
+    ) as Stream.Stream<
+      Schema.WithResult.Success<TReq>,
+      UnexpectedError | Schema.WithResult.Failure<TReq>,
+      Schema.WithResult.Context<TReq>
+    >
 
   const resetCurrentWorkerCtx = Effect.gen(function* () {
     const prevWorker = yield* SubscriptionRef.get(leaderWorkerContextSubRef)
@@ -231,7 +258,10 @@ const makeWorkerRunner = Effect.gen(function* () {
     ExportEventlog: forwardRequest,
     Setup: forwardRequest,
     GetLeaderSyncState: forwardRequest,
+    SyncStateStream: forwardRequestStream,
     GetLeaderHead: forwardRequest,
+    GetNetworkStatus: forwardRequest,
+    NetworkStatusStream: forwardRequestStream,
     Shutdown: forwardRequest,
     ExtraDevtoolsMessage: forwardRequest,
 
@@ -241,8 +271,11 @@ const makeWorkerRunner = Effect.gen(function* () {
 }).pipe(Layer.unwrapScoped)
 
 export const makeWorker = () => {
-  // Extract from `livestore-shared-worker-${storeId}`
-  const storeId = self.name.replace('livestore-shared-worker-', '')
+  const layer = Layer.mergeAll(
+    Logger.prettyWithThread(self.name),
+    FetchHttpClient.layer,
+    WebmeshWorker.CacheService.layer({ nodeName: DevtoolsWeb.makeNodeName.sharedWorker({ storeId }) }),
+  )
 
   makeWorkerRunner.pipe(
     Layer.provide(BrowserWorkerRunner.layer),
@@ -251,9 +284,7 @@ export const makeWorker = () => {
     Effect.scoped,
     Effect.tapCauseLogPretty,
     Effect.annotateLogs({ thread: self.name }),
-    Effect.provide(Logger.prettyWithThread(self.name)),
-    Effect.provide(FetchHttpClient.layer),
-    Effect.provide(WebmeshWorker.CacheService.layer({ nodeName: DevtoolsWeb.makeNodeName.sharedWorker({ storeId }) })),
+    Effect.provide(layer),
     LS_DEV ? TaskTracing.withAsyncTaggingTracing((name) => (console as any).createTask(name)) : identity,
     // TODO remove type-cast (currently needed to silence a tsc bug)
     (_) => _ as any as Effect.Effect<void, any>,

package/src/web-worker/vite-dev-polyfill.ts

@@ -5,6 +5,7 @@ globalThis.$RefreshReg$ = () => {}
 // @ts-expect-error TODO remove when Vite does proper treeshaking during dev
 globalThis.$RefreshSig$ = () => (type: any) => type
 
+// biome-ignore lint/suspicious/noTsIgnore: sometimes @types/node is there, sometimes not.
 // @ts-ignore
 globalThis.process = globalThis.process ?? { env: {} }