@dxos/client-services 0.14.0 → 0.16.0

package/src/internal/spaces/spaces-service.ts

@@ -66,6 +66,7 @@ import * as SpacesContract from '../../contracts/spaces.ts';
 import * as Readiness from '../../Readiness.ts';
 import {
   SpaceArchiveWriter,
+  collectFeedMessages,
   detectSpaceArchiveFormat,
   extractSpaceArchive,
   objJsonToObjectStructure,
@@ -395,8 +396,13 @@ export class SpacesServiceImpl implements SpacesService.Handlers {
         const space = dataSpaceManager.getSpaceById(request.spaceId) ?? raise(new Error('Space not found'));
 
         const format = request.format ?? SpacesService.SpaceArchiveFormat.enums.BINARY;
-        if (format === SpacesService.SpaceArchiveFormat.enums.JSON) {
-          const archive = await writeSerializedSpaceArchive({ space, echoHost: this._echoHost });
+        // A binary archive holds whole automerge documents, which cannot be narrowed to a few feeds.
+        if (format === SpacesService.SpaceArchiveFormat.enums.JSON || request.feedIds) {
+          const archive = await writeSerializedSpaceArchive({
+            space,
+            echoHost: this._echoHost,
+            ...(request.feedIds ? { feedIds: request.feedIds } : {}),
+          });
           return { archive };
         }
 
@@ -445,6 +451,20 @@ export class SpacesServiceImpl implements SpacesService.Handlers {
         const dataSpaceManager = await this._getDataSpaceManager();
 
         const format = request.archive.format ?? detectSpaceArchiveFormat(request.archive);
+        if (request.targetSpaceId) {
+          assertArgument(
+            format === SpacesService.SpaceArchiveFormat.enums.JSON,
+            'archive',
+            'Only a JSON archive can be imported into an existing space',
+          );
+          assertArgument(SpaceId.isValid(request.targetSpaceId), 'targetSpaceId', 'Invalid space ID');
+          const target =
+            dataSpaceManager.getSpaceById(request.targetSpaceId) ?? raise(new Error('Target space not found'));
+          await this._hydrateSpaceFromSerialized(target, readSerializedSpaceArchive(request.archive), {
+            merge: true,
+          });
+          return { newSpaceId: target.id };
+        }
         if (format === SpacesService.SpaceArchiveFormat.enums.JSON) {
           const serialized = readSerializedSpaceArchive(request.archive);
           const space = await dataSpaceManager.createSpace(ctx, { tags: request.tags });
@@ -477,6 +497,7 @@ export class SpacesServiceImpl implements SpacesService.Handlers {
   private async '_hydrateSpaceFromSerialized'(
     space: DataSpace,
     serialized: ReturnType<typeof readSerializedSpaceArchive>,
+    { merge = false }: { merge?: boolean } = {},
   ): Promise<void> {
     const databaseRoot = space.databaseRoot;
     assertState(databaseRoot, 'Space database root is not ready');
@@ -486,17 +507,30 @@ export class SpacesServiceImpl implements SpacesService.Handlers {
         doc.objects = {};
       }
       for (const obj of serialized.objects) {
+        // Merging into a live space keeps its own copy of an object the archive also holds, whether it is inline in
+        // the root document or in a linked document of its own.
+        if (merge && (doc.objects[obj.id] || doc.links?.[obj.id])) {
+          continue;
+        }
         doc.objects[obj.id] = objJsonToObjectStructure(obj);
       }
     });
 
     for (const feed of serialized.feeds ?? []) {
-      if (feed.messages.length === 0) {
+      const namespace =
+        feed.namespace === 'trace' ? FeedProtocol.WellKnownNamespaces.trace : FeedProtocol.WellKnownNamespaces.data;
+      const held = merge
+        ? new Set(
+            (await collectFeedMessages(this._echoHost, space.id, feed.feedObjectId, feed.namespace)).map(
+              (message) => message.id,
+            ),
+          )
+        : new Set<string>();
+      const messages = feed.messages.filter((message) => !held.has(message.id));
+      if (messages.length === 0) {
         continue;
       }
 
-      const namespace =
-        feed.namespace === 'trace' ? FeedProtocol.WellKnownNamespaces.trace : FeedProtocol.WellKnownNamespaces.data;
       try {
         // Handlers are served, not called directly; bridge to a client in-process for this write.
         const feedService = this._echoHost.feedService;
@@ -507,11 +541,15 @@ export class SpacesServiceImpl implements SpacesService.Handlers {
               spaceId: space.id,
               feedId: feed.feedObjectId,
               subspaceTag: namespace,
-              objects: feed.messages.map((message) => JSON.stringify(message)),
+              objects: messages.map((message) => JSON.stringify(message)),
             });
           }).pipe(Effect.scoped),
         );
       } catch (err) {
+        // A merge into an existing space must fail rather than report success with messages missing.
+        if (merge) {
+          throw err;
+        }
         log.warn('failed to import feed data', { feedObjectId: feed.feedObjectId, error: err });
       }
     }

package/src/internal/storage/index.ts

@@ -5,3 +5,4 @@
 export * from './storage.ts';
 export * from './profile-archive.ts';
 export * from './profile-archive-sqlite.ts';
+export * from './storage-watchdog.ts';

package/src/internal/storage/storage-watchdog.test.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Deferred from 'effect/Deferred';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as SqlClient from 'effect/sql/SqlClient';
+import { describe, test } from 'vitest';
+
+import * as Hook from '@dxos/effect/Hook';
+import { layerMemory } from '@dxos/sql-sqlite/Platform';
+
+import * as Events from '../../Events.ts';
+import { storageWatchdogLayer } from './storage-watchdog.ts';
+
+/** Opens a stack watched by the watchdog, optionally with a transaction holding the connection, and reports whether it was declared stuck. */
+const reportsStuck = ({ hold }: { hold: boolean }) =>
+  Effect.gen(function* () {
+    const stuck = yield* Deferred.make<void>();
+    yield* Hook.on(Events.StorageStuck, () => Deferred.succeed(stuck, undefined));
+    yield* Layer.build(storageWatchdogLayer({ interval: '10 millis', stuckAfter: '50 millis' }));
+    if (hold) {
+      const sql = yield* SqlClient.SqlClient;
+      yield* Effect.forkScoped(sql.withTransaction(Effect.never));
+    }
+    yield* Hook.emit(Events.StackOpened, undefined);
+    return Option.isSome(yield* Deferred.await(stuck).pipe(Effect.timeoutOption('500 millis')));
+  }).pipe(
+    Effect.scoped,
+    Effect.provide(layerMemory),
+    Effect.provideService(Hook.Controller, Hook.makeController()),
+    Effect.orDie,
+    Effect.runPromise,
+  );
+
+describe('storageWatchdogLayer', () => {
+  test('a connection held past the limit is reported stuck', async ({ expect }) => {
+    expect(await reportsStuck({ hold: true })).toBe(true);
+  });
+
+  test('a free connection is not', async ({ expect }) => {
+    expect(await reportsStuck({ hold: false })).toBe(false);
+  });
+});

package/src/internal/storage/storage-watchdog.ts

@@ -0,0 +1,58 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as SqlClient from 'effect/sql/SqlClient';
+
+import * as Hook from '@dxos/effect/Hook';
+import { log } from '@dxos/log';
+
+import * as Events from '../../Events.ts';
+
+export type StorageWatchdogOptions = {
+  /** How often the connection is asked for. */
+  interval?: Duration.Input;
+  /** How long a probe may wait for the connection before storage counts as stuck: above any one statement a healthy store runs. */
+  stuckAfter?: Duration.Input;
+};
+
+/**
+ * Once the stack is open, asks for the SQLite connection on an interval. Every statement and index pass
+ * goes through that one connection, so a probe that cannot get it means nothing more will be saved:
+ * {@link Events.StorageStuck} is emitted once, for the embedder to replace the process.
+ */
+export const storageWatchdogLayer = ({
+  interval = Duration.seconds(30),
+  stuckAfter = Duration.minutes(2),
+}: StorageWatchdogOptions = {}) =>
+  Layer.effectDiscard(
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+      const controller = yield* Hook.Controller;
+      const scope = yield* Effect.scope;
+
+      const watch = Effect.gen(function* () {
+        while (true) {
+          yield* Effect.sleep(interval);
+          const startedAt = Date.now();
+          // A failing statement still got the connection; only a probe that never runs is stuck.
+          const answered = yield* sql`SELECT 1`.pipe(
+            Effect.timeoutOption(stuckAfter),
+            Effect.catch(() => Effect.succeed(Option.some([]))),
+          );
+          if (Option.isNone(answered)) {
+            log.error('sqlite connection stuck, requesting a new process', { waitedMs: Date.now() - startedAt });
+            yield* Hook.emit(Events.StorageStuck, undefined);
+            return;
+          }
+        }
+      }).pipe(Effect.provideService(Hook.Controller, controller));
+
+      // Forked: `emit` waits on its handlers, and this one runs for the life of the stack.
+      yield* Hook.on(Events.StackOpened, () => Effect.asVoid(Effect.forkIn(watch, scope)));
+    }),
+  );

package/src/internal/worker/worker-runtime.ts

@@ -168,6 +168,8 @@ export const makeWorkerRuntime = ({
       SqliteStorage.wipeSqliteStorage.pipe(Effect.provide(sqlite), Effect.orDie),
     );
     yield* Hook.on(Events.Reset, () => requestShutdown);
+    // The tabs hold every change this process has not saved, and reconnect to a replacement with them.
+    yield* Hook.on(Events.StorageStuck, () => requestShutdown);
     yield* Hook.on(
       SessionClosed,
       Effect.fn('WorkerRuntime.onSessionClosed')(function* ({ session }) {

package/src/migrations/blobs/0001_init.sql

@@ -0,0 +1,17 @@
+--
+-- Device-local, content-addressed blob bytes, kept outside Automerge so large blobs do not bloat
+-- documents. `uploaded_at` is NULL while the hosted store does not yet hold the bytes; the partial
+-- index makes that upload ledger cheap to scan.
+--
+-- Immutable: recorded in `blobs_migrations` and never re-run.
+--
+CREATE TABLE IF NOT EXISTS blobs (
+  hash TEXT PRIMARY KEY,
+  type TEXT,
+  size INTEGER NOT NULL,
+  data BLOB NOT NULL,
+  created_at INTEGER NOT NULL,
+  uploaded_at INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS blobs_pending ON blobs (created_at) WHERE uploaded_at IS NULL;

package/src/migrations/blobs/index.ts

@@ -0,0 +1,18 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as SqlMigrations from '@dxos/sql-sqlite/SqlMigrations';
+
+import init from './0001_init.sql?raw';
+
+/**
+ * Local blob store migrations, keyed `<id>_<name>` as `Migrator.fromRecord` expects. Ids must only ever increase,
+ * and an applied migration must never be edited — nothing verifies its contents.
+ */
+export const MIGRATIONS = {
+  '0001_init': SqlMigrations.apply(init),
+};
+
+/** Own history table per store, since many stores share the client database. */
+export const MIGRATIONS_TABLE = 'blobs_migrations';

package/src/migrations/migrations.test.ts

@@ -11,12 +11,15 @@ import * as EffectEx from '@dxos/effect/EffectEx';
 import { layerMemory } from '@dxos/sql-sqlite/Platform';
 import * as SqlMigrations from '@dxos/sql-sqlite/SqlMigrations';
 
+import blobsInit from './blobs/0001_init.sql?raw';
+import { MIGRATIONS as BLOBS, MIGRATIONS_TABLE as BLOBS_TABLE } from './blobs/index.ts';
 import hypercoreInit from './hypercore/0001_init.sql?raw';
 import { MIGRATIONS as HYPERCORE, MIGRATIONS_TABLE as HYPERCORE_TABLE } from './hypercore/index.ts';
 import metadataInit from './metadata/0001_init.sql?raw';
 import { MIGRATIONS as METADATA, MIGRATIONS_TABLE as METADATA_TABLE } from './metadata/index.ts';
 
 const STORES = [
+  { name: 'blobs', init: blobsInit, manifest: BLOBS, table: BLOBS_TABLE },
   { name: 'hypercore', init: hypercoreInit, manifest: HYPERCORE, table: HYPERCORE_TABLE },
   { name: 'metadata', init: metadataInit, manifest: METADATA, table: METADATA_TABLE },
 ];

package/src/version.ts

@@ -2,4 +2,4 @@
 // Copyright 2023 DXOS.org
 //
 
-export const DXOS_VERSION = '0.14.0';
+export const DXOS_VERSION = '0.16.0';