@dxos/client-services 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/lib/chunk-client-services-stack.mjs +799 -573
  2. package/dist/lib/chunk-client-services-stack.mjs.map +1 -1
  3. package/dist/lib/index.mjs +139 -131
  4. package/dist/lib/index.mjs.map +1 -1
  5. package/dist/lib/testing.mjs +1 -1
  6. package/dist/lib/testing.mjs.map +1 -1
  7. package/dist/types/src/Events.d.ts +2 -0
  8. package/dist/types/src/Events.d.ts.map +1 -1
  9. package/dist/types/src/internal/blobs/blob-replication.d.ts +28 -0
  10. package/dist/types/src/internal/blobs/blob-replication.d.ts.map +1 -0
  11. package/dist/types/src/internal/blobs/blob-store-service.d.ts +10 -0
  12. package/dist/types/src/internal/blobs/blob-store-service.d.ts.map +1 -0
  13. package/dist/types/src/internal/blobs/blob-store-service.test.d.ts +2 -0
  14. package/dist/types/src/internal/blobs/blob-store-service.test.d.ts.map +1 -0
  15. package/dist/types/src/internal/blobs/index.d.ts +4 -0
  16. package/dist/types/src/internal/blobs/index.d.ts.map +1 -0
  17. package/dist/types/src/internal/blobs/local-blob-store.d.ts +21 -0
  18. package/dist/types/src/internal/blobs/local-blob-store.d.ts.map +1 -0
  19. package/dist/types/src/internal/blobs/local-blob-store.test.d.ts +2 -0
  20. package/dist/types/src/internal/blobs/local-blob-store.test.d.ts.map +1 -0
  21. package/dist/types/src/internal/services/layer-specs.d.ts +6 -0
  22. package/dist/types/src/internal/services/layer-specs.d.ts.map +1 -1
  23. package/dist/types/src/internal/services/service-stack.d.ts.map +1 -1
  24. package/dist/types/src/internal/space-export/serialized-space-writer.d.ts +4 -1
  25. package/dist/types/src/internal/space-export/serialized-space-writer.d.ts.map +1 -1
  26. package/dist/types/src/internal/spaces/spaces-service.d.ts.map +1 -1
  27. package/dist/types/src/internal/storage/index.d.ts +1 -0
  28. package/dist/types/src/internal/storage/index.d.ts.map +1 -1
  29. package/dist/types/src/internal/storage/storage-watchdog.d.ts +17 -0
  30. package/dist/types/src/internal/storage/storage-watchdog.d.ts.map +1 -0
  31. package/dist/types/src/internal/storage/storage-watchdog.test.d.ts +2 -0
  32. package/dist/types/src/internal/storage/storage-watchdog.test.d.ts.map +1 -0
  33. package/dist/types/src/internal/worker/worker-runtime.d.ts.map +1 -1
  34. package/dist/types/src/migrations/blobs/index.d.ts +10 -0
  35. package/dist/types/src/migrations/blobs/index.d.ts.map +1 -0
  36. package/dist/types/src/version.d.ts +1 -1
  37. package/dist/types/tsconfig.tsbuildinfo +1 -1
  38. package/package.json +43 -42
  39. package/src/Events.ts +3 -0
  40. package/src/internal/blobs/blob-replication.ts +80 -0
  41. package/src/internal/blobs/blob-store-service.test.ts +76 -0
  42. package/src/internal/blobs/blob-store-service.ts +45 -0
  43. package/src/internal/blobs/index.ts +7 -0
  44. package/src/internal/blobs/local-blob-store.test.ts +53 -0
  45. package/src/internal/blobs/local-blob-store.ts +89 -0
  46. package/src/internal/services/layer-specs.ts +55 -0
  47. package/src/internal/services/service-stack.ts +2 -0
  48. package/src/internal/space-export/serialized-space-writer.ts +16 -9
  49. package/src/internal/spaces/spaces-service.ts +44 -6
  50. package/src/internal/storage/index.ts +1 -0
  51. package/src/internal/storage/storage-watchdog.test.ts +46 -0
  52. package/src/internal/storage/storage-watchdog.ts +58 -0
  53. package/src/internal/worker/worker-runtime.ts +2 -0
  54. package/src/migrations/blobs/0001_init.sql +17 -0
  55. package/src/migrations/blobs/index.ts +18 -0
  56. package/src/migrations/migrations.test.ts +3 -0
  57. package/src/version.ts +1 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/client-services",
3
- "version": "0.14.0",
3
+ "version": "0.16.0",
4
4
  "description": "DXOS client services implementation",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -34,54 +34,55 @@
34
34
  "@obsidize/tar-browserify": "^5.2.0",
35
35
  "cbor-x": "^1.5.4",
36
36
  "crc-32": "^1.2.2",
37
- "effect": "4.0.0",
37
+ "effect": "4.0.4",
38
38
  "platform": "^1.3.6",
39
- "@dxos/async": "0.14.0",
40
- "@dxos/client-protocol": "0.14.0",
41
- "@dxos/compute": "0.14.0",
42
- "@dxos/compute-runtime": "0.14.0",
43
- "@dxos/config": "0.14.0",
44
- "@dxos/context": "0.14.0",
45
- "@dxos/credentials": "0.14.0",
46
- "@dxos/crypto": "0.14.0",
47
- "@dxos/debug": "0.14.0",
48
- "@dxos/echo": "0.14.0",
49
- "@dxos/echo-client": "0.14.0",
50
- "@dxos/echo-doc": "0.14.0",
51
- "@dxos/echo-host": "0.14.0",
52
- "@dxos/echo-protocol": "0.14.0",
53
- "@dxos/edge-client": "0.14.0",
54
- "@dxos/effect": "0.14.0",
55
- "@dxos/errors": "0.14.0",
56
- "@dxos/feed": "0.14.0",
57
- "@dxos/feed-store": "0.14.0",
58
- "@dxos/hypercore": "0.14.0",
59
- "@dxos/invariant": "0.14.0",
60
- "@dxos/keyring": "0.14.0",
61
- "@dxos/keys": "0.14.0",
62
- "@dxos/log": "0.14.0",
63
- "@dxos/messaging": "0.14.0",
64
- "@dxos/network-manager": "0.14.0",
65
- "@dxos/node-std": "0.14.0",
66
- "@dxos/protocols": "0.14.0",
67
- "@dxos/random-access-storage": "0.14.0",
68
- "@dxos/rpc": "0.14.0",
69
- "@dxos/sql-sqlite": "0.14.0",
70
- "@dxos/teleport": "0.14.0",
71
- "@dxos/teleport-extension-gossip": "0.14.0",
72
- "@dxos/teleport-extension-replicator": "0.14.0",
73
- "@dxos/timeframe": "0.14.0",
74
- "@dxos/tracing": "0.14.0",
75
- "@dxos/types": "0.14.0",
76
- "@dxos/util": "0.14.0"
39
+ "@dxos/async": "0.16.0",
40
+ "@dxos/blob": "0.16.0",
41
+ "@dxos/client-protocol": "0.16.0",
42
+ "@dxos/compute": "0.16.0",
43
+ "@dxos/compute-runtime": "0.16.0",
44
+ "@dxos/config": "0.16.0",
45
+ "@dxos/context": "0.16.0",
46
+ "@dxos/credentials": "0.16.0",
47
+ "@dxos/crypto": "0.16.0",
48
+ "@dxos/debug": "0.16.0",
49
+ "@dxos/echo": "0.16.0",
50
+ "@dxos/echo-client": "0.16.0",
51
+ "@dxos/echo-doc": "0.16.0",
52
+ "@dxos/echo-host": "0.16.0",
53
+ "@dxos/echo-protocol": "0.16.0",
54
+ "@dxos/edge-client": "0.16.0",
55
+ "@dxos/effect": "0.16.0",
56
+ "@dxos/errors": "0.16.0",
57
+ "@dxos/feed": "0.16.0",
58
+ "@dxos/feed-store": "0.16.0",
59
+ "@dxos/hypercore": "0.16.0",
60
+ "@dxos/invariant": "0.16.0",
61
+ "@dxos/keyring": "0.16.0",
62
+ "@dxos/keys": "0.16.0",
63
+ "@dxos/log": "0.16.0",
64
+ "@dxos/messaging": "0.16.0",
65
+ "@dxos/network-manager": "0.16.0",
66
+ "@dxos/node-std": "0.16.0",
67
+ "@dxos/protocols": "0.16.0",
68
+ "@dxos/random-access-storage": "0.16.0",
69
+ "@dxos/rpc": "0.16.0",
70
+ "@dxos/sql-sqlite": "0.16.0",
71
+ "@dxos/teleport": "0.16.0",
72
+ "@dxos/teleport-extension-gossip": "0.16.0",
73
+ "@dxos/teleport-extension-replicator": "0.16.0",
74
+ "@dxos/timeframe": "0.16.0",
75
+ "@dxos/tracing": "0.16.0",
76
+ "@dxos/types": "0.16.0",
77
+ "@dxos/util": "0.16.0"
77
78
  },
78
79
  "devDependencies": {
79
- "@effect/platform-browser": "4.0.0",
80
+ "@effect/platform-browser": "4.0.4",
80
81
  "@types/platform": "^1.3.4",
81
82
  "fast-check": "^3.19.0",
82
83
  "get-port-please": "^3.1.1",
83
84
  "random-access-storage": "^1.3.0",
84
- "@dxos/test-utils": "0.14.0"
85
+ "@dxos/test-utils": "0.16.0"
85
86
  },
86
87
  "publishConfig": {
87
88
  "access": "public"
package/src/Events.ts CHANGED
@@ -71,5 +71,8 @@ export const WipingStorage = Hook.make<void>()('client-services/WipingStorage');
71
71
  /** Storage is wiped; the embedder typically reloads. */
72
72
  export const Reset = Hook.make<void>()('client-services/Reset');
73
73
 
74
+ /** Storage stopped answering, so nothing more will be saved; the embedder replaces the process. */
75
+ export const StorageStuck = Hook.make<void>()('client-services/StorageStuck');
76
+
74
77
  /** The local profile changed and should be broadcast to every open space. */
75
78
  export const ProfileUpdated = Hook.make<{ profile: ProfileDocument }>()('client-services/ProfileUpdated');
@@ -0,0 +1,80 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as EffectContext from 'effect/Context';
6
+ import * as Effect from 'effect/Effect';
7
+ import * as Layer from 'effect/Layer';
8
+
9
+ import { type BlobStore, type BlobTransport } from '@dxos/blob';
10
+ import { type BlobReplicator, createBlobReplicator, withReadThrough } from '@dxos/blob/hosted';
11
+ import { Context } from '@dxos/context';
12
+ import { type EdgeHttpClient, EdgeHttpClientService } from '@dxos/edge-client';
13
+ import * as Hook from '@dxos/effect/Hook';
14
+
15
+ import * as Events from '../../Events.ts';
16
+ import { LocalBlobStoreService } from './local-blob-store.ts';
17
+
18
+ /**
19
+ * Replication of the local blob store with the EDGE blob service: pushes stored blobs asynchronously
20
+ * and fills local misses. Present only when an EDGE endpoint is configured; without it the local
21
+ * store is complete on its own.
22
+ */
23
+ export interface BlobReplication {
24
+ /** The local store, filling misses from EDGE and caching them. */
25
+ readonly store: BlobStore;
26
+ /** Signals that a blob was written locally and may need pushing; never waits on the network. */
27
+ wake(): void;
28
+ }
29
+
30
+ export class BlobReplicationService extends EffectContext.Service<BlobReplicationService, BlobReplication>()(
31
+ '@dxos/client-services/BlobReplication',
32
+ ) {}
33
+
34
+ /**
35
+ * Replication for a stack with no EDGE endpoint: reads are served by the local store alone and writes
36
+ * stay in its ledger, to be pushed by a later session that has one.
37
+ */
38
+ export const BlobReplicationNoopLayer: Layer.Layer<BlobReplicationService, never, LocalBlobStoreService> = Layer.effect(
39
+ BlobReplicationService,
40
+ Effect.gen(function* () {
41
+ const local = yield* LocalBlobStoreService;
42
+ return { store: local, wake: () => {} };
43
+ }),
44
+ );
45
+
46
+ const edgeBlobTransport = (edge: EdgeHttpClient): BlobTransport => ({
47
+ url: (key) => edge.getBlobUrl(key),
48
+ put: (key, data, options) => edge.putBlob(Context.default(), key, data, options),
49
+ get: (key) => edge.getBlob(Context.default(), key),
50
+ has: (key) => edge.hasBlob(Context.default(), key),
51
+ });
52
+
53
+ export const BlobReplicationLayer: Layer.Layer<
54
+ BlobReplicationService,
55
+ never,
56
+ LocalBlobStoreService | EdgeHttpClientService | Hook.Controller
57
+ > = Layer.effect(
58
+ BlobReplicationService,
59
+ Effect.gen(function* () {
60
+ const local = yield* LocalBlobStoreService;
61
+ const transport = edgeBlobTransport(yield* EdgeHttpClientService);
62
+
63
+ // Started once the network identity is set, since EDGE authenticates uploads; writes before that
64
+ // wait in the ledger.
65
+ let replicator: BlobReplicator | undefined;
66
+ yield* Hook.on(
67
+ Events.NetworkReady,
68
+ Effect.fn('BlobReplication.onNetworkReady')(function* () {
69
+ replicator ??= createBlobReplicator({ local, transport });
70
+ replicator.wake();
71
+ }),
72
+ );
73
+ yield* Effect.addFinalizer(() => Effect.promise(async () => replicator?.close()));
74
+
75
+ return {
76
+ store: withReadThrough({ local, transport }),
77
+ wake: () => replicator?.wake(),
78
+ };
79
+ }),
80
+ );
@@ -0,0 +1,76 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as Effect from 'effect/Effect';
6
+ import * as Layer from 'effect/Layer';
7
+ import { describe, expect, test, vi } from 'vitest';
8
+
9
+ import { type LocalBlobStore, createMemoryBlobStore } from '@dxos/blob';
10
+ import * as EffectEx from '@dxos/effect/EffectEx';
11
+ import { makeInProcessClient } from '@dxos/protocols';
12
+ import { BlobStoreService } from '@dxos/protocols/rpc';
13
+
14
+ import { type BlobReplication, BlobReplicationNoopLayer, BlobReplicationService } from './blob-replication.ts';
15
+ import { BlobStoreServiceLayer } from './blob-store-service.ts';
16
+ import { LocalBlobStoreService } from './local-blob-store.ts';
17
+
18
+ /** Runs `body` against the service over an in-process RPC client, with replication only if given. */
19
+ const withService = (
20
+ local: LocalBlobStore,
21
+ replication: BlobReplication | undefined,
22
+ body: (client: BlobStoreService.Client) => Effect.Effect<void, unknown>,
23
+ ) => {
24
+ const localLayer = Layer.succeed(LocalBlobStoreService, local);
25
+ const dependencies = Layer.mergeAll(
26
+ localLayer,
27
+ replication
28
+ ? Layer.succeed(BlobReplicationService, replication)
29
+ : BlobReplicationNoopLayer.pipe(Layer.provide(localLayer)),
30
+ );
31
+ return EffectEx.runPromise(
32
+ Effect.gen(function* () {
33
+ const handlers = yield* BlobStoreService.Tag;
34
+ yield* body(yield* makeInProcessClient(BlobStoreService.Rpcs, handlers));
35
+ }).pipe(Effect.provide(BlobStoreServiceLayer.pipe(Layer.provide(dependencies))), Effect.scoped),
36
+ );
37
+ };
38
+
39
+ describe('BlobStoreService', () => {
40
+ test('with the noop replication, writes and reads are served by the local store alone', async () => {
41
+ const local = createMemoryBlobStore();
42
+ await withService(local, undefined, (client) =>
43
+ Effect.gen(function* () {
44
+ yield* client['BlobStoreService.put']({ key: 'aa', data: new Uint8Array([1]) });
45
+ expect(yield* client['BlobStoreService.get']({ key: 'aa' })).toEqual({ blob: { data: new Uint8Array([1]) } });
46
+ expect(yield* client['BlobStoreService.get']({ key: 'bb' })).toEqual({});
47
+ expect(yield* client['BlobStoreService.has']({ key: 'aa' })).toEqual({ exists: true });
48
+ }),
49
+ );
50
+ expect(await local.listPending({ limit: 10 })).toEqual(['aa']);
51
+ });
52
+
53
+ test('with replication, a write is stored locally before replication is woken, and reads go through it', async () => {
54
+ const local = createMemoryBlobStore();
55
+ const remote = createMemoryBlobStore();
56
+ await remote.put('bb', new Uint8Array([2]));
57
+ // What the local store held at the moment replication was told, to show confirmation comes first.
58
+ const storedWhenWoken: Promise<boolean>[] = [];
59
+ const wake = vi.fn(() => {
60
+ storedWhenWoken.push(local.has('aa'));
61
+ });
62
+ const replication: BlobReplication = {
63
+ store: { put: local.put, get: async (key) => (await local.get(key)) ?? remote.get(key), has: local.has },
64
+ wake,
65
+ };
66
+
67
+ await withService(local, replication, (client) =>
68
+ Effect.gen(function* () {
69
+ yield* client['BlobStoreService.put']({ key: 'aa', data: new Uint8Array([1]) });
70
+ expect(wake).toHaveBeenCalledTimes(1);
71
+ expect(yield* Effect.promise(() => Promise.all(storedWhenWoken))).toEqual([true]);
72
+ expect(yield* client['BlobStoreService.get']({ key: 'bb' })).toEqual({ blob: { data: new Uint8Array([2]) } });
73
+ }),
74
+ );
75
+ });
76
+ });
@@ -0,0 +1,45 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as Effect from 'effect/Effect';
6
+ import * as Layer from 'effect/Layer';
7
+
8
+ import { toServiceError } from '@dxos/protocols';
9
+ import { BlobStoreService } from '@dxos/protocols/rpc';
10
+
11
+ import { BlobReplicationService } from './blob-replication.ts';
12
+ import { LocalBlobStoreService } from './local-blob-store.ts';
13
+
14
+ /**
15
+ * Serves `BlobStoreService` RPCs from the local blob store. A `put` is confirmed once the bytes are
16
+ * stored locally; replication is only told about it afterwards.
17
+ */
18
+ export const BlobStoreServiceLayer: Layer.Layer<
19
+ BlobStoreService.Tag,
20
+ never,
21
+ LocalBlobStoreService | BlobReplicationService
22
+ > = Layer.effect(
23
+ BlobStoreService.Tag,
24
+ Effect.gen(function* () {
25
+ const local = yield* LocalBlobStoreService;
26
+ const replication = yield* BlobReplicationService;
27
+ const store = replication.store;
28
+
29
+ const handlers: BlobStoreService.Handlers = {
30
+ 'BlobStoreService.put': ({ key, data, contentType }) =>
31
+ Effect.tryPromise({ try: () => local.put(key, data, { contentType }), catch: toServiceError }).pipe(
32
+ Effect.tap(() => Effect.sync(() => replication.wake())),
33
+ ),
34
+ 'BlobStoreService.get': ({ key }) =>
35
+ Effect.tryPromise({ try: () => store.get(key), catch: toServiceError }).pipe(
36
+ Effect.map((blob) => (blob ? { blob } : {})),
37
+ ),
38
+ 'BlobStoreService.has': ({ key }) =>
39
+ Effect.tryPromise({ try: () => store.has(key), catch: toServiceError }).pipe(
40
+ Effect.map((exists) => ({ exists })),
41
+ ),
42
+ };
43
+ return handlers;
44
+ }),
45
+ );
@@ -0,0 +1,7 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ export * from './blob-replication.ts';
6
+ export * from './blob-store-service.ts';
7
+ export * from './local-blob-store.ts';
@@ -0,0 +1,53 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as Effect from 'effect/Effect';
6
+ import { describe, expect, test } from 'vitest';
7
+
8
+ import { type LocalBlobStore } from '@dxos/blob';
9
+ import * as EffectEx from '@dxos/effect/EffectEx';
10
+ import { layerMemory } from '@dxos/sql-sqlite/Platform';
11
+
12
+ import { makeLocalBlobStore, migrateBlobStore } from './local-blob-store.ts';
13
+
14
+ const withStore = (body: (store: LocalBlobStore) => Promise<void>) =>
15
+ EffectEx.runPromise(
16
+ Effect.gen(function* () {
17
+ yield* migrateBlobStore;
18
+ const store = yield* makeLocalBlobStore;
19
+ yield* Effect.promise(() => body(store));
20
+ }).pipe(Effect.provide(layerMemory), Effect.orDie),
21
+ );
22
+
23
+ describe('LocalBlobStore', () => {
24
+ test('stores bytes and reports them pending until uploaded', () =>
25
+ withStore(async (store) => {
26
+ const data = new Uint8Array([1, 2, 3]);
27
+ await store.put('aa', data, { contentType: 'image/png' });
28
+ await store.put('bb', new Uint8Array([4]));
29
+
30
+ const blob = await store.get('aa');
31
+ expect(blob?.contentType).toBe('image/png');
32
+ expect(new Uint8Array(blob?.data ?? [])).toEqual(data);
33
+ expect((await store.get('bb'))?.contentType).toBeUndefined();
34
+ expect(await store.get('cc')).toBeUndefined();
35
+ expect(await store.has('aa')).toBe(true);
36
+ expect(await store.has('cc')).toBe(false);
37
+
38
+ expect(await store.listPending({ limit: 10 })).toEqual(['aa', 'bb']);
39
+ expect(await store.listPending({ limit: 1 })).toEqual(['aa']);
40
+
41
+ await store.markUploaded('aa');
42
+ expect(await store.listPending({ limit: 10 })).toEqual(['bb']);
43
+ }));
44
+
45
+ test('a repeated write never clears an upload, and a read-through fill is not pending', () =>
46
+ withStore(async (store) => {
47
+ await store.put('aa', new Uint8Array([1]), { uploaded: true });
48
+ await store.put('aa', new Uint8Array([1]), { contentType: 'text/plain' });
49
+
50
+ expect(await store.listPending({ limit: 10 })).toEqual([]);
51
+ expect((await store.get('aa'))?.contentType).toBe('text/plain');
52
+ }));
53
+ });
@@ -0,0 +1,89 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as EffectContext from 'effect/Context';
6
+ import * as Effect from 'effect/Effect';
7
+ import * as Layer from 'effect/Layer';
8
+ import * as Migrator from 'effect/sql/Migrator';
9
+ import * as SqlClient from 'effect/sql/SqlClient';
10
+ import type * as SqlError from 'effect/sql/SqlError';
11
+
12
+ import { type LocalBlobStore } from '@dxos/blob';
13
+ import * as EffectEx from '@dxos/effect/EffectEx';
14
+
15
+ import { MIGRATIONS, MIGRATIONS_TABLE } from '../../migrations/blobs/index.ts';
16
+
17
+ /**
18
+ * Applies any local blob store migrations this database has not recorded yet.
19
+ */
20
+ export const migrateBlobStore: Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> = Migrator.make({})({
21
+ loader: Migrator.fromRecord(MIGRATIONS),
22
+ table: MIGRATIONS_TABLE,
23
+ }).pipe(
24
+ // A malformed bundled manifest is a defect, not something a caller can recover from.
25
+ Effect.catchTag('MigrationError', (error) => Effect.die(error)),
26
+ Effect.asVoid,
27
+ Effect.withSpan('BlobStore.migrate'),
28
+ );
29
+
30
+ type BlobRow = { data: Uint8Array; type: string | null };
31
+
32
+ /**
33
+ * The `blobs` table of the host's database: blob bytes on this device, outside Automerge, with the
34
+ * ledger of those the hosted store does not hold yet. Knows nothing of replication.
35
+ */
36
+ export const makeLocalBlobStore: Effect.Effect<LocalBlobStore, never, SqlClient.SqlClient> = Effect.gen(function* () {
37
+ const sql = yield* SqlClient.SqlClient;
38
+ return {
39
+ put: (key, data, { contentType, uploaded = false } = {}) => {
40
+ const now = Date.now();
41
+ // Content addressing makes the stored bytes identical, so a conflict only ever learns an
42
+ // upload or a content type — it never clears either.
43
+ return EffectEx.runPromise(
44
+ sql`
45
+ INSERT INTO blobs (hash, type, size, data, created_at, uploaded_at)
46
+ VALUES (${key}, ${contentType ?? null}, ${data.byteLength}, ${data}, ${now}, ${uploaded ? now : null})
47
+ ON CONFLICT (hash) DO UPDATE SET
48
+ type = COALESCE(blobs.type, excluded.type),
49
+ uploaded_at = COALESCE(blobs.uploaded_at, excluded.uploaded_at)
50
+ `.pipe(Effect.asVoid),
51
+ );
52
+ },
53
+
54
+ get: async (key) => {
55
+ const [row] = await EffectEx.runPromise(sql<BlobRow>`SELECT data, type FROM blobs WHERE hash = ${key}`);
56
+ if (!row) {
57
+ return undefined;
58
+ }
59
+ return row.type === null ? { data: row.data } : { data: row.data, contentType: row.type };
60
+ },
61
+
62
+ has: async (key) => (await EffectEx.runPromise(sql`SELECT 1 AS found FROM blobs WHERE hash = ${key}`)).length > 0,
63
+
64
+ listPending: async ({ limit }) =>
65
+ (
66
+ await EffectEx.runPromise(
67
+ sql<{ hash: string }>`
68
+ SELECT hash FROM blobs WHERE uploaded_at IS NULL ORDER BY created_at, rowid LIMIT ${limit}
69
+ `,
70
+ )
71
+ ).map(({ hash }) => hash),
72
+
73
+ markUploaded: (key) =>
74
+ EffectEx.runPromise(
75
+ sql`UPDATE blobs SET uploaded_at = ${Date.now()} WHERE hash = ${key} AND uploaded_at IS NULL`.pipe(
76
+ Effect.asVoid,
77
+ ),
78
+ ),
79
+ };
80
+ });
81
+
82
+ export class LocalBlobStoreService extends EffectContext.Service<LocalBlobStoreService, LocalBlobStore>()(
83
+ '@dxos/client-services/LocalBlobStore',
84
+ ) {}
85
+
86
+ export const LocalBlobStoreLayer: Layer.Layer<LocalBlobStoreService, never, SqlClient.SqlClient> = Layer.effect(
87
+ LocalBlobStoreService,
88
+ makeLocalBlobStore,
89
+ );
@@ -41,6 +41,7 @@ import {
41
41
  import { SwarmNetworkManagerService, createIceProvider, createRtcTransportFactory } from '@dxos/network-manager';
42
42
  import { FeedProtocol } from '@dxos/protocols';
43
43
  import {
44
+ BlobStoreService,
44
45
  ContactsService,
45
46
  DataService,
46
47
  DevicesService,
@@ -66,6 +67,14 @@ import * as SpacesContract from '../../contracts/spaces.ts';
66
67
  import * as Readiness from '../../Readiness.ts';
67
68
  import * as SqliteStorage from '../../SqliteStorage.ts';
68
69
  import { EdgeAgentManagerLayer, EdgeAgentManagerService, EdgeAgentServiceLayer } from '../agents/index.ts';
70
+ import {
71
+ BlobReplicationLayer,
72
+ BlobReplicationNoopLayer,
73
+ BlobReplicationService,
74
+ BlobStoreServiceLayer,
75
+ LocalBlobStoreLayer,
76
+ LocalBlobStoreService,
77
+ } from '../blobs/index.ts';
69
78
  import { DevicesServiceLayer } from '../devices/index.ts';
70
79
  import { DevtoolsHostLayer, DevtoolsHostService } from '../devtools/index.ts';
71
80
  import {
@@ -93,6 +102,7 @@ import { valueEncoding } from '../pipeline/index.ts';
93
102
  import { SpaceManagerLayer, SpaceManagerService } from '../space/index.ts';
94
103
  import { DataSpaceManagerLayer, SigningContextProviderLayer, SpacesServiceLayer } from '../spaces/index.ts';
95
104
  import { SqlServiceLayer } from '../sql/index.ts';
105
+ import { storageWatchdogLayer } from '../storage/index.ts';
96
106
  import { SystemServiceLayer } from '../system/index.ts';
97
107
  import { TransportFactoryService } from './client-platform.ts';
98
108
  import {
@@ -281,6 +291,11 @@ export const StorageMigrationSpec = LayerSpec.make(
281
291
  () => storageMigrationLayer,
282
292
  );
283
293
 
294
+ export const StorageWatchdogSpec = LayerSpec.make(
295
+ { affinity: 'application', requires: [SqlClient.SqlClient, Hook.Controller], provides: [], eager: true },
296
+ () => storageWatchdogLayer(),
297
+ );
298
+
284
299
  export const StorageLifecycleSpec = LayerSpec.make(
285
300
  {
286
301
  affinity: 'application',
@@ -790,6 +805,41 @@ export const SqlServiceRegistrationSpec = LayerSpec.make(
790
805
  () => RegisterService(SqlService.Rpcs, SqlService.Tag),
791
806
  );
792
807
 
808
+ export const LocalBlobStoreSpec = LayerSpec.make(
809
+ { affinity: 'application', requires: [SqlClient.SqlClient], provides: [LocalBlobStoreService] },
810
+ () => LocalBlobStoreLayer,
811
+ );
812
+
813
+ /** Replicates with the edge only when one is configured; without it, blobs live on this device alone. */
814
+ export const BlobReplicationSpec = (options: ServiceStackServices) =>
815
+ options.edgeAvailable
816
+ ? LayerSpec.make(
817
+ {
818
+ affinity: 'application',
819
+ requires: [LocalBlobStoreService, EdgeHttpClientService, Hook.Controller],
820
+ provides: [BlobReplicationService],
821
+ },
822
+ () => BlobReplicationLayer,
823
+ )
824
+ : LayerSpec.make(
825
+ { affinity: 'application', requires: [LocalBlobStoreService], provides: [BlobReplicationService] },
826
+ () => BlobReplicationNoopLayer,
827
+ );
828
+
829
+ export const BlobStoreServiceSpec = LayerSpec.make(
830
+ {
831
+ affinity: 'application',
832
+ requires: [LocalBlobStoreService, BlobReplicationService],
833
+ provides: [BlobStoreService.Tag],
834
+ },
835
+ () => BlobStoreServiceLayer,
836
+ );
837
+
838
+ export const BlobStoreServiceRegistrationSpec = LayerSpec.make(
839
+ { affinity: 'application', requires: [BlobStoreService.Tag, RpcRouter.RpcRouter], provides: [], eager: true },
840
+ () => RegisterService(BlobStoreService.Rpcs, BlobStoreService.Tag),
841
+ );
842
+
793
843
  export const DevtoolsHostRegistrationSpec = LayerSpec.make(
794
844
  { affinity: 'application', requires: [DevtoolsHost.Tag, RpcRouter.RpcRouter], provides: [], eager: true },
795
845
  () => RegisterService(DevtoolsHost.Rpcs, DevtoolsHost.Tag),
@@ -811,6 +861,7 @@ export const clientServiceSpecs = (options: ServiceStackServices): LayerSpec.Lay
811
861
  HypercoreStoreSpec,
812
862
  StorageMigrationSpec,
813
863
  StorageLifecycleSpec,
864
+ StorageWatchdogSpec,
814
865
 
815
866
  StackReadinessSpec,
816
867
  SwarmNetworkManagerSpec(options),
@@ -863,6 +914,10 @@ export const clientServiceSpecs = (options: ServiceStackServices): LayerSpec.Lay
863
914
  LoggingServiceRegistrationSpec,
864
915
  SqlServiceSpec,
865
916
  SqlServiceRegistrationSpec,
917
+ LocalBlobStoreSpec,
918
+ BlobReplicationSpec(options),
919
+ BlobStoreServiceSpec,
920
+ BlobStoreServiceRegistrationSpec,
866
921
  DevtoolsHostSpec,
867
922
  DevtoolsHostRegistrationSpec,
868
923
  ];
@@ -29,6 +29,7 @@ import { type Runtime_Client_EdgeFeatures } from '@dxos/protocols/buf/dxos/confi
29
29
  import * as IdentityContract from '../../contracts/identity.ts';
30
30
  import * as Events from '../../Events.ts';
31
31
  import * as SqliteStorage from '../../SqliteStorage.ts';
32
+ import { migrateBlobStore } from '../blobs/index.ts';
32
33
  import { type IdentityManagerProps, type InboxRelay, identityProviderFromManager } from '../identity/index.ts';
33
34
  import { type InvitationConnectionProps } from '../invitations/index.ts';
34
35
  import { IMetadataStoreService, SqliteMetadataStore } from '../metadata/index.ts';
@@ -126,6 +127,7 @@ export const storageMigrationLayer = Layer.effect(
126
127
  new SqliteMetadataStore({ runtime }).migrate,
127
128
  new SqliteKeyring({ runtime }).migrate,
128
129
  new SqliteStorage.SqliteStorage({ runtime }).migrate,
130
+ migrateBlobStore,
129
131
  ],
130
132
  { discard: true },
131
133
  );
@@ -90,6 +90,8 @@ export type WriteSerializedSpaceArchiveOptions = {
90
90
  space: ExportableSpace;
91
91
  echoHost: EchoHost;
92
92
  exportedBy?: IdentityDid;
93
+ /** Export only these feeds: their `Feed` objects and messages, and no other object. */
94
+ feedIds?: readonly string[];
93
95
  };
94
96
 
95
97
  /**
@@ -103,7 +105,7 @@ export type WriteSerializedSpaceArchiveOptions = {
103
105
  export const writeSerializedSpaceArchive = async (
104
106
  options: WriteSerializedSpaceArchiveOptions,
105
107
  ): Promise<SpacesService.SpaceArchive> => {
106
- const { space, echoHost, exportedBy } = options;
108
+ const { space, echoHost, exportedBy, feedIds } = options;
107
109
 
108
110
  const rootUrl = space.automergeSpaceState.lastEpoch?.assertion.automergeRoot;
109
111
  assertState(rootUrl, 'Space does not have a root URL');
@@ -126,6 +128,12 @@ export const writeSerializedSpaceArchive = async (
126
128
  collectObjectsFromDoc(lease.doc(), objects);
127
129
  }
128
130
 
131
+ if (feedIds) {
132
+ const selected = new Set(feedIds);
133
+ // Only `Feed` objects: an id naming any other object must not pull it into a feeds-only archive.
134
+ objects.splice(0, objects.length, ...objects.filter((obj) => selected.has(obj.id) && isFeedObject(obj)));
135
+ }
136
+
129
137
  // Export queue/feed messages for every Feed object in the space.
130
138
  const feeds = await exportFeedData(space, echoHost, objects);
131
139
 
@@ -201,6 +209,11 @@ export const objectStructureToObjJson = (objectId: string, structure: EntityStru
201
209
  return result as Obj.JSON;
202
210
  };
203
211
 
212
+ const isFeedObject = (obj: Obj.JSON): boolean => {
213
+ const typeDxn = obj[ATTR_TYPE];
214
+ return typeDxn != null && DXN.isDXN(typeDxn) && DXN.getName(typeDxn) === FEED_TYPENAME;
215
+ };
216
+
204
217
  const exportFeedData = async (
205
218
  space: ExportableSpace,
206
219
  echoHost: EchoHost,
@@ -210,13 +223,7 @@ const exportFeedData = async (
210
223
  const spaceId: SpaceId = space.id;
211
224
 
212
225
  for (const obj of objects) {
213
- if (obj[ATTR_TYPE] == null) {
214
- continue;
215
- }
216
-
217
- const typeDxn = obj[ATTR_TYPE];
218
- const typeDxnNsid = DXN.isDXN(typeDxn) ? DXN.getName(typeDxn) : undefined;
219
- if (typeDxnNsid !== FEED_TYPENAME) {
226
+ if (!isFeedObject(obj)) {
220
227
  continue;
221
228
  }
222
229
 
@@ -239,7 +246,7 @@ const exportFeedData = async (
239
246
  return feeds;
240
247
  };
241
248
 
242
- const collectFeedMessages = async (
249
+ export const collectFeedMessages = async (
243
250
  echoHost: EchoHost,
244
251
  spaceId: SpaceId,
245
252
  feedId: EntityId,