@dxos/client-services 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/chunk-client-services-stack.mjs +799 -573
- package/dist/lib/chunk-client-services-stack.mjs.map +1 -1
- package/dist/lib/index.mjs +139 -131
- package/dist/lib/index.mjs.map +1 -1
- package/dist/lib/testing.mjs +1 -1
- package/dist/lib/testing.mjs.map +1 -1
- package/dist/types/src/Events.d.ts +2 -0
- package/dist/types/src/Events.d.ts.map +1 -1
- package/dist/types/src/internal/blobs/blob-replication.d.ts +28 -0
- package/dist/types/src/internal/blobs/blob-replication.d.ts.map +1 -0
- package/dist/types/src/internal/blobs/blob-store-service.d.ts +10 -0
- package/dist/types/src/internal/blobs/blob-store-service.d.ts.map +1 -0
- package/dist/types/src/internal/blobs/blob-store-service.test.d.ts +2 -0
- package/dist/types/src/internal/blobs/blob-store-service.test.d.ts.map +1 -0
- package/dist/types/src/internal/blobs/index.d.ts +4 -0
- package/dist/types/src/internal/blobs/index.d.ts.map +1 -0
- package/dist/types/src/internal/blobs/local-blob-store.d.ts +21 -0
- package/dist/types/src/internal/blobs/local-blob-store.d.ts.map +1 -0
- package/dist/types/src/internal/blobs/local-blob-store.test.d.ts +2 -0
- package/dist/types/src/internal/blobs/local-blob-store.test.d.ts.map +1 -0
- package/dist/types/src/internal/services/layer-specs.d.ts +6 -0
- package/dist/types/src/internal/services/layer-specs.d.ts.map +1 -1
- package/dist/types/src/internal/services/service-stack.d.ts.map +1 -1
- package/dist/types/src/internal/space-export/serialized-space-writer.d.ts +4 -1
- package/dist/types/src/internal/space-export/serialized-space-writer.d.ts.map +1 -1
- package/dist/types/src/internal/spaces/spaces-service.d.ts.map +1 -1
- package/dist/types/src/internal/storage/index.d.ts +1 -0
- package/dist/types/src/internal/storage/index.d.ts.map +1 -1
- package/dist/types/src/internal/storage/storage-watchdog.d.ts +17 -0
- package/dist/types/src/internal/storage/storage-watchdog.d.ts.map +1 -0
- package/dist/types/src/internal/storage/storage-watchdog.test.d.ts +2 -0
- package/dist/types/src/internal/storage/storage-watchdog.test.d.ts.map +1 -0
- package/dist/types/src/internal/worker/worker-runtime.d.ts.map +1 -1
- package/dist/types/src/migrations/blobs/index.d.ts +10 -0
- package/dist/types/src/migrations/blobs/index.d.ts.map +1 -0
- package/dist/types/src/version.d.ts +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +43 -42
- package/src/Events.ts +3 -0
- package/src/internal/blobs/blob-replication.ts +80 -0
- package/src/internal/blobs/blob-store-service.test.ts +76 -0
- package/src/internal/blobs/blob-store-service.ts +45 -0
- package/src/internal/blobs/index.ts +7 -0
- package/src/internal/blobs/local-blob-store.test.ts +53 -0
- package/src/internal/blobs/local-blob-store.ts +89 -0
- package/src/internal/services/layer-specs.ts +55 -0
- package/src/internal/services/service-stack.ts +2 -0
- package/src/internal/space-export/serialized-space-writer.ts +16 -9
- package/src/internal/spaces/spaces-service.ts +44 -6
- package/src/internal/storage/index.ts +1 -0
- package/src/internal/storage/storage-watchdog.test.ts +46 -0
- package/src/internal/storage/storage-watchdog.ts +58 -0
- package/src/internal/worker/worker-runtime.ts +2 -0
- package/src/migrations/blobs/0001_init.sql +17 -0
- package/src/migrations/blobs/index.ts +18 -0
- package/src/migrations/migrations.test.ts +3 -0
- package/src/version.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/client-services",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "DXOS client services implementation",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -34,54 +34,55 @@
|
|
|
34
34
|
"@obsidize/tar-browserify": "^5.2.0",
|
|
35
35
|
"cbor-x": "^1.5.4",
|
|
36
36
|
"crc-32": "^1.2.2",
|
|
37
|
-
"effect": "4.0.
|
|
37
|
+
"effect": "4.0.4",
|
|
38
38
|
"platform": "^1.3.6",
|
|
39
|
-
"@dxos/async": "0.
|
|
40
|
-
"@dxos/
|
|
41
|
-
"@dxos/
|
|
42
|
-
"@dxos/compute
|
|
43
|
-
"@dxos/
|
|
44
|
-
"@dxos/
|
|
45
|
-
"@dxos/
|
|
46
|
-
"@dxos/
|
|
47
|
-
"@dxos/
|
|
48
|
-
"@dxos/
|
|
49
|
-
"@dxos/echo
|
|
50
|
-
"@dxos/echo-
|
|
51
|
-
"@dxos/echo-
|
|
52
|
-
"@dxos/echo-
|
|
53
|
-
"@dxos/
|
|
54
|
-
"@dxos/
|
|
55
|
-
"@dxos/
|
|
56
|
-
"@dxos/
|
|
57
|
-
"@dxos/feed
|
|
58
|
-
"@dxos/
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/
|
|
63
|
-
"@dxos/
|
|
64
|
-
"@dxos/
|
|
65
|
-
"@dxos/
|
|
66
|
-
"@dxos/
|
|
67
|
-
"@dxos/
|
|
68
|
-
"@dxos/
|
|
69
|
-
"@dxos/
|
|
70
|
-
"@dxos/
|
|
71
|
-
"@dxos/teleport
|
|
72
|
-
"@dxos/teleport-extension-
|
|
73
|
-
"@dxos/
|
|
74
|
-
"@dxos/
|
|
75
|
-
"@dxos/
|
|
76
|
-
"@dxos/
|
|
39
|
+
"@dxos/async": "0.16.0",
|
|
40
|
+
"@dxos/blob": "0.16.0",
|
|
41
|
+
"@dxos/client-protocol": "0.16.0",
|
|
42
|
+
"@dxos/compute": "0.16.0",
|
|
43
|
+
"@dxos/compute-runtime": "0.16.0",
|
|
44
|
+
"@dxos/config": "0.16.0",
|
|
45
|
+
"@dxos/context": "0.16.0",
|
|
46
|
+
"@dxos/credentials": "0.16.0",
|
|
47
|
+
"@dxos/crypto": "0.16.0",
|
|
48
|
+
"@dxos/debug": "0.16.0",
|
|
49
|
+
"@dxos/echo": "0.16.0",
|
|
50
|
+
"@dxos/echo-client": "0.16.0",
|
|
51
|
+
"@dxos/echo-doc": "0.16.0",
|
|
52
|
+
"@dxos/echo-host": "0.16.0",
|
|
53
|
+
"@dxos/echo-protocol": "0.16.0",
|
|
54
|
+
"@dxos/edge-client": "0.16.0",
|
|
55
|
+
"@dxos/effect": "0.16.0",
|
|
56
|
+
"@dxos/errors": "0.16.0",
|
|
57
|
+
"@dxos/feed": "0.16.0",
|
|
58
|
+
"@dxos/feed-store": "0.16.0",
|
|
59
|
+
"@dxos/hypercore": "0.16.0",
|
|
60
|
+
"@dxos/invariant": "0.16.0",
|
|
61
|
+
"@dxos/keyring": "0.16.0",
|
|
62
|
+
"@dxos/keys": "0.16.0",
|
|
63
|
+
"@dxos/log": "0.16.0",
|
|
64
|
+
"@dxos/messaging": "0.16.0",
|
|
65
|
+
"@dxos/network-manager": "0.16.0",
|
|
66
|
+
"@dxos/node-std": "0.16.0",
|
|
67
|
+
"@dxos/protocols": "0.16.0",
|
|
68
|
+
"@dxos/random-access-storage": "0.16.0",
|
|
69
|
+
"@dxos/rpc": "0.16.0",
|
|
70
|
+
"@dxos/sql-sqlite": "0.16.0",
|
|
71
|
+
"@dxos/teleport": "0.16.0",
|
|
72
|
+
"@dxos/teleport-extension-gossip": "0.16.0",
|
|
73
|
+
"@dxos/teleport-extension-replicator": "0.16.0",
|
|
74
|
+
"@dxos/timeframe": "0.16.0",
|
|
75
|
+
"@dxos/tracing": "0.16.0",
|
|
76
|
+
"@dxos/types": "0.16.0",
|
|
77
|
+
"@dxos/util": "0.16.0"
|
|
77
78
|
},
|
|
78
79
|
"devDependencies": {
|
|
79
|
-
"@effect/platform-browser": "4.0.
|
|
80
|
+
"@effect/platform-browser": "4.0.4",
|
|
80
81
|
"@types/platform": "^1.3.4",
|
|
81
82
|
"fast-check": "^3.19.0",
|
|
82
83
|
"get-port-please": "^3.1.1",
|
|
83
84
|
"random-access-storage": "^1.3.0",
|
|
84
|
-
"@dxos/test-utils": "0.
|
|
85
|
+
"@dxos/test-utils": "0.16.0"
|
|
85
86
|
},
|
|
86
87
|
"publishConfig": {
|
|
87
88
|
"access": "public"
|
package/src/Events.ts
CHANGED
|
@@ -71,5 +71,8 @@ export const WipingStorage = Hook.make<void>()('client-services/WipingStorage');
|
|
|
71
71
|
/** Storage is wiped; the embedder typically reloads. */
|
|
72
72
|
export const Reset = Hook.make<void>()('client-services/Reset');
|
|
73
73
|
|
|
74
|
+
/** Storage stopped answering, so nothing more will be saved; the embedder replaces the process. */
|
|
75
|
+
export const StorageStuck = Hook.make<void>()('client-services/StorageStuck');
|
|
76
|
+
|
|
74
77
|
/** The local profile changed and should be broadcast to every open space. */
|
|
75
78
|
export const ProfileUpdated = Hook.make<{ profile: ProfileDocument }>()('client-services/ProfileUpdated');
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as EffectContext from 'effect/Context';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import * as Layer from 'effect/Layer';
|
|
8
|
+
|
|
9
|
+
import { type BlobStore, type BlobTransport } from '@dxos/blob';
|
|
10
|
+
import { type BlobReplicator, createBlobReplicator, withReadThrough } from '@dxos/blob/hosted';
|
|
11
|
+
import { Context } from '@dxos/context';
|
|
12
|
+
import { type EdgeHttpClient, EdgeHttpClientService } from '@dxos/edge-client';
|
|
13
|
+
import * as Hook from '@dxos/effect/Hook';
|
|
14
|
+
|
|
15
|
+
import * as Events from '../../Events.ts';
|
|
16
|
+
import { LocalBlobStoreService } from './local-blob-store.ts';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Replication of the local blob store with the EDGE blob service: pushes stored blobs asynchronously
|
|
20
|
+
* and fills local misses. Present only when an EDGE endpoint is configured; without it the local
|
|
21
|
+
* store is complete on its own.
|
|
22
|
+
*/
|
|
23
|
+
export interface BlobReplication {
|
|
24
|
+
/** The local store, filling misses from EDGE and caching them. */
|
|
25
|
+
readonly store: BlobStore;
|
|
26
|
+
/** Signals that a blob was written locally and may need pushing; never waits on the network. */
|
|
27
|
+
wake(): void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class BlobReplicationService extends EffectContext.Service<BlobReplicationService, BlobReplication>()(
|
|
31
|
+
'@dxos/client-services/BlobReplication',
|
|
32
|
+
) {}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Replication for a stack with no EDGE endpoint: reads are served by the local store alone and writes
|
|
36
|
+
* stay in its ledger, to be pushed by a later session that has one.
|
|
37
|
+
*/
|
|
38
|
+
export const BlobReplicationNoopLayer: Layer.Layer<BlobReplicationService, never, LocalBlobStoreService> = Layer.effect(
|
|
39
|
+
BlobReplicationService,
|
|
40
|
+
Effect.gen(function* () {
|
|
41
|
+
const local = yield* LocalBlobStoreService;
|
|
42
|
+
return { store: local, wake: () => {} };
|
|
43
|
+
}),
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
const edgeBlobTransport = (edge: EdgeHttpClient): BlobTransport => ({
|
|
47
|
+
url: (key) => edge.getBlobUrl(key),
|
|
48
|
+
put: (key, data, options) => edge.putBlob(Context.default(), key, data, options),
|
|
49
|
+
get: (key) => edge.getBlob(Context.default(), key),
|
|
50
|
+
has: (key) => edge.hasBlob(Context.default(), key),
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
export const BlobReplicationLayer: Layer.Layer<
|
|
54
|
+
BlobReplicationService,
|
|
55
|
+
never,
|
|
56
|
+
LocalBlobStoreService | EdgeHttpClientService | Hook.Controller
|
|
57
|
+
> = Layer.effect(
|
|
58
|
+
BlobReplicationService,
|
|
59
|
+
Effect.gen(function* () {
|
|
60
|
+
const local = yield* LocalBlobStoreService;
|
|
61
|
+
const transport = edgeBlobTransport(yield* EdgeHttpClientService);
|
|
62
|
+
|
|
63
|
+
// Started once the network identity is set, since EDGE authenticates uploads; writes before that
|
|
64
|
+
// wait in the ledger.
|
|
65
|
+
let replicator: BlobReplicator | undefined;
|
|
66
|
+
yield* Hook.on(
|
|
67
|
+
Events.NetworkReady,
|
|
68
|
+
Effect.fn('BlobReplication.onNetworkReady')(function* () {
|
|
69
|
+
replicator ??= createBlobReplicator({ local, transport });
|
|
70
|
+
replicator.wake();
|
|
71
|
+
}),
|
|
72
|
+
);
|
|
73
|
+
yield* Effect.addFinalizer(() => Effect.promise(async () => replicator?.close()));
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
store: withReadThrough({ local, transport }),
|
|
77
|
+
wake: () => replicator?.wake(),
|
|
78
|
+
};
|
|
79
|
+
}),
|
|
80
|
+
);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import * as Layer from 'effect/Layer';
|
|
7
|
+
import { describe, expect, test, vi } from 'vitest';
|
|
8
|
+
|
|
9
|
+
import { type LocalBlobStore, createMemoryBlobStore } from '@dxos/blob';
|
|
10
|
+
import * as EffectEx from '@dxos/effect/EffectEx';
|
|
11
|
+
import { makeInProcessClient } from '@dxos/protocols';
|
|
12
|
+
import { BlobStoreService } from '@dxos/protocols/rpc';
|
|
13
|
+
|
|
14
|
+
import { type BlobReplication, BlobReplicationNoopLayer, BlobReplicationService } from './blob-replication.ts';
|
|
15
|
+
import { BlobStoreServiceLayer } from './blob-store-service.ts';
|
|
16
|
+
import { LocalBlobStoreService } from './local-blob-store.ts';
|
|
17
|
+
|
|
18
|
+
/** Runs `body` against the service over an in-process RPC client, with replication only if given. */
|
|
19
|
+
const withService = (
|
|
20
|
+
local: LocalBlobStore,
|
|
21
|
+
replication: BlobReplication | undefined,
|
|
22
|
+
body: (client: BlobStoreService.Client) => Effect.Effect<void, unknown>,
|
|
23
|
+
) => {
|
|
24
|
+
const localLayer = Layer.succeed(LocalBlobStoreService, local);
|
|
25
|
+
const dependencies = Layer.mergeAll(
|
|
26
|
+
localLayer,
|
|
27
|
+
replication
|
|
28
|
+
? Layer.succeed(BlobReplicationService, replication)
|
|
29
|
+
: BlobReplicationNoopLayer.pipe(Layer.provide(localLayer)),
|
|
30
|
+
);
|
|
31
|
+
return EffectEx.runPromise(
|
|
32
|
+
Effect.gen(function* () {
|
|
33
|
+
const handlers = yield* BlobStoreService.Tag;
|
|
34
|
+
yield* body(yield* makeInProcessClient(BlobStoreService.Rpcs, handlers));
|
|
35
|
+
}).pipe(Effect.provide(BlobStoreServiceLayer.pipe(Layer.provide(dependencies))), Effect.scoped),
|
|
36
|
+
);
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
describe('BlobStoreService', () => {
|
|
40
|
+
test('with the noop replication, writes and reads are served by the local store alone', async () => {
|
|
41
|
+
const local = createMemoryBlobStore();
|
|
42
|
+
await withService(local, undefined, (client) =>
|
|
43
|
+
Effect.gen(function* () {
|
|
44
|
+
yield* client['BlobStoreService.put']({ key: 'aa', data: new Uint8Array([1]) });
|
|
45
|
+
expect(yield* client['BlobStoreService.get']({ key: 'aa' })).toEqual({ blob: { data: new Uint8Array([1]) } });
|
|
46
|
+
expect(yield* client['BlobStoreService.get']({ key: 'bb' })).toEqual({});
|
|
47
|
+
expect(yield* client['BlobStoreService.has']({ key: 'aa' })).toEqual({ exists: true });
|
|
48
|
+
}),
|
|
49
|
+
);
|
|
50
|
+
expect(await local.listPending({ limit: 10 })).toEqual(['aa']);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test('with replication, a write is stored locally before replication is woken, and reads go through it', async () => {
|
|
54
|
+
const local = createMemoryBlobStore();
|
|
55
|
+
const remote = createMemoryBlobStore();
|
|
56
|
+
await remote.put('bb', new Uint8Array([2]));
|
|
57
|
+
// What the local store held at the moment replication was told, to show confirmation comes first.
|
|
58
|
+
const storedWhenWoken: Promise<boolean>[] = [];
|
|
59
|
+
const wake = vi.fn(() => {
|
|
60
|
+
storedWhenWoken.push(local.has('aa'));
|
|
61
|
+
});
|
|
62
|
+
const replication: BlobReplication = {
|
|
63
|
+
store: { put: local.put, get: async (key) => (await local.get(key)) ?? remote.get(key), has: local.has },
|
|
64
|
+
wake,
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
await withService(local, replication, (client) =>
|
|
68
|
+
Effect.gen(function* () {
|
|
69
|
+
yield* client['BlobStoreService.put']({ key: 'aa', data: new Uint8Array([1]) });
|
|
70
|
+
expect(wake).toHaveBeenCalledTimes(1);
|
|
71
|
+
expect(yield* Effect.promise(() => Promise.all(storedWhenWoken))).toEqual([true]);
|
|
72
|
+
expect(yield* client['BlobStoreService.get']({ key: 'bb' })).toEqual({ blob: { data: new Uint8Array([2]) } });
|
|
73
|
+
}),
|
|
74
|
+
);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import * as Layer from 'effect/Layer';
|
|
7
|
+
|
|
8
|
+
import { toServiceError } from '@dxos/protocols';
|
|
9
|
+
import { BlobStoreService } from '@dxos/protocols/rpc';
|
|
10
|
+
|
|
11
|
+
import { BlobReplicationService } from './blob-replication.ts';
|
|
12
|
+
import { LocalBlobStoreService } from './local-blob-store.ts';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Serves `BlobStoreService` RPCs from the local blob store. A `put` is confirmed once the bytes are
|
|
16
|
+
* stored locally; replication is only told about it afterwards.
|
|
17
|
+
*/
|
|
18
|
+
export const BlobStoreServiceLayer: Layer.Layer<
|
|
19
|
+
BlobStoreService.Tag,
|
|
20
|
+
never,
|
|
21
|
+
LocalBlobStoreService | BlobReplicationService
|
|
22
|
+
> = Layer.effect(
|
|
23
|
+
BlobStoreService.Tag,
|
|
24
|
+
Effect.gen(function* () {
|
|
25
|
+
const local = yield* LocalBlobStoreService;
|
|
26
|
+
const replication = yield* BlobReplicationService;
|
|
27
|
+
const store = replication.store;
|
|
28
|
+
|
|
29
|
+
const handlers: BlobStoreService.Handlers = {
|
|
30
|
+
'BlobStoreService.put': ({ key, data, contentType }) =>
|
|
31
|
+
Effect.tryPromise({ try: () => local.put(key, data, { contentType }), catch: toServiceError }).pipe(
|
|
32
|
+
Effect.tap(() => Effect.sync(() => replication.wake())),
|
|
33
|
+
),
|
|
34
|
+
'BlobStoreService.get': ({ key }) =>
|
|
35
|
+
Effect.tryPromise({ try: () => store.get(key), catch: toServiceError }).pipe(
|
|
36
|
+
Effect.map((blob) => (blob ? { blob } : {})),
|
|
37
|
+
),
|
|
38
|
+
'BlobStoreService.has': ({ key }) =>
|
|
39
|
+
Effect.tryPromise({ try: () => store.has(key), catch: toServiceError }).pipe(
|
|
40
|
+
Effect.map((exists) => ({ exists })),
|
|
41
|
+
),
|
|
42
|
+
};
|
|
43
|
+
return handlers;
|
|
44
|
+
}),
|
|
45
|
+
);
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import { describe, expect, test } from 'vitest';
|
|
7
|
+
|
|
8
|
+
import { type LocalBlobStore } from '@dxos/blob';
|
|
9
|
+
import * as EffectEx from '@dxos/effect/EffectEx';
|
|
10
|
+
import { layerMemory } from '@dxos/sql-sqlite/Platform';
|
|
11
|
+
|
|
12
|
+
import { makeLocalBlobStore, migrateBlobStore } from './local-blob-store.ts';
|
|
13
|
+
|
|
14
|
+
const withStore = (body: (store: LocalBlobStore) => Promise<void>) =>
|
|
15
|
+
EffectEx.runPromise(
|
|
16
|
+
Effect.gen(function* () {
|
|
17
|
+
yield* migrateBlobStore;
|
|
18
|
+
const store = yield* makeLocalBlobStore;
|
|
19
|
+
yield* Effect.promise(() => body(store));
|
|
20
|
+
}).pipe(Effect.provide(layerMemory), Effect.orDie),
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
describe('LocalBlobStore', () => {
|
|
24
|
+
test('stores bytes and reports them pending until uploaded', () =>
|
|
25
|
+
withStore(async (store) => {
|
|
26
|
+
const data = new Uint8Array([1, 2, 3]);
|
|
27
|
+
await store.put('aa', data, { contentType: 'image/png' });
|
|
28
|
+
await store.put('bb', new Uint8Array([4]));
|
|
29
|
+
|
|
30
|
+
const blob = await store.get('aa');
|
|
31
|
+
expect(blob?.contentType).toBe('image/png');
|
|
32
|
+
expect(new Uint8Array(blob?.data ?? [])).toEqual(data);
|
|
33
|
+
expect((await store.get('bb'))?.contentType).toBeUndefined();
|
|
34
|
+
expect(await store.get('cc')).toBeUndefined();
|
|
35
|
+
expect(await store.has('aa')).toBe(true);
|
|
36
|
+
expect(await store.has('cc')).toBe(false);
|
|
37
|
+
|
|
38
|
+
expect(await store.listPending({ limit: 10 })).toEqual(['aa', 'bb']);
|
|
39
|
+
expect(await store.listPending({ limit: 1 })).toEqual(['aa']);
|
|
40
|
+
|
|
41
|
+
await store.markUploaded('aa');
|
|
42
|
+
expect(await store.listPending({ limit: 10 })).toEqual(['bb']);
|
|
43
|
+
}));
|
|
44
|
+
|
|
45
|
+
test('a repeated write never clears an upload, and a read-through fill is not pending', () =>
|
|
46
|
+
withStore(async (store) => {
|
|
47
|
+
await store.put('aa', new Uint8Array([1]), { uploaded: true });
|
|
48
|
+
await store.put('aa', new Uint8Array([1]), { contentType: 'text/plain' });
|
|
49
|
+
|
|
50
|
+
expect(await store.listPending({ limit: 10 })).toEqual([]);
|
|
51
|
+
expect((await store.get('aa'))?.contentType).toBe('text/plain');
|
|
52
|
+
}));
|
|
53
|
+
});
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as EffectContext from 'effect/Context';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import * as Layer from 'effect/Layer';
|
|
8
|
+
import * as Migrator from 'effect/sql/Migrator';
|
|
9
|
+
import * as SqlClient from 'effect/sql/SqlClient';
|
|
10
|
+
import type * as SqlError from 'effect/sql/SqlError';
|
|
11
|
+
|
|
12
|
+
import { type LocalBlobStore } from '@dxos/blob';
|
|
13
|
+
import * as EffectEx from '@dxos/effect/EffectEx';
|
|
14
|
+
|
|
15
|
+
import { MIGRATIONS, MIGRATIONS_TABLE } from '../../migrations/blobs/index.ts';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Applies any local blob store migrations this database has not recorded yet.
|
|
19
|
+
*/
|
|
20
|
+
export const migrateBlobStore: Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> = Migrator.make({})({
|
|
21
|
+
loader: Migrator.fromRecord(MIGRATIONS),
|
|
22
|
+
table: MIGRATIONS_TABLE,
|
|
23
|
+
}).pipe(
|
|
24
|
+
// A malformed bundled manifest is a defect, not something a caller can recover from.
|
|
25
|
+
Effect.catchTag('MigrationError', (error) => Effect.die(error)),
|
|
26
|
+
Effect.asVoid,
|
|
27
|
+
Effect.withSpan('BlobStore.migrate'),
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
type BlobRow = { data: Uint8Array; type: string | null };
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The `blobs` table of the host's database: blob bytes on this device, outside Automerge, with the
|
|
34
|
+
* ledger of those the hosted store does not hold yet. Knows nothing of replication.
|
|
35
|
+
*/
|
|
36
|
+
export const makeLocalBlobStore: Effect.Effect<LocalBlobStore, never, SqlClient.SqlClient> = Effect.gen(function* () {
|
|
37
|
+
const sql = yield* SqlClient.SqlClient;
|
|
38
|
+
return {
|
|
39
|
+
put: (key, data, { contentType, uploaded = false } = {}) => {
|
|
40
|
+
const now = Date.now();
|
|
41
|
+
// Content addressing makes the stored bytes identical, so a conflict only ever learns an
|
|
42
|
+
// upload or a content type — it never clears either.
|
|
43
|
+
return EffectEx.runPromise(
|
|
44
|
+
sql`
|
|
45
|
+
INSERT INTO blobs (hash, type, size, data, created_at, uploaded_at)
|
|
46
|
+
VALUES (${key}, ${contentType ?? null}, ${data.byteLength}, ${data}, ${now}, ${uploaded ? now : null})
|
|
47
|
+
ON CONFLICT (hash) DO UPDATE SET
|
|
48
|
+
type = COALESCE(blobs.type, excluded.type),
|
|
49
|
+
uploaded_at = COALESCE(blobs.uploaded_at, excluded.uploaded_at)
|
|
50
|
+
`.pipe(Effect.asVoid),
|
|
51
|
+
);
|
|
52
|
+
},
|
|
53
|
+
|
|
54
|
+
get: async (key) => {
|
|
55
|
+
const [row] = await EffectEx.runPromise(sql<BlobRow>`SELECT data, type FROM blobs WHERE hash = ${key}`);
|
|
56
|
+
if (!row) {
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
return row.type === null ? { data: row.data } : { data: row.data, contentType: row.type };
|
|
60
|
+
},
|
|
61
|
+
|
|
62
|
+
has: async (key) => (await EffectEx.runPromise(sql`SELECT 1 AS found FROM blobs WHERE hash = ${key}`)).length > 0,
|
|
63
|
+
|
|
64
|
+
listPending: async ({ limit }) =>
|
|
65
|
+
(
|
|
66
|
+
await EffectEx.runPromise(
|
|
67
|
+
sql<{ hash: string }>`
|
|
68
|
+
SELECT hash FROM blobs WHERE uploaded_at IS NULL ORDER BY created_at, rowid LIMIT ${limit}
|
|
69
|
+
`,
|
|
70
|
+
)
|
|
71
|
+
).map(({ hash }) => hash),
|
|
72
|
+
|
|
73
|
+
markUploaded: (key) =>
|
|
74
|
+
EffectEx.runPromise(
|
|
75
|
+
sql`UPDATE blobs SET uploaded_at = ${Date.now()} WHERE hash = ${key} AND uploaded_at IS NULL`.pipe(
|
|
76
|
+
Effect.asVoid,
|
|
77
|
+
),
|
|
78
|
+
),
|
|
79
|
+
};
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
export class LocalBlobStoreService extends EffectContext.Service<LocalBlobStoreService, LocalBlobStore>()(
|
|
83
|
+
'@dxos/client-services/LocalBlobStore',
|
|
84
|
+
) {}
|
|
85
|
+
|
|
86
|
+
export const LocalBlobStoreLayer: Layer.Layer<LocalBlobStoreService, never, SqlClient.SqlClient> = Layer.effect(
|
|
87
|
+
LocalBlobStoreService,
|
|
88
|
+
makeLocalBlobStore,
|
|
89
|
+
);
|
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
import { SwarmNetworkManagerService, createIceProvider, createRtcTransportFactory } from '@dxos/network-manager';
|
|
42
42
|
import { FeedProtocol } from '@dxos/protocols';
|
|
43
43
|
import {
|
|
44
|
+
BlobStoreService,
|
|
44
45
|
ContactsService,
|
|
45
46
|
DataService,
|
|
46
47
|
DevicesService,
|
|
@@ -66,6 +67,14 @@ import * as SpacesContract from '../../contracts/spaces.ts';
|
|
|
66
67
|
import * as Readiness from '../../Readiness.ts';
|
|
67
68
|
import * as SqliteStorage from '../../SqliteStorage.ts';
|
|
68
69
|
import { EdgeAgentManagerLayer, EdgeAgentManagerService, EdgeAgentServiceLayer } from '../agents/index.ts';
|
|
70
|
+
import {
|
|
71
|
+
BlobReplicationLayer,
|
|
72
|
+
BlobReplicationNoopLayer,
|
|
73
|
+
BlobReplicationService,
|
|
74
|
+
BlobStoreServiceLayer,
|
|
75
|
+
LocalBlobStoreLayer,
|
|
76
|
+
LocalBlobStoreService,
|
|
77
|
+
} from '../blobs/index.ts';
|
|
69
78
|
import { DevicesServiceLayer } from '../devices/index.ts';
|
|
70
79
|
import { DevtoolsHostLayer, DevtoolsHostService } from '../devtools/index.ts';
|
|
71
80
|
import {
|
|
@@ -93,6 +102,7 @@ import { valueEncoding } from '../pipeline/index.ts';
|
|
|
93
102
|
import { SpaceManagerLayer, SpaceManagerService } from '../space/index.ts';
|
|
94
103
|
import { DataSpaceManagerLayer, SigningContextProviderLayer, SpacesServiceLayer } from '../spaces/index.ts';
|
|
95
104
|
import { SqlServiceLayer } from '../sql/index.ts';
|
|
105
|
+
import { storageWatchdogLayer } from '../storage/index.ts';
|
|
96
106
|
import { SystemServiceLayer } from '../system/index.ts';
|
|
97
107
|
import { TransportFactoryService } from './client-platform.ts';
|
|
98
108
|
import {
|
|
@@ -281,6 +291,11 @@ export const StorageMigrationSpec = LayerSpec.make(
|
|
|
281
291
|
() => storageMigrationLayer,
|
|
282
292
|
);
|
|
283
293
|
|
|
294
|
+
export const StorageWatchdogSpec = LayerSpec.make(
|
|
295
|
+
{ affinity: 'application', requires: [SqlClient.SqlClient, Hook.Controller], provides: [], eager: true },
|
|
296
|
+
() => storageWatchdogLayer(),
|
|
297
|
+
);
|
|
298
|
+
|
|
284
299
|
export const StorageLifecycleSpec = LayerSpec.make(
|
|
285
300
|
{
|
|
286
301
|
affinity: 'application',
|
|
@@ -790,6 +805,41 @@ export const SqlServiceRegistrationSpec = LayerSpec.make(
|
|
|
790
805
|
() => RegisterService(SqlService.Rpcs, SqlService.Tag),
|
|
791
806
|
);
|
|
792
807
|
|
|
808
|
+
export const LocalBlobStoreSpec = LayerSpec.make(
|
|
809
|
+
{ affinity: 'application', requires: [SqlClient.SqlClient], provides: [LocalBlobStoreService] },
|
|
810
|
+
() => LocalBlobStoreLayer,
|
|
811
|
+
);
|
|
812
|
+
|
|
813
|
+
/** Replicates with the edge only when one is configured; without it, blobs live on this device alone. */
|
|
814
|
+
export const BlobReplicationSpec = (options: ServiceStackServices) =>
|
|
815
|
+
options.edgeAvailable
|
|
816
|
+
? LayerSpec.make(
|
|
817
|
+
{
|
|
818
|
+
affinity: 'application',
|
|
819
|
+
requires: [LocalBlobStoreService, EdgeHttpClientService, Hook.Controller],
|
|
820
|
+
provides: [BlobReplicationService],
|
|
821
|
+
},
|
|
822
|
+
() => BlobReplicationLayer,
|
|
823
|
+
)
|
|
824
|
+
: LayerSpec.make(
|
|
825
|
+
{ affinity: 'application', requires: [LocalBlobStoreService], provides: [BlobReplicationService] },
|
|
826
|
+
() => BlobReplicationNoopLayer,
|
|
827
|
+
);
|
|
828
|
+
|
|
829
|
+
export const BlobStoreServiceSpec = LayerSpec.make(
|
|
830
|
+
{
|
|
831
|
+
affinity: 'application',
|
|
832
|
+
requires: [LocalBlobStoreService, BlobReplicationService],
|
|
833
|
+
provides: [BlobStoreService.Tag],
|
|
834
|
+
},
|
|
835
|
+
() => BlobStoreServiceLayer,
|
|
836
|
+
);
|
|
837
|
+
|
|
838
|
+
export const BlobStoreServiceRegistrationSpec = LayerSpec.make(
|
|
839
|
+
{ affinity: 'application', requires: [BlobStoreService.Tag, RpcRouter.RpcRouter], provides: [], eager: true },
|
|
840
|
+
() => RegisterService(BlobStoreService.Rpcs, BlobStoreService.Tag),
|
|
841
|
+
);
|
|
842
|
+
|
|
793
843
|
export const DevtoolsHostRegistrationSpec = LayerSpec.make(
|
|
794
844
|
{ affinity: 'application', requires: [DevtoolsHost.Tag, RpcRouter.RpcRouter], provides: [], eager: true },
|
|
795
845
|
() => RegisterService(DevtoolsHost.Rpcs, DevtoolsHost.Tag),
|
|
@@ -811,6 +861,7 @@ export const clientServiceSpecs = (options: ServiceStackServices): LayerSpec.Lay
|
|
|
811
861
|
HypercoreStoreSpec,
|
|
812
862
|
StorageMigrationSpec,
|
|
813
863
|
StorageLifecycleSpec,
|
|
864
|
+
StorageWatchdogSpec,
|
|
814
865
|
|
|
815
866
|
StackReadinessSpec,
|
|
816
867
|
SwarmNetworkManagerSpec(options),
|
|
@@ -863,6 +914,10 @@ export const clientServiceSpecs = (options: ServiceStackServices): LayerSpec.Lay
|
|
|
863
914
|
LoggingServiceRegistrationSpec,
|
|
864
915
|
SqlServiceSpec,
|
|
865
916
|
SqlServiceRegistrationSpec,
|
|
917
|
+
LocalBlobStoreSpec,
|
|
918
|
+
BlobReplicationSpec(options),
|
|
919
|
+
BlobStoreServiceSpec,
|
|
920
|
+
BlobStoreServiceRegistrationSpec,
|
|
866
921
|
DevtoolsHostSpec,
|
|
867
922
|
DevtoolsHostRegistrationSpec,
|
|
868
923
|
];
|
|
@@ -29,6 +29,7 @@ import { type Runtime_Client_EdgeFeatures } from '@dxos/protocols/buf/dxos/confi
|
|
|
29
29
|
import * as IdentityContract from '../../contracts/identity.ts';
|
|
30
30
|
import * as Events from '../../Events.ts';
|
|
31
31
|
import * as SqliteStorage from '../../SqliteStorage.ts';
|
|
32
|
+
import { migrateBlobStore } from '../blobs/index.ts';
|
|
32
33
|
import { type IdentityManagerProps, type InboxRelay, identityProviderFromManager } from '../identity/index.ts';
|
|
33
34
|
import { type InvitationConnectionProps } from '../invitations/index.ts';
|
|
34
35
|
import { IMetadataStoreService, SqliteMetadataStore } from '../metadata/index.ts';
|
|
@@ -126,6 +127,7 @@ export const storageMigrationLayer = Layer.effect(
|
|
|
126
127
|
new SqliteMetadataStore({ runtime }).migrate,
|
|
127
128
|
new SqliteKeyring({ runtime }).migrate,
|
|
128
129
|
new SqliteStorage.SqliteStorage({ runtime }).migrate,
|
|
130
|
+
migrateBlobStore,
|
|
129
131
|
],
|
|
130
132
|
{ discard: true },
|
|
131
133
|
);
|
|
@@ -90,6 +90,8 @@ export type WriteSerializedSpaceArchiveOptions = {
|
|
|
90
90
|
space: ExportableSpace;
|
|
91
91
|
echoHost: EchoHost;
|
|
92
92
|
exportedBy?: IdentityDid;
|
|
93
|
+
/** Export only these feeds: their `Feed` objects and messages, and no other object. */
|
|
94
|
+
feedIds?: readonly string[];
|
|
93
95
|
};
|
|
94
96
|
|
|
95
97
|
/**
|
|
@@ -103,7 +105,7 @@ export type WriteSerializedSpaceArchiveOptions = {
|
|
|
103
105
|
export const writeSerializedSpaceArchive = async (
|
|
104
106
|
options: WriteSerializedSpaceArchiveOptions,
|
|
105
107
|
): Promise<SpacesService.SpaceArchive> => {
|
|
106
|
-
const { space, echoHost, exportedBy } = options;
|
|
108
|
+
const { space, echoHost, exportedBy, feedIds } = options;
|
|
107
109
|
|
|
108
110
|
const rootUrl = space.automergeSpaceState.lastEpoch?.assertion.automergeRoot;
|
|
109
111
|
assertState(rootUrl, 'Space does not have a root URL');
|
|
@@ -126,6 +128,12 @@ export const writeSerializedSpaceArchive = async (
|
|
|
126
128
|
collectObjectsFromDoc(lease.doc(), objects);
|
|
127
129
|
}
|
|
128
130
|
|
|
131
|
+
if (feedIds) {
|
|
132
|
+
const selected = new Set(feedIds);
|
|
133
|
+
// Only `Feed` objects: an id naming any other object must not pull it into a feeds-only archive.
|
|
134
|
+
objects.splice(0, objects.length, ...objects.filter((obj) => selected.has(obj.id) && isFeedObject(obj)));
|
|
135
|
+
}
|
|
136
|
+
|
|
129
137
|
// Export queue/feed messages for every Feed object in the space.
|
|
130
138
|
const feeds = await exportFeedData(space, echoHost, objects);
|
|
131
139
|
|
|
@@ -201,6 +209,11 @@ export const objectStructureToObjJson = (objectId: string, structure: EntityStru
|
|
|
201
209
|
return result as Obj.JSON;
|
|
202
210
|
};
|
|
203
211
|
|
|
212
|
+
const isFeedObject = (obj: Obj.JSON): boolean => {
|
|
213
|
+
const typeDxn = obj[ATTR_TYPE];
|
|
214
|
+
return typeDxn != null && DXN.isDXN(typeDxn) && DXN.getName(typeDxn) === FEED_TYPENAME;
|
|
215
|
+
};
|
|
216
|
+
|
|
204
217
|
const exportFeedData = async (
|
|
205
218
|
space: ExportableSpace,
|
|
206
219
|
echoHost: EchoHost,
|
|
@@ -210,13 +223,7 @@ const exportFeedData = async (
|
|
|
210
223
|
const spaceId: SpaceId = space.id;
|
|
211
224
|
|
|
212
225
|
for (const obj of objects) {
|
|
213
|
-
if (obj
|
|
214
|
-
continue;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const typeDxn = obj[ATTR_TYPE];
|
|
218
|
-
const typeDxnNsid = DXN.isDXN(typeDxn) ? DXN.getName(typeDxn) : undefined;
|
|
219
|
-
if (typeDxnNsid !== FEED_TYPENAME) {
|
|
226
|
+
if (!isFeedObject(obj)) {
|
|
220
227
|
continue;
|
|
221
228
|
}
|
|
222
229
|
|
|
@@ -239,7 +246,7 @@ const exportFeedData = async (
|
|
|
239
246
|
return feeds;
|
|
240
247
|
};
|
|
241
248
|
|
|
242
|
-
const collectFeedMessages = async (
|
|
249
|
+
export const collectFeedMessages = async (
|
|
243
250
|
echoHost: EchoHost,
|
|
244
251
|
spaceId: SpaceId,
|
|
245
252
|
feedId: EntityId,
|