@livestore/sync-electric 0.0.58-dev.1 → 0.0.58-dev.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/index.d.ts +93 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +79 -31
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
- package/src/index.ts +156 -35
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,95 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common';
|
|
2
2
|
import type { Scope } from '@livestore/utils/effect';
|
|
3
|
-
import { Effect } from '@livestore/utils/effect';
|
|
4
|
-
export declare const
|
|
3
|
+
import { Effect, Schema } from '@livestore/utils/effect';
|
|
4
|
+
export declare const syncBackend: any;
|
|
5
|
+
export declare const ApiPushEventPayload: Schema.TaggedStruct<"sync-electric.PushEvent", {
|
|
6
|
+
roomId: typeof Schema.String;
|
|
7
|
+
mutationEventEncoded: Schema.SchemaClass<{
|
|
8
|
+
readonly id: {
|
|
9
|
+
readonly global: number;
|
|
10
|
+
readonly local: number;
|
|
11
|
+
};
|
|
12
|
+
readonly mutation: string;
|
|
13
|
+
readonly args: any;
|
|
14
|
+
readonly parentId: {
|
|
15
|
+
readonly global: number;
|
|
16
|
+
readonly local: number;
|
|
17
|
+
};
|
|
18
|
+
}, {
|
|
19
|
+
readonly id: {
|
|
20
|
+
readonly global: number;
|
|
21
|
+
readonly local: number;
|
|
22
|
+
};
|
|
23
|
+
readonly mutation: string;
|
|
24
|
+
readonly args: any;
|
|
25
|
+
readonly parentId: {
|
|
26
|
+
readonly global: number;
|
|
27
|
+
readonly local: number;
|
|
28
|
+
};
|
|
29
|
+
}, never>;
|
|
30
|
+
persisted: typeof Schema.Boolean;
|
|
31
|
+
}>;
|
|
32
|
+
export declare const ApiInitRoomPayload: Schema.TaggedStruct<"sync-electric.InitRoom", {
|
|
33
|
+
roomId: typeof Schema.String;
|
|
34
|
+
}>;
|
|
35
|
+
export declare const ApiPayload: Schema.Union<[Schema.TaggedStruct<"sync-electric.PushEvent", {
|
|
36
|
+
roomId: typeof Schema.String;
|
|
37
|
+
mutationEventEncoded: Schema.SchemaClass<{
|
|
38
|
+
readonly id: {
|
|
39
|
+
readonly global: number;
|
|
40
|
+
readonly local: number;
|
|
41
|
+
};
|
|
42
|
+
readonly mutation: string;
|
|
43
|
+
readonly args: any;
|
|
44
|
+
readonly parentId: {
|
|
45
|
+
readonly global: number;
|
|
46
|
+
readonly local: number;
|
|
47
|
+
};
|
|
48
|
+
}, {
|
|
49
|
+
readonly id: {
|
|
50
|
+
readonly global: number;
|
|
51
|
+
readonly local: number;
|
|
52
|
+
};
|
|
53
|
+
readonly mutation: string;
|
|
54
|
+
readonly args: any;
|
|
55
|
+
readonly parentId: {
|
|
56
|
+
readonly global: number;
|
|
57
|
+
readonly local: number;
|
|
58
|
+
};
|
|
59
|
+
}, never>;
|
|
60
|
+
persisted: typeof Schema.Boolean;
|
|
61
|
+
}>, Schema.TaggedStruct<"sync-electric.InitRoom", {
|
|
62
|
+
roomId: typeof Schema.String;
|
|
63
|
+
}>]>;
|
|
64
|
+
export declare const syncBackendOptions: <TOptions extends SyncBackendOptions>(options: TOptions) => TOptions;
|
|
65
|
+
export interface SyncBackendOptions extends SyncBackendOptionsBase {
|
|
66
|
+
type: 'electric';
|
|
67
|
+
/**
|
|
68
|
+
* The host of the Electric server
|
|
69
|
+
*
|
|
70
|
+
* @example "https://localhost:3000"
|
|
71
|
+
*/
|
|
72
|
+
electricHost: string;
|
|
73
|
+
roomId: string;
|
|
74
|
+
/**
|
|
75
|
+
* The POST endpoint to push events to
|
|
76
|
+
*
|
|
77
|
+
* @example "/api/push-event"
|
|
78
|
+
* @example "https://api.myapp.com/push-event"
|
|
79
|
+
*/
|
|
80
|
+
pushEventEndpoint: string;
|
|
81
|
+
}
|
|
82
|
+
interface LiveStoreGlobalElectric {
|
|
83
|
+
syncBackend: SyncBackendOptions;
|
|
84
|
+
}
|
|
85
|
+
declare global {
|
|
86
|
+
interface LiveStoreGlobal extends LiveStoreGlobalElectric {
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
type SyncMetadata = {
|
|
90
|
+
offset: string;
|
|
91
|
+
shapeId: string;
|
|
92
|
+
};
|
|
93
|
+
export declare const makeSyncBackend: ({ electricHost, roomId, pushEventEndpoint, }: SyncBackendOptions) => Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope>;
|
|
94
|
+
export {};
|
|
5
95
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,WAAW,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAA;AAGrF,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AACpD,OAAO,EAGL,MAAM,EAKN,MAAM,EAGP,MAAM,yBAAyB,CAAA;AA4BhC,eAAO,MAAM,WAAW,EAAS,GAAG,CAAA;AAEpC,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;EAI9B,CAAA;AAEF,eAAO,MAAM,kBAAkB;;EAE7B,CAAA;AAEF,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAAwD,CAAA;AAE/E,eAAO,MAAM,kBAAkB,GAAI,QAAQ,SAAS,kBAAkB,WAAW,QAAQ,aAAY,CAAA;AAErG,MAAM,WAAW,kBAAmB,SAAQ,sBAAsB;IAChE,IAAI,EAAE,UAAU,CAAA;IAChB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;;;;OAKG;IACH,iBAAiB,EAAE,MAAM,CAAA;CAC1B;AAED,UAAU,uBAAuB;IAC/B,WAAW,EAAE,kBAAkB,CAAA;CAChC;AAED,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,eAAgB,SAAQ,uBAAuB;KAAG;CAC7D;AAED,KAAK,YAAY,GAAG;IAClB,MAAM,EAAE,MAAM,CAAA;IAEd,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,eAAO,MAAM,eAAe,iDAIzB,kBAAkB,KAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAiH/E,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { InvalidPullError } from '@livestore/common';
|
|
1
|
+
import { InvalidPullError, InvalidPushError } from '@livestore/common';
|
|
2
2
|
import { mutationEventSchemaEncodedAny } from '@livestore/common/schema';
|
|
3
|
-
import {
|
|
4
|
-
import { Chunk, Effect, HttpClient, HttpClientRequest, HttpClientResponse, Option, Schema, Stream, SubscriptionRef, } from '@livestore/utils/effect';
|
|
3
|
+
import { Chunk, Deferred, Effect, HttpClient, HttpClientRequest, HttpClientResponse, Option, Schema, Stream, SubscriptionRef, } from '@livestore/utils/effect';
|
|
5
4
|
/*
|
|
6
5
|
Example data:
|
|
7
6
|
|
|
@@ -9,6 +8,8 @@ Example data:
|
|
|
9
8
|
,{"key":"\"public\".\"events\"/\"1725703554783\"","value":{"id":"1725703554783","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:54.783Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
|
|
10
9
|
,{"headers":{"control":"up-to-date"}}]
|
|
11
10
|
|
|
11
|
+
Also see: https://github.com/electric-sql/electric/blob/main/packages/typescript-client/src/client.ts
|
|
12
|
+
|
|
12
13
|
*/
|
|
13
14
|
const ResponseItem = Schema.Struct({
|
|
14
15
|
/** Postgres path (e.g. "public.events/1") */
|
|
@@ -19,36 +20,83 @@ const ResponseItem = Schema.Struct({
|
|
|
19
20
|
});
|
|
20
21
|
const ResponseHeaders = Schema.Struct({
|
|
21
22
|
'x-electric-shape-id': Schema.String,
|
|
22
|
-
'x-electric-schema': Schema.parseJson(Schema.Any),
|
|
23
|
+
// 'x-electric-schema': Schema.parseJson(Schema.Any),
|
|
23
24
|
/** e.g. 26799576_0 */
|
|
24
25
|
'x-electric-chunk-last-offset': Schema.String,
|
|
25
26
|
});
|
|
26
|
-
export const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
27
|
+
export const syncBackend = {};
|
|
28
|
+
export const ApiPushEventPayload = Schema.TaggedStruct('sync-electric.PushEvent', {
|
|
29
|
+
roomId: Schema.String,
|
|
30
|
+
mutationEventEncoded: mutationEventSchemaEncodedAny,
|
|
31
|
+
persisted: Schema.Boolean,
|
|
32
|
+
});
|
|
33
|
+
export const ApiInitRoomPayload = Schema.TaggedStruct('sync-electric.InitRoom', {
|
|
34
|
+
roomId: Schema.String,
|
|
35
|
+
});
|
|
36
|
+
export const ApiPayload = Schema.Union(ApiPushEventPayload, ApiInitRoomPayload);
|
|
37
|
+
export const syncBackendOptions = (options) => options;
|
|
38
|
+
export const makeSyncBackend = ({ electricHost, roomId, pushEventEndpoint, }) => Effect.gen(function* () {
|
|
39
|
+
const endpointUrl = `${electricHost}/v1/shape/events_${roomId}`;
|
|
40
|
+
const isConnected = yield* SubscriptionRef.make(true);
|
|
41
|
+
const initRoom = HttpClientRequest.schemaBodyJson(ApiInitRoomPayload)(HttpClientRequest.post(pushEventEndpoint), ApiInitRoomPayload.make({ roomId })).pipe(Effect.andThen(HttpClient.execute));
|
|
42
|
+
const pendingPushDeferredMap = new Map();
|
|
43
|
+
const pull = (args, { listenForNew }) => Effect.gen(function* () {
|
|
44
|
+
const liveParam = listenForNew ? '&live=true' : '';
|
|
45
|
+
const url = args._tag === 'None'
|
|
46
|
+
? `${endpointUrl}?offset=-1`
|
|
47
|
+
: `${endpointUrl}?offset=${args.value.offset}&shape_id=${args.value.shapeId}${liveParam}`;
|
|
48
|
+
const resp = yield* HttpClient.get(url).pipe(Effect.tapErrorTag('ResponseError', (error) =>
|
|
49
|
+
// TODO handle 409 error when the shapeId you request no longer exists for whatever reason.
|
|
50
|
+
// The correct behavior here is to refetch the shape from scratch and to reset the local state.
|
|
51
|
+
error.response.status === 400 ? initRoom : Effect.fail(error)), Effect.retry({ times: 1 }));
|
|
52
|
+
const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp);
|
|
53
|
+
const nextCursor = {
|
|
54
|
+
offset: headers['x-electric-chunk-last-offset'],
|
|
55
|
+
shapeId: headers['x-electric-shape-id'],
|
|
51
56
|
};
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
// Electric completes the long-poll request after ~20 seconds with a 204 status
|
|
58
|
+
// In this case we just retry where we left off
|
|
59
|
+
if (resp.status === 204) {
|
|
60
|
+
return Option.some([Chunk.empty(), Option.some(nextCursor)]);
|
|
61
|
+
}
|
|
62
|
+
const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp);
|
|
63
|
+
const items = body
|
|
64
|
+
.filter((item) => item.value !== undefined)
|
|
65
|
+
.map((item) => ({
|
|
66
|
+
metadata: Option.some({ offset: item.offset, shapeId: nextCursor.shapeId }),
|
|
67
|
+
mutationEventEncoded: {
|
|
68
|
+
mutation: item.value.mutation,
|
|
69
|
+
args: JSON.parse(item.value.args),
|
|
70
|
+
id: item.value.id,
|
|
71
|
+
parentId: item.value.parentId,
|
|
72
|
+
},
|
|
73
|
+
persisted: true,
|
|
74
|
+
}));
|
|
75
|
+
if (listenForNew === false && items.length === 0) {
|
|
76
|
+
return Option.none();
|
|
77
|
+
}
|
|
78
|
+
const [newItems, pendingPushItems] = Chunk.fromIterable(items).pipe(Chunk.partition((item) => pendingPushDeferredMap.has(eventIdToString(item.mutationEventEncoded.id))));
|
|
79
|
+
for (const item of pendingPushItems) {
|
|
80
|
+
const deferred = pendingPushDeferredMap.get(eventIdToString(item.mutationEventEncoded.id));
|
|
81
|
+
yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata));
|
|
82
|
+
}
|
|
83
|
+
return Option.some([newItems, Option.some(nextCursor)]);
|
|
84
|
+
}).pipe(Effect.scoped, Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })));
|
|
85
|
+
return {
|
|
86
|
+
pull: (args, { listenForNew }) => Stream.unfoldChunkEffect(args.pipe(Option.map((_) => _.metadata), Option.flatten), (metadataOption) => pull(metadataOption, { listenForNew })),
|
|
87
|
+
push: (mutationEventEncoded, persisted) => Effect.gen(function* () {
|
|
88
|
+
const deferred = yield* Deferred.make();
|
|
89
|
+
pendingPushDeferredMap.set(eventIdToString(mutationEventEncoded.id), deferred);
|
|
90
|
+
const resp = yield* HttpClientRequest.schemaBodyJson(ApiPushEventPayload)(HttpClientRequest.post(pushEventEndpoint), ApiPushEventPayload.make({ roomId, mutationEventEncoded, persisted })).pipe(Effect.andThen(HttpClient.execute), Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))), Effect.scoped, Effect.mapError((cause) => InvalidPushError.make({ message: cause.toString() })));
|
|
91
|
+
if (!resp.success) {
|
|
92
|
+
yield* InvalidPushError.make({ message: 'Push failed' });
|
|
93
|
+
}
|
|
94
|
+
const metadata = yield* Deferred.await(deferred);
|
|
95
|
+
pendingPushDeferredMap.delete(eventIdToString(mutationEventEncoded.id));
|
|
96
|
+
return { metadata: Option.some(metadata) };
|
|
97
|
+
}),
|
|
98
|
+
isConnected,
|
|
99
|
+
};
|
|
100
|
+
});
|
|
101
|
+
const eventIdToString = (eventId) => `${eventId.global}_${eventId.local}`;
|
|
54
102
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AAExE,OAAO,EACL,KAAK,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EAClB,MAAM,EACN,MAAM,EACN,MAAM,EACN,eAAe,GAChB,MAAM,yBAAyB,CAAA;AAEhC;;;;;;;;;EASE;AAEF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,6CAA6C;IAC7C,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;CACvC,CAAC,CAAA;AAEF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC,MAAM;IACpC,qDAAqD;IACrD,sBAAsB;IACtB,8BAA8B,EAAE,MAAM,CAAC,MAAM;CAC9C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,EAAS,CAAA;AAEpC,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC,YAAY,CAAC,yBAAyB,EAAE;IAChF,MAAM,EAAE,MAAM,CAAC,MAAM;IACrB,oBAAoB,EAAE,6BAA6B;IACnD,SAAS,EAAE,MAAM,CAAC,OAAO;CAC1B,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,YAAY,CAAC,wBAAwB,EAAE;IAC9E,MAAM,EAAE,MAAM,CAAC,MAAM;CACtB,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,kBAAkB,CAAC,CAAA;AAE/E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAsC,OAAiB,EAAE,EAAE,CAAC,OAAO,CAAA;AAkCrG,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAC9B,YAAY,EACZ,MAAM,EACN,iBAAiB,GACE,EAAgE,EAAE,CACrF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,WAAW,GAAG,GAAG,YAAY,oBAAoB,MAAM,EAAE,CAAA;IAE/D,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAErD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,cAAc,CAAC,kBAAkB,CAAC,CACnE,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,EACzC,kBAAkB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CACpC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAA;IAE1C,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA2C,CAAA;IAEjF,MAAM,IAAI,GAAG,CAAC,IAAiC,EAAE,EAAE,YAAY,EAA6B,EAAE,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAA;QAClD,MAAM,GAAG,GACP,IAAI,CAAC,IAAI,KAAK,MAAM;YAClB,CAAC,CAAC,GAAG,WAAW,YAAY;YAC5B,CAAC,CAAC,GAAG,WAAW,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,aAAa,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,SAAS,EAAE,CAAA;QAE7F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAC1C,MAAM,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;QAC5C,2FAA2F;QAC3F,+FAA+F;QAC/F,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAC9D,EACD,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAC3B,CAAA;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAA;QAC9E,MAAM,UAAU,GAAG;YACjB,MAAM,EAAE,OAAO,CAAC,8BAA8B,CAAC;YAC/C,OAAO,EAAE,OAAO,CAAC,qBAAqB,CAAC;SACxC,CAAA;QAED,+EAA+E;QAC/E,+CAA+C;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAU,CAAC,CAAA;QACvE,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEvF,MAAM,KAAK,GAAG,IAAI;aACf,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC;aAC1C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;YAC5E,oBAAoB,EAAE;gBACpB,QAAQ,EAAE,IAAI,CAAC,KAAM,CAAC,QAAQ;gBAC9B,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAM,CAAC,IAAI,CAAC;gBAClC,EAAE,EAAE,IAAI,CAAC,KAAM,CAAC,EAAE;gBAClB,QAAQ,EAAE,IAAI,CAAC,KAAM,CAAC,QAAQ;aAC/B;YACD,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC,CAAA;QAEL,IAAI,YAAY,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;QACtB,CAAC;QAED,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CACjE,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,CACrG,CAAA;QAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAE,CAAA;YAC3F,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAA;QACrE,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAU,CAAC,CAAA;IAClE,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACjF,CAAA;IAEH,OAAO;QACL,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAC/B,MAAM,CAAC,iBAAiB,CACtB,IAAI,CAAC,IAAI,CACP,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC7B,MAAM,CAAC,OAAO,CACf,EACD,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,YAAY,EAAE,CAAC,CAC3D;QAEH,IAAI,EAAE,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,CACxC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAgB,CAAA;YACrD,sBAAsB,CAAC,GAAG,CAAC,eAAe,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;YAE9E,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,cAAc,CAAC,mBAAmB,CAAC,CACvE,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,EACzC,mBAAmB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,SAAS,EAAE,CAAC,CACtE,CAAC,IAAI,CACJ,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAClC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAC7F,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACjF,CAAA;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,KAAK,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAA;YAC1D,CAAC;YAED,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YAEhD,sBAAsB,CAAC,MAAM,CAAC,eAAe,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAA;YAEvE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC5C,CAAC,CAAC;QACJ,WAAW;KACwB,CAAA;AACvC,CAAC,CAAC,CAAA;AAEJ,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@livestore/sync-electric",
|
|
3
|
-
"version": "0.0.58-dev.
|
|
3
|
+
"version": "0.0.58-dev.10",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
}
|
|
10
10
|
},
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@livestore/
|
|
13
|
-
"@livestore/
|
|
12
|
+
"@livestore/utils": "0.0.58-dev.10",
|
|
13
|
+
"@livestore/common": "0.0.58-dev.10"
|
|
14
14
|
},
|
|
15
15
|
"devDependencies": {},
|
|
16
16
|
"publishConfig": {
|
package/src/index.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { EventId, SyncBackend, SyncBackendOptionsBase } from '@livestore/common'
|
|
2
2
|
import { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
3
3
|
import { mutationEventSchemaEncodedAny } from '@livestore/common/schema'
|
|
4
|
-
import { isNotUndefined } from '@livestore/utils'
|
|
5
|
-
import { cuid } from '@livestore/utils/cuid'
|
|
6
4
|
import type { Scope } from '@livestore/utils/effect'
|
|
7
5
|
import {
|
|
8
6
|
Chunk,
|
|
@@ -12,8 +10,6 @@ import {
|
|
|
12
10
|
HttpClientRequest,
|
|
13
11
|
HttpClientResponse,
|
|
14
12
|
Option,
|
|
15
|
-
Queue,
|
|
16
|
-
ReadonlyArray,
|
|
17
13
|
Schema,
|
|
18
14
|
Stream,
|
|
19
15
|
SubscriptionRef,
|
|
@@ -26,6 +22,8 @@ Example data:
|
|
|
26
22
|
,{"key":"\"public\".\"events\"/\"1725703554783\"","value":{"id":"1725703554783","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:54.783Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
|
|
27
23
|
,{"headers":{"control":"up-to-date"}}]
|
|
28
24
|
|
|
25
|
+
Also see: https://github.com/electric-sql/electric/blob/main/packages/typescript-client/src/client.ts
|
|
26
|
+
|
|
29
27
|
*/
|
|
30
28
|
|
|
31
29
|
const ResponseItem = Schema.Struct({
|
|
@@ -38,53 +36,176 @@ const ResponseItem = Schema.Struct({
|
|
|
38
36
|
|
|
39
37
|
const ResponseHeaders = Schema.Struct({
|
|
40
38
|
'x-electric-shape-id': Schema.String,
|
|
41
|
-
'x-electric-schema': Schema.parseJson(Schema.Any),
|
|
39
|
+
// 'x-electric-schema': Schema.parseJson(Schema.Any),
|
|
42
40
|
/** e.g. 26799576_0 */
|
|
43
41
|
'x-electric-chunk-last-offset': Schema.String,
|
|
44
42
|
})
|
|
45
43
|
|
|
46
|
-
export const
|
|
44
|
+
export const syncBackend = {} as any
|
|
45
|
+
|
|
46
|
+
export const ApiPushEventPayload = Schema.TaggedStruct('sync-electric.PushEvent', {
|
|
47
|
+
roomId: Schema.String,
|
|
48
|
+
mutationEventEncoded: mutationEventSchemaEncodedAny,
|
|
49
|
+
persisted: Schema.Boolean,
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
export const ApiInitRoomPayload = Schema.TaggedStruct('sync-electric.InitRoom', {
|
|
53
|
+
roomId: Schema.String,
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
export const ApiPayload = Schema.Union(ApiPushEventPayload, ApiInitRoomPayload)
|
|
57
|
+
|
|
58
|
+
export const syncBackendOptions = <TOptions extends SyncBackendOptions>(options: TOptions) => options
|
|
59
|
+
|
|
60
|
+
export interface SyncBackendOptions extends SyncBackendOptionsBase {
|
|
61
|
+
type: 'electric'
|
|
47
62
|
/**
|
|
48
63
|
* The host of the Electric server
|
|
49
64
|
*
|
|
50
|
-
* @example "https://
|
|
65
|
+
* @example "https://localhost:3000"
|
|
66
|
+
*/
|
|
67
|
+
electricHost: string
|
|
68
|
+
roomId: string
|
|
69
|
+
/**
|
|
70
|
+
* The POST endpoint to push events to
|
|
71
|
+
*
|
|
72
|
+
* @example "/api/push-event"
|
|
73
|
+
* @example "https://api.myapp.com/push-event"
|
|
51
74
|
*/
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
75
|
+
pushEventEndpoint: string
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
interface LiveStoreGlobalElectric {
|
|
79
|
+
syncBackend: SyncBackendOptions
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
declare global {
|
|
83
|
+
interface LiveStoreGlobal extends LiveStoreGlobalElectric {}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
type SyncMetadata = {
|
|
87
|
+
offset: string
|
|
88
|
+
// TODO move this into "global" sync metadata as it's the same for each event
|
|
89
|
+
shapeId: string
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export const makeSyncBackend = ({
|
|
93
|
+
electricHost,
|
|
94
|
+
roomId,
|
|
95
|
+
pushEventEndpoint,
|
|
96
|
+
}: SyncBackendOptions): Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope> =>
|
|
97
|
+
Effect.gen(function* () {
|
|
98
|
+
const endpointUrl = `${electricHost}/v1/shape/events_${roomId}`
|
|
58
99
|
|
|
59
100
|
const isConnected = yield* SubscriptionRef.make(true)
|
|
60
101
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const resp = yield* HttpClientRequest.get(`${endpointUrl}?offset=${cursor ?? '-1'}`).pipe(
|
|
66
|
-
HttpClient.fetchOk,
|
|
67
|
-
)
|
|
102
|
+
const initRoom = HttpClientRequest.schemaBodyJson(ApiInitRoomPayload)(
|
|
103
|
+
HttpClientRequest.post(pushEventEndpoint),
|
|
104
|
+
ApiInitRoomPayload.make({ roomId }),
|
|
105
|
+
).pipe(Effect.andThen(HttpClient.execute))
|
|
68
106
|
|
|
69
|
-
|
|
70
|
-
const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp)
|
|
107
|
+
const pendingPushDeferredMap = new Map<string, Deferred.Deferred<SyncMetadata>>()
|
|
71
108
|
|
|
72
|
-
|
|
109
|
+
const pull = (args: Option.Option<SyncMetadata>, { listenForNew }: { listenForNew: boolean }) =>
|
|
110
|
+
Effect.gen(function* () {
|
|
111
|
+
const liveParam = listenForNew ? '&live=true' : ''
|
|
112
|
+
const url =
|
|
113
|
+
args._tag === 'None'
|
|
114
|
+
? `${endpointUrl}?offset=-1`
|
|
115
|
+
: `${endpointUrl}?offset=${args.value.offset}&shape_id=${args.value.shapeId}${liveParam}`
|
|
73
116
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
//
|
|
117
|
+
const resp = yield* HttpClient.get(url).pipe(
|
|
118
|
+
Effect.tapErrorTag('ResponseError', (error) =>
|
|
119
|
+
// TODO handle 409 error when the shapeId you request no longer exists for whatever reason.
|
|
120
|
+
// The correct behavior here is to refetch the shape from scratch and to reset the local state.
|
|
121
|
+
error.response.status === 400 ? initRoom : Effect.fail(error),
|
|
122
|
+
),
|
|
123
|
+
Effect.retry({ times: 1 }),
|
|
124
|
+
)
|
|
77
125
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
126
|
+
const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
|
|
127
|
+
const nextCursor = {
|
|
128
|
+
offset: headers['x-electric-chunk-last-offset'],
|
|
129
|
+
shapeId: headers['x-electric-shape-id'],
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Electric completes the long-poll request after ~20 seconds with a 204 status
|
|
133
|
+
// In this case we just retry where we left off
|
|
134
|
+
if (resp.status === 204) {
|
|
135
|
+
return Option.some([Chunk.empty(), Option.some(nextCursor)] as const)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp)
|
|
139
|
+
|
|
140
|
+
const items = body
|
|
141
|
+
.filter((item) => item.value !== undefined)
|
|
142
|
+
.map((item) => ({
|
|
143
|
+
metadata: Option.some({ offset: item.offset!, shapeId: nextCursor.shapeId }),
|
|
144
|
+
mutationEventEncoded: {
|
|
145
|
+
mutation: item.value!.mutation,
|
|
146
|
+
args: JSON.parse(item.value!.args),
|
|
147
|
+
id: item.value!.id,
|
|
148
|
+
parentId: item.value!.parentId,
|
|
149
|
+
},
|
|
150
|
+
persisted: true,
|
|
151
|
+
}))
|
|
152
|
+
|
|
153
|
+
if (listenForNew === false && items.length === 0) {
|
|
154
|
+
return Option.none()
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const [newItems, pendingPushItems] = Chunk.fromIterable(items).pipe(
|
|
158
|
+
Chunk.partition((item) => pendingPushDeferredMap.has(eventIdToString(item.mutationEventEncoded.id))),
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
for (const item of pendingPushItems) {
|
|
162
|
+
const deferred = pendingPushDeferredMap.get(eventIdToString(item.mutationEventEncoded.id))!
|
|
163
|
+
yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return Option.some([newItems, Option.some(nextCursor)] as const)
|
|
167
|
+
}).pipe(
|
|
168
|
+
Effect.scoped,
|
|
169
|
+
Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
pull: (args, { listenForNew }) =>
|
|
174
|
+
Stream.unfoldChunkEffect(
|
|
175
|
+
args.pipe(
|
|
176
|
+
Option.map((_) => _.metadata),
|
|
177
|
+
Option.flatten,
|
|
82
178
|
),
|
|
179
|
+
(metadataOption) => pull(metadataOption, { listenForNew }),
|
|
83
180
|
),
|
|
84
181
|
|
|
85
|
-
|
|
86
|
-
|
|
182
|
+
push: (mutationEventEncoded, persisted) =>
|
|
183
|
+
Effect.gen(function* () {
|
|
184
|
+
const deferred = yield* Deferred.make<SyncMetadata>()
|
|
185
|
+
pendingPushDeferredMap.set(eventIdToString(mutationEventEncoded.id), deferred)
|
|
186
|
+
|
|
187
|
+
const resp = yield* HttpClientRequest.schemaBodyJson(ApiPushEventPayload)(
|
|
188
|
+
HttpClientRequest.post(pushEventEndpoint),
|
|
189
|
+
ApiPushEventPayload.make({ roomId, mutationEventEncoded, persisted }),
|
|
190
|
+
).pipe(
|
|
191
|
+
Effect.andThen(HttpClient.execute),
|
|
192
|
+
Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
|
|
193
|
+
Effect.scoped,
|
|
194
|
+
Effect.mapError((cause) => InvalidPushError.make({ message: cause.toString() })),
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
if (!resp.success) {
|
|
198
|
+
yield* InvalidPushError.make({ message: 'Push failed' })
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const metadata = yield* Deferred.await(deferred)
|
|
202
|
+
|
|
203
|
+
pendingPushDeferredMap.delete(eventIdToString(mutationEventEncoded.id))
|
|
204
|
+
|
|
205
|
+
return { metadata: Option.some(metadata) }
|
|
206
|
+
}),
|
|
87
207
|
isConnected,
|
|
88
|
-
} satisfies
|
|
208
|
+
} satisfies SyncBackend<SyncMetadata>
|
|
89
209
|
})
|
|
90
|
-
|
|
210
|
+
|
|
211
|
+
const eventIdToString = (eventId: EventId) => `${eventId.global}_${eventId.local}`
|