@spooky-sync/core 0.0.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +21 -0
  2. package/dist/index.d.ts +590 -0
  3. package/dist/index.js +3082 -0
  4. package/package.json +46 -0
  5. package/src/events/events.test.ts +242 -0
  6. package/src/events/index.ts +261 -0
  7. package/src/index.ts +3 -0
  8. package/src/modules/auth/events/index.ts +18 -0
  9. package/src/modules/auth/index.ts +267 -0
  10. package/src/modules/cache/index.ts +241 -0
  11. package/src/modules/cache/types.ts +19 -0
  12. package/src/modules/data/data.test.ts +58 -0
  13. package/src/modules/data/index.ts +777 -0
  14. package/src/modules/devtools/index.ts +364 -0
  15. package/src/modules/sync/engine.ts +163 -0
  16. package/src/modules/sync/events/index.ts +77 -0
  17. package/src/modules/sync/index.ts +3 -0
  18. package/src/modules/sync/queue/index.ts +2 -0
  19. package/src/modules/sync/queue/queue-down.ts +89 -0
  20. package/src/modules/sync/queue/queue-up.ts +223 -0
  21. package/src/modules/sync/scheduler.ts +84 -0
  22. package/src/modules/sync/sync.ts +407 -0
  23. package/src/modules/sync/utils.test.ts +311 -0
  24. package/src/modules/sync/utils.ts +171 -0
  25. package/src/services/database/database.ts +108 -0
  26. package/src/services/database/events/index.ts +32 -0
  27. package/src/services/database/index.ts +5 -0
  28. package/src/services/database/local-migrator.ts +203 -0
  29. package/src/services/database/local.ts +99 -0
  30. package/src/services/database/remote.ts +110 -0
  31. package/src/services/logger/index.ts +118 -0
  32. package/src/services/persistence/localstorage.ts +26 -0
  33. package/src/services/persistence/surrealdb.ts +62 -0
  34. package/src/services/stream-processor/index.ts +364 -0
  35. package/src/services/stream-processor/stream-processor.test.ts +140 -0
  36. package/src/services/stream-processor/wasm-types.ts +31 -0
  37. package/src/spooky.ts +346 -0
  38. package/src/types.ts +237 -0
  39. package/src/utils/error-classification.ts +28 -0
  40. package/src/utils/index.ts +172 -0
  41. package/src/utils/parser.test.ts +125 -0
  42. package/src/utils/parser.ts +46 -0
  43. package/src/utils/surql.ts +182 -0
  44. package/src/utils/utils.test.ts +152 -0
  45. package/src/utils/withRetry.test.ts +153 -0
  46. package/tsconfig.json +14 -0
  47. package/tsdown.config.ts +9 -0
package/package.json ADDED
@@ -0,0 +1,46 @@
1
+ {
2
+ "name": "@spooky-sync/core",
3
+ "version": "0.0.0-canary.1",
4
+ "type": "module",
5
+ "main": "./dist/index.js",
6
+ "types": "./dist/index.d.ts",
7
+ "exports": {
8
+ ".": {
9
+ "types": "./dist/index.d.ts",
10
+ "import": "./dist/index.js"
11
+ }
12
+ },
13
+ "scripts": {
14
+ "dev": "tsdown --watch",
15
+ "dev:example": "pnpm dev",
16
+ "build": "tsdown",
17
+ "test": "vitest"
18
+ },
19
+ "repository": {
20
+ "type": "git",
21
+ "url": "https://github.com/mono424/spooky.git",
22
+ "directory": "packages/core"
23
+ },
24
+ "publishConfig": {
25
+ "access": "public"
26
+ },
27
+ "dependencies": {
28
+ "@opentelemetry/api": "^1.9.0",
29
+ "@opentelemetry/exporter-logs-otlp-proto": "^0.211.0",
30
+ "@opentelemetry/resources": "^2.5.0",
31
+ "@opentelemetry/sdk-logs": "^0.211.0",
32
+ "@opentelemetry/semantic-conventions": "^1.39.0",
33
+ "@spooky-sync/query-builder": "workspace:*",
34
+ "@spooky-sync/ssp-wasm": "workspace:*",
35
+ "@surrealdb/wasm": "^3.0.0",
36
+ "fast-json-patch": "^3.1.1",
37
+ "pino": "^10.1.0",
38
+ "surrealdb": "2.0.0"
39
+ },
40
+ "devDependencies": {
41
+ "@types/node": "^22.5.2",
42
+ "tsdown": "^0.12.4",
43
+ "typescript": "^5.6.2",
44
+ "vitest": "^3.2.0"
45
+ }
46
+ }
@@ -0,0 +1,242 @@
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
2
+ import { EventSystem, EventDefinition } from './index';
3
+
4
+ // Define test event types
5
+ type TestEvents = {
6
+ userCreated: EventDefinition<'userCreated', { id: string; name: string }>;
7
+ userUpdated: EventDefinition<'userUpdated', { id: string; changes: string[] }>;
8
+ ping: EventDefinition<'ping', undefined>;
9
+ };
10
+
11
+ function createTestSystem(): EventSystem<TestEvents> {
12
+ return new EventSystem<TestEvents>(['userCreated', 'userUpdated', 'ping']);
13
+ }
14
+
15
+ /** Flush microtasks so queueMicrotask-scheduled processing runs. */
16
+ async function flush() {
17
+ await new Promise<void>((resolve) => queueMicrotask(resolve));
18
+ }
19
+
20
+ describe('EventSystem', () => {
21
+ beforeEach(() => {
22
+ vi.useFakeTimers();
23
+ });
24
+
25
+ afterEach(() => {
26
+ vi.useRealTimers();
27
+ });
28
+
29
+ describe('subscribe + emit', () => {
30
+ it('handler receives event with correct type and payload', async () => {
31
+ const system = createTestSystem();
32
+ const handler = vi.fn();
33
+
34
+ system.subscribe('userCreated', handler);
35
+ system.emit('userCreated', { id: '1', name: 'Alice' });
36
+
37
+ await flush();
38
+
39
+ expect(handler).toHaveBeenCalledOnce();
40
+ expect(handler).toHaveBeenCalledWith({
41
+ type: 'userCreated',
42
+ payload: { id: '1', name: 'Alice' },
43
+ });
44
+ });
45
+
46
+ it('does not call handler for different event type', async () => {
47
+ const system = createTestSystem();
48
+ const handler = vi.fn();
49
+
50
+ system.subscribe('userCreated', handler);
51
+ system.emit('userUpdated', { id: '1', changes: ['name'] });
52
+
53
+ await flush();
54
+
55
+ expect(handler).not.toHaveBeenCalled();
56
+ });
57
+ });
58
+
59
+ describe('multiple subscribers', () => {
60
+ it('all handlers are called', async () => {
61
+ const system = createTestSystem();
62
+ const handler1 = vi.fn();
63
+ const handler2 = vi.fn();
64
+
65
+ system.subscribe('userCreated', handler1);
66
+ system.subscribe('userCreated', handler2);
67
+ system.emit('userCreated', { id: '1', name: 'Alice' });
68
+
69
+ await flush();
70
+
71
+ expect(handler1).toHaveBeenCalledOnce();
72
+ expect(handler2).toHaveBeenCalledOnce();
73
+ });
74
+ });
75
+
76
+ describe('unsubscribe', () => {
77
+ it('handler no longer called after unsubscribe', async () => {
78
+ const system = createTestSystem();
79
+ const handler = vi.fn();
80
+
81
+ const id = system.subscribe('userCreated', handler);
82
+ system.unsubscribe(id);
83
+ system.emit('userCreated', { id: '1', name: 'Alice' });
84
+
85
+ await flush();
86
+
87
+ expect(handler).not.toHaveBeenCalled();
88
+ });
89
+
90
+ it('returns true when subscription exists', () => {
91
+ const system = createTestSystem();
92
+ const id = system.subscribe('userCreated', vi.fn());
93
+ expect(system.unsubscribe(id)).toBe(true);
94
+ });
95
+
96
+ it('returns false when subscription does not exist', () => {
97
+ const system = createTestSystem();
98
+ expect(system.unsubscribe(999)).toBe(false);
99
+ });
100
+ });
101
+
102
+ describe('subscribeMany', () => {
103
+ it('subscribes to multiple event types', async () => {
104
+ const system = createTestSystem();
105
+ const handler = vi.fn();
106
+
107
+ const ids = system.subscribeMany(['userCreated', 'userUpdated'], handler);
108
+ expect(ids).toHaveLength(2);
109
+
110
+ system.emit('userCreated', { id: '1', name: 'Alice' });
111
+ system.emit('userUpdated', { id: '1', changes: ['name'] });
112
+
113
+ await flush();
114
+
115
+ expect(handler).toHaveBeenCalledTimes(2);
116
+ });
117
+ });
118
+
119
+ describe('once option', () => {
120
+ it('handler called only once then auto-removed', async () => {
121
+ const system = createTestSystem();
122
+ const handler = vi.fn();
123
+
124
+ system.subscribe('userCreated', handler, { once: true });
125
+
126
+ system.emit('userCreated', { id: '1', name: 'Alice' });
127
+ await flush();
128
+
129
+ system.emit('userCreated', { id: '2', name: 'Bob' });
130
+ await flush();
131
+
132
+ expect(handler).toHaveBeenCalledOnce();
133
+ });
134
+ });
135
+
136
+ describe('immediately option', () => {
137
+ it('handler called with last event on subscribe', async () => {
138
+ const system = createTestSystem();
139
+
140
+ // Emit an event first
141
+ system.emit('userCreated', { id: '1', name: 'Alice' });
142
+ await flush();
143
+
144
+ // Subscribe with immediately option
145
+ const handler = vi.fn();
146
+ system.subscribe('userCreated', handler, { immediately: true });
147
+
148
+ // Handler should be called synchronously with last event
149
+ expect(handler).toHaveBeenCalledOnce();
150
+ expect(handler).toHaveBeenCalledWith({
151
+ type: 'userCreated',
152
+ payload: { id: '1', name: 'Alice' },
153
+ });
154
+ });
155
+
156
+ it('does not call handler if no prior event', () => {
157
+ const system = createTestSystem();
158
+ const handler = vi.fn();
159
+
160
+ system.subscribe('userCreated', handler, { immediately: true });
161
+
162
+ expect(handler).not.toHaveBeenCalled();
163
+ });
164
+ });
165
+
166
+ describe('debounced events', () => {
167
+ it('only fires the last event after delay', async () => {
168
+ const system = createTestSystem();
169
+ const handler = vi.fn();
170
+
171
+ system.subscribe('userUpdated', handler);
172
+
173
+ // Add debounced events rapidly
174
+ system.addEvent(
175
+ { type: 'userUpdated', payload: { id: '1', changes: ['a'] } },
176
+ { debounced: { key: 'update-1', delay: 100 } }
177
+ );
178
+ system.addEvent(
179
+ { type: 'userUpdated', payload: { id: '1', changes: ['b'] } },
180
+ { debounced: { key: 'update-1', delay: 100 } }
181
+ );
182
+ system.addEvent(
183
+ { type: 'userUpdated', payload: { id: '1', changes: ['c'] } },
184
+ { debounced: { key: 'update-1', delay: 100 } }
185
+ );
186
+
187
+ // Before delay, nothing should fire
188
+ await flush();
189
+ expect(handler).not.toHaveBeenCalled();
190
+
191
+ // After delay
192
+ await vi.advanceTimersByTimeAsync(100);
193
+ await flush();
194
+
195
+ expect(handler).toHaveBeenCalledOnce();
196
+ expect(handler).toHaveBeenCalledWith({
197
+ type: 'userUpdated',
198
+ payload: { id: '1', changes: ['c'] },
199
+ });
200
+ });
201
+
202
+ it('different keys debounce independently', async () => {
203
+ const system = createTestSystem();
204
+ const handler = vi.fn();
205
+
206
+ system.subscribe('userUpdated', handler);
207
+
208
+ system.addEvent(
209
+ { type: 'userUpdated', payload: { id: '1', changes: ['a'] } },
210
+ { debounced: { key: 'key-1', delay: 100 } }
211
+ );
212
+ system.addEvent(
213
+ { type: 'userUpdated', payload: { id: '2', changes: ['b'] } },
214
+ { debounced: { key: 'key-2', delay: 100 } }
215
+ );
216
+
217
+ await vi.advanceTimersByTimeAsync(100);
218
+ await flush();
219
+
220
+ expect(handler).toHaveBeenCalledTimes(2);
221
+ });
222
+ });
223
+
224
+ describe('event buffering', () => {
225
+ it('multiple emits in same tick processed in order', async () => {
226
+ const system = createTestSystem();
227
+ const received: string[] = [];
228
+
229
+ system.subscribe('userCreated', (event) => {
230
+ received.push(event.payload.name);
231
+ });
232
+
233
+ system.emit('userCreated', { id: '1', name: 'Alice' });
234
+ system.emit('userCreated', { id: '2', name: 'Bob' });
235
+ system.emit('userCreated', { id: '3', name: 'Charlie' });
236
+
237
+ await flush();
238
+
239
+ expect(received).toEqual(['Alice', 'Bob', 'Charlie']);
240
+ });
241
+ });
242
+ });
@@ -0,0 +1,261 @@
1
+ /**
2
+ * Utility type to define the payload structure of an event.
3
+ * If the payload type P is never, it defines payload as undefined.
4
+ */
5
+ export type EventPayloadDefinition<P> = [P] extends [never]
6
+ ? { payload: undefined }
7
+ : { payload: P };
8
+
9
+ /**
10
+ * Defines the structure of an event with a specific type and payload.
11
+ * @template T The string literal type of the event.
12
+ * @template P The type of the event payload.
13
+ */
14
+ export type EventDefinition<T extends string, P> = {
15
+ type: T;
16
+ } & EventPayloadDefinition<P>;
17
+
18
+ /**
19
+ * A map of event types to their definitions.
20
+ * Keys are event names, values are EventDefinitions.
21
+ */
22
+ export type EventTypeMap = Record<
23
+ string,
24
+ EventDefinition<any, unknown> | EventDefinition<any, never>
25
+ >;
26
+
27
+ /**
28
+ * Options for pushing/emitting events.
29
+ */
30
+ export interface PushEventOptions {
31
+ /** Configuration for debouncing the event. */
32
+ debounced?: { key: string; delay: number };
33
+ }
34
+
35
+ /**
36
+ * Extracts the full Event object type from the map for a given key.
37
+ */
38
+ export type Event<E extends EventTypeMap, T extends EventType<E>> = E[T];
39
+
40
+ /**
41
+ * Extracts the payload type from the map for a given key.
42
+ */
43
+ export type EventPayload<E extends EventTypeMap, T extends EventType<E>> = E[T]['payload'];
44
+
45
+ /**
46
+ * Array of available event type keys.
47
+ */
48
+ export type EventTypes<E extends EventTypeMap> = (keyof E)[];
49
+
50
+ /**
51
+ * Represents a valid key (event name) from the EventTypeMap.
52
+ */
53
+ export type EventType<E extends EventTypeMap> = keyof E;
54
+
55
+ /**
56
+ * Function signature for an event handler.
57
+ */
58
+ export type EventHandler<E extends EventTypeMap, T extends EventType<E>> = (
59
+ event: Event<E, T>
60
+ ) => void;
61
+
62
+ type InnerEventHandler<E extends EventTypeMap, T extends EventType<E>> = {
63
+ id: number;
64
+ handler: EventHandler<E, T>;
65
+ once?: boolean;
66
+ };
67
+
68
+ /**
69
+ * Options when subscribing to an event.
70
+ */
71
+ export type EventSubscriptionOptions = {
72
+ /** If true, the handler will be called immediately with the last emitted event of this type (if any). */
73
+ immediately?: boolean;
74
+ /** If true, the subscription will be automatically removed after the first event is handled. */
75
+ once?: boolean;
76
+ };
77
+
78
+ /**
79
+ * A type-safe event system that handles subscription, emission (including debouncing), and buffering of events.
80
+ * @template E The EventTypeMap defining all supported events.
81
+ */
82
+ export class EventSystem<E extends EventTypeMap> {
83
+ private subscriberId: number = 0;
84
+ private isProcessing: boolean = false;
85
+ private buffer: Event<E, EventType<E>>[];
86
+ private subscribers: {
87
+ [K in EventType<E>]: Map<number, InnerEventHandler<E, K>>;
88
+ };
89
+ private subscribersTypeMap: Map<number, EventType<E>>;
90
+ private lastEvents: {
91
+ [K in EventType<E>]?: Event<E, K>;
92
+ };
93
+
94
+ private debouncedEvents: Map<string, { timer: any; resolve: (val: any) => void }>;
95
+
96
+ constructor(private _eventTypes: EventTypes<E>) {
97
+ this.buffer = [];
98
+ this.subscribers = this._eventTypes.reduce(
99
+ (acc, key) => {
100
+ return Object.assign(acc, { [key]: new Map() });
101
+ },
102
+ {} as { [K in EventType<E>]: Map<number, InnerEventHandler<E, K>> }
103
+ );
104
+ this.lastEvents = {};
105
+ this.subscribersTypeMap = new Map();
106
+ this.debouncedEvents = new Map();
107
+ }
108
+
109
+ get eventTypes(): EventTypes<E> {
110
+ return this._eventTypes;
111
+ }
112
+
113
+ /**
114
+ * Subscribes a handler to a specific event type.
115
+ * @param type The event type to subscribe to.
116
+ * @param handler The function to call when the event occurs.
117
+ * @param options Subscription options (once, immediately).
118
+ * @returns A subscription ID that can be used to unsubscribe.
119
+ */
120
+ subscribe<T extends EventType<E>>(
121
+ type: T,
122
+ handler: EventHandler<E, T>,
123
+ options?: EventSubscriptionOptions
124
+ ): number {
125
+ const id = this.subscriberId++;
126
+ this.subscribers[type].set(id, {
127
+ id,
128
+ handler,
129
+ once: options?.once ?? false,
130
+ });
131
+ this.subscribersTypeMap.set(id, type);
132
+ if (options?.immediately) {
133
+ const lastEvent = this.lastEvents[type];
134
+ if (lastEvent) handler(lastEvent);
135
+ }
136
+ return id;
137
+ }
138
+
139
+ /**
140
+ * Subscribes a handler to multiple event types.
141
+ * @param types An array of event types to subscribe to.
142
+ * @param handler The function to call when any of the events occur.
143
+ * @param options Subscription options.
144
+ * @returns An array of subscription IDs.
145
+ */
146
+ subscribeMany<T extends EventType<E>>(
147
+ types: T[],
148
+ handler: EventHandler<E, T>,
149
+ options?: EventSubscriptionOptions
150
+ ): number[] {
151
+ return types.map((type) => this.subscribe(type, handler, options));
152
+ }
153
+
154
+ /**
155
+ * Unsubscribes a specific subscription by ID.
156
+ * @param id The subscription ID returned by subscribe().
157
+ * @returns True if the subscription was found and removed, false otherwise.
158
+ */
159
+ unsubscribe(id: number): boolean {
160
+ const type = this.subscribersTypeMap.get(id);
161
+ if (type) {
162
+ this.subscribers[type].delete(id);
163
+ this.subscribersTypeMap.delete(id);
164
+ return true;
165
+ }
166
+ return false;
167
+ }
168
+
169
+ /**
170
+ * Emits an event with the given type and payload.
171
+ * @param type The type of event to emit.
172
+ * @param payload The data associated with the event.
173
+ */
174
+ emit<T extends EventType<E>, P extends EventPayload<E, T>>(type: T, payload: P): void {
175
+ const event = {
176
+ type,
177
+ payload,
178
+ } as unknown as Event<E, T>;
179
+ this.addEvent(event);
180
+ }
181
+
182
+ /**
183
+ * Adds a fully constructed event object to the system.
184
+ * Similar to emit, but takes the full event object directly.
185
+ * Supports debouncing if options are provided.
186
+ * @param event The event object.
187
+ * @param options Options for the event push (e.g., debouncing).
188
+ */
189
+ addEvent<T extends EventType<E>>(event: Event<E, T>, options?: PushEventOptions): void {
190
+ if (options?.debounced) {
191
+ this.handleDebouncedEvent(event, options.debounced.key, options.debounced.delay);
192
+ return;
193
+ }
194
+ this.buffer.push(event);
195
+ this.scheduleProcessing();
196
+ }
197
+
198
+ private handleDebouncedEvent<T extends EventType<E>>(
199
+ event: Event<E, T>,
200
+ key: string,
201
+ delay: number
202
+ ): void {
203
+ if (this.debouncedEvents.has(key)) {
204
+ clearTimeout(this.debouncedEvents.get(key)?.timer);
205
+ }
206
+
207
+ const timer = setTimeout(() => {
208
+ this.debouncedEvents.delete(key);
209
+ this.buffer.push(event);
210
+ this.scheduleProcessing();
211
+ }, delay);
212
+
213
+ this.debouncedEvents.set(key, { timer, resolve: () => {} });
214
+ }
215
+
216
+ private scheduleProcessing(): void {
217
+ if (!this.isProcessing) {
218
+ queueMicrotask(() => this.processEvents());
219
+ }
220
+ }
221
+
222
+ private async processEvents(): Promise<void> {
223
+ if (this.isProcessing) return;
224
+ this.isProcessing = true;
225
+
226
+ try {
227
+ while (this.dequeue());
228
+ } finally {
229
+ this.isProcessing = false;
230
+ }
231
+ }
232
+
233
+ private dequeue(): boolean {
234
+ const event = this.buffer.shift();
235
+ if (!event) return false;
236
+
237
+ this.setLastEvent(event.type, event);
238
+ this.broadcastEvent(event.type, event);
239
+ return true;
240
+ }
241
+
242
+ private setLastEvent<T extends EventType<E>>(type: T, event: Event<E, T>): void {
243
+ this.lastEvents[type] = event;
244
+ }
245
+
246
+ private broadcastEvent<T extends EventType<E>>(type: T, event: Event<E, T>): void {
247
+ const subscribers = this.subscribers[type].values();
248
+ for (const subscriber of subscribers) {
249
+ subscriber.handler(event);
250
+ if (subscriber.once) {
251
+ this.unsubscribe(subscriber.id);
252
+ }
253
+ }
254
+ }
255
+ }
256
+
257
+ export function createEventSystem<E extends EventTypeMap>(
258
+ eventTypes: EventTypes<E>
259
+ ): EventSystem<E> {
260
+ return new EventSystem(eventTypes);
261
+ }
package/src/index.ts ADDED
@@ -0,0 +1,3 @@
1
+ export * from './types';
2
+ export * from './spooky';
3
+ export * from './modules/auth/index';
@@ -0,0 +1,18 @@
1
+ import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
2
+
3
+ export const AuthEventTypes = {
4
+ AuthStateChanged: 'AUTH_STATE_CHANGED',
5
+ } as const;
6
+
7
+ export type AuthEventTypeMap = {
8
+ [AuthEventTypes.AuthStateChanged]: EventDefinition<
9
+ typeof AuthEventTypes.AuthStateChanged,
10
+ string | null
11
+ >;
12
+ };
13
+
14
+ export type AuthEventSystem = EventSystem<AuthEventTypeMap>;
15
+
16
+ export function createAuthEventSystem(): AuthEventSystem {
17
+ return createEventSystem([AuthEventTypes.AuthStateChanged]);
18
+ }