@spooky-sync/core 0.0.1-canary.8 → 0.0.1-canary.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +854 -339
- package/dist/index.js +2633 -396
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +494 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +726 -108
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +89 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +56 -0
- package/src/modules/ref-tables.ts +57 -0
- package/src/modules/sync/engine.ts +97 -37
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +676 -58
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +625 -0
- package/src/types.ts +140 -13
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { u as PinoTransmit } from "../types.js";
|
|
2
|
+
import { Level } from "pino";
|
|
3
|
+
|
|
4
|
+
//#region src/otel/index.d.ts
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Creates a pino browser transmit object that forwards logs to an OpenTelemetry collector.
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```ts
|
|
11
|
+
* import { createOtelTransmit } from '@spooky-sync/core/otel';
|
|
12
|
+
*
|
|
13
|
+
* new Sp00kyClient({
|
|
14
|
+
* // ...
|
|
15
|
+
* otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
|
|
16
|
+
* });
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
declare function createOtelTransmit(endpoint: string, level?: Level): PinoTransmit;
|
|
20
|
+
//#endregion
|
|
21
|
+
export { createOtelTransmit };
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
//#region src/otel/index.ts
|
|
2
|
+
function mapLevelToSeverityNumber(level) {
|
|
3
|
+
switch (level) {
|
|
4
|
+
case "trace": return 1;
|
|
5
|
+
case "debug": return 5;
|
|
6
|
+
case "info": return 9;
|
|
7
|
+
case "warn": return 13;
|
|
8
|
+
case "error": return 17;
|
|
9
|
+
case "fatal": return 21;
|
|
10
|
+
default: return 9;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
async function loadOtelModules(otelEndpoint) {
|
|
14
|
+
const [{ LoggerProvider, BatchLogRecordProcessor }, { OTLPLogExporter }, { resourceFromAttributes }, { ATTR_SERVICE_NAME }] = await Promise.all([
|
|
15
|
+
import("@opentelemetry/sdk-logs"),
|
|
16
|
+
import("@opentelemetry/exporter-logs-otlp-proto"),
|
|
17
|
+
import("@opentelemetry/resources"),
|
|
18
|
+
import("@opentelemetry/semantic-conventions")
|
|
19
|
+
]);
|
|
20
|
+
const loggerProvider = new LoggerProvider({
|
|
21
|
+
resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: "sp00ky-client" }),
|
|
22
|
+
processors: [new BatchLogRecordProcessor(new OTLPLogExporter({ url: otelEndpoint }))]
|
|
23
|
+
});
|
|
24
|
+
const otelLoggerCache = {};
|
|
25
|
+
return (category) => {
|
|
26
|
+
if (!otelLoggerCache[category]) otelLoggerCache[category] = loggerProvider.getLogger(category);
|
|
27
|
+
return otelLoggerCache[category];
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Creates a pino browser transmit object that forwards logs to an OpenTelemetry collector.
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* ```ts
|
|
35
|
+
* import { createOtelTransmit } from '@spooky-sync/core/otel';
|
|
36
|
+
*
|
|
37
|
+
* new Sp00kyClient({
|
|
38
|
+
* // ...
|
|
39
|
+
* otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
|
|
40
|
+
* });
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
function createOtelTransmit(endpoint, level = "info") {
|
|
44
|
+
const otelReady = loadOtelModules(endpoint);
|
|
45
|
+
return {
|
|
46
|
+
level,
|
|
47
|
+
send: (levelLabel, logEvent) => {
|
|
48
|
+
otelReady.then((getOtelLogger) => {
|
|
49
|
+
try {
|
|
50
|
+
const messages = [...logEvent.messages];
|
|
51
|
+
const severityNumber = mapLevelToSeverityNumber(levelLabel);
|
|
52
|
+
let body = "";
|
|
53
|
+
const lastMsg = messages.pop();
|
|
54
|
+
if (typeof lastMsg === "string") body = lastMsg;
|
|
55
|
+
else if (lastMsg) body = JSON.stringify(lastMsg);
|
|
56
|
+
let category = "sp00ky-client::unknown";
|
|
57
|
+
const attributes = {};
|
|
58
|
+
for (const msg of messages) if (typeof msg === "object") {
|
|
59
|
+
if (msg.Category) {
|
|
60
|
+
category = msg.Category;
|
|
61
|
+
delete msg.Category;
|
|
62
|
+
}
|
|
63
|
+
Object.assign(attributes, msg);
|
|
64
|
+
}
|
|
65
|
+
getOtelLogger(category).emit({
|
|
66
|
+
severityNumber,
|
|
67
|
+
severityText: levelLabel.toUpperCase(),
|
|
68
|
+
body,
|
|
69
|
+
attributes: {
|
|
70
|
+
...logEvent.bindings[0],
|
|
71
|
+
...attributes
|
|
72
|
+
},
|
|
73
|
+
timestamp: new Date(logEvent.ts)
|
|
74
|
+
});
|
|
75
|
+
} catch (e) {
|
|
76
|
+
console.warn("Failed to transmit log to OTEL endpoint", e);
|
|
77
|
+
}
|
|
78
|
+
}).catch((e) => {
|
|
79
|
+
console.warn("Failed to load OpenTelemetry modules", e);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
//#endregion
|
|
86
|
+
export { createOtelTransmit };
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
import { RecordId } from "surrealdb";
|
|
2
|
+
import { RecordId as RecordId$1, SchemaStructure } from "@spooky-sync/query-builder";
|
|
3
|
+
import { Level, Level as Level$1, Logger, LoggerOptions } from "pino";
|
|
4
|
+
|
|
5
|
+
//#region src/events/index.d.ts
|
|
6
|
+
/**
|
|
7
|
+
* Utility type to define the payload structure of an event.
|
|
8
|
+
* If the payload type P is never, it defines payload as undefined.
|
|
9
|
+
*/
|
|
10
|
+
type EventPayloadDefinition<P> = [P] extends [never] ? {
|
|
11
|
+
payload: undefined;
|
|
12
|
+
} : {
|
|
13
|
+
payload: P;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Defines the structure of an event with a specific type and payload.
|
|
17
|
+
* @template T The string literal type of the event.
|
|
18
|
+
* @template P The type of the event payload.
|
|
19
|
+
*/
|
|
20
|
+
type EventDefinition<T extends string, P> = {
|
|
21
|
+
type: T;
|
|
22
|
+
} & EventPayloadDefinition<P>;
|
|
23
|
+
/**
|
|
24
|
+
* A map of event types to their definitions.
|
|
25
|
+
* Keys are event names, values are EventDefinitions.
|
|
26
|
+
*/
|
|
27
|
+
type EventTypeMap = Record<string, EventDefinition<any, unknown> | EventDefinition<any, never>>;
|
|
28
|
+
/**
|
|
29
|
+
* Options for pushing/emitting events.
|
|
30
|
+
*/
|
|
31
|
+
interface PushEventOptions {
|
|
32
|
+
/** Configuration for debouncing the event. */
|
|
33
|
+
debounced?: {
|
|
34
|
+
key: string;
|
|
35
|
+
delay: number;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Extracts the full Event object type from the map for a given key.
|
|
40
|
+
*/
|
|
41
|
+
type Event<E extends EventTypeMap, T extends EventType<E>> = E[T];
|
|
42
|
+
/**
|
|
43
|
+
* Extracts the payload type from the map for a given key.
|
|
44
|
+
*/
|
|
45
|
+
type EventPayload<E extends EventTypeMap, T extends EventType<E>> = E[T]['payload'];
|
|
46
|
+
/**
|
|
47
|
+
* Array of available event type keys.
|
|
48
|
+
*/
|
|
49
|
+
type EventTypes<E extends EventTypeMap> = (keyof E)[];
|
|
50
|
+
/**
|
|
51
|
+
* Represents a valid key (event name) from the EventTypeMap.
|
|
52
|
+
*/
|
|
53
|
+
type EventType<E extends EventTypeMap> = keyof E;
|
|
54
|
+
/**
|
|
55
|
+
* Function signature for an event handler.
|
|
56
|
+
*/
|
|
57
|
+
type EventHandler<E extends EventTypeMap, T extends EventType<E>> = (event: Event<E, T>) => void;
|
|
58
|
+
/**
|
|
59
|
+
* Options when subscribing to an event.
|
|
60
|
+
*/
|
|
61
|
+
type EventSubscriptionOptions$1 = {
|
|
62
|
+
/** If true, the handler will be called immediately with the last emitted event of this type (if any). */
|
|
63
|
+
immediately?: boolean;
|
|
64
|
+
/** If true, the subscription will be automatically removed after the first event is handled. */
|
|
65
|
+
once?: boolean;
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* A type-safe event system that handles subscription, emission (including debouncing), and buffering of events.
|
|
69
|
+
* @template E The EventTypeMap defining all supported events.
|
|
70
|
+
*/
|
|
71
|
+
declare class EventSystem<E extends EventTypeMap> {
|
|
72
|
+
private _eventTypes;
|
|
73
|
+
private subscriberId;
|
|
74
|
+
private isProcessing;
|
|
75
|
+
private buffer;
|
|
76
|
+
private subscribers;
|
|
77
|
+
private subscribersTypeMap;
|
|
78
|
+
private lastEvents;
|
|
79
|
+
private debouncedEvents;
|
|
80
|
+
constructor(_eventTypes: EventTypes<E>);
|
|
81
|
+
get eventTypes(): EventTypes<E>;
|
|
82
|
+
/**
|
|
83
|
+
* Subscribes a handler to a specific event type.
|
|
84
|
+
* @param type The event type to subscribe to.
|
|
85
|
+
* @param handler The function to call when the event occurs.
|
|
86
|
+
* @param options Subscription options (once, immediately).
|
|
87
|
+
* @returns A subscription ID that can be used to unsubscribe.
|
|
88
|
+
*/
|
|
89
|
+
subscribe<T extends EventType<E>>(type: T, handler: EventHandler<E, T>, options?: EventSubscriptionOptions$1): number;
|
|
90
|
+
/**
|
|
91
|
+
* Subscribes a handler to multiple event types.
|
|
92
|
+
* @param types An array of event types to subscribe to.
|
|
93
|
+
* @param handler The function to call when any of the events occur.
|
|
94
|
+
* @param options Subscription options.
|
|
95
|
+
* @returns An array of subscription IDs.
|
|
96
|
+
*/
|
|
97
|
+
subscribeMany<T extends EventType<E>>(types: T[], handler: EventHandler<E, T>, options?: EventSubscriptionOptions$1): number[];
|
|
98
|
+
/**
|
|
99
|
+
* Unsubscribes a specific subscription by ID.
|
|
100
|
+
* @param id The subscription ID returned by subscribe().
|
|
101
|
+
* @returns True if the subscription was found and removed, false otherwise.
|
|
102
|
+
*/
|
|
103
|
+
unsubscribe(id: number): boolean;
|
|
104
|
+
/**
|
|
105
|
+
* Emits an event with the given type and payload.
|
|
106
|
+
* @param type The type of event to emit.
|
|
107
|
+
* @param payload The data associated with the event.
|
|
108
|
+
*/
|
|
109
|
+
emit<T extends EventType<E>, P extends EventPayload<E, T>>(type: T, payload: P): void;
|
|
110
|
+
/**
|
|
111
|
+
* Adds a fully constructed event object to the system.
|
|
112
|
+
* Similar to emit, but takes the full event object directly.
|
|
113
|
+
* Supports debouncing if options are provided.
|
|
114
|
+
* @param event The event object.
|
|
115
|
+
* @param options Options for the event push (e.g., debouncing).
|
|
116
|
+
*/
|
|
117
|
+
addEvent<T extends EventType<E>>(event: Event<E, T>, options?: PushEventOptions): void;
|
|
118
|
+
private handleDebouncedEvent;
|
|
119
|
+
private scheduleProcessing;
|
|
120
|
+
private processEvents;
|
|
121
|
+
private dequeue;
|
|
122
|
+
private setLastEvent;
|
|
123
|
+
private broadcastEvent;
|
|
124
|
+
}
|
|
125
|
+
//#endregion
|
|
126
|
+
//#region src/modules/sync/events/index.d.ts
|
|
127
|
+
declare const SyncEventTypes: {
|
|
128
|
+
readonly QueryUpdated: "SYNC_QUERY_UPDATED";
|
|
129
|
+
readonly RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED";
|
|
130
|
+
readonly MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK";
|
|
131
|
+
};
|
|
132
|
+
type SyncEventTypeMap = {
|
|
133
|
+
[SyncEventTypes.QueryUpdated]: EventDefinition<typeof SyncEventTypes.QueryUpdated, {
|
|
134
|
+
queryId: any;
|
|
135
|
+
localHash?: string;
|
|
136
|
+
localArray?: RecordVersionArray;
|
|
137
|
+
remoteHash?: string;
|
|
138
|
+
remoteArray?: RecordVersionArray;
|
|
139
|
+
records: Record<string, any>[];
|
|
140
|
+
}>;
|
|
141
|
+
[SyncEventTypes.RemoteDataIngested]: EventDefinition<typeof SyncEventTypes.RemoteDataIngested, {
|
|
142
|
+
records: Record<string, any>[];
|
|
143
|
+
}>;
|
|
144
|
+
[SyncEventTypes.MutationRolledBack]: EventDefinition<typeof SyncEventTypes.MutationRolledBack, {
|
|
145
|
+
eventType: string;
|
|
146
|
+
recordId: string;
|
|
147
|
+
error: string;
|
|
148
|
+
}>;
|
|
149
|
+
};
|
|
150
|
+
type SyncEventSystem = EventSystem<SyncEventTypeMap>;
|
|
151
|
+
//#endregion
|
|
152
|
+
//#region src/services/logger/index.d.ts
|
|
153
|
+
type Logger$1 = Logger;
|
|
154
|
+
//#endregion
|
|
155
|
+
//#region src/modules/sync/queue/queue-up.d.ts
|
|
156
|
+
type CreateEvent = {
|
|
157
|
+
type: 'create';
|
|
158
|
+
mutation_id: RecordId;
|
|
159
|
+
record_id: RecordId;
|
|
160
|
+
data: Record<string, unknown>;
|
|
161
|
+
record?: Record<string, unknown>;
|
|
162
|
+
tableName?: string;
|
|
163
|
+
options?: PushEventOptions;
|
|
164
|
+
};
|
|
165
|
+
type UpdateEvent = {
|
|
166
|
+
type: 'update';
|
|
167
|
+
mutation_id: RecordId;
|
|
168
|
+
record_id: RecordId;
|
|
169
|
+
data: Record<string, unknown>;
|
|
170
|
+
record?: Record<string, unknown>;
|
|
171
|
+
beforeRecord?: Record<string, unknown>;
|
|
172
|
+
options?: PushEventOptions;
|
|
173
|
+
};
|
|
174
|
+
type DeleteEvent = {
|
|
175
|
+
type: 'delete';
|
|
176
|
+
mutation_id: RecordId;
|
|
177
|
+
record_id: RecordId;
|
|
178
|
+
options?: PushEventOptions;
|
|
179
|
+
};
|
|
180
|
+
type UpEvent = CreateEvent | UpdateEvent | DeleteEvent;
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/types.d.ts
|
|
183
|
+
/**
|
|
184
|
+
* A pino browser transmit object for forwarding logs to an external sink (e.g. OpenTelemetry).
|
|
185
|
+
*/
|
|
186
|
+
type PinoTransmit = NonNullable<NonNullable<LoggerOptions['browser']>['transmit']>;
|
|
187
|
+
/**
|
|
188
|
+
* The type of storage backend to use for the local database.
|
|
189
|
+
* - 'memory': In-memory storage (transient).
|
|
190
|
+
* - 'indexeddb': IndexedDB storage (persistent).
|
|
191
|
+
*/
|
|
192
|
+
type StoreType = 'memory' | 'indexeddb';
|
|
193
|
+
/**
|
|
194
|
+
* Interface for a custom persistence client.
|
|
195
|
+
* Allows providing a custom storage mechanism for the local database.
|
|
196
|
+
*/
|
|
197
|
+
interface PersistenceClient {
|
|
198
|
+
/**
|
|
199
|
+
* Sets a value in the storage.
|
|
200
|
+
* @param key The key to set.
|
|
201
|
+
* @param value The value to store.
|
|
202
|
+
*/
|
|
203
|
+
set<T>(key: string, value: T): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* Gets a value from the storage.
|
|
206
|
+
* @param key The key to retrieve.
|
|
207
|
+
* @returns The stored value or null if not found.
|
|
208
|
+
*/
|
|
209
|
+
get<T>(key: string): Promise<T | null>;
|
|
210
|
+
/**
|
|
211
|
+
* Removes a value from the storage.
|
|
212
|
+
* @param key The key to remove.
|
|
213
|
+
*/
|
|
214
|
+
remove(key: string): Promise<void>;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Supported Time-To-Live (TTL) values for cached queries.
|
|
218
|
+
* Format: number + unit (m=minutes, h=hours, d=days).
|
|
219
|
+
*/
|
|
220
|
+
type QueryTimeToLive = '1m' | '5m' | '10m' | '15m' | '20m' | '25m' | '30m' | '1h' | '2h' | '3h' | '4h' | '5h' | '6h' | '7h' | '8h' | '9h' | '10h' | '11h' | '12h' | '1d';
|
|
221
|
+
/**
|
|
222
|
+
* Result object returned when a query is registered or executed.
|
|
223
|
+
*/
|
|
224
|
+
interface Sp00kyQueryResult {
|
|
225
|
+
/** The unique hash identifier for the query. */
|
|
226
|
+
hash: string;
|
|
227
|
+
}
|
|
228
|
+
type Sp00kyQueryResultPromise = Promise<Sp00kyQueryResult>;
|
|
229
|
+
interface EventSubscriptionOptions {
|
|
230
|
+
priority?: number;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Configuration options for the Sp00ky client.
|
|
234
|
+
* @template S The schema structure type.
|
|
235
|
+
*/
|
|
236
|
+
interface Sp00kyConfig<S extends SchemaStructure> {
|
|
237
|
+
/** Database connection configuration. */
|
|
238
|
+
database: {
|
|
239
|
+
/** The SurrealDB endpoint URL. */
|
|
240
|
+
endpoint?: string;
|
|
241
|
+
/** The namespace to use. */
|
|
242
|
+
namespace: string;
|
|
243
|
+
/** The database name. */
|
|
244
|
+
database: string;
|
|
245
|
+
/** The local store type implementation. */
|
|
246
|
+
store?: StoreType;
|
|
247
|
+
/** Authentication token. */
|
|
248
|
+
token?: string;
|
|
249
|
+
};
|
|
250
|
+
/** The schema definition. */
|
|
251
|
+
schema: S;
|
|
252
|
+
/** The compiled SURQL schema string. */
|
|
253
|
+
schemaSurql: string;
|
|
254
|
+
/** Logging level. */
|
|
255
|
+
logLevel: Level$1;
|
|
256
|
+
/**
|
|
257
|
+
* Persistence client to use.
|
|
258
|
+
* Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
|
|
259
|
+
*/
|
|
260
|
+
persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
|
|
261
|
+
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
262
|
+
otelTransmit?: PinoTransmit;
|
|
263
|
+
/**
|
|
264
|
+
* Debounce time in milliseconds for stream updates.
|
|
265
|
+
* Defaults to 100ms.
|
|
266
|
+
*/
|
|
267
|
+
streamDebounceTime?: number;
|
|
268
|
+
/**
|
|
269
|
+
* Debounce time in milliseconds for syncing collaborative (CRDT) field
|
|
270
|
+
* changes to the remote database. Local writes happen immediately on
|
|
271
|
+
* every keystroke (so reload/offline works), but the remote UPSERT is
|
|
272
|
+
* coalesced over this window. Lower = snappier remote propagation +
|
|
273
|
+
* more network traffic; higher = less traffic + more lag for other
|
|
274
|
+
* collaborators. Defaults to 500ms.
|
|
275
|
+
*/
|
|
276
|
+
crdtDebounceMs?: number;
|
|
277
|
+
/**
|
|
278
|
+
* Cadence (ms) for the `_00_list_ref` poll that catches cross-session
|
|
279
|
+
* UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
|
|
280
|
+
* convergence + more query load; higher = the inverse. Non-positive
|
|
281
|
+
* values fall back to the default (500ms).
|
|
282
|
+
*/
|
|
283
|
+
refSyncIntervalMs?: number;
|
|
284
|
+
/**
|
|
285
|
+
* Instant-hydrate cold queries: when a query is registered with no local
|
|
286
|
+
* data yet, first run its surql directly on the remote (one-shot) and display
|
|
287
|
+
* the result immediately, THEN do the full realtime registration in the
|
|
288
|
+
* background. The hydrated rows are ingested with their versions so the
|
|
289
|
+
* registration's `syncRecords` skips re-pulling unchanged bodies. Cuts cold
|
|
290
|
+
* first-paint from ~one full registration round-trip to ~one query.
|
|
291
|
+
* Defaults to `true`; set `false` to keep the old wait-for-registration path.
|
|
292
|
+
*/
|
|
293
|
+
instantHydrate?: boolean;
|
|
294
|
+
}
|
|
295
|
+
type QueryHash = string;
|
|
296
|
+
type RecordVersionArray = Array<[string, number]>;
|
|
297
|
+
/**
|
|
298
|
+
* Represents the difference between two record version sets.
|
|
299
|
+
* Used for synchronizing local and remote states.
|
|
300
|
+
*/
|
|
301
|
+
interface RecordVersionDiff {
|
|
302
|
+
/** List of records added. */
|
|
303
|
+
added: Array<{
|
|
304
|
+
id: RecordId$1<string>;
|
|
305
|
+
version: number;
|
|
306
|
+
}>;
|
|
307
|
+
/** List of records updated. */
|
|
308
|
+
updated: Array<{
|
|
309
|
+
id: RecordId$1<string>;
|
|
310
|
+
version: number;
|
|
311
|
+
}>;
|
|
312
|
+
/** List of record IDs removed. */
|
|
313
|
+
removed: RecordId$1<string>[];
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Configuration for a specific query instance.
|
|
317
|
+
* Stores metadata about the query's state, parameters, and versioning.
|
|
318
|
+
*/
|
|
319
|
+
interface QueryConfig {
|
|
320
|
+
/** The unique ID of the query config record. */
|
|
321
|
+
id: RecordId$1<string>;
|
|
322
|
+
/** The SURQL query string. */
|
|
323
|
+
surql: string;
|
|
324
|
+
/** Parameters used in the query. */
|
|
325
|
+
params: Record<string, any>;
|
|
326
|
+
/** The version array representing the local state of results. */
|
|
327
|
+
localArray: RecordVersionArray;
|
|
328
|
+
/** The version array representing the remote (server) state of results. */
|
|
329
|
+
remoteArray: RecordVersionArray;
|
|
330
|
+
/**
|
|
331
|
+
* In-memory only (never persisted to `_00_query`): version array of the
|
|
332
|
+
* subquery CHILD rows pulled via `parent IS NOT NONE` edges, so the
|
|
333
|
+
* child-body sync is idempotent across polls. Kept separate from
|
|
334
|
+
* `remoteArray` so related child rows never enter the primary window /
|
|
335
|
+
* `rowCount` / `localArray`.
|
|
336
|
+
*/
|
|
337
|
+
subqueryRemoteArray?: RecordVersionArray;
|
|
338
|
+
/** Time-To-Live for this query. */
|
|
339
|
+
ttl: QueryTimeToLive;
|
|
340
|
+
/** Timestamp when the query was last accessed/active. */
|
|
341
|
+
lastActiveAt: Date;
|
|
342
|
+
/** The name of the table this query targets (if applicable). */
|
|
343
|
+
tableName: string;
|
|
344
|
+
}
|
|
345
|
+
type QueryConfigRecord = QueryConfig & {
|
|
346
|
+
id: string;
|
|
347
|
+
};
|
|
348
|
+
/**
|
|
349
|
+
* Runtime fetch status of a live query.
|
|
350
|
+
* - `idle`: not currently fetching missing records.
|
|
351
|
+
* - `fetching`: the sync engine is fetching/ingesting missing records for this
|
|
352
|
+
* query. UI notifications are coalesced so the result lands as a single
|
|
353
|
+
* update once fetching completes.
|
|
354
|
+
*/
|
|
355
|
+
type QueryStatus = 'idle' | 'fetching';
|
|
356
|
+
/**
|
|
357
|
+
* Internal state of a live query.
|
|
358
|
+
*/
|
|
359
|
+
interface QueryState {
|
|
360
|
+
/** The configuration for this query. */
|
|
361
|
+
config: QueryConfig;
|
|
362
|
+
/** The current cached records for this query. */
|
|
363
|
+
records: Record<string, any>[];
|
|
364
|
+
/** Set once `applyHydration` has run for this query, so the cold instant-hydrate
|
|
365
|
+
* path fires at most once per query (see DataModule.isCold/applyHydration). */
|
|
366
|
+
hydrated?: boolean;
|
|
367
|
+
/** Timer for TTL expiration. */
|
|
368
|
+
ttlTimer: NodeJS.Timeout | null;
|
|
369
|
+
/** TTL duration in milliseconds. */
|
|
370
|
+
ttlDurationMs: number;
|
|
371
|
+
/** Number of times the query has been updated. */
|
|
372
|
+
updateCount: number;
|
|
373
|
+
/**
|
|
374
|
+
* Rolling window of the most recent materialization-step latencies (ms).
|
|
375
|
+
* Capped at MATERIALIZATION_SAMPLE_WINDOW; used to recompute p55/p90/p99
|
|
376
|
+
* before each persist to `_00_query`. Samples themselves are not persisted.
|
|
377
|
+
*/
|
|
378
|
+
materializationSamples: number[];
|
|
379
|
+
/** Most recent end-to-end ingest latency in ms, or null until the first ingest. */
|
|
380
|
+
lastIngestLatencyMs: number | null;
|
|
381
|
+
/** Cumulative count of ingest/materialization errors observed for this query. */
|
|
382
|
+
errorCount: number;
|
|
383
|
+
/**
|
|
384
|
+
* Ephemeral runtime fetch status. Not persisted to `_00_query`; observable
|
|
385
|
+
* via DevTools and the `useQuery` hook. `fetching` while the sync engine is
|
|
386
|
+
* pulling missing records for this query, otherwise `idle`.
|
|
387
|
+
*/
|
|
388
|
+
status: QueryStatus;
|
|
389
|
+
/**
|
|
390
|
+
* Rolling per-phase timing samples (ms), in addition to `materializationSamples`
|
|
391
|
+
* (which holds the SSP whole-ingest wall time). Keyed by `TimingPhase` minus
|
|
392
|
+
* `ssp`. Not persisted — surfaced live to DevTools + MCP via `phaseTimings`.
|
|
393
|
+
*/
|
|
394
|
+
phaseSamples: Record<string, number[]>;
|
|
395
|
+
/** Most recent sample (ms) per phase, or null. */
|
|
396
|
+
phaseLast: Record<string, number | null>;
|
|
397
|
+
/** One-shot SSP registration timings (ms). */
|
|
398
|
+
registrationTimings: RegistrationTimings;
|
|
399
|
+
}
|
|
400
|
+
/** Cap on the rolling materialization-sample window kept per query in memory. */
|
|
401
|
+
declare const MATERIALIZATION_SAMPLE_WINDOW = 100;
|
|
402
|
+
/** Timed processing phases surfaced per query. `ssp` is the WASM-ingest wall
|
|
403
|
+
* time; the `ssp*` phases are its internal breakdown from the SSP binding. */
|
|
404
|
+
type TimingPhase = 'ssp' | 'sspStoreApply' | 'sspCircuitStep' | 'sspTransform' | 'localFetch' | 'remoteFetch' | 'frontend';
|
|
405
|
+
/** One-shot registration timings (ms), captured once when a query registers. */
|
|
406
|
+
interface RegistrationTimings {
|
|
407
|
+
/** SSP surql→plan parse + permission injection. */
|
|
408
|
+
parseMs: number | null;
|
|
409
|
+
/** SSP operator-DAG build. */
|
|
410
|
+
planMs: number | null;
|
|
411
|
+
/** SSP initial snapshot evaluation. */
|
|
412
|
+
snapshotMs: number | null;
|
|
413
|
+
/** Wall time of `cache.registerQuery` (register_view round-trip). */
|
|
414
|
+
wallMs: number | null;
|
|
415
|
+
}
|
|
416
|
+
/** Percentile summary for one timed phase, surfaced to DevTools + MCP. */
|
|
417
|
+
interface PhaseStat {
|
|
418
|
+
lastMs: number | null;
|
|
419
|
+
p50: number | null;
|
|
420
|
+
p90: number | null;
|
|
421
|
+
p99: number | null;
|
|
422
|
+
count: number;
|
|
423
|
+
}
|
|
424
|
+
/** Per-query processing-time breakdown surfaced via DevTools panel + MCP. */
|
|
425
|
+
interface QueryTimings {
|
|
426
|
+
ssp: PhaseStat;
|
|
427
|
+
sspStoreApply: PhaseStat;
|
|
428
|
+
sspCircuitStep: PhaseStat;
|
|
429
|
+
sspTransform: PhaseStat;
|
|
430
|
+
localFetch: PhaseStat;
|
|
431
|
+
remoteFetch: PhaseStat;
|
|
432
|
+
frontend: PhaseStat;
|
|
433
|
+
registration: RegistrationTimings;
|
|
434
|
+
updateCount: number;
|
|
435
|
+
errorCount: number;
|
|
436
|
+
}
|
|
437
|
+
type QueryUpdateCallback = (records: Record<string, any>[]) => void;
|
|
438
|
+
type QueryStatusCallback = (status: QueryStatus) => void;
|
|
439
|
+
type MutationCallback = (mutations: UpEvent[]) => void;
|
|
440
|
+
type MutationEventType = 'create' | 'update' | 'delete';
|
|
441
|
+
/**
|
|
442
|
+
* Represents a mutation event (create, update, delete) to be synchronized.
|
|
443
|
+
*/
|
|
444
|
+
interface MutationEvent {
|
|
445
|
+
/** Example: 'create', 'update', or 'delete'. */
|
|
446
|
+
type: MutationEventType;
|
|
447
|
+
/** unique id of the mutation */
|
|
448
|
+
mutation_id: RecordId$1<string>;
|
|
449
|
+
/** The ID of the record being mutated. */
|
|
450
|
+
record_id: RecordId$1<string>;
|
|
451
|
+
/** The data payload for create/update operations. */
|
|
452
|
+
data?: any;
|
|
453
|
+
/** The full record data (optional context). */
|
|
454
|
+
record?: any;
|
|
455
|
+
/** Options for the mutation event (e.g., debounce settings). */
|
|
456
|
+
options?: PushEventOptions;
|
|
457
|
+
/** Timestamp when the event was created. */
|
|
458
|
+
createdAt: Date;
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Options for run operations.
|
|
462
|
+
*/
|
|
463
|
+
interface RunOptions {
|
|
464
|
+
assignedTo?: string;
|
|
465
|
+
max_retries?: number;
|
|
466
|
+
retry_strategy?: 'linear' | 'exponential';
|
|
467
|
+
/** Timeout in seconds for the backend HTTP call. Only used if the backend allows timeout override. */
|
|
468
|
+
timeout?: number;
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Options for update operations.
|
|
472
|
+
*/
|
|
473
|
+
interface UpdateOptions {
|
|
474
|
+
/**
|
|
475
|
+
* Debounce configuration for the update.
|
|
476
|
+
* If boolean, enables default debounce behavior.
|
|
477
|
+
*/
|
|
478
|
+
debounced?: boolean | DebounceOptions;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Configuration options for debouncing updates.
|
|
482
|
+
*/
|
|
483
|
+
interface DebounceOptions {
|
|
484
|
+
/**
|
|
485
|
+
* The key to use for debouncing.
|
|
486
|
+
* - 'recordId': Debounce based on the specific record ID. WARNING: IT WILL ONLY ACCEPT THE LATEST CHANGE AND DOES *NOT* MERGE THE PREVIOUS ONCES. IF YOU ARE UNSURE JUST USE 'recordId_x_fields'.
|
|
487
|
+
* - 'recordId_x_fields': Debounce based on record ID and specific fields.
|
|
488
|
+
*/
|
|
489
|
+
key?: 'recordId' | 'recordId_x_fields';
|
|
490
|
+
/** The debounce delay in milliseconds. */
|
|
491
|
+
delay?: number;
|
|
492
|
+
}
|
|
493
|
+
//#endregion
|
|
494
|
+
export { UpEvent as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SyncEventSystem as M, EventDefinition as N, TimingPhase as O, EventSystem as P, RegistrationTimings as S, Sp00kyQueryResult as T, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, Logger$1 as j, UpdateOptions as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.81",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
".": {
|
|
9
9
|
"types": "./dist/index.d.ts",
|
|
10
10
|
"import": "./dist/index.js"
|
|
11
|
+
},
|
|
12
|
+
"./otel": {
|
|
13
|
+
"types": "./dist/otel/index.d.ts",
|
|
14
|
+
"import": "./dist/otel/index.js"
|
|
11
15
|
}
|
|
12
16
|
},
|
|
13
17
|
"scripts": {
|
|
@@ -18,24 +22,50 @@
|
|
|
18
22
|
},
|
|
19
23
|
"repository": {
|
|
20
24
|
"type": "git",
|
|
21
|
-
"url": "https://github.com/mono424/
|
|
25
|
+
"url": "https://github.com/mono424/sp00ky.git",
|
|
22
26
|
"directory": "packages/core"
|
|
23
27
|
},
|
|
28
|
+
"keywords": [
|
|
29
|
+
"surrealdb",
|
|
30
|
+
"local-first",
|
|
31
|
+
"sync",
|
|
32
|
+
"tanstack-intent"
|
|
33
|
+
],
|
|
24
34
|
"publishConfig": {
|
|
25
35
|
"access": "public"
|
|
26
36
|
},
|
|
27
|
-
"
|
|
37
|
+
"peerDependencies": {
|
|
28
38
|
"@opentelemetry/api": "^1.9.0",
|
|
29
39
|
"@opentelemetry/exporter-logs-otlp-proto": "^0.211.0",
|
|
30
|
-
"@opentelemetry/resources": "^2.
|
|
40
|
+
"@opentelemetry/resources": "^2.6.0",
|
|
31
41
|
"@opentelemetry/sdk-logs": "^0.211.0",
|
|
32
|
-
"@opentelemetry/semantic-conventions": "^1.39.0"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
"@
|
|
42
|
+
"@opentelemetry/semantic-conventions": "^1.39.0"
|
|
43
|
+
},
|
|
44
|
+
"peerDependenciesMeta": {
|
|
45
|
+
"@opentelemetry/api": {
|
|
46
|
+
"optional": true
|
|
47
|
+
},
|
|
48
|
+
"@opentelemetry/exporter-logs-otlp-proto": {
|
|
49
|
+
"optional": true
|
|
50
|
+
},
|
|
51
|
+
"@opentelemetry/resources": {
|
|
52
|
+
"optional": true
|
|
53
|
+
},
|
|
54
|
+
"@opentelemetry/sdk-logs": {
|
|
55
|
+
"optional": true
|
|
56
|
+
},
|
|
57
|
+
"@opentelemetry/semantic-conventions": {
|
|
58
|
+
"optional": true
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
"dependencies": {
|
|
62
|
+
"@spooky-sync/query-builder": "0.0.1-canary.81",
|
|
63
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.81",
|
|
64
|
+
"@surrealdb/wasm": "^3.0.3",
|
|
36
65
|
"fast-json-patch": "^3.1.1",
|
|
66
|
+
"loro-crdt": "^1.5.6",
|
|
37
67
|
"pino": "^10.1.0",
|
|
38
|
-
"surrealdb": "2.0.
|
|
68
|
+
"surrealdb": "2.0.3"
|
|
39
69
|
},
|
|
40
70
|
"devDependencies": {
|
|
41
71
|
"@types/node": "^22.5.2",
|