@statezero/core 0.2.29 → 0.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wraps a promise with a timeout.
|
|
3
|
+
* @param {Promise} promise - The promise to wrap
|
|
4
|
+
* @param {number} ms - Timeout in milliseconds (default: 30000)
|
|
5
|
+
* @returns {Promise} - Resolves with promise result or rejects on timeout
|
|
6
|
+
*/
|
|
7
|
+
export function withTimeout(promise: Promise<any>, ms?: number): Promise<any>;
|
|
1
8
|
/**
|
|
2
9
|
* Process included entities from a response and register them in the model store.
|
|
3
10
|
* Uses the model registry to find the appropriate model class for each entity type.
|
|
@@ -18,6 +25,12 @@ export function processIncludedEntities(modelStoreRegistry: ModelStoreRegistry,
|
|
|
18
25
|
* @param {string} operationId - A unique id for the operation
|
|
19
26
|
* @param {Function} beforeExit - Optional callback before returning
|
|
20
27
|
* @param {string} canonicalId - Optional canonical_id for cache sharing
|
|
28
|
+
* @param {Object} options - Additional options
|
|
29
|
+
* @param {string} options.namespace - Queue namespace ('default' for app ops, 'sync' for background sync)
|
|
30
|
+
* @param {number} options.timeout - Timeout in ms (default: no timeout)
|
|
21
31
|
* @returns {Promise<Object>} The API response.
|
|
22
32
|
*/
|
|
23
|
-
export function makeApiCall(querySet: QuerySet, operationType: string, args: Object | undefined, operationId: string, beforeExit?: Function, canonicalId?: string
|
|
33
|
+
export function makeApiCall(querySet: QuerySet, operationType: string, args: Object | undefined, operationId: string, beforeExit?: Function, canonicalId?: string, options?: {
|
|
34
|
+
namespace: string;
|
|
35
|
+
timeout: number;
|
|
36
|
+
}): Promise<Object>;
|
|
@@ -5,7 +5,27 @@ import { replaceTempPks } from './tempPk.js';
|
|
|
5
5
|
import { parseStateZeroError, MultipleObjectsReturned, DoesNotExist } from './errors.js';
|
|
6
6
|
import { FileObject } from './files.js';
|
|
7
7
|
import { querysetStoreRegistry } from '../../syncEngine/registries/querysetStoreRegistry.js';
|
|
8
|
-
|
|
8
|
+
// Namespace-based queues: separate queues for different operation types
|
|
9
|
+
// This prevents sync operations from blocking user-initiated app operations
|
|
10
|
+
const queues = new Map();
|
|
11
|
+
function getQueue(namespace = 'default') {
|
|
12
|
+
if (!queues.has(namespace)) {
|
|
13
|
+
queues.set(namespace, new PQueue({ concurrency: 1 }));
|
|
14
|
+
}
|
|
15
|
+
return queues.get(namespace);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Wraps a promise with a timeout.
|
|
19
|
+
* @param {Promise} promise - The promise to wrap
|
|
20
|
+
* @param {number} ms - Timeout in milliseconds (default: 30000)
|
|
21
|
+
* @returns {Promise} - Resolves with promise result or rejects on timeout
|
|
22
|
+
*/
|
|
23
|
+
export function withTimeout(promise, ms = 30000) {
|
|
24
|
+
return Promise.race([
|
|
25
|
+
promise,
|
|
26
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Request timeout after ${ms}ms`)), ms)),
|
|
27
|
+
]);
|
|
28
|
+
}
|
|
9
29
|
/**
|
|
10
30
|
* Process included entities from a response and register them in the model store.
|
|
11
31
|
* Uses the model registry to find the appropriate model class for each entity type.
|
|
@@ -64,9 +84,13 @@ export function processIncludedEntities(modelStoreRegistry, included, ModelClass
|
|
|
64
84
|
* @param {string} operationId - A unique id for the operation
|
|
65
85
|
* @param {Function} beforeExit - Optional callback before returning
|
|
66
86
|
* @param {string} canonicalId - Optional canonical_id for cache sharing
|
|
87
|
+
* @param {Object} options - Additional options
|
|
88
|
+
* @param {string} options.namespace - Queue namespace ('default' for app ops, 'sync' for background sync)
|
|
89
|
+
* @param {number} options.timeout - Timeout in ms (default: no timeout)
|
|
67
90
|
* @returns {Promise<Object>} The API response.
|
|
68
91
|
*/
|
|
69
|
-
export async function makeApiCall(querySet, operationType, args = {}, operationId, beforeExit = null, canonicalId = null) {
|
|
92
|
+
export async function makeApiCall(querySet, operationType, args = {}, operationId, beforeExit = null, canonicalId = null, options = {}) {
|
|
93
|
+
const { namespace = 'default', timeout } = options;
|
|
70
94
|
const ModelClass = querySet.ModelClass;
|
|
71
95
|
const config = configInstance.getConfig();
|
|
72
96
|
const backend = config.backendConfigs[ModelClass.configKey];
|
|
@@ -133,5 +157,9 @@ export async function makeApiCall(querySet, operationType, args = {}, operationI
|
|
|
133
157
|
}
|
|
134
158
|
};
|
|
135
159
|
// Queue write operations, execute read operations immediately
|
|
136
|
-
|
|
160
|
+
// Use namespace-based queues to separate sync ops from app ops
|
|
161
|
+
const queue = getQueue(namespace);
|
|
162
|
+
const queuedCall = isWriteOperation ? queue.add(apiCall) : apiCall();
|
|
163
|
+
// Apply timeout if specified
|
|
164
|
+
return timeout ? withTimeout(queuedCall, timeout) : queuedCall;
|
|
137
165
|
}
|
|
@@ -184,7 +184,8 @@ export class QuerysetStoreRegistry {
|
|
|
184
184
|
};
|
|
185
185
|
const response = await makeApiCall(queryset, 'list', payload, null, // operationId
|
|
186
186
|
null, // beforeExit
|
|
187
|
-
canonical_id // canonical_id for caching
|
|
187
|
+
canonical_id, // canonical_id for caching
|
|
188
|
+
{ namespace: 'sync', timeout: 30000 } // Sync ops on separate queue
|
|
188
189
|
);
|
|
189
190
|
return response.data;
|
|
190
191
|
};
|
|
@@ -22,15 +22,13 @@ export class SyncManager {
|
|
|
22
22
|
debounceMs: number;
|
|
23
23
|
maxWaitMs: number;
|
|
24
24
|
batchStartTime: number | null;
|
|
25
|
-
/** @type {PQueue} */
|
|
26
|
-
syncQueue: PQueue;
|
|
27
25
|
withTimeout(promise: any, ms: any): Promise<any>;
|
|
28
26
|
/**
|
|
29
27
|
* Initialize event handlers for all event receivers
|
|
30
28
|
*/
|
|
31
29
|
initialize(): void;
|
|
32
30
|
startPeriodicSync(): void;
|
|
33
|
-
syncStaleQuerysets(): void
|
|
31
|
+
syncStaleQuerysets(): Promise<void>;
|
|
34
32
|
pruneUnreferencedModels(): void;
|
|
35
33
|
isStoreFollowed(registry: any, semanticKey: any): boolean;
|
|
36
34
|
cleanup(): void;
|
|
@@ -45,5 +43,4 @@ export class SyncManager {
|
|
|
45
43
|
processMetrics(event: any): void;
|
|
46
44
|
processModels(event: any): void;
|
|
47
45
|
}
|
|
48
|
-
import PQueue from "p-queue";
|
|
49
46
|
export const syncManager: SyncManager;
|
package/dist/syncEngine/sync.js
CHANGED
|
@@ -8,7 +8,6 @@ import { metricRegistry, MetricRegistry } from "./registries/metricRegistry.js";
|
|
|
8
8
|
import { getModelClass, getConfig } from "../config.js";
|
|
9
9
|
import { isNil } from "lodash-es";
|
|
10
10
|
import { QuerysetStore } from "./stores/querysetStore.js";
|
|
11
|
-
import PQueue from "p-queue";
|
|
12
11
|
import { v7 as uuidv7 } from "uuid";
|
|
13
12
|
export class EventPayload {
|
|
14
13
|
constructor(data) {
|
|
@@ -96,9 +95,6 @@ export class SyncManager {
|
|
|
96
95
|
this.debounceMs = 100; // Wait for rapid events to settle
|
|
97
96
|
this.maxWaitMs = 2000; // Maximum time to hold events
|
|
98
97
|
this.batchStartTime = null;
|
|
99
|
-
// SyncQueue
|
|
100
|
-
/** @type {PQueue} */
|
|
101
|
-
this.syncQueue = new PQueue({ concurrency: 1 });
|
|
102
98
|
}
|
|
103
99
|
withTimeout(promise, ms) {
|
|
104
100
|
// If no timeout specified, use 2x the periodic sync interval, or 30s as fallback
|
|
@@ -156,8 +152,7 @@ export class SyncManager {
|
|
|
156
152
|
console.log("[SyncManager] No config found, periodic sync disabled by default");
|
|
157
153
|
}
|
|
158
154
|
}
|
|
159
|
-
syncStaleQuerysets() {
|
|
160
|
-
let syncedCount = 0;
|
|
155
|
+
async syncStaleQuerysets() {
|
|
161
156
|
const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
|
|
162
157
|
if (!querysetRegistry)
|
|
163
158
|
return;
|
|
@@ -165,16 +160,18 @@ export class SyncManager {
|
|
|
165
160
|
const operationId = `periodic-sync-${uuidv7()}`;
|
|
166
161
|
// Get dbSynced keys (followed querysets)
|
|
167
162
|
const dbSyncedKeys = new Set([...this.followedQuerysets].map(qs => qs.semanticKey));
|
|
163
|
+
// Collect all stores to sync
|
|
164
|
+
const storesToSync = [];
|
|
168
165
|
for (const [semanticKey, store] of querysetRegistry._stores.entries()) {
|
|
169
|
-
// Only sync if this store is actually being followed
|
|
170
166
|
const isFollowed = this.isStoreFollowed(querysetRegistry, semanticKey);
|
|
171
167
|
if (this.followAllQuerysets || isFollowed) {
|
|
172
|
-
|
|
173
|
-
syncedCount++;
|
|
168
|
+
storesToSync.push(store);
|
|
174
169
|
}
|
|
175
170
|
}
|
|
176
|
-
if (
|
|
177
|
-
console.log(`[SyncManager] Periodic sync: ${
|
|
171
|
+
if (storesToSync.length > 0) {
|
|
172
|
+
console.log(`[SyncManager] Periodic sync: syncing ${storesToSync.length} stores`);
|
|
173
|
+
// Run all groupSync calls in parallel - they coordinate via shared promise cache
|
|
174
|
+
await Promise.all(storesToSync.map(store => querysetRegistry.groupSync(store.queryset, operationId, dbSyncedKeys)));
|
|
178
175
|
}
|
|
179
176
|
// Prune unreferenced model instances
|
|
180
177
|
this.pruneUnreferencedModels();
|
|
@@ -310,15 +307,16 @@ export class SyncManager {
|
|
|
310
307
|
}
|
|
311
308
|
}
|
|
312
309
|
}
|
|
310
|
+
if (storesToSync.length === 0)
|
|
311
|
+
return;
|
|
313
312
|
// Sync all relevant stores for this model
|
|
314
313
|
console.log(`[SyncManager] Syncing ${storesToSync.length} queryset stores for ${representativeEvent.model}`);
|
|
315
314
|
// Generate operationId for this batch - querysets in same chain will coordinate
|
|
316
315
|
const operationId = `remote-event-${uuidv7()}`;
|
|
317
316
|
// Get dbSynced keys (followed querysets)
|
|
318
317
|
const dbSyncedKeys = new Set([...this.followedQuerysets].map(qs => qs.semanticKey));
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
});
|
|
318
|
+
// Run all groupSync calls in parallel - they coordinate via shared promise cache
|
|
319
|
+
Promise.all(storesToSync.map(store => registry.groupSync(store.queryset, operationId, dbSyncedKeys)));
|
|
322
320
|
}
|
|
323
321
|
processMetrics(event) {
|
|
324
322
|
const registry = this.registries.get(MetricRegistry);
|
|
@@ -380,7 +378,7 @@ export class SyncManager {
|
|
|
380
378
|
});
|
|
381
379
|
if (pksToSync.length > 0) {
|
|
382
380
|
console.log(`[SyncManager] Syncing ${pksToSync.length} nested-only PKs for ${event.model}: ${pksToSync}`);
|
|
383
|
-
|
|
381
|
+
modelStore.sync(pksToSync);
|
|
384
382
|
}
|
|
385
383
|
}
|
|
386
384
|
}
|
package/package.json
CHANGED