@statezero/core 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adaptors/vue/composables.js +12 -0
- package/dist/core/eventReceivers.js +15 -0
- package/dist/flavours/django/makeApiCall.d.ts +5 -2
- package/dist/flavours/django/makeApiCall.js +25 -2
- package/dist/flavours/django/queryExecutor.js +6 -6
- package/dist/flavours/django/querySet.js +0 -1
- package/dist/syncEngine/namespaceUtils.d.ts +16 -0
- package/dist/syncEngine/namespaceUtils.js +32 -0
- package/dist/syncEngine/registries/querysetStoreRegistry.js +5 -2
- package/dist/syncEngine/stores/modelStore.d.ts +7 -0
- package/dist/syncEngine/stores/modelStore.js +37 -0
- package/dist/syncEngine/stores/querysetStore.d.ts +2 -1
- package/dist/syncEngine/stores/querysetStore.js +17 -4
- package/dist/syncEngine/sync.d.ts +8 -0
- package/dist/syncEngine/sync.js +173 -3
- package/package.json +2 -1
|
@@ -9,6 +9,18 @@ export const querysets = new Map(); // Map of composableId -> queryset
|
|
|
9
9
|
function updateSyncManager() {
|
|
10
10
|
// Get unique querysets from all active composables
|
|
11
11
|
const uniqueQuerysets = new Set(querysets.values());
|
|
12
|
+
// Unsubscribe from querysets no longer followed
|
|
13
|
+
for (const [key, info] of syncManager.activeSubscriptions) {
|
|
14
|
+
if (!uniqueQuerysets.has(info.queryset)) {
|
|
15
|
+
syncManager.unsubscribeFromNamespace(info.queryset);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
// Subscribe to new querysets
|
|
19
|
+
for (const queryset of uniqueQuerysets) {
|
|
20
|
+
if (!syncManager.activeSubscriptions.has(queryset.semanticKey)) {
|
|
21
|
+
syncManager.subscribeToNamespace(queryset);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
12
24
|
syncManager.followedQuerysets = uniqueQuerysets;
|
|
13
25
|
}
|
|
14
26
|
export function useQueryset(querysetFactory) {
|
|
@@ -83,6 +83,21 @@ export class PusherEventReceiver {
|
|
|
83
83
|
clearTimeout(this.connectionTimeoutId);
|
|
84
84
|
this.connectionTimeoutId = null;
|
|
85
85
|
}
|
|
86
|
+
// Notify connected event
|
|
87
|
+
this.eventHandlers.forEach((handler) => {
|
|
88
|
+
if (handler.onReconnected) {
|
|
89
|
+
handler.onReconnected();
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
this.pusherClient.connection.bind("disconnected", () => {
|
|
94
|
+
console.log(`Pusher client disconnected for backend: ${this.configKey}.`);
|
|
95
|
+
// Notify disconnected event
|
|
96
|
+
this.eventHandlers.forEach((handler) => {
|
|
97
|
+
if (handler.onDisconnected) {
|
|
98
|
+
handler.onDisconnected();
|
|
99
|
+
}
|
|
100
|
+
});
|
|
86
101
|
});
|
|
87
102
|
this.pusherClient.connection.bind("failed", () => {
|
|
88
103
|
this._logConnectionError("Pusher connection explicitly failed.");
|
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
* @param {ModelStoreRegistry} modelStoreRegistry - The model store registry to use
|
|
6
6
|
* @param {Object} included - The included entities object from the response
|
|
7
7
|
* @param {Function} ModelClass - The base model class to get the configKey from
|
|
8
|
+
* @param {QuerySet} [queryset] - Optional queryset to track which PKs came from this fetch
|
|
8
9
|
*/
|
|
9
|
-
export function processIncludedEntities(modelStoreRegistry: ModelStoreRegistry, included: Object, ModelClass: Function): void;
|
|
10
|
+
export function processIncludedEntities(modelStoreRegistry: ModelStoreRegistry, included: Object, ModelClass: Function, queryset?: QuerySet): void;
|
|
10
11
|
/**
|
|
11
12
|
* Makes an API call to the backend with the given QuerySet.
|
|
12
13
|
* Automatically handles FileObject replacement with file paths for write operations.
|
|
@@ -15,6 +16,8 @@ export function processIncludedEntities(modelStoreRegistry: ModelStoreRegistry,
|
|
|
15
16
|
* @param {string} operationType - The type of operation to perform.
|
|
16
17
|
* @param {Object} args - Additional arguments for the operation.
|
|
17
18
|
* @param {string} operationId - A unique id for the operation
|
|
19
|
+
* @param {Function} beforeExit - Optional callback before returning
|
|
20
|
+
* @param {string} canonicalId - Optional canonical_id for cache sharing
|
|
18
21
|
* @returns {Promise<Object>} The API response.
|
|
19
22
|
*/
|
|
20
|
-
export function makeApiCall(querySet: QuerySet, operationType: string, args: Object | undefined, operationId: string, beforeExit?:
|
|
23
|
+
export function makeApiCall(querySet: QuerySet, operationType: string, args: Object | undefined, operationId: string, beforeExit?: Function, canonicalId?: string): Promise<Object>;
|
|
@@ -4,6 +4,7 @@ import { configInstance } from '../../config.js';
|
|
|
4
4
|
import { replaceTempPks } from './tempPk.js';
|
|
5
5
|
import { parseStateZeroError, MultipleObjectsReturned, DoesNotExist } from './errors.js';
|
|
6
6
|
import { FileObject } from './files.js';
|
|
7
|
+
import { querysetStoreRegistry } from '../../syncEngine/registries/querysetStoreRegistry.js';
|
|
7
8
|
const apiCallQueue = new PQueue({ concurrency: 1 });
|
|
8
9
|
/**
|
|
9
10
|
* Process included entities from a response and register them in the model store.
|
|
@@ -12,11 +13,17 @@ const apiCallQueue = new PQueue({ concurrency: 1 });
|
|
|
12
13
|
* @param {ModelStoreRegistry} modelStoreRegistry - The model store registry to use
|
|
13
14
|
* @param {Object} included - The included entities object from the response
|
|
14
15
|
* @param {Function} ModelClass - The base model class to get the configKey from
|
|
16
|
+
* @param {QuerySet} [queryset] - Optional queryset to track which PKs came from this fetch
|
|
15
17
|
*/
|
|
16
|
-
export function processIncludedEntities(modelStoreRegistry, included, ModelClass) {
|
|
18
|
+
export function processIncludedEntities(modelStoreRegistry, included, ModelClass, queryset = null) {
|
|
17
19
|
if (!included)
|
|
18
20
|
return;
|
|
19
21
|
const configKey = ModelClass.configKey;
|
|
22
|
+
// Get the queryset store if a queryset is provided
|
|
23
|
+
let querysetStore = null;
|
|
24
|
+
if (queryset) {
|
|
25
|
+
querysetStore = querysetStoreRegistry.getStore(queryset);
|
|
26
|
+
}
|
|
20
27
|
try {
|
|
21
28
|
// Process each model type
|
|
22
29
|
for (const [modelName, entityMap] of Object.entries(included)) {
|
|
@@ -26,6 +33,17 @@ export function processIncludedEntities(modelStoreRegistry, included, ModelClass
|
|
|
26
33
|
console.error(`Model class not found for ${modelName} in config ${configKey}`);
|
|
27
34
|
throw new Error(`Model class not found for ${modelName}`);
|
|
28
35
|
}
|
|
36
|
+
// Track which PKs are included if a queryset store is available
|
|
37
|
+
if (querysetStore) {
|
|
38
|
+
if (!querysetStore.includedPks.has(modelName)) {
|
|
39
|
+
querysetStore.includedPks.set(modelName, new Set());
|
|
40
|
+
}
|
|
41
|
+
const pksSet = querysetStore.includedPks.get(modelName);
|
|
42
|
+
// Add all PKs from this model to the set
|
|
43
|
+
for (const pk of Object.keys(entityMap)) {
|
|
44
|
+
pksSet.add(Number(pk));
|
|
45
|
+
}
|
|
46
|
+
}
|
|
29
47
|
// Register each entity in the model store
|
|
30
48
|
for (const [pk, entity] of Object.entries(entityMap)) {
|
|
31
49
|
modelStoreRegistry.setEntity(EntityClass, pk, entity);
|
|
@@ -45,9 +63,11 @@ export function processIncludedEntities(modelStoreRegistry, included, ModelClass
|
|
|
45
63
|
* @param {string} operationType - The type of operation to perform.
|
|
46
64
|
* @param {Object} args - Additional arguments for the operation.
|
|
47
65
|
* @param {string} operationId - A unique id for the operation
|
|
66
|
+
* @param {Function} beforeExit - Optional callback before returning
|
|
67
|
+
* @param {string} canonicalId - Optional canonical_id for cache sharing
|
|
48
68
|
* @returns {Promise<Object>} The API response.
|
|
49
69
|
*/
|
|
50
|
-
export async function makeApiCall(querySet, operationType, args = {}, operationId, beforeExit = null) {
|
|
70
|
+
export async function makeApiCall(querySet, operationType, args = {}, operationId, beforeExit = null, canonicalId = null) {
|
|
51
71
|
const ModelClass = querySet.ModelClass;
|
|
52
72
|
const config = configInstance.getConfig();
|
|
53
73
|
const backend = config.backendConfigs[ModelClass.configKey];
|
|
@@ -90,6 +110,9 @@ export async function makeApiCall(querySet, operationType, args = {}, operationI
|
|
|
90
110
|
if (operationId) {
|
|
91
111
|
headers["X-Operation-ID"] = operationId;
|
|
92
112
|
}
|
|
113
|
+
if (canonicalId) {
|
|
114
|
+
headers["X-Canonical-ID"] = canonicalId;
|
|
115
|
+
}
|
|
93
116
|
// Use the queue for write operations, bypass for read operations
|
|
94
117
|
const apiCall = async () => {
|
|
95
118
|
try {
|
|
@@ -62,7 +62,7 @@ export class QueryExecutor {
|
|
|
62
62
|
if (isNil(data) || (Array.isArray(data) && data.length === 0)) {
|
|
63
63
|
return null;
|
|
64
64
|
}
|
|
65
|
-
processIncludedEntities(modelStoreRegistry, included, ModelClass);
|
|
65
|
+
processIncludedEntities(modelStoreRegistry, included, ModelClass, querySet);
|
|
66
66
|
const realPk = Array.isArray(data) ? data[0] : data;
|
|
67
67
|
// swap in the real PK on the same instance, will trigger reactivity
|
|
68
68
|
live.pk = realPk;
|
|
@@ -90,7 +90,7 @@ export class QueryExecutor {
|
|
|
90
90
|
const live = querysetStoreRegistry.getEntity(qs);
|
|
91
91
|
const promise = makeApiCall(qs, op, args).then((resp) => {
|
|
92
92
|
const { data, included } = resp.data;
|
|
93
|
-
processIncludedEntities(modelStoreRegistry, included, qs.ModelClass);
|
|
93
|
+
processIncludedEntities(modelStoreRegistry, included, qs.ModelClass, qs);
|
|
94
94
|
const pks = Array.isArray(data) ? data : [];
|
|
95
95
|
querysetStoreRegistry.setEntity(qs, pks);
|
|
96
96
|
return querysetStoreRegistry.getEntity(qs);
|
|
@@ -129,7 +129,7 @@ export class QueryExecutor {
|
|
|
129
129
|
const { data, included, model_name } = response.data;
|
|
130
130
|
const created = response.metadata.created;
|
|
131
131
|
// Process included entities
|
|
132
|
-
processIncludedEntities(modelStoreRegistry, included, ModelClass);
|
|
132
|
+
processIncludedEntities(modelStoreRegistry, included, ModelClass, querySet);
|
|
133
133
|
// Get the real PK
|
|
134
134
|
const pk = Array.isArray(data) ? data[0] : data;
|
|
135
135
|
// Update PK if we created a new instance
|
|
@@ -307,7 +307,7 @@ export class QueryExecutor {
|
|
|
307
307
|
.then((response) => {
|
|
308
308
|
const { data, included, model_name } = response.data;
|
|
309
309
|
// Process included entities
|
|
310
|
-
processIncludedEntities(modelStoreRegistry, included, ModelClass);
|
|
310
|
+
processIncludedEntities(modelStoreRegistry, included, ModelClass, querySet);
|
|
311
311
|
// Get the real PK
|
|
312
312
|
const pk = Array.isArray(data) ? data[0] : data;
|
|
313
313
|
live.pk = pk;
|
|
@@ -372,7 +372,7 @@ export class QueryExecutor {
|
|
|
372
372
|
.then((response) => {
|
|
373
373
|
const { data, included, model_name } = response.data;
|
|
374
374
|
// Process included entities
|
|
375
|
-
processIncludedEntities(modelStoreRegistry, included, ModelClass);
|
|
375
|
+
processIncludedEntities(modelStoreRegistry, included, ModelClass, querySet);
|
|
376
376
|
// Get the real PKs
|
|
377
377
|
const pks = Array.isArray(data) ? data : [];
|
|
378
378
|
// Update each live instance with its real PK
|
|
@@ -426,7 +426,7 @@ export class QueryExecutor {
|
|
|
426
426
|
.then((response) => {
|
|
427
427
|
const { data: raw, included, model_name } = response.data;
|
|
428
428
|
// Process included entities
|
|
429
|
-
processIncludedEntities(modelStoreRegistry, included, ModelClass);
|
|
429
|
+
processIncludedEntities(modelStoreRegistry, included, ModelClass, querySet);
|
|
430
430
|
// Swap in the real PK
|
|
431
431
|
const pk = Array.isArray(raw) ? raw[0] : raw;
|
|
432
432
|
live.pk = pk;
|
|
@@ -3,7 +3,6 @@ import { Model } from "./model.js";
|
|
|
3
3
|
import { ModelSerializer, relationshipFieldSerializer, dateFieldSerializer } from "./serializers.js";
|
|
4
4
|
import axios from "axios";
|
|
5
5
|
import { QueryExecutor } from "./queryExecutor.js";
|
|
6
|
-
import { json } from "stream/consumers";
|
|
7
6
|
import { v7 } from "uuid";
|
|
8
7
|
import hash from "object-hash";
|
|
9
8
|
import rfdc from "rfdc";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generate a stable hash for a namespace dictionary.
|
|
3
|
+
* Must match the backend implementation in statezero/core/namespace_utils.py
|
|
4
|
+
*
|
|
5
|
+
* @param {Object} namespace - Namespace dictionary (e.g., {room_id: 5})
|
|
6
|
+
* @returns {string} MD5 hash of the namespace
|
|
7
|
+
*/
|
|
8
|
+
export function getNamespaceHash(namespace: Object): string;
|
|
9
|
+
/**
|
|
10
|
+
* Extract namespace fields from a queryset filter.
|
|
11
|
+
* Namespace fields are those that use exact equality (not __gt, __lt, etc.)
|
|
12
|
+
*
|
|
13
|
+
* @param {Object} filters - Queryset filters
|
|
14
|
+
* @returns {Object} Namespace dictionary
|
|
15
|
+
*/
|
|
16
|
+
export function extractNamespaceFromFilter(filters: Object): Object;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import crypto from 'crypto-js';
|
|
2
|
+
/**
|
|
3
|
+
* Generate a stable hash for a namespace dictionary.
|
|
4
|
+
* Must match the backend implementation in statezero/core/namespace_utils.py
|
|
5
|
+
*
|
|
6
|
+
* @param {Object} namespace - Namespace dictionary (e.g., {room_id: 5})
|
|
7
|
+
* @returns {string} MD5 hash of the namespace
|
|
8
|
+
*/
|
|
9
|
+
export function getNamespaceHash(namespace) {
|
|
10
|
+
// Sort keys to ensure stable hash
|
|
11
|
+
const namespaceJson = JSON.stringify(namespace, Object.keys(namespace).sort());
|
|
12
|
+
return crypto.MD5(namespaceJson).toString();
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Extract namespace fields from a queryset filter.
|
|
16
|
+
* Namespace fields are those that use exact equality (not __gt, __lt, etc.)
|
|
17
|
+
*
|
|
18
|
+
* @param {Object} filters - Queryset filters
|
|
19
|
+
* @returns {Object} Namespace dictionary
|
|
20
|
+
*/
|
|
21
|
+
export function extractNamespaceFromFilter(filters) {
|
|
22
|
+
if (!filters)
|
|
23
|
+
return {};
|
|
24
|
+
const namespace = {};
|
|
25
|
+
for (const [key, value] of Object.entries(filters)) {
|
|
26
|
+
// Only include exact matches (not lookups like __gt, __in, etc.)
|
|
27
|
+
if (!key.includes('__')) {
|
|
28
|
+
namespace[key] = value;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return namespace;
|
|
32
|
+
}
|
|
@@ -174,14 +174,17 @@ export class QuerysetStoreRegistry {
|
|
|
174
174
|
current = current.__parent;
|
|
175
175
|
}
|
|
176
176
|
// Create a new temporary store
|
|
177
|
-
const fetchQueryset = async ({ ast, modelClass }) => {
|
|
177
|
+
const fetchQueryset = async ({ ast, modelClass, canonical_id }) => {
|
|
178
178
|
// Directly assemble the request and call the API to avoid recursive logic from the
|
|
179
179
|
// queryset back to the registry / store
|
|
180
180
|
const payload = {
|
|
181
181
|
...ast,
|
|
182
182
|
type: 'list'
|
|
183
183
|
};
|
|
184
|
-
const response = await makeApiCall(queryset, 'list', payload
|
|
184
|
+
const response = await makeApiCall(queryset, 'list', payload, null, // operationId
|
|
185
|
+
null, // beforeExit
|
|
186
|
+
canonical_id // canonical_id for caching
|
|
187
|
+
);
|
|
185
188
|
return response.data;
|
|
186
189
|
};
|
|
187
190
|
let initialGroundTruthPks = null;
|
|
@@ -37,6 +37,13 @@ export class ModelStore {
|
|
|
37
37
|
getTrimmedOperations(): any[];
|
|
38
38
|
getInflightOperations(): any[];
|
|
39
39
|
prune(): void;
|
|
40
|
+
/**
|
|
41
|
+
* Prune model instances that aren't referenced by any queryset store.
|
|
42
|
+
* This prevents unbounded cache growth from included nested models.
|
|
43
|
+
*
|
|
44
|
+
* @param {QuerysetStoreRegistry} querysetStoreRegistry - The registry to check for queryset references
|
|
45
|
+
*/
|
|
46
|
+
pruneUnreferencedInstances(querysetStoreRegistry: QuerysetStoreRegistry): void;
|
|
40
47
|
render(pks?: null, optimistic?: boolean): any[];
|
|
41
48
|
sync(pks?: null): Promise<void>;
|
|
42
49
|
}
|
|
@@ -396,6 +396,43 @@ export class ModelStore {
|
|
|
396
396
|
this.setOperations(this.getInflightOperations());
|
|
397
397
|
this.setCache(renderedPks);
|
|
398
398
|
}
|
|
399
|
+
/**
|
|
400
|
+
* Prune model instances that aren't referenced by any queryset store.
|
|
401
|
+
* This prevents unbounded cache growth from included nested models.
|
|
402
|
+
*
|
|
403
|
+
* @param {QuerysetStoreRegistry} querysetStoreRegistry - The registry to check for queryset references
|
|
404
|
+
*/
|
|
405
|
+
pruneUnreferencedInstances(querysetStoreRegistry) {
|
|
406
|
+
const pkField = this.pkField;
|
|
407
|
+
const modelName = this.modelClass.modelName;
|
|
408
|
+
// Collect all PKs that are needed by ANY queryset (permanent or temporary)
|
|
409
|
+
const neededPks = new Set();
|
|
410
|
+
for (const [semanticKey, store] of querysetStoreRegistry._stores.entries()) {
|
|
411
|
+
// Check top-level PKs (groundTruthPks)
|
|
412
|
+
if (store.modelClass.modelName === modelName) {
|
|
413
|
+
store.groundTruthPks.forEach(pk => neededPks.add(pk));
|
|
414
|
+
}
|
|
415
|
+
// Check included PKs (nested models)
|
|
416
|
+
if (store.includedPks.has(modelName)) {
|
|
417
|
+
const includedPkSet = store.includedPks.get(modelName);
|
|
418
|
+
includedPkSet.forEach(pk => neededPks.add(pk));
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
// Filter ground truth to only keep needed PKs
|
|
422
|
+
const filteredGroundTruth = this.groundTruthArray.filter(instance => {
|
|
423
|
+
if (!instance || typeof instance !== 'object' || !(pkField in instance)) {
|
|
424
|
+
return false;
|
|
425
|
+
}
|
|
426
|
+
return neededPks.has(instance[pkField]);
|
|
427
|
+
});
|
|
428
|
+
const removedCount = this.groundTruthArray.length - filteredGroundTruth.length;
|
|
429
|
+
if (removedCount > 0) {
|
|
430
|
+
console.log(`[ModelStore ${modelName}] Pruned ${removedCount} unreferenced instances (${filteredGroundTruth.length} remaining)`);
|
|
431
|
+
this.groundTruthArray = filteredGroundTruth;
|
|
432
|
+
// Update the cache to reflect the pruned data
|
|
433
|
+
this.setCache(filteredGroundTruth);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
399
436
|
// Render methods
|
|
400
437
|
render(pks = null, optimistic = true) {
|
|
401
438
|
const pksSet = pks === null
|
|
@@ -10,6 +10,7 @@ export class QuerysetStore {
|
|
|
10
10
|
isTemp: any;
|
|
11
11
|
pruneThreshold: any;
|
|
12
12
|
getRootStore: any;
|
|
13
|
+
includedPks: Map<any, any>;
|
|
13
14
|
qsCache: Cache;
|
|
14
15
|
_lastRenderedPks: any[] | null;
|
|
15
16
|
renderCallbacks: Set<any>;
|
|
@@ -45,6 +46,6 @@ export class QuerysetStore {
|
|
|
45
46
|
renderFromRoot(optimistic: boolean | undefined, rootStore: any): any[];
|
|
46
47
|
renderFromData(optimistic?: boolean): any[];
|
|
47
48
|
applyOperation(operation: any, currentPks: any): any;
|
|
48
|
-
sync(
|
|
49
|
+
sync(options?: {}): Promise<void>;
|
|
49
50
|
}
|
|
50
51
|
import { Cache } from '../cache/cache.js';
|
|
@@ -20,6 +20,9 @@ export class QuerysetStore {
|
|
|
20
20
|
this.getRootStore = options.getRootStore || null;
|
|
21
21
|
this.groundTruthPks = initialGroundTruthPks || [];
|
|
22
22
|
this.operationsMap = new Map();
|
|
23
|
+
// Track which model PKs are in this queryset's included data
|
|
24
|
+
// Map<modelName, Set<pk>>
|
|
25
|
+
this.includedPks = new Map();
|
|
23
26
|
if (Array.isArray(initialOperations)) {
|
|
24
27
|
for (const opData of initialOperations) {
|
|
25
28
|
const existing = operationRegistry.get(opData.operationId);
|
|
@@ -266,7 +269,8 @@ export class QuerysetStore {
|
|
|
266
269
|
}
|
|
267
270
|
return currentPks;
|
|
268
271
|
}
|
|
269
|
-
async sync(
|
|
272
|
+
async sync(options = {}) {
|
|
273
|
+
const { canonical_id, forceFromDb = false } = typeof options === 'boolean' ? { forceFromDb: options } : options;
|
|
270
274
|
const id = this.modelClass.modelName;
|
|
271
275
|
if (this.isSyncing) {
|
|
272
276
|
console.warn(`[QuerysetStore ${id}] Already syncing, request ignored.`);
|
|
@@ -289,17 +293,26 @@ export class QuerysetStore {
|
|
|
289
293
|
this.isSyncing = true;
|
|
290
294
|
console.log(`[${id}] Starting sync...`);
|
|
291
295
|
try {
|
|
292
|
-
|
|
296
|
+
// Build request with canonical_id
|
|
297
|
+
const requestBody = {
|
|
293
298
|
ast: this.queryset.build(),
|
|
294
299
|
modelClass: this.modelClass,
|
|
295
|
-
}
|
|
300
|
+
};
|
|
301
|
+
// Include canonical_id if provided (for cache sharing)
|
|
302
|
+
if (canonical_id) {
|
|
303
|
+
requestBody.canonical_id = canonical_id;
|
|
304
|
+
}
|
|
305
|
+
const response = await this.fetchFn(requestBody);
|
|
296
306
|
const { data, included } = response;
|
|
297
307
|
if (isNil(data)) {
|
|
298
308
|
return;
|
|
299
309
|
}
|
|
300
310
|
console.log(`[${id}] Sync fetch completed. Received: ${JSON.stringify(data)}.`);
|
|
311
|
+
// Clear previous included PKs tracking before processing new data
|
|
312
|
+
this.includedPks.clear();
|
|
301
313
|
// Persists all the instances (including nested instances) to the model store
|
|
302
|
-
|
|
314
|
+
// Pass this queryset to track which PKs are in the included data
|
|
315
|
+
processIncludedEntities(modelStoreRegistry, included, this.modelClass, this.queryset);
|
|
303
316
|
this.setGroundTruth(data);
|
|
304
317
|
this.setOperations(this.getInflightOperations());
|
|
305
318
|
this.lastSync = Date.now();
|
|
@@ -5,6 +5,7 @@ export class EventPayload {
|
|
|
5
5
|
operation_id: any;
|
|
6
6
|
pk_field_name: any;
|
|
7
7
|
configKey: any;
|
|
8
|
+
canonical_id: any;
|
|
8
9
|
instances: any;
|
|
9
10
|
_cachedInstances: any;
|
|
10
11
|
get modelClass(): Function | null;
|
|
@@ -23,15 +24,22 @@ export class SyncManager {
|
|
|
23
24
|
maxWaitMs: number;
|
|
24
25
|
batchStartTime: number | null;
|
|
25
26
|
syncQueue: PQueue<import("p-queue/dist/priority-queue").default, import("p-queue").QueueAddOptions>;
|
|
27
|
+
activeSubscriptions: Map<any, any>;
|
|
28
|
+
currentCanonicalId: any;
|
|
26
29
|
withTimeout(promise: any, ms: any): Promise<any>;
|
|
27
30
|
/**
|
|
28
31
|
* Initialize event handlers for all event receivers
|
|
29
32
|
*/
|
|
30
33
|
initialize(): void;
|
|
34
|
+
handleDisconnect(): void;
|
|
35
|
+
handleReconnect(): void;
|
|
31
36
|
startPeriodicSync(): void;
|
|
32
37
|
syncStaleQuerysets(): void;
|
|
38
|
+
pruneUnreferencedModels(): void;
|
|
33
39
|
isStoreFollowed(registry: any, semanticKey: any): boolean;
|
|
34
40
|
cleanup(): void;
|
|
41
|
+
subscribeToNamespace(queryset: any): Promise<void>;
|
|
42
|
+
unsubscribeFromNamespace(queryset: any): Promise<void>;
|
|
35
43
|
followModel(registry: any, modelClass: any): void;
|
|
36
44
|
unfollowModel(registry: any, modelClass: any): void;
|
|
37
45
|
manageRegistry(registry: any): void;
|
package/dist/syncEngine/sync.js
CHANGED
|
@@ -16,6 +16,7 @@ export class EventPayload {
|
|
|
16
16
|
this.operation_id = data.operation_id;
|
|
17
17
|
this.pk_field_name = data.pk_field_name;
|
|
18
18
|
this.configKey = data.configKey;
|
|
19
|
+
this.canonical_id = data.canonical_id;
|
|
19
20
|
// Parse PK fields to numbers in instances
|
|
20
21
|
this.instances = data.instances?.map(instance => {
|
|
21
22
|
if (instance && this.pk_field_name && instance[this.pk_field_name] != null) {
|
|
@@ -50,6 +51,10 @@ export class SyncManager {
|
|
|
50
51
|
this.handleEvent = (event) => {
|
|
51
52
|
let payload = new EventPayload(event);
|
|
52
53
|
let isLocalOperation = operationRegistry.has(payload.operation_id);
|
|
54
|
+
// Store canonical_id for upcoming refetches
|
|
55
|
+
if (event.canonical_id) {
|
|
56
|
+
this.currentCanonicalId = event.canonical_id;
|
|
57
|
+
}
|
|
53
58
|
// Always process metrics immediately (they're lightweight)
|
|
54
59
|
if (this.registries.has(MetricRegistry)) {
|
|
55
60
|
this.processMetrics(payload);
|
|
@@ -97,6 +102,9 @@ export class SyncManager {
|
|
|
97
102
|
this.batchStartTime = null;
|
|
98
103
|
// SyncQueue
|
|
99
104
|
this.syncQueue = new PQueue({ concurrency: 1 });
|
|
105
|
+
// Namespace subscription tracking
|
|
106
|
+
this.activeSubscriptions = new Map(); // semanticKey -> {queryset, subscribedAt}
|
|
107
|
+
this.currentCanonicalId = null; // Store canonical_id from events
|
|
100
108
|
}
|
|
101
109
|
withTimeout(promise, ms) {
|
|
102
110
|
// If no timeout specified, use 2x the periodic sync interval, or 30s as fallback
|
|
@@ -123,15 +131,31 @@ export class SyncManager {
|
|
|
123
131
|
initializeAllEventReceivers();
|
|
124
132
|
// Get all registered event receivers
|
|
125
133
|
const eventReceivers = getAllEventReceivers();
|
|
134
|
+
// Create event handler with connection lifecycle methods
|
|
135
|
+
const eventHandlerWithLifecycle = this.handleEvent.bind(this);
|
|
136
|
+
eventHandlerWithLifecycle.onReconnected = this.handleReconnect.bind(this);
|
|
137
|
+
eventHandlerWithLifecycle.onDisconnected = this.handleDisconnect.bind(this);
|
|
126
138
|
// Register the event handlers with each receiver
|
|
127
139
|
eventReceivers.forEach((receiver, configKey) => {
|
|
128
140
|
if (receiver) {
|
|
129
141
|
// Model events go to handleEvent
|
|
130
|
-
receiver.addModelEventHandler(
|
|
142
|
+
receiver.addModelEventHandler(eventHandlerWithLifecycle);
|
|
131
143
|
}
|
|
132
144
|
});
|
|
133
145
|
this.startPeriodicSync();
|
|
134
146
|
}
|
|
147
|
+
handleDisconnect() {
|
|
148
|
+
// Clear local subscription state on disconnect
|
|
149
|
+
this.activeSubscriptions.clear();
|
|
150
|
+
console.log('[SyncManager] Cleared subscriptions on disconnect');
|
|
151
|
+
}
|
|
152
|
+
handleReconnect() {
|
|
153
|
+
// Resubscribe to all active querysets after reconnect
|
|
154
|
+
console.log('[SyncManager] Resubscribing to active querysets after reconnect');
|
|
155
|
+
for (const queryset of this.followedQuerysets) {
|
|
156
|
+
this.subscribeToNamespace(queryset);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
135
159
|
startPeriodicSync() {
|
|
136
160
|
if (this.periodicSyncTimer)
|
|
137
161
|
return;
|
|
@@ -171,6 +195,18 @@ export class SyncManager {
|
|
|
171
195
|
if (syncedCount > 0) {
|
|
172
196
|
console.log(`[SyncManager] Periodic sync: ${syncedCount} stores pushed to the sync queue`);
|
|
173
197
|
}
|
|
198
|
+
// Prune unreferenced model instances
|
|
199
|
+
this.pruneUnreferencedModels();
|
|
200
|
+
}
|
|
201
|
+
pruneUnreferencedModels() {
|
|
202
|
+
const modelRegistry = this.registries.get(ModelStoreRegistry);
|
|
203
|
+
const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
|
|
204
|
+
if (!modelRegistry || !querysetRegistry)
|
|
205
|
+
return;
|
|
206
|
+
// Prune each model store
|
|
207
|
+
for (const [modelClass, store] of modelRegistry._stores.entries()) {
|
|
208
|
+
store.pruneUnreferencedInstances(querysetRegistry);
|
|
209
|
+
}
|
|
174
210
|
}
|
|
175
211
|
isStoreFollowed(registry, semanticKey) {
|
|
176
212
|
const followingQuerysets = registry.followingQuerysets.get(semanticKey);
|
|
@@ -195,6 +231,97 @@ export class SyncManager {
|
|
|
195
231
|
this.maxWaitTimer = null;
|
|
196
232
|
}
|
|
197
233
|
}
|
|
234
|
+
async subscribeToNamespace(queryset) {
|
|
235
|
+
const key = queryset.semanticKey;
|
|
236
|
+
// Skip if already subscribed
|
|
237
|
+
if (this.activeSubscriptions.has(key))
|
|
238
|
+
return;
|
|
239
|
+
try {
|
|
240
|
+
const config = getConfig();
|
|
241
|
+
const backendConfig = config.backendConfigs[queryset.modelClass.configKey];
|
|
242
|
+
if (!backendConfig) {
|
|
243
|
+
console.error(`[SyncManager] Backend config not found for: ${queryset.modelClass.configKey}`);
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
const apiUrl = backendConfig.API_URL;
|
|
247
|
+
// Get Pusher socket ID
|
|
248
|
+
const eventReceiver = getEventReceiver(queryset.modelClass.configKey);
|
|
249
|
+
const socketId = eventReceiver?.pusherClient?.connection?.socket_id;
|
|
250
|
+
if (!socketId) {
|
|
251
|
+
console.warn('[SyncManager] No socket_id available for subscription');
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
// Send subscription request to backend
|
|
255
|
+
const response = await fetch(`${apiUrl}/namespaces/subscribe/`, {
|
|
256
|
+
method: 'POST',
|
|
257
|
+
headers: {
|
|
258
|
+
'Content-Type': 'application/json',
|
|
259
|
+
},
|
|
260
|
+
body: JSON.stringify({
|
|
261
|
+
socket_id: socketId,
|
|
262
|
+
model_name: queryset.modelClass.modelName,
|
|
263
|
+
filter: queryset._filters // Backend extracts namespace from this
|
|
264
|
+
})
|
|
265
|
+
});
|
|
266
|
+
const result = await response.json();
|
|
267
|
+
if (!result.success) {
|
|
268
|
+
throw new Error('Subscription failed');
|
|
269
|
+
}
|
|
270
|
+
// Subscribe to the namespace-specific Pusher channel
|
|
271
|
+
const namespaceChannel = result.channel; // e.g., "django_app.Message:abc123hash"
|
|
272
|
+
eventReceiver?.subscribe(namespaceChannel);
|
|
273
|
+
this.activeSubscriptions.set(key, {
|
|
274
|
+
queryset,
|
|
275
|
+
subscribedAt: Date.now(),
|
|
276
|
+
channel: namespaceChannel,
|
|
277
|
+
namespace_hash: result.namespace_hash
|
|
278
|
+
});
|
|
279
|
+
console.log(`[SyncManager] Subscribed to namespace channel: ${namespaceChannel}`);
|
|
280
|
+
}
|
|
281
|
+
catch (error) {
|
|
282
|
+
console.error('[SyncManager] Namespace subscription failed:', error);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
async unsubscribeFromNamespace(queryset) {
|
|
286
|
+
const key = queryset.semanticKey;
|
|
287
|
+
if (!this.activeSubscriptions.has(key))
|
|
288
|
+
return;
|
|
289
|
+
const subscription = this.activeSubscriptions.get(key);
|
|
290
|
+
try {
|
|
291
|
+
const config = getConfig();
|
|
292
|
+
const backendConfig = config.backendConfigs[queryset.modelClass.configKey];
|
|
293
|
+
if (!backendConfig) {
|
|
294
|
+
console.error(`[SyncManager] Backend config not found for: ${queryset.modelClass.configKey}`);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const apiUrl = backendConfig.API_URL;
|
|
298
|
+
const eventReceiver = getEventReceiver(queryset.modelClass.configKey);
|
|
299
|
+
const socketId = eventReceiver?.pusherClient?.connection?.socket_id;
|
|
300
|
+
if (!socketId)
|
|
301
|
+
return;
|
|
302
|
+
// Unsubscribe from the Pusher channel
|
|
303
|
+
if (subscription.channel) {
|
|
304
|
+
eventReceiver?.unsubscribe(subscription.channel);
|
|
305
|
+
}
|
|
306
|
+
// Notify backend to remove subscription
|
|
307
|
+
await fetch(`${apiUrl}/namespaces/unsubscribe/`, {
|
|
308
|
+
method: 'POST',
|
|
309
|
+
headers: {
|
|
310
|
+
'Content-Type': 'application/json',
|
|
311
|
+
},
|
|
312
|
+
body: JSON.stringify({
|
|
313
|
+
socket_id: socketId,
|
|
314
|
+
model_name: queryset.modelClass.modelName,
|
|
315
|
+
filter: queryset._filters
|
|
316
|
+
})
|
|
317
|
+
});
|
|
318
|
+
this.activeSubscriptions.delete(key);
|
|
319
|
+
console.log(`[SyncManager] Unsubscribed from namespace channel: ${subscription.channel}`);
|
|
320
|
+
}
|
|
321
|
+
catch (error) {
|
|
322
|
+
console.error('[SyncManager] Namespace unsubscription failed:', error);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
198
325
|
followModel(registry, modelClass) {
|
|
199
326
|
const models = this.followedModels.get(registry) || new Set();
|
|
200
327
|
this.followedModels.set(registry, models);
|
|
@@ -296,7 +423,9 @@ export class SyncManager {
|
|
|
296
423
|
// Sync all relevant stores for this model
|
|
297
424
|
console.log(`[SyncManager] Syncing ${storesToSync.length} queryset stores for ${representativeEvent.model}`);
|
|
298
425
|
storesToSync.forEach((store) => {
|
|
299
|
-
this.syncQueue.add(() => this.withTimeout(store.sync(
|
|
426
|
+
this.syncQueue.add(() => this.withTimeout(store.sync({
|
|
427
|
+
canonical_id: representativeEvent.canonical_id // Pass canonical_id
|
|
428
|
+
})));
|
|
300
429
|
});
|
|
301
430
|
}
|
|
302
431
|
processMetrics(event) {
|
|
@@ -319,7 +448,48 @@ export class SyncManager {
|
|
|
319
448
|
}
|
|
320
449
|
}
|
|
321
450
|
processModels(event) {
|
|
322
|
-
|
|
451
|
+
const registry = this.registries.get(ModelStoreRegistry);
|
|
452
|
+
if (!registry)
|
|
453
|
+
return;
|
|
454
|
+
const modelStore = registry.getStore(event.modelClass);
|
|
455
|
+
if (!modelStore)
|
|
456
|
+
return;
|
|
457
|
+
// Get PKs from the event
|
|
458
|
+
const eventPks = new Set((event.instances || [])
|
|
459
|
+
.filter(inst => inst && inst[event.pk_field_name] != null)
|
|
460
|
+
.map(inst => inst[event.pk_field_name]));
|
|
461
|
+
if (eventPks.size === 0)
|
|
462
|
+
return;
|
|
463
|
+
// Get PKs that are already in the model store's ground truth
|
|
464
|
+
const groundTruthPks = new Set(modelStore.groundTruthPks);
|
|
465
|
+
// Get PKs that are top-level in ANY queryset for this model (not just followed ones)
|
|
466
|
+
const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
|
|
467
|
+
const topLevelQuerysetPks = new Set();
|
|
468
|
+
if (querysetRegistry) {
|
|
469
|
+
for (const [semanticKey, store] of querysetRegistry._stores.entries()) {
|
|
470
|
+
// Check ALL querysets for this model (permanent and temporary)
|
|
471
|
+
if (store.modelClass.modelName === event.model &&
|
|
472
|
+
store.modelClass.configKey === event.configKey) {
|
|
473
|
+
// Get this queryset's ground truth PKs (top-level instances)
|
|
474
|
+
// Don't use render() as it applies operations - we just want ground truth
|
|
475
|
+
store.groundTruthPks.forEach(pk => topLevelQuerysetPks.add(pk));
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
// Find PKs that are:
|
|
480
|
+
// 1. In the event
|
|
481
|
+
// 2. Already in ground truth (we're tracking them)
|
|
482
|
+
// 3. NOT top-level in any queryset (they're only nested/included)
|
|
483
|
+
const pksToSync = [];
|
|
484
|
+
eventPks.forEach(pk => {
|
|
485
|
+
if (groundTruthPks.has(pk) && !topLevelQuerysetPks.has(pk)) {
|
|
486
|
+
pksToSync.push(pk);
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
if (pksToSync.length > 0) {
|
|
490
|
+
console.log(`[SyncManager] Syncing ${pksToSync.length} nested-only PKs for ${event.model}: ${pksToSync}`);
|
|
491
|
+
this.syncQueue.add(() => this.withTimeout(modelStore.sync(pksToSync)));
|
|
492
|
+
}
|
|
323
493
|
}
|
|
324
494
|
}
|
|
325
495
|
const syncManager = new SyncManager();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statezero/core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"module": "ESNext",
|
|
6
6
|
"description": "The type-safe frontend client for StateZero - connect directly to your backend models with zero boilerplate",
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"test:e2e": "vitest run --config=vitest.sequential.config.ts tests/e2e",
|
|
33
33
|
"generate:test-apps": "ts-node scripts/generate-test-apps.js",
|
|
34
34
|
"test:adaptors": "playwright test tests/adaptors",
|
|
35
|
+
"test:integration": "playwright test --config=playwright.integration.config.ts",
|
|
35
36
|
"test:coverage": "vitest run --coverage",
|
|
36
37
|
"build": "tsc",
|
|
37
38
|
"parse-queries": "node scripts/perfect-query-parser.js",
|