@abloatai/humans 0.38.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/core.d.ts +1 -1
  2. package/dist/core.js +1 -1
  3. package/dist/local/BaseSyncedStore.d.ts +9 -0
  4. package/dist/local/BaseSyncedStore.js +28 -1
  5. package/dist/local/Database.d.ts +20 -0
  6. package/dist/local/Database.js +83 -49
  7. package/dist/local/InstanceCache.d.ts +40 -8
  8. package/dist/local/InstanceCache.js +156 -83
  9. package/dist/local/Model.d.ts +28 -0
  10. package/dist/local/Model.js +102 -35
  11. package/dist/local/SyncClient.d.ts +1 -4
  12. package/dist/local/SyncClient.js +63 -62
  13. package/dist/local/client/reactiveEngine.js +18 -0
  14. package/dist/local/sync/SyncWebSocket.js +3 -6
  15. package/dist/local/sync/deltaPipeline.d.ts +32 -1
  16. package/dist/local/sync/deltaPipeline.js +116 -8
  17. package/dist/local/sync/drainProfile.d.ts +45 -0
  18. package/dist/local/sync/drainProfile.js +55 -0
  19. package/dist/local/transactions/mutations/MutationQueue.js +11 -3
  20. package/dist/local/utils/mobxSetup.d.ts +1 -0
  21. package/dist/local/utils/mobxSetup.js +5 -0
  22. package/package.json +2 -2
  23. package/src/core.ts +3 -0
  24. package/src/local/BaseSyncedStore.ts +37 -1
  25. package/src/local/Database.ts +87 -50
  26. package/src/local/InstanceCache.ts +162 -80
  27. package/src/local/Model.ts +117 -42
  28. package/src/local/SyncClient.ts +66 -63
  29. package/src/local/client/reactiveEngine.ts +18 -0
  30. package/src/local/sync/SyncWebSocket.ts +3 -6
  31. package/src/local/sync/deltaPipeline.ts +135 -9
  32. package/src/local/sync/drainProfile.ts +93 -0
  33. package/src/local/transactions/mutations/MutationQueue.ts +14 -4
  34. package/src/local/utils/mobxSetup.ts +5 -0
@@ -275,10 +275,6 @@ export declare class SyncClient extends EventEmitter {
275
275
  * conflict resolver reads exactly these fields and no others.
276
276
  */
277
277
  private extractCriticalState;
278
- /**
279
- * Check if critical state changes exist that require forcing server state
280
- */
281
- private hasCriticalStateChange;
282
278
  /**
283
279
  * Handle network reconnection
284
280
  */
@@ -487,6 +483,7 @@ export declare class SyncClient extends EventEmitter {
487
483
  */
488
484
  getMutationQueue(): MutationQueue;
489
485
  applyDeltaBatchToPool(dbResults: readonly AppliedChange[], enrichRelations: (modelName: string, data: Record<string, unknown>) => Record<string, unknown>): void;
486
+ private applyDeltaBatchToPoolInAction;
490
487
  /**
491
488
  * Apply bootstrap data to the InstanceCache with ghost removal.
492
489
  * Owns: model creation, batch upsert, ghost detection + removal.
@@ -1038,31 +1038,22 @@ export class SyncClient extends EventEmitter {
1038
1038
  * local model has unsynced changes, so the two sides stay consistent.
1039
1039
  */
1040
1040
  resolveConflicts(localModel, serverData) {
1041
- const hasLocalChanges = localModel.hasChanges;
1042
- // Safely get timestamp, handling both Date objects and strings
1043
- const localUpdatedAt = localModel.updatedAt
1044
- ? localModel.updatedAt instanceof Date
1045
- ? localModel.updatedAt.getTime()
1046
- : new Date(localModel.updatedAt).getTime()
1047
- : 0;
1048
- const serverUpdatedAt = toEpochMs(serverData.updatedAt);
1049
- this.runtime.logger.debug('Conflict resolution', {
1050
- modelId: localModel.id,
1051
- modelType: localModel.getModelName(),
1052
- hasLocalChanges,
1053
- localUpdatedAt: localModel.updatedAt?.toString(),
1054
- serverUpdatedAt: serverData.updatedAt,
1055
- localChanges: localModel.getChanges(),
1056
- serverState: this.extractCriticalState(serverData),
1057
- });
1058
- // PRIORITY 1: Check for critical server states that must be respected
1059
- // These states override any local changes to maintain data consistency
1060
- const criticalServerStates = this.extractCriticalState(serverData);
1061
- const shouldForceAcceptServer = this.hasCriticalStateChange(criticalServerStates);
1041
+ // No entry-point debug here: this runs once per incoming update delta,
1042
+ // and a debug call's payload is BUILT even when the logger discards it —
1043
+ // a per-delta model scan on the apply hot path. The outcome branches
1044
+ // below log the cases worth reading.
1045
+ // PRIORITY 1: Check for critical server states that must be respected.
1046
+ // These states override any local changes to maintain data consistency.
1047
+ // Checked inline — the collected-object form (`extractCriticalState`)
1048
+ // only materializes on the rare force-accept branch, for its log line.
1049
+ const shouldForceAcceptServer = (serverData.deletedAt !== undefined && serverData.deletedAt !== null) ||
1050
+ (serverData.archivedAt !== undefined && serverData.archivedAt !== null) ||
1051
+ serverData.isActive === false ||
1052
+ (serverData.unassignedAt !== undefined && serverData.unassignedAt !== null);
1062
1053
  if (shouldForceAcceptServer) {
1063
1054
  this.runtime.logger.debug('Accepting server update - critical state change detected', {
1064
1055
  modelId: localModel.id,
1065
- criticalStates: criticalServerStates,
1056
+ criticalStates: this.extractCriticalState(serverData),
1066
1057
  });
1067
1058
  // Force accept server state for critical changes
1068
1059
  localModel.updateFromData(serverData);
@@ -1072,7 +1063,7 @@ export class SyncClient extends EventEmitter {
1072
1063
  }
1073
1064
  // Local-first: if we have local dirty fields, merge by field.
1074
1065
  // Keep locally changed fields; apply server for the rest.
1075
- if (hasLocalChanges) {
1066
+ if (localModel.hasChanges) {
1076
1067
  const localChanges = localModel.getChanges();
1077
1068
  this.runtime.logger.debug('Merging server update with local dirty fields', {
1078
1069
  modelId: localModel.id,
@@ -1082,6 +1073,13 @@ export class SyncClient extends EventEmitter {
1082
1073
  const merged = { ...serverData, ...(localChanges || {}) };
1083
1074
  // Preserve the most recent updatedAt without clearing dirty flags
1084
1075
  if (serverData.updatedAt || localModel.updatedAt) {
1076
+ // Safely get timestamp, handling both Date objects and strings
1077
+ const localUpdatedAt = localModel.updatedAt
1078
+ ? localModel.updatedAt instanceof Date
1079
+ ? localModel.updatedAt.getTime()
1080
+ : new Date(localModel.updatedAt).getTime()
1081
+ : 0;
1082
+ const serverUpdatedAt = toEpochMs(serverData.updatedAt);
1085
1083
  const mergedUpdatedAt = new Date(Math.max(localUpdatedAt, serverUpdatedAt));
1086
1084
  // updateFromData accepts Date or ISO string for dates
1087
1085
  merged.updatedAt = mergedUpdatedAt;
@@ -1090,10 +1088,9 @@ export class SyncClient extends EventEmitter {
1090
1088
  // Intentionally DO NOT clearChanges here; pending tx will confirm and clear
1091
1089
  return localModel;
1092
1090
  }
1093
- // No local changes: fall back to LWW to converge
1094
- // Accept server regardless of timestamp equality to stay in sync
1095
- const acceptReason = serverUpdatedAt > localUpdatedAt ? 'server is newer' : 'no local changes';
1096
- this.runtime.logger.debug(`Accepting server update - ${acceptReason}`);
1091
+ // No local changes: fall back to LWW to converge. Accept server
1092
+ // regardless of timestamp equality to stay in sync. Not logged — this is
1093
+ // the common path for every collaborator update a client receives.
1097
1094
  localModel.updateFromData(serverData);
1098
1095
  localModel.clearChanges();
1099
1096
  localModel.markAsSynced();
@@ -1125,14 +1122,6 @@ export class SyncClient extends EventEmitter {
1125
1122
  }
1126
1123
  return critical;
1127
1124
  }
1128
- /**
1129
- * Check if critical state changes exist that require forcing server state
1130
- */
1131
- hasCriticalStateChange(criticalStates) {
1132
- // Any critical state present means we should force accept server
1133
- return (Object.keys(criticalStates).length > 0 &&
1134
- Object.values(criticalStates).some((v) => v !== null && v !== undefined));
1135
- }
1136
1125
  /**
1137
1126
  * Handle network reconnection
1138
1127
  */
@@ -1536,6 +1525,18 @@ export class SyncClient extends EventEmitter {
1536
1525
  return this.mutationQueue;
1537
1526
  }
1538
1527
  applyDeltaBatchToPool(dbResults, enrichRelations) {
1528
+ // The WHOLE batch — conflict resolution and model mutation included, not
1529
+ // just the pool bookkeeping at the end — runs in one MobX action.
1530
+ // `resolveConflicts` writes model fields via `updateFromData`; when those
1531
+ // writes ran before the action, every delta opened its own top-level
1532
+ // action and flushed reactions at its boundary, so a large frame paid one
1533
+ // reaction pass per delta and an observer could see a partially applied
1534
+ // frame between them.
1535
+ runInAction(() => {
1536
+ this.applyDeltaBatchToPoolInAction(dbResults, enrichRelations);
1537
+ });
1538
+ }
1539
+ applyDeltaBatchToPoolInAction(dbResults, enrichRelations) {
1539
1540
  const modelsToAdd = [];
1540
1541
  const modelsToUpsert = [];
1541
1542
  const idsToRemove = [];
@@ -1578,7 +1579,13 @@ export class SyncClient extends EventEmitter {
1578
1579
  }
1579
1580
  switch (action) {
1580
1581
  case 'add': {
1581
- const existing = this.objectPool.get(modelId);
1582
+ // `peek`, not `get`: this loop is ingestion, not a consumer read.
1583
+ // `get()` activates deferred MobX instrumentation, so reading
1584
+ // through it here made the delta stream itself instrument every
1585
+ // row it touched — the dominant term of apply cost for rows no
1586
+ // consumer observes. Activation belongs to the consumer-facing
1587
+ // reads (`get`, views, subscribers), which are unchanged.
1588
+ const existing = this.objectPool.peek(modelId);
1582
1589
  if (existing) {
1583
1590
  existing.markAsSynced();
1584
1591
  }
@@ -1593,7 +1600,7 @@ export class SyncClient extends EventEmitter {
1593
1600
  break;
1594
1601
  }
1595
1602
  case 'update': {
1596
- const existing = this.objectPool.get(modelId);
1603
+ const existing = this.objectPool.peek(modelId);
1597
1604
  if (existing && !existing.disposed && result.data) {
1598
1605
  enrichRelations(modelName, result.data);
1599
1606
  const resolved = this.resolveConflicts(existing, result.data);
@@ -1633,31 +1640,25 @@ export class SyncClient extends EventEmitter {
1633
1640
  break;
1634
1641
  }
1635
1642
  }
1636
- // Reveal the whole frame in a single MobX action. `addBatch`,
1637
- // `upsertBatch`, `removeBatch`, and `updateScope` are each individually
1638
- // wrapped in an action, so calling them in sequence flushes reactions at
1639
- // every action boundary a catch-up frame that adds, updates, and removes
1640
- // would fire every dependent reaction several times in a row, re-rendering
1641
- // and re-sorting on each. Wrapping them in one outer `runInAction` defers
1642
- // all reaction flushes to a single boundary, so dependents recompute
1643
- // exactly once regardless of how many models or operation kinds the frame
1644
- // touched. The app therefore never observes a partially applied frame.
1645
- runInAction(() => {
1646
- if (modelsToAdd.length > 0)
1647
- this.objectPool.addBatch(modelsToAdd, ModelScope.live);
1648
- if (modelsToUpsert.length > 0)
1649
- this.objectPool.upsertBatch(modelsToUpsert, ModelScope.live);
1650
- if (idsToRemove.length > 0)
1651
- this.objectPool.removeBatch(idsToRemove);
1652
- for (const id of idsToArchive)
1653
- this.objectPool.updateScope(id, ModelScope.archived);
1654
- // Emit changed model types so QueryProcessor can auto-invalidate.
1655
- // Kept inside the action so any observable query-cache state it
1656
- // flips is part of the same atomic reveal.
1657
- const changedTypes = new Set(dbResults.map(r => r.modelName));
1658
- if (changedTypes.size > 0)
1659
- this.emit('models:changed', changedTypes);
1660
- });
1643
+ // Reveal the whole frame at one reaction boundary: the caller's single
1644
+ // action covers the collection loop above and these batch pool writes, so
1645
+ // dependents recompute exactly once regardless of how many models or
1646
+ // operation kinds the frame touched, and the app never observes a
1647
+ // partially applied frame.
1648
+ if (modelsToAdd.length > 0)
1649
+ this.objectPool.addBatch(modelsToAdd, ModelScope.live);
1650
+ if (modelsToUpsert.length > 0)
1651
+ this.objectPool.upsertBatch(modelsToUpsert, ModelScope.live);
1652
+ if (idsToRemove.length > 0)
1653
+ this.objectPool.removeBatch(idsToRemove);
1654
+ for (const id of idsToArchive)
1655
+ this.objectPool.updateScope(id, ModelScope.archived);
1656
+ // Emit changed model types so QueryProcessor can auto-invalidate.
1657
+ // Kept inside the action so any observable query-cache state it
1658
+ // flips is part of the same atomic reveal.
1659
+ const changedTypes = new Set(dbResults.map(r => r.modelName));
1660
+ if (changedTypes.size > 0)
1661
+ this.emit('models:changed', changedTypes);
1661
1662
  }
1662
1663
  /**
1663
1664
  * Apply bootstrap data to the InstanceCache with ghost removal.
@@ -11,6 +11,7 @@
11
11
  * host-built core client with the per-model surface split — the cut's own
12
12
  * design step (docs/plans/package-split.md).
13
13
  */
14
+ import { omittedModelError } from '@abloatai/transaction/schema/select';
14
15
  import { durableCommitOperationSchema, } from '@abloatai/transaction/transactions/settlement/commitEnvelope';
15
16
  import { AbloAuthenticationError, AbloConnectionError, AbloValidationError, claimedError } from '@abloatai/transaction/errors';
16
17
  import { batchFence, fenceTokenFor, modelTarget, streamTarget, subTarget, } from '@abloatai/transaction/coordination';
@@ -703,5 +704,22 @@ export function buildReactiveEngine(inputs) {
703
704
  });
704
705
  },
705
706
  };
707
+ // A model the schema projection left out answers with an error naming the
708
+ // model and the fix, not `undefined`. An app can compile against the full
709
+ // source schema while running a projection, so the type system never sees
710
+ // this gap; without the stub the caller crashes one property later with a
711
+ // bare TypeError ("reading 'local'") that names neither. Non-enumerable so
712
+ // spread, Object.keys, and JSON.stringify walk past the stubs untriggered.
713
+ for (const name of schema.omittedModels ?? []) {
714
+ if (name in engine)
715
+ continue;
716
+ Object.defineProperty(engine, name, {
717
+ get() {
718
+ throw omittedModelError(name);
719
+ },
720
+ enumerable: false,
721
+ configurable: true,
722
+ });
723
+ }
706
724
  return engine;
707
725
  }
@@ -178,12 +178,9 @@ export class SyncWebSocket extends WsTransport {
178
178
  const delta = this.normalizeWireDelta(rawDelta);
179
179
  if (!delta)
180
180
  return;
181
- getContext().logger.debug('Received delta', {
182
- action: delta.actionType,
183
- model: delta.modelName,
184
- id: delta.modelId,
185
- syncId: delta.id,
186
- });
181
+ // No per-delta debug here: the payload object is built even when the
182
+ // logger discards it, and this runs at the full live wire rate. Dropped
183
+ // malformed deltas are still logged by `normalizeWireDelta`.
187
184
  // Do not advance `this.cursor.lastSyncId` on receipt. The runtime cursor
188
185
  // must stay consistent with what has been persisted locally; otherwise the
189
186
  // next `requestIncrementalSync()` (and the connect-time handshake) would
@@ -38,6 +38,7 @@ export interface DeltaPipelineContext {
38
38
  readonly smartSyncOptions: {
39
39
  readonly batchingDelay: number;
40
40
  readonly maxBatchSize: number;
41
+ readonly applySliceDeltas: number;
41
42
  };
42
43
  /** Pool-applied cursor (`syncClient.position.applied`). */
43
44
  readonly highestProcessedSyncId: number;
@@ -62,7 +63,8 @@ export interface DeltaPipelineContext {
62
63
  /** Acknowledges a sync id back to the server; a no-op when the socket is down. */
63
64
  acknowledge(syncId: number): void;
64
65
  readonly objectPool: {
65
- get(id: string): Model | undefined;
66
+ /** Ingestion-side lookup resolves without activating observability. */
67
+ peek(id: string): Model | undefined;
66
68
  add(model: Model, scope: ModelScope): void;
67
69
  remove(id: string): boolean;
68
70
  /** Full in-memory clear — the revocation-failure fallback (see
@@ -114,3 +116,32 @@ export declare function applyDeltaFrame(ctx: DeltaPipelineContext, deltas: SyncD
114
116
  * and advances the acknowledgement cursor once the store write has committed.
115
117
  */
116
118
  export declare function flushPendingDeltas(ctx: DeltaPipelineContext): Promise<void>;
119
+ /**
120
+ * Wedge forensics: where the pipeline currently is, updated synchronously at
121
+ * every stage boundary. A hang diagnoses itself by which counter pair
122
+ * diverged and which phase the active flush froze in. Mirrored onto
123
+ * `globalThis.__abloPipelineDebug` so a bench watchdog in the same thread
124
+ * can read it without an import path into SDK internals — diagnostics only,
125
+ * a handful of numbers, no payload data.
126
+ */
127
+ export declare const pipelineDebug: {
128
+ flushesStarted: number;
129
+ flushesSettled: number;
130
+ persistsStarted: number;
131
+ persistsSettled: number;
132
+ applySlices: number;
133
+ applyYields: number;
134
+ enqueued: number;
135
+ phase: string;
136
+ };
137
+ /**
138
+ * Split applied changes into slices of at most `maxDeltas`, never splitting a
139
+ * transaction: consecutive changes sharing a `transactionId` form one
140
+ * indivisible group (a commit reveals whole), while changes without one are
141
+ * individually splittable. A single transaction larger than the bound forms
142
+ * its own oversized slice, so apply always advances rather than stalling on
143
+ * an oversized commit — the same rule the server's publication chunking uses.
144
+ */
145
+ export declare function sliceApplyChanges<T extends {
146
+ readonly transactionId?: string;
147
+ }>(changes: readonly T[], maxDeltas: number): readonly T[][];
@@ -14,7 +14,7 @@ import { runInAction } from 'mobx';
14
14
  import { globalRuntime } from '../context.js';
15
15
  import { ModelScope } from '../InstanceCache.js';
16
16
  import { runStage, pluginsForStage, } from '../../plugin.js';
17
- import { observeDrainBatch, timeDrainStage, timeDrainStageAsync } from './drainProfile.js';
17
+ import { observeDrainBatch, observeDrainAcknowledge, timeDrainStage, timeDrainStageAsync, openDrainBatchRow, closeDrainBatchRow, } from './drainProfile.js';
18
18
  /**
19
19
  * One drain per store. Incoming WebSocket frames may arrive while persistence
20
20
  * and pool application are awaiting. Without a single-flight guard every
@@ -199,6 +199,7 @@ export function enqueueDelta(ctx, delta, options = {}) {
199
199
  // The delta is accepted and queued — the `receive` stage boundary.
200
200
  runStage(ctx.stagePlugins ?? [], 'receive', { delta });
201
201
  ctx.pendingDeltas.push(delta);
202
+ pipelineDebug.enqueued += 1;
202
203
  return true;
203
204
  }
204
205
  /** Debounce a flush for live single-delta traffic. */
@@ -285,6 +286,68 @@ async function drainPendingDeltas(ctx) {
285
286
  ctx.batchTimer = null;
286
287
  }
287
288
  }
289
+ /**
290
+ * Uninterrupted apply time allowed before the sliced loop yields — the
291
+ * "no visible stall" bound. Yields are amortized against it because one host
292
+ * yield costs milliseconds under load; at the measured per-delta apply cost
293
+ * this works out to roughly one yield per one to two 600-delta slices.
294
+ */
295
+ const APPLY_YIELD_BUDGET_MS = 12;
296
+ /**
297
+ * Wedge forensics: where the pipeline currently is, updated synchronously at
298
+ * every stage boundary. A hang diagnoses itself by which counter pair
299
+ * diverged and which phase the active flush froze in. Mirrored onto
300
+ * `globalThis.__abloPipelineDebug` so a bench watchdog in the same thread
301
+ * can read it without an import path into SDK internals — diagnostics only,
302
+ * a handful of numbers, no payload data.
303
+ */
304
+ export const pipelineDebug = {
305
+ flushesStarted: 0,
306
+ flushesSettled: 0,
307
+ persistsStarted: 0,
308
+ persistsSettled: 0,
309
+ applySlices: 0,
310
+ applyYields: 0,
311
+ enqueued: 0,
312
+ phase: 'idle',
313
+ };
314
+ globalThis.__abloPipelineDebug =
315
+ pipelineDebug;
316
+ /**
317
+ * Split applied changes into slices of at most `maxDeltas`, never splitting a
318
+ * transaction: consecutive changes sharing a `transactionId` form one
319
+ * indivisible group (a commit reveals whole), while changes without one are
320
+ * individually splittable. A single transaction larger than the bound forms
321
+ * its own oversized slice, so apply always advances rather than stalling on
322
+ * an oversized commit — the same rule the server's publication chunking uses.
323
+ */
324
+ export function sliceApplyChanges(changes, maxDeltas) {
325
+ if (changes.length <= maxDeltas)
326
+ return changes.length > 0 ? [[...changes]] : [];
327
+ const slices = [];
328
+ let current = [];
329
+ let index = 0;
330
+ while (index < changes.length) {
331
+ // The indivisible unit starting here: one transaction's run, or a single
332
+ // untransacted change.
333
+ const transactionId = changes[index].transactionId;
334
+ let end = index + 1;
335
+ if (transactionId !== undefined) {
336
+ while (end < changes.length && changes[end].transactionId === transactionId)
337
+ end += 1;
338
+ }
339
+ const groupSize = end - index;
340
+ if (current.length > 0 && current.length + groupSize > maxDeltas) {
341
+ slices.push(current);
342
+ current = [];
343
+ }
344
+ current.push(...changes.slice(index, end));
345
+ index = end;
346
+ }
347
+ if (current.length > 0)
348
+ slices.push(current);
349
+ return slices;
350
+ }
288
351
  function yieldToHost() {
289
352
  const immediate = globalThis.setImmediate;
290
353
  return new Promise((resolve) => {
@@ -295,7 +358,20 @@ function yieldToHost() {
295
358
  });
296
359
  }
297
360
  async function flushDeltaBatch(ctx, queuedDeltas) {
361
+ openDrainBatchRow(queuedDeltas.length);
362
+ pipelineDebug.flushesStarted += 1;
363
+ try {
364
+ await flushDeltaBatchInner(ctx, queuedDeltas);
365
+ }
366
+ finally {
367
+ pipelineDebug.flushesSettled += 1;
368
+ pipelineDebug.phase = 'idle';
369
+ closeDrainBatchRow();
370
+ }
371
+ }
372
+ async function flushDeltaBatchInner(ctx, queuedDeltas) {
298
373
  const stagePlugins = ctx.stagePlugins ?? [];
374
+ pipelineDebug.phase = 'dedupe';
299
375
  const deduplicatedDeltas = timeDrainStage('dedupe', () => ctx.deduplicateDeltas(queuedDeltas));
300
376
  observeDrainBatch(queuedDeltas.length, deduplicatedDeltas.length);
301
377
  runStage(stagePlugins, 'dedupe', { deltas: deduplicatedDeltas });
@@ -311,7 +387,7 @@ async function flushDeltaBatch(ctx, queuedDeltas) {
311
387
  // gained permission to see the entity, so we insert it into the
312
388
  // pool as if newly created.
313
389
  if (delta.actionType === 'I' || delta.actionType === 'U' || delta.actionType === 'C') {
314
- const existing = ctx.objectPool.get(delta.modelId);
390
+ const existing = ctx.objectPool.peek(delta.modelId);
315
391
  if (existing) {
316
392
  existing.updateFromData(data);
317
393
  }
@@ -334,6 +410,8 @@ async function flushDeltaBatch(ctx, queuedDeltas) {
334
410
  // handleGroupRemoved) and never reach here, though the persistence
335
411
  // signature accepts them defensively.
336
412
  const regularDeltas = deduplicatedDeltas.filter((d) => !ctx.isCustomEntity(d.modelName));
413
+ pipelineDebug.phase = 'persist';
414
+ pipelineDebug.persistsStarted += 1;
337
415
  const batch = await timeDrainStageAsync('persist', () => ctx.processDeltaBatch(regularDeltas.map((d) => ({
338
416
  syncId: d.id,
339
417
  actionType: d.actionType,
@@ -344,6 +422,7 @@ async function flushDeltaBatch(ctx, queuedDeltas) {
344
422
  // echoes of locally-applied transactions and skip the pool mutation.
345
423
  transactionId: d.transactionId,
346
424
  }))));
425
+ pipelineDebug.persistsSettled += 1;
347
426
  const dbResults = batch.results;
348
427
  runStage(stagePlugins, 'persist', { deltas: regularDeltas });
349
428
  // Apply the batch results to the in-memory graph. When a plugin has
@@ -351,14 +430,42 @@ async function flushDeltaBatch(ctx, queuedDeltas) {
351
430
  // materialiser attached where it said it would. The direct call is the
352
431
  // bridge for stores constructed without plugins (subclasses, tests),
353
432
  // whose own apply is the whole pipeline.
354
- timeDrainStage('apply', () => {
355
- if (pluginsForStage(stagePlugins, 'apply').length > 0) {
356
- runStage(stagePlugins, 'apply', { changes: dbResults });
357
- }
358
- else {
359
- ctx.applyDeltaBatchToPool(dbResults);
433
+ //
434
+ // Large batches apply in TIME SLICES: split at transaction boundaries into
435
+ // bounded chunks with the event loop yielded between them, so a catch-up
436
+ // wave reveals commit-by-commit instead of holding the thread for one long
437
+ // synchronous block. Each slice is still one MobX action (reactions fire
438
+ // once per slice), and a transaction never splits across slices — the
439
+ // commit remains the atomic unit of visibility.
440
+ const slices = sliceApplyChanges(dbResults, ctx.smartSyncOptions.applySliceDeltas);
441
+ await timeDrainStageAsync('apply', async () => {
442
+ const hasApplyPlugins = pluginsForStage(stagePlugins, 'apply').length > 0;
443
+ // Yield on a TIME budget, not per slice: a host yield costs milliseconds
444
+ // under load (measured ~7x throughput collapse when yielding every few
445
+ // deltas), so the yield decision amortizes it — only after the budget of
446
+ // uninterrupted apply work has been spent, and never before the first
447
+ // slice. Slices stay the atomicity unit; the budget only decides where
448
+ // the loop breathes.
449
+ let sliceStartedAt = performance.now();
450
+ for (let index = 0; index < slices.length; index++) {
451
+ if (index > 0 && performance.now() - sliceStartedAt > APPLY_YIELD_BUDGET_MS) {
452
+ pipelineDebug.phase = `apply-yield-${index}`;
453
+ pipelineDebug.applyYields += 1;
454
+ await yieldToHost();
455
+ sliceStartedAt = performance.now();
456
+ }
457
+ pipelineDebug.phase = `apply-slice-${index}`;
458
+ pipelineDebug.applySlices += 1;
459
+ const slice = slices[index];
460
+ if (hasApplyPlugins) {
461
+ runStage(stagePlugins, 'apply', { changes: slice });
462
+ }
463
+ else {
464
+ ctx.applyDeltaBatchToPool(slice);
465
+ }
360
466
  }
361
467
  });
468
+ pipelineDebug.phase = 'acknowledge';
362
469
  // Acknowledge and advance the sync cursor, gated on persistence.
363
470
  //
364
471
  // We must acknowledge `persistedSyncId` — the high-water mark of deltas whose
@@ -372,6 +479,7 @@ async function flushDeltaBatch(ctx, queuedDeltas) {
372
479
  timeDrainStage('acknowledge', () => {
373
480
  ctx.acknowledge(persistedSyncId);
374
481
  ctx.advancePersisted(persistedSyncId);
482
+ observeDrainAcknowledge(persistedSyncId);
375
483
  runStage(stagePlugins, 'acknowledge', { syncId: persistedSyncId });
376
484
  });
377
485
  }
@@ -28,6 +28,18 @@ export interface DrainStageTotals {
28
28
  /** How many times the stage ran. Per-delta for `parse`, per-batch for the rest. */
29
29
  readonly calls: number;
30
30
  }
31
+ /**
32
+ * One flush batch on the wall clock. Wall time (`Date.now`) rather than
33
+ * `performance.now`, because rows cross the worker boundary and each thread
34
+ * has its own `performance` origin — the drain-tail stamps learned the same
35
+ * lesson. Stage entries are the batch's own share of each pipeline stage.
36
+ */
37
+ export interface DrainBatchRow {
38
+ readonly startedAtWallMs: number;
39
+ readonly endedAtWallMs: number;
40
+ readonly deltas: number;
41
+ readonly stages: Readonly<Partial<Record<DrainStage, number>>>;
42
+ }
31
43
  export interface DrainProfile {
32
44
  /** Flush batches drained. The per-batch fixed cost multiplies by this. */
33
45
  readonly batches: number;
@@ -38,7 +50,40 @@ export interface DrainProfile {
38
50
  /** Wall time from the first observed stage to the last. */
39
51
  readonly spanMs: number;
40
52
  readonly stages: Readonly<Record<DrainStage, DrainStageTotals>>;
53
+ /**
54
+ * The most recent flush batches, oldest first, capped — enough to cover a
55
+ * drain tail. Optional because derived profiles (window subtraction, fleet
56
+ * merges) drop it; only a worker's own snapshot carries rows.
57
+ */
58
+ readonly recentBatches?: readonly DrainBatchRow[];
41
59
  }
60
+ /**
61
+ * Wall-stamped persisted-cursor advances, oldest first. The benchmark's drain
62
+ * gate reads THESE rather than observing the cursor from a timer or a
63
+ * cross-thread poll: any observation that has to be scheduled onto the
64
+ * worker's event loop queues behind the very drain burst it is measuring and
65
+ * reports the queue's latency as drain. A stamp taken synchronously inside
66
+ * the acknowledge stage cannot be deferred by anything.
67
+ */
68
+ export interface AcknowledgeStamp {
69
+ readonly syncId: number;
70
+ readonly atWallMs: number;
71
+ }
72
+ /**
73
+ * Record a persisted-cursor advance. Called by the pipeline's acknowledge
74
+ * stage. Unlike every stage timer here, this is NOT gated on the profiler
75
+ * flag: it is one wall-clock read and one bounded push per flush batch —
76
+ * nothing against the batch's own work — and the certification benchmark
77
+ * runs unprofiled (the profiler costs ~15%), so the honest drain stamp must
78
+ * exist without it.
79
+ */
80
+ export declare function observeDrainAcknowledge(syncId: number): void;
81
+ /** The recorded persisted-advance stamps, oldest first. */
82
+ export declare function drainAcknowledgeStamps(): readonly AcknowledgeStamp[];
83
+ /** Begin a batch row. Called by the pipeline at flush entry when profiling. */
84
+ export declare function openDrainBatchRow(deltaCount: number): void;
85
+ /** Close the open batch row and commit it to the ring. */
86
+ export declare function closeDrainBatchRow(): void;
42
87
  /** Whether drain profiling is on. Callers skip their own bookkeeping when it is not. */
43
88
  export declare function drainProfilingEnabled(): boolean;
44
89
  /** Attribute already-measured wall time to a stage. */
@@ -40,6 +40,54 @@ let deltas = 0;
40
40
  let deduplicated = 0;
41
41
  let firstMark;
42
42
  let lastMark = 0;
43
+ /** Ring of recent batch rows. ~50 batches/sec at benchmark rates, so this covers seconds of tail. */
44
+ const BATCH_ROW_CAP = 128;
45
+ let batchRows = [];
46
+ /**
47
+ * The batch currently being flushed. Module-global like the totals above, so
48
+ * an isolate hosting several stores attributes interleaved awaits to whichever
49
+ * batch is open — the same per-isolate approximation the totals already make.
50
+ */
51
+ let currentRow = null;
52
+ const ACK_STAMP_CAP = 512;
53
+ let ackStamps = [];
54
+ /**
55
+ * Record a persisted-cursor advance. Called by the pipeline's acknowledge
56
+ * stage. Unlike every stage timer here, this is NOT gated on the profiler
57
+ * flag: it is one wall-clock read and one bounded push per flush batch —
58
+ * nothing against the batch's own work — and the certification benchmark
59
+ * runs unprofiled (the profiler costs ~15%), so the honest drain stamp must
60
+ * exist without it.
61
+ */
62
+ export function observeDrainAcknowledge(syncId) {
63
+ ackStamps.push({ syncId, atWallMs: Date.now() });
64
+ if (ackStamps.length > ACK_STAMP_CAP)
65
+ ackStamps.shift();
66
+ }
67
+ /** The recorded persisted-advance stamps, oldest first. */
68
+ export function drainAcknowledgeStamps() {
69
+ return [...ackStamps];
70
+ }
71
+ /** Begin a batch row. Called by the pipeline at flush entry when profiling. */
72
+ export function openDrainBatchRow(deltaCount) {
73
+ if (!enabled)
74
+ return;
75
+ currentRow = { startedAtWallMs: Date.now(), deltas: deltaCount, stages: {} };
76
+ }
77
+ /** Close the open batch row and commit it to the ring. */
78
+ export function closeDrainBatchRow() {
79
+ if (!enabled || currentRow === null)
80
+ return;
81
+ batchRows.push({
82
+ startedAtWallMs: currentRow.startedAtWallMs,
83
+ endedAtWallMs: Date.now(),
84
+ deltas: currentRow.deltas,
85
+ stages: currentRow.stages,
86
+ });
87
+ if (batchRows.length > BATCH_ROW_CAP)
88
+ batchRows.shift();
89
+ currentRow = null;
90
+ }
43
91
  /**
44
92
  * Read once. A profiler that consults the environment on every delta would
45
93
  * itself become a per-delta cost in the path it is measuring.
@@ -64,6 +112,9 @@ export function observeDrainStage(stage, elapsedMs) {
64
112
  const entry = totals[stage];
65
113
  entry.totalMs += elapsedMs;
66
114
  entry.calls += 1;
115
+ if (currentRow !== null) {
116
+ currentRow.stages[stage] = (currentRow.stages[stage] ?? 0) + elapsedMs;
117
+ }
67
118
  mark(elapsedMs);
68
119
  }
69
120
  /** Time a synchronous stage. Returns the callback's value untouched. */
@@ -114,6 +165,7 @@ export function drainProfileSnapshot() {
114
165
  deduplicated,
115
166
  spanMs: firstMark === undefined ? 0 : lastMark - firstMark,
116
167
  stages,
168
+ recentBatches: [...batchRows],
117
169
  };
118
170
  }
119
171
  /** Clear the totals so a phase measures only its own traffic. */
@@ -124,4 +176,7 @@ export function resetDrainProfile() {
124
176
  deduplicated = 0;
125
177
  firstMark = undefined;
126
178
  lastMark = 0;
179
+ batchRows = [];
180
+ currentRow = null;
181
+ ackStamps = [];
127
182
  }