@abloatai/humans 0.64.4 → 0.64.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -472,6 +472,9 @@ export class Database {
472
472
  }));
473
473
  // Use batch processing for better performance
474
474
  const batch = await this.processDeltaBatch(formattedDeltas);
475
+ if (batch.persistedSyncId < Math.max(...formattedDeltas.map(delta => delta.syncId))) {
476
+ throw new Error('Could not persist all bootstrap changes; local storage must recover before retrying.');
477
+ }
475
478
  deltaResults = batch.results;
476
479
  deltasApplied = formattedDeltas.length;
477
480
  onProgress?.(deltasApplied);
@@ -518,38 +521,17 @@ export class Database {
518
521
  this.runtime.logger.debug(`[Bootstrap] NO IDB STORE for ${modelName} — ${modelData.length} items DROPPED`);
519
522
  continue;
520
523
  }
521
- let writeErrors = 0;
522
- // Store all items to IndexedDB (compacted)
524
+ // Fail before marking the model or snapshot persisted. A dropped row
525
+ // must be retried, not hidden behind an advanced bootstrap cursor.
523
526
  for (const item of modelData) {
524
- try {
525
- const compacted = this.compactRecord(modelName, item);
526
- await store.put(compacted);
527
- modelsStored++;
528
- modelsLoaded++;
529
- // Report progress every 10 items
530
- if (modelsLoaded % 10 === 0) {
531
- onProgress?.(modelsLoaded);
532
- }
533
- }
534
- catch (error) {
535
- writeErrors++;
536
- this.runtime.observability.breadcrumb(`Failed to store ${modelName} item`, 'sync.database', 'error', {
537
- error: error instanceof Error ? error.message : String(error),
538
- });
539
- }
540
- }
541
- // The model is marked persisted below whether or not every item landed,
542
- // because a partial store is still what the next sync reconciles
543
- // against. Counted and surfaced here so a partial does not read as a
544
- // clean bootstrap.
545
- if (writeErrors > 0) {
546
- this.runtime.observability.breadcrumb(`Stored ${modelName} with ${writeErrors} of ${modelData.length} items dropped`, 'sync.database', 'warning');
547
- }
548
- // Mark model as persisted after successful write
549
- try {
550
- await this.setModelPersisted(modelName, true);
527
+ const compacted = this.compactRecord(modelName, item);
528
+ await store.put(compacted);
529
+ modelsStored++;
530
+ modelsLoaded++;
531
+ if (modelsLoaded % 10 === 0)
532
+ onProgress?.(modelsLoaded);
551
533
  }
552
- catch { }
534
+ await this.setModelPersisted(modelName, true);
553
535
  }
554
536
  // Update workspace metadata with bootstrap snapshot's lastSyncId
555
537
  // Note: This method is only called for 'full' bootstrap (not 'local')
@@ -1142,6 +1124,7 @@ export class Database {
1142
1124
  // Wait for transaction to complete
1143
1125
  await new Promise((resolve, reject) => {
1144
1126
  tx.oncomplete = () => { resolve(); };
1127
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
1145
1128
  tx.onerror = () => { reject(tx.error); };
1146
1129
  });
1147
1130
  // Only commit staged results to the global results if the transaction
@@ -142,6 +142,7 @@ export class DatabaseManager {
142
142
  tx.oncomplete = () => {
143
143
  resolve();
144
144
  };
145
+ tx.onabort = () => { reject(indexedDbError(tx.error, 'registry transaction')); };
145
146
  tx.onerror = () => { reject(indexedDbError(tx.error, 'registry transaction')); };
146
147
  request.onerror = () => { reject(indexedDbError(request.error, 'registry write')); };
147
148
  });
@@ -270,6 +271,7 @@ export class DatabaseManager {
270
271
  const store = tx.objectStore('__meta');
271
272
  const request = store.put(metadata, 'metadata');
272
273
  tx.oncomplete = () => { resolve(); };
274
+ tx.onabort = () => { reject(indexedDbError(tx.error, 'workspace metadata transaction')); };
273
275
  tx.onerror = () => { reject(indexedDbError(tx.error, 'workspace metadata transaction')); };
274
276
  request.onerror = () => { reject(indexedDbError(request.error, 'workspace metadata write')); };
275
277
  });
@@ -304,6 +306,7 @@ export class DatabaseManager {
304
306
  };
305
307
  const request = store.put(persistenceData, modelName);
306
308
  tx.oncomplete = () => { resolve(); };
309
+ tx.onabort = () => { reject(indexedDbError(tx.error, 'model persistence transaction')); };
307
310
  tx.onerror = () => { reject(indexedDbError(tx.error, 'model persistence transaction')); };
308
311
  request.onerror = () => { reject(indexedDbError(request.error, 'model persistence write')); };
309
312
  });
@@ -41,6 +41,7 @@ export class ObjectStore {
41
41
  const store = tx.objectStore(this.storeName);
42
42
  const request = store.add(data);
43
43
  tx.oncomplete = () => { resolve(); };
44
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
44
45
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
45
46
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
46
47
  }
@@ -93,6 +94,7 @@ export class ObjectStore {
93
94
  const store = tx.objectStore(this.storeName);
94
95
  const request = store.put(data);
95
96
  tx.oncomplete = () => { resolve(); };
97
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
96
98
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
97
99
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
98
100
  }
@@ -197,6 +199,7 @@ export class ObjectStore {
197
199
  const store = tx.objectStore(this.storeName);
198
200
  const request = store.delete(id);
199
201
  tx.oncomplete = () => { resolve(); };
202
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
200
203
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
201
204
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
202
205
  }
@@ -233,6 +236,7 @@ export class ObjectStore {
233
236
  const store = tx.objectStore(this.storeName);
234
237
  const request = store.clear();
235
238
  tx.oncomplete = () => { resolve(); };
239
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
236
240
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
237
241
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
238
242
  }
@@ -141,6 +141,7 @@ export declare class BootstrapFetcher {
141
141
  * through its models {@link CHUNK_CONCURRENCY} at a time; each request may
142
142
  * spend `fetchTimeout` waiting for response headers and `stallTimeout`
143
143
  * waiting for the next body chunk, and may be retried `maxRetries` times.
144
+ * Capacity recovery has a separate `fetchTimeout` window.
144
145
  */
145
146
  get budgetMs(): number;
146
147
  get baseUrl(): string;
@@ -9,6 +9,7 @@
9
9
  import { globalRuntime } from '../context.js';
10
10
  import { AbloError, AbloSessionError, AbloConnectionError, translateHttpError, toAbloError, isRetryableCode } from '@abloatai/transaction/errors';
11
11
  import { withAuthHeaders } from '@abloatai/transaction/auth/credentialSource';
12
+ import { retryAfterSecondsFromHeader } from '@abloatai/transaction/wire/rateLimit';
12
13
  import { classifySchemaDrift, describeSchemaDrift, } from './schemaDrift.js';
13
14
  // SyncObservability replaced by this.runtime.observability
14
15
  import { parseBootstrapResponse } from './schemas.js';
@@ -111,11 +112,13 @@ export class BootstrapFetcher {
111
112
  * through its models {@link CHUNK_CONCURRENCY} at a time; each request may
112
113
  * spend `fetchTimeout` waiting for response headers and `stallTimeout`
113
114
  * waiting for the next body chunk, and may be retried `maxRetries` times.
115
+ * Capacity recovery has a separate `fetchTimeout` window.
114
116
  */
115
117
  get budgetMs() {
116
118
  const models = Math.max(this.options.instantModels?.length ?? 1, 1);
117
119
  const waves = Math.ceil(models / CHUNK_CONCURRENCY);
118
- return (waves * (this.options.fetchTimeout + this.options.stallTimeout) * this.options.maxRetries);
120
+ return (waves * ((this.options.fetchTimeout + this.options.stallTimeout) * this.options.maxRetries +
121
+ this.options.fetchTimeout));
119
122
  }
120
123
  get baseUrl() {
121
124
  return this.options.baseUrl;
@@ -447,7 +450,8 @@ export class BootstrapFetcher {
447
450
  */
448
451
  async fetchWithRetries(url, lane) {
449
452
  let lastError = null;
450
- for (let attempt = 0; attempt < this.options.maxRetries; attempt++) {
453
+ const capacityDeadline = Date.now() + this.options.fetchTimeout;
454
+ for (let attempt = 0; attempt < this.options.maxRetries;) {
451
455
  try {
452
456
  return await this.fetchOnce(url, lane);
453
457
  }
@@ -474,8 +478,17 @@ export class BootstrapFetcher {
474
478
  this.runtime.observability.breadcrumb('Bootstrap fetch failed', 'sync.bootstrap', 'warning', {
475
479
  attempt: attempt + 1,
476
480
  });
477
- if (attempt < this.options.maxRetries - 1) {
478
- await this.delay(this.options.retryDelay * Math.pow(2, attempt));
481
+ const delayMs = Math.max(this.options.retryDelay * Math.pow(2, attempt), (ablo.retryAfterSeconds ?? 0) * 1_000);
482
+ const capacity = ablo.code === 'instance_at_capacity' && ablo.retryAfterSeconds !== undefined;
483
+ // Admission has not run the request yet. Allow its recovery window
484
+ // without spending the attempts reserved for actual fetch failures.
485
+ if (ablo.retryAfterSeconds !== undefined && Date.now() + delayMs >= capacityDeadline) {
486
+ throw ablo;
487
+ }
488
+ if (!capacity)
489
+ attempt++;
490
+ if (capacity || attempt < this.options.maxRetries) {
491
+ await this.delay(delayMs, lane);
479
492
  }
480
493
  }
481
494
  }
@@ -609,7 +622,7 @@ export class BootstrapFetcher {
609
622
  // Translate the canonical envelope first so the server's specific code
610
623
  // and message survive (for example `api_key_required` or
611
624
  // `jwt_issuer_untrusted`).
612
- const translated = translateHttpError(res.status, parsed ?? `Bootstrap fetch failed: ${res.status} ${res.statusText}`, res.headers.get('x-request-id') ?? undefined);
625
+ const translated = translateHttpError(res.status, parsed ?? `Bootstrap fetch failed: ${res.status} ${res.statusText}`, res.headers.get('x-request-id') ?? undefined, { retryAfterSeconds: retryAfterSecondsFromHeader(res.headers.get('retry-after')) });
613
626
  // Only a genuine session or JWT expiry — or a bare auth failure carrying
614
627
  // no structured code — should drive the sign-in redirect. A specific auth
615
628
  // code like `api_key_required` is not an expired session: signing in again
@@ -764,7 +777,7 @@ export class BootstrapFetcher {
764
777
  // Same code-aware handling as the primary bootstrap fetch: preserve the
765
778
  // server's specific code/message; only a genuine expiry (or a bare,
766
779
  // code-less auth failure) drives the sign-in redirect.
767
- const translated = translateHttpError(response.status, parsed ?? `Bootstrap fetch failed: ${response.status} ${response.statusText}`, response.headers.get('x-request-id') ?? undefined);
780
+ const translated = translateHttpError(response.status, parsed ?? `Bootstrap fetch failed: ${response.status} ${response.statusText}`, response.headers.get('x-request-id') ?? undefined, { retryAfterSeconds: retryAfterSecondsFromHeader(response.headers.get('retry-after')) });
768
781
  if (translated.code === 'session_expired' ||
769
782
  translated.code === 'jwt_expired' ||
770
783
  ((response.status === 401 || response.status === 403) &&
@@ -828,7 +841,7 @@ export class BootstrapFetcher {
828
841
  // Keep as string.
829
842
  }
830
843
  }
831
- throw translateHttpError(response.status, parsed ?? `Entity fetch failed: ${response.status} ${response.statusText}`, response.headers.get('x-request-id') ?? undefined);
844
+ throw translateHttpError(response.status, parsed ?? `Entity fetch failed: ${response.status} ${response.statusText}`, response.headers.get('x-request-id') ?? undefined, { retryAfterSeconds: retryAfterSecondsFromHeader(response.headers.get('retry-after')) });
832
845
  }
833
846
  return (await response.json());
834
847
  }
@@ -901,8 +914,22 @@ export class BootstrapFetcher {
901
914
  /**
902
915
  * Helper to delay execution
903
916
  */
904
- delay(ms) {
905
- return new Promise((resolve) => setTimeout(resolve, ms));
917
+ async delay(ms, lane) {
918
+ const controller = new AbortController();
919
+ this.activeControllers.set(controller, lane);
920
+ let timer;
921
+ try {
922
+ await new Promise((resolve, reject) => {
923
+ timer = setTimeout(resolve, ms);
924
+ controller.signal.addEventListener('abort', () => {
925
+ reject(classifyRequestFailure(undefined, controller, 'Bootstrap retry aborted'));
926
+ }, { once: true });
927
+ });
928
+ }
929
+ finally {
930
+ clearTimeout(timer);
931
+ this.activeControllers.delete(controller);
932
+ }
906
933
  }
907
934
  /**
908
935
  * Get health status of sync engine
@@ -81,9 +81,8 @@ export interface FetchOptions<T> {
81
81
  };
82
82
  readonly limit?: number;
83
83
  /**
84
- * Freshness mode. When omitted, the default is derived from the model's
85
- * load strategy: `lazy` models default to `'unknown'` (local-first), while
86
- * `instant`/`partial` models default to `'complete'`.
84
+ * Freshness mode. Omitted and `'unknown'` use local-first reads for every
85
+ * load strategy. A previously hydrated query relies on the live stream.
87
86
  *
88
87
  * `'complete'`: wait for the network round-trip even if local data exists,
89
88
  * so the caller observes server-confirmed state (read-after-write).
@@ -117,19 +117,20 @@ export class OnDemandLoader {
117
117
  }
118
118
  const clauses = normalizeWhere(options?.where);
119
119
  const queryKey = stableKey(modelName, clauses, options?.orderBy, options?.limit, options?.expand);
120
- // Single-flight: an identical hydration is already in flight.
121
- const inFlight = this.inFlight.get(queryKey);
120
+ // A local-first read cannot satisfy an authoritative caller.
121
+ const flightKey = `${options?.type === 'complete' ? 'complete' : 'unknown'}:${queryKey}`;
122
+ const inFlight = this.inFlight.get(flightKey);
122
123
  if (inFlight)
123
124
  return inFlight;
124
125
  const work = this.runFetch(modelName, typename, ModelClass, clauses, options, queryKey);
125
- this.inFlight.set(queryKey, work);
126
+ this.inFlight.set(flightKey, work);
126
127
  // The rejection (if any) reaches callers via the returned `work`; this
127
128
  // side-chain only clears the single-flight slot. Without the trailing
128
129
  // catch, `.finally()` mirrors the rejection into a second, unhandled
129
130
  // promise even when every caller handles theirs.
130
131
  void work
131
132
  .finally(() => {
132
- this.inFlight.delete(queryKey);
133
+ this.inFlight.delete(flightKey);
133
134
  })
134
135
  .catch(() => undefined);
135
136
  return work;
@@ -175,11 +176,8 @@ export class OnDemandLoader {
175
176
  // network, then mark this query hydrated so future reads serve local.
176
177
  const networkModels = await this.fetchFromNetwork(modelName, typename, clauses, options);
177
178
  this.hydratedKeys.add(queryKey);
178
- if (networkModels.length > 0)
179
- return applyLimit(networkModels, options?.limit);
180
- // Network returned nothing — fall back to whatever's local (e.g. a
181
- // complete read whose server result was empty but IDB still holds rows).
182
- return applyLimit(await this.readLocal(modelName, typename, ModelClass, clauses, hasExpand, expand), options?.limit);
179
+ // An empty authoritative answer is absence, not permission to serve stale rows.
180
+ return applyLimit(networkModels, options?.limit);
183
181
  }
184
182
  /**
185
183
  * Read a query's rows from local storage only — pool first, then IndexedDB
@@ -223,12 +221,13 @@ export class OnDemandLoader {
223
221
  const network = await this.queryNetwork(modelName, clauses, options);
224
222
  const networkRows = network.rows;
225
223
  const evidenceById = new Map(network.evidence.map((entry) => [entry.id, entry.stamp]));
224
+ const acceptedRows = [];
226
225
  const networkModels = networkRows
227
226
  // Strict: a row the server returned whose type name this client never
228
227
  // registered is a genuine schema collision (the pushed schema differs
229
228
  // from the local one). Throw here, naming the cause, rather than silently
230
229
  // dropping the row and failing downstream as `entity_not_found`.
231
- .map((raw) => this.hydrateOne(raw, { kind: 'network', position: snapshotPosition(raw, evidenceById, network.position) }, typename, { strict: true }))
230
+ .map((raw) => this.hydrateOne(raw, { kind: 'network', position: snapshotPosition(raw, evidenceById, network.position) }, typename, { strict: true, acceptedRows }))
232
231
  .filter((m) => m !== null);
233
232
  for (const model of networkModels) {
234
233
  const stamp = evidenceById.get(model.id);
@@ -241,9 +240,8 @@ export class OnDemandLoader {
241
240
  }
242
241
  if (networkModels.length > 0) {
243
242
  this.opts.objectPool.addBatch(networkModels, ModelScope.live);
244
- // Background IDB write — don't block the caller. Expanded children are
245
- // persisted to their own stores inside `queryNetwork`/`hydrateExpanded`.
246
- void this.persistToIdb(modelName, networkRows);
243
+ // Persist only accepted snapshots: a stale response must not roll disk back.
244
+ await this.persistToIdb(modelName, acceptedRows);
247
245
  }
248
246
  return networkModels;
249
247
  }
@@ -377,6 +375,7 @@ export class OnDemandLoader {
377
375
  if (this.opts.objectPool.watermarks.isAheadOf(existing, origin.position))
378
376
  return existing;
379
377
  const stamped = this.stampTypename(obj, typename);
378
+ opts?.acceptedRows?.push(stamped);
380
379
  // Retain pending local fields while accepting the server's others —
381
380
  // the same local-first merge contract SyncClient's delta resolver uses.
382
381
  const localChanges = existing.getChanges();
@@ -397,7 +396,12 @@ export class OnDemandLoader {
397
396
  // re-populate it). The typename comes from the schema relation
398
397
  // (`'Block'`, `'Section'`, etc.) so no guessing involved.
399
398
  const stamped = this.stampTypename(obj, typename);
400
- return this.opts.objectPool.createFromData(stamped, undefined, opts);
399
+ const model = this.opts.objectPool.createFromData(stamped, undefined, opts);
400
+ if (model && origin.kind === 'network') {
401
+ this.opts.objectPool.watermarks.advance(model, origin.position);
402
+ opts?.acceptedRows?.push(stamped);
403
+ }
404
+ return model;
401
405
  }
402
406
  /**
403
407
  * Stamp `__typename` onto a row when it's known (from the schema's
@@ -472,7 +476,7 @@ export class OnDemandLoader {
472
476
  // own typed pool, then leave the nested arrays in place on the
473
477
  // primary row.
474
478
  if (options?.expand && options.expand.length > 0) {
475
- this.hydrateExpanded(modelName, normalized, options.expand, position);
479
+ await this.hydrateExpanded(modelName, normalized, options.expand, position);
476
480
  }
477
481
  return { rows: normalized, evidence, position };
478
482
  }
@@ -483,7 +487,8 @@ export class OnDemandLoader {
483
487
  * `__typename` field gets mangled by `postgres.camel` (`__typename`
484
488
  * → `_Typename`), so the SDK can't trust whatever string lands.
485
489
  */
486
- hydrateExpanded(parentModelName, rows, relationNames, position) {
490
+ async hydrateExpanded(parentModelName, rows, relationNames, position) {
491
+ const writes = [];
487
492
  const parentDef = this.getModelDef(parentModelName);
488
493
  // Nested rows carry no evidence of their own; the read floor at issue
489
494
  // time is what they provably reflect. A floor of zero says nothing.
@@ -505,8 +510,7 @@ export class OnDemandLoader {
505
510
  const stampedItems = [];
506
511
  for (const item of items) {
507
512
  const stamped = this.stampTypename(item, targetTypename);
508
- stampedItems.push(stamped);
509
- const m = this.hydrateOne(stamped, origin);
513
+ const m = this.hydrateOne(stamped, origin, targetTypename, { acceptedRows: stampedItems });
510
514
  if (m)
511
515
  models.push(m);
512
516
  }
@@ -518,24 +522,22 @@ export class OnDemandLoader {
518
522
  // this, expand-fetched relations live only inside the parent's row
519
523
  // and are lost to a lazy child query after a cold start.
520
524
  if (stampedItems.length > 0 && targetKey) {
521
- void this.persistToIdb(targetKey, stampedItems);
525
+ writes.push(this.persistToIdb(targetKey, stampedItems));
522
526
  }
523
527
  }
524
528
  }
529
+ await Promise.all(writes);
525
530
  }
526
531
  async persistToIdb(modelName, rows) {
532
+ if (rows.length === 0)
533
+ return;
527
534
  const store = this.opts.database.getStore(this.resolveTypename(modelName));
535
+ // Before ready(), callers may have a graph but no initialized storage.
528
536
  if (!store)
529
537
  return;
530
- for (const row of rows) {
531
- try {
532
- await store.put(row);
533
- }
534
- catch {
535
- // IDB writes are best-effort — a transient quota/transaction
536
- // failure shouldn't break the hydration's primary purpose.
537
- }
538
- }
538
+ // Enqueue together, before yielding: a later delta must not be followed by
539
+ // an older row from the tail of this query's sequential write loop.
540
+ await Promise.all(rows.map((row) => store.put(row)));
539
541
  }
540
542
  resolveTypename(modelName) {
541
543
  // Schema is the source of truth for wire typenames. The model proxy
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@abloatai/humans",
3
- "version": "0.64.4",
3
+ "version": "0.64.5",
4
4
  "description": "The optional human-facing local-state package for Ablo: presence, live queries, and React bindings.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -85,7 +85,7 @@
85
85
  "directory": "packages/humans"
86
86
  },
87
87
  "dependencies": {
88
- "@abloatai/transaction": "0.64.4",
88
+ "@abloatai/transaction": "0.64.5",
89
89
  "events": "^3.3.0",
90
90
  "mobx": "^6.13.7",
91
91
  "uuid": "^11.1.0",
@@ -621,6 +621,9 @@ export class Database {
621
621
 
622
622
  // Use batch processing for better performance
623
623
  const batch = await this.processDeltaBatch(formattedDeltas);
624
+ if (batch.persistedSyncId < Math.max(...formattedDeltas.map(delta => delta.syncId))) {
625
+ throw new Error('Could not persist all bootstrap changes; local storage must recover before retrying.');
626
+ }
624
627
  deltaResults = batch.results;
625
628
  deltasApplied = formattedDeltas.length;
626
629
  onProgress?.(deltasApplied);
@@ -687,48 +690,16 @@ export class Database {
687
690
  );
688
691
  continue;
689
692
  }
690
- let writeErrors = 0;
691
- // Store all items to IndexedDB (compacted)
693
+ // Fail before marking the model or snapshot persisted. A dropped row
694
+ // must be retried, not hidden behind an advanced bootstrap cursor.
692
695
  for (const item of modelData) {
693
- try {
694
- const compacted = this.compactRecord(modelName, item as ModelData);
695
- await store.put(compacted);
696
- modelsStored++;
697
- modelsLoaded++;
698
-
699
- // Report progress every 10 items
700
- if (modelsLoaded % 10 === 0) {
701
- onProgress?.(modelsLoaded);
702
- }
703
- } catch (error) {
704
- writeErrors++;
705
- this.runtime.observability.breadcrumb(
706
- `Failed to store ${modelName} item`,
707
- 'sync.database',
708
- 'error',
709
- {
710
- error: error instanceof Error ? error.message : String(error),
711
- }
712
- );
713
- }
714
- }
715
-
716
- // The model is marked persisted below whether or not every item landed,
717
- // because a partial store is still what the next sync reconciles
718
- // against. Counted and surfaced here so a partial does not read as a
719
- // clean bootstrap.
720
- if (writeErrors > 0) {
721
- this.runtime.observability.breadcrumb(
722
- `Stored ${modelName} with ${writeErrors} of ${modelData.length} items dropped`,
723
- 'sync.database',
724
- 'warning',
725
- );
696
+ const compacted = this.compactRecord(modelName, item as ModelData);
697
+ await store.put(compacted);
698
+ modelsStored++;
699
+ modelsLoaded++;
700
+ if (modelsLoaded % 10 === 0) onProgress?.(modelsLoaded);
726
701
  }
727
-
728
- // Mark model as persisted after successful write
729
- try {
730
- await this.setModelPersisted(modelName, true);
731
- } catch {}
702
+ await this.setModelPersisted(modelName, true);
732
703
  }
733
704
 
734
705
  // Update workspace metadata with bootstrap snapshot's lastSyncId
@@ -1490,6 +1461,7 @@ export class Database {
1490
1461
  // Wait for transaction to complete
1491
1462
  await new Promise<void>((resolve, reject) => {
1492
1463
  tx.oncomplete = () => { resolve(); };
1464
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
1493
1465
  tx.onerror = () => { reject(tx.error); };
1494
1466
  });
1495
1467
  // Only commit staged results to the global results if the transaction
@@ -208,6 +208,7 @@ export class DatabaseManager {
208
208
  resolve();
209
209
  };
210
210
 
211
+ tx.onabort = () => { reject(indexedDbError(tx.error, 'registry transaction')); };
211
212
  tx.onerror = () => { reject(indexedDbError(tx.error, 'registry transaction')); };
212
213
  request.onerror = () => { reject(indexedDbError(request.error, 'registry write')); };
213
214
  });
@@ -347,6 +348,7 @@ export class DatabaseManager {
347
348
  const request = store.put(metadata, 'metadata');
348
349
 
349
350
  tx.oncomplete = () => { resolve(); };
351
+ tx.onabort = () => { reject(indexedDbError(tx.error, 'workspace metadata transaction')); };
350
352
  tx.onerror = () => { reject(indexedDbError(tx.error, 'workspace metadata transaction')); };
351
353
  request.onerror = () => { reject(indexedDbError(request.error, 'workspace metadata write')); };
352
354
  });
@@ -388,6 +390,7 @@ export class DatabaseManager {
388
390
  const request = store.put(persistenceData, modelName);
389
391
 
390
392
  tx.oncomplete = () => { resolve(); };
393
+ tx.onabort = () => { reject(indexedDbError(tx.error, 'model persistence transaction')); };
391
394
  tx.onerror = () => { reject(indexedDbError(tx.error, 'model persistence transaction')); };
392
395
  request.onerror = () => { reject(indexedDbError(request.error, 'model persistence write')); };
393
396
  });
@@ -54,6 +54,7 @@ export class ObjectStore implements ObjectStoreContract {
54
54
  const request = store.add(data);
55
55
 
56
56
  tx.oncomplete = () => { resolve(); };
57
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
57
58
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
58
59
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
59
60
  } catch (error) {
@@ -110,6 +111,7 @@ export class ObjectStore implements ObjectStoreContract {
110
111
  const request = store.put(data);
111
112
 
112
113
  tx.oncomplete = () => { resolve(); };
114
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
113
115
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
114
116
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
115
117
  } catch (error) {
@@ -223,6 +225,7 @@ export class ObjectStore implements ObjectStoreContract {
223
225
  const request = store.delete(id);
224
226
 
225
227
  tx.oncomplete = () => { resolve(); };
228
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
226
229
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
227
230
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
228
231
  } catch (error) {
@@ -261,6 +264,7 @@ export class ObjectStore implements ObjectStoreContract {
261
264
  const request = store.clear();
262
265
 
263
266
  tx.oncomplete = () => { resolve(); };
267
+ tx.onabort = () => { reject(tx.error ?? new DOMException('IndexedDB transaction aborted', 'AbortError')); };
264
268
  tx.onerror = () => { reject(tx.error ?? new Error('IndexedDB transaction error')); };
265
269
  request.onerror = () => { reject(request.error ?? new Error('IndexedDB request error')); };
266
270
  } catch (error) {
@@ -107,6 +107,7 @@ import { globalRuntime } from '../context.js';
107
107
  import type { RuntimeContext } from '../RuntimeContext.js';
108
108
  import { AbloError, AbloSessionError, AbloConnectionError, translateHttpError, toAbloError, isRetryableCode } from '@abloatai/transaction/errors';
109
109
  import { withAuthHeaders, type AuthTokenGetter } from '@abloatai/transaction/auth/credentialSource';
110
+ import { retryAfterSecondsFromHeader } from '@abloatai/transaction/wire/rateLimit';
110
111
  import {
111
112
  classifySchemaDrift,
112
113
  describeSchemaDrift,
@@ -244,12 +245,16 @@ export class BootstrapFetcher {
244
245
  * through its models {@link CHUNK_CONCURRENCY} at a time; each request may
245
246
  * spend `fetchTimeout` waiting for response headers and `stallTimeout`
246
247
  * waiting for the next body chunk, and may be retried `maxRetries` times.
248
+ * Capacity recovery has a separate `fetchTimeout` window.
247
249
  */
248
250
  get budgetMs(): number {
249
251
  const models = Math.max(this.options.instantModels?.length ?? 1, 1);
250
252
  const waves = Math.ceil(models / CHUNK_CONCURRENCY);
251
253
  return (
252
- waves * (this.options.fetchTimeout + this.options.stallTimeout) * this.options.maxRetries
254
+ waves * (
255
+ (this.options.fetchTimeout + this.options.stallTimeout) * this.options.maxRetries +
256
+ this.options.fetchTimeout
257
+ )
253
258
  );
254
259
  }
255
260
 
@@ -625,7 +630,8 @@ export class BootstrapFetcher {
625
630
  */
626
631
  private async fetchWithRetries(url: string, lane: CancelLane): Promise<BootstrapData> {
627
632
  let lastError: Error | null = null;
628
- for (let attempt = 0; attempt < this.options.maxRetries; attempt++) {
633
+ const capacityDeadline = Date.now() + this.options.fetchTimeout;
634
+ for (let attempt = 0; attempt < this.options.maxRetries;) {
629
635
  try {
630
636
  return await this.fetchOnce(url, lane);
631
637
  } catch (error) {
@@ -664,8 +670,19 @@ export class BootstrapFetcher {
664
670
  attempt: attempt + 1,
665
671
  });
666
672
 
667
- if (attempt < this.options.maxRetries - 1) {
668
- await this.delay(this.options.retryDelay * Math.pow(2, attempt));
673
+ const delayMs = Math.max(
674
+ this.options.retryDelay * Math.pow(2, attempt),
675
+ (ablo.retryAfterSeconds ?? 0) * 1_000,
676
+ );
677
+ const capacity = ablo.code === 'instance_at_capacity' && ablo.retryAfterSeconds !== undefined;
678
+ // Admission has not run the request yet. Allow its recovery window
679
+ // without spending the attempts reserved for actual fetch failures.
680
+ if (ablo.retryAfterSeconds !== undefined && Date.now() + delayMs >= capacityDeadline) {
681
+ throw ablo;
682
+ }
683
+ if (!capacity) attempt++;
684
+ if (capacity || attempt < this.options.maxRetries) {
685
+ await this.delay(delayMs, lane);
669
686
  }
670
687
  }
671
688
  }
@@ -827,6 +844,7 @@ export class BootstrapFetcher {
827
844
  res.status,
828
845
  parsed ?? `Bootstrap fetch failed: ${res.status} ${res.statusText}`,
829
846
  res.headers.get('x-request-id') ?? undefined,
847
+ { retryAfterSeconds: retryAfterSecondsFromHeader(res.headers.get('retry-after')) },
830
848
  );
831
849
  // Only a genuine session or JWT expiry — or a bare auth failure carrying
832
850
  // no structured code — should drive the sign-in redirect. A specific auth
@@ -1019,6 +1037,7 @@ export class BootstrapFetcher {
1019
1037
  response.status,
1020
1038
  parsed ?? `Bootstrap fetch failed: ${response.status} ${response.statusText}`,
1021
1039
  response.headers.get('x-request-id') ?? undefined,
1040
+ { retryAfterSeconds: retryAfterSecondsFromHeader(response.headers.get('retry-after')) },
1022
1041
  );
1023
1042
  if (
1024
1043
  translated.code === 'session_expired' ||
@@ -1095,6 +1114,7 @@ export class BootstrapFetcher {
1095
1114
  response.status,
1096
1115
  parsed ?? `Entity fetch failed: ${response.status} ${response.statusText}`,
1097
1116
  response.headers.get('x-request-id') ?? undefined,
1117
+ { retryAfterSeconds: retryAfterSecondsFromHeader(response.headers.get('retry-after')) },
1098
1118
  );
1099
1119
  }
1100
1120
 
@@ -1168,8 +1188,21 @@ export class BootstrapFetcher {
1168
1188
  /**
1169
1189
  * Helper to delay execution
1170
1190
  */
1171
- private delay(ms: number): Promise<void> {
1172
- return new Promise((resolve) => setTimeout(resolve, ms));
1191
+ private async delay(ms: number, lane: CancelLane): Promise<void> {
1192
+ const controller = new AbortController();
1193
+ this.activeControllers.set(controller, lane);
1194
+ let timer: ReturnType<typeof setTimeout> | undefined;
1195
+ try {
1196
+ await new Promise<void>((resolve, reject) => {
1197
+ timer = setTimeout(resolve, ms);
1198
+ controller.signal.addEventListener('abort', () => {
1199
+ reject(classifyRequestFailure(undefined, controller, 'Bootstrap retry aborted'));
1200
+ }, { once: true });
1201
+ });
1202
+ } finally {
1203
+ clearTimeout(timer);
1204
+ this.activeControllers.delete(controller);
1205
+ }
1173
1206
  }
1174
1207
 
1175
1208
  /**
@@ -85,9 +85,8 @@ export interface FetchOptions<T> {
85
85
  readonly orderBy?: { [K in keyof T]?: 'asc' | 'desc' };
86
86
  readonly limit?: number;
87
87
  /**
88
- * Freshness mode. When omitted, the default is derived from the model's
89
- * load strategy: `lazy` models default to `'unknown'` (local-first), while
90
- * `instant`/`partial` models default to `'complete'`.
88
+ * Freshness mode. Omitted and `'unknown'` use local-first reads for every
89
+ * load strategy. A previously hydrated query relies on the live stream.
91
90
  *
92
91
  * `'complete'`: wait for the network round-trip even if local data exists,
93
92
  * so the caller observes server-confirmed state (read-after-write).
@@ -240,19 +239,20 @@ export class OnDemandLoader {
240
239
  const clauses = normalizeWhere(options?.where);
241
240
  const queryKey = stableKey(modelName, clauses, options?.orderBy, options?.limit, options?.expand);
242
241
 
243
- // Single-flight: an identical hydration is already in flight.
244
- const inFlight = this.inFlight.get(queryKey);
242
+ // A local-first read cannot satisfy an authoritative caller.
243
+ const flightKey = `${options?.type === 'complete' ? 'complete' : 'unknown'}:${queryKey}`;
244
+ const inFlight = this.inFlight.get(flightKey);
245
245
  if (inFlight) return inFlight;
246
246
 
247
247
  const work = this.runFetch(modelName, typename, ModelClass, clauses, options, queryKey);
248
- this.inFlight.set(queryKey, work);
248
+ this.inFlight.set(flightKey, work);
249
249
  // The rejection (if any) reaches callers via the returned `work`; this
250
250
  // side-chain only clears the single-flight slot. Without the trailing
251
251
  // catch, `.finally()` mirrors the rejection into a second, unhandled
252
252
  // promise even when every caller handles theirs.
253
253
  void work
254
254
  .finally(() => {
255
- this.inFlight.delete(queryKey);
255
+ this.inFlight.delete(flightKey);
256
256
  })
257
257
  .catch(() => undefined);
258
258
  return work;
@@ -313,14 +313,8 @@ export class OnDemandLoader {
313
313
  // network, then mark this query hydrated so future reads serve local.
314
314
  const networkModels = await this.fetchFromNetwork(modelName, typename, clauses, options);
315
315
  this.hydratedKeys.add(queryKey);
316
- if (networkModels.length > 0) return applyLimit(networkModels, options?.limit);
317
-
318
- // Network returned nothing — fall back to whatever's local (e.g. a
319
- // complete read whose server result was empty but IDB still holds rows).
320
- return applyLimit(
321
- await this.readLocal(modelName, typename, ModelClass, clauses, hasExpand, expand),
322
- options?.limit,
323
- );
316
+ // An empty authoritative answer is absence, not permission to serve stale rows.
317
+ return applyLimit(networkModels, options?.limit);
324
318
  }
325
319
 
326
320
  /**
@@ -379,6 +373,7 @@ export class OnDemandLoader {
379
373
  const network = await this.queryNetwork(modelName, clauses, options);
380
374
  const networkRows = network.rows;
381
375
  const evidenceById = new Map(network.evidence.map((entry) => [entry.id, entry.stamp]));
376
+ const acceptedRows: unknown[] = [];
382
377
  const networkModels = networkRows
383
378
  // Strict: a row the server returned whose type name this client never
384
379
  // registered is a genuine schema collision (the pushed schema differs
@@ -389,7 +384,7 @@ export class OnDemandLoader {
389
384
  raw,
390
385
  { kind: 'network', position: snapshotPosition(raw, evidenceById, network.position) },
391
386
  typename,
392
- { strict: true },
387
+ { strict: true, acceptedRows },
393
388
  ),
394
389
  )
395
390
  .filter((m): m is Model => m !== null);
@@ -405,9 +400,8 @@ export class OnDemandLoader {
405
400
 
406
401
  if (networkModels.length > 0) {
407
402
  this.opts.objectPool.addBatch(networkModels, ModelScope.live);
408
- // Background IDB write — don't block the caller. Expanded children are
409
- // persisted to their own stores inside `queryNetwork`/`hydrateExpanded`.
410
- void this.persistToIdb(modelName, networkRows);
403
+ // Persist only accepted snapshots: a stale response must not roll disk back.
404
+ await this.persistToIdb(modelName, acceptedRows);
411
405
  }
412
406
 
413
407
  return networkModels;
@@ -538,7 +532,7 @@ export class OnDemandLoader {
538
532
  raw: unknown,
539
533
  origin: HydrationOrigin,
540
534
  typename?: string,
541
- opts?: { strict?: boolean },
535
+ opts?: { strict?: boolean; acceptedRows?: unknown[] },
542
536
  ): Model | null {
543
537
  if (!raw || typeof raw !== 'object') return null;
544
538
  const obj = raw as Record<string, unknown>;
@@ -559,6 +553,7 @@ export class OnDemandLoader {
559
553
  if (this.opts.objectPool.watermarks.isAheadOf(existing, origin.position)) return existing;
560
554
 
561
555
  const stamped = this.stampTypename(obj, typename) as Record<string, unknown>;
556
+ opts?.acceptedRows?.push(stamped);
562
557
  // Retain pending local fields while accepting the server's others —
563
558
  // the same local-first merge contract SyncClient's delta resolver uses.
564
559
  const localChanges = existing.getChanges();
@@ -581,7 +576,12 @@ export class OnDemandLoader {
581
576
  // re-populate it). The typename comes from the schema relation
582
577
  // (`'Block'`, `'Section'`, etc.) so no guessing involved.
583
578
  const stamped = this.stampTypename(obj, typename) as Record<string, unknown>;
584
- return this.opts.objectPool.createFromData(stamped, undefined, opts);
579
+ const model = this.opts.objectPool.createFromData(stamped, undefined, opts);
580
+ if (model && origin.kind === 'network') {
581
+ this.opts.objectPool.watermarks.advance(model, origin.position);
582
+ opts?.acceptedRows?.push(stamped);
583
+ }
584
+ return model;
585
585
  }
586
586
 
587
587
  /**
@@ -672,7 +672,7 @@ export class OnDemandLoader {
672
672
  // own typed pool, then leave the nested arrays in place on the
673
673
  // primary row.
674
674
  if (options?.expand && options.expand.length > 0) {
675
- this.hydrateExpanded(modelName, normalized, options.expand, position);
675
+ await this.hydrateExpanded(modelName, normalized, options.expand, position);
676
676
  }
677
677
  return { rows: normalized, evidence, position };
678
678
  }
@@ -684,12 +684,13 @@ export class OnDemandLoader {
684
684
  * `__typename` field gets mangled by `postgres.camel` (`__typename`
685
685
  * → `_Typename`), so the SDK can't trust whatever string lands.
686
686
  */
687
- private hydrateExpanded(
687
+ private async hydrateExpanded(
688
688
  parentModelName: string,
689
689
  rows: unknown[],
690
690
  relationNames: readonly string[],
691
691
  position: number,
692
- ): void {
692
+ ): Promise<void> {
693
+ const writes: Promise<void>[] = [];
693
694
  const parentDef = this.getModelDef(parentModelName);
694
695
  // Nested rows carry no evidence of their own; the read floor at issue
695
696
  // time is what they provably reflect. A floor of zero says nothing.
@@ -710,8 +711,7 @@ export class OnDemandLoader {
710
711
  const stampedItems: unknown[] = [];
711
712
  for (const item of items) {
712
713
  const stamped = this.stampTypename(item, targetTypename);
713
- stampedItems.push(stamped);
714
- const m = this.hydrateOne(stamped, origin);
714
+ const m = this.hydrateOne(stamped, origin, targetTypename, { acceptedRows: stampedItems });
715
715
  if (m) models.push(m);
716
716
  }
717
717
  if (models.length > 0) {
@@ -722,23 +722,21 @@ export class OnDemandLoader {
722
722
  // this, expand-fetched relations live only inside the parent's row
723
723
  // and are lost to a lazy child query after a cold start.
724
724
  if (stampedItems.length > 0 && targetKey) {
725
- void this.persistToIdb(targetKey, stampedItems);
725
+ writes.push(this.persistToIdb(targetKey, stampedItems));
726
726
  }
727
727
  }
728
728
  }
729
+ await Promise.all(writes);
729
730
  }
730
731
 
731
732
  private async persistToIdb(modelName: string, rows: unknown[]): Promise<void> {
733
+ if (rows.length === 0) return;
732
734
  const store = this.opts.database.getStore(this.resolveTypename(modelName));
735
+ // Before ready(), callers may have a graph but no initialized storage.
733
736
  if (!store) return;
734
- for (const row of rows) {
735
- try {
736
- await store.put(row as Record<string, unknown>);
737
- } catch {
738
- // IDB writes are best-effort — a transient quota/transaction
739
- // failure shouldn't break the hydration's primary purpose.
740
- }
741
- }
737
+ // Enqueue together, before yielding: a later delta must not be followed by
738
+ // an older row from the tail of this query's sequential write loop.
739
+ await Promise.all(rows.map((row) => store.put(row as Record<string, unknown>)));
742
740
  }
743
741
 
744
742
  private resolveTypename(modelName: string): string {