@pod-network/trade-sdk 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +6 -4
  2. package/dist/client.d.ts +20 -10
  3. package/dist/client.d.ts.map +1 -1
  4. package/dist/client.js +26 -11
  5. package/dist/client.js.map +1 -1
  6. package/dist/codec/activity.d.ts +11 -0
  7. package/dist/codec/activity.d.ts.map +1 -0
  8. package/dist/codec/activity.js +33 -0
  9. package/dist/codec/activity.js.map +1 -0
  10. package/dist/codec/decode.d.ts +14 -6
  11. package/dist/codec/decode.d.ts.map +1 -1
  12. package/dist/codec/decode.js +58 -24
  13. package/dist/codec/decode.js.map +1 -1
  14. package/dist/codec/orders-v2.d.ts +9 -3
  15. package/dist/codec/orders-v2.d.ts.map +1 -1
  16. package/dist/codec/orders-v2.js +12 -9
  17. package/dist/codec/orders-v2.js.map +1 -1
  18. package/dist/index.d.ts +2 -1
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +1 -0
  21. package/dist/index.js.map +1 -1
  22. package/dist/sync/activity.d.ts +58 -0
  23. package/dist/sync/activity.d.ts.map +1 -0
  24. package/dist/sync/activity.js +259 -0
  25. package/dist/sync/activity.js.map +1 -0
  26. package/dist/sync/orders.d.ts +4 -64
  27. package/dist/sync/orders.d.ts.map +1 -1
  28. package/dist/sync/orders.js +40 -190
  29. package/dist/sync/orders.js.map +1 -1
  30. package/dist/sync/pnl-history.d.ts +33 -14
  31. package/dist/sync/pnl-history.d.ts.map +1 -1
  32. package/dist/sync/pnl-history.js +255 -65
  33. package/dist/sync/pnl-history.js.map +1 -1
  34. package/dist/sync/stream.d.ts +101 -0
  35. package/dist/sync/stream.d.ts.map +1 -0
  36. package/dist/sync/stream.js +243 -0
  37. package/dist/sync/stream.js.map +1 -0
  38. package/dist/transport/jsonrpc.d.ts +0 -5
  39. package/dist/transport/jsonrpc.d.ts.map +1 -1
  40. package/dist/transport/jsonrpc.js +0 -20
  41. package/dist/transport/jsonrpc.js.map +1 -1
  42. package/dist/transport/rest.d.ts +36 -4
  43. package/dist/transport/rest.d.ts.map +1 -1
  44. package/dist/transport/rest.js +43 -10
  45. package/dist/transport/rest.js.map +1 -1
  46. package/dist/transport/ws.d.ts +5 -1
  47. package/dist/transport/ws.d.ts.map +1 -1
  48. package/dist/transport/ws.js.map +1 -1
  49. package/dist/types/public.d.ts +62 -23
  50. package/dist/types/public.d.ts.map +1 -1
  51. package/dist/types/wire.d.ts +98 -16
  52. package/dist/types/wire.d.ts.map +1 -1
  53. package/dist/write/index.d.ts +3 -3
  54. package/dist/write/index.d.ts.map +1 -1
  55. package/dist/write/index.js +0 -3
  56. package/dist/write/index.js.map +1 -1
  57. package/package.json +1 -1
  58. package/src/client.ts +28 -13
  59. package/src/codec/activity.ts +46 -0
  60. package/src/codec/decode.ts +65 -30
  61. package/src/codec/orders-v2.ts +24 -11
  62. package/src/index.ts +3 -1
  63. package/src/sync/activity.ts +266 -0
  64. package/src/sync/orders.ts +41 -187
  65. package/src/sync/pnl-history.ts +244 -75
  66. package/src/sync/stream.ts +270 -0
  67. package/src/transport/jsonrpc.ts +0 -23
  68. package/src/transport/rest.ts +62 -14
  69. package/src/transport/ws.ts +5 -1
  70. package/src/types/public.ts +62 -26
  71. package/src/types/wire.ts +105 -18
  72. package/src/write/index.ts +3 -6
@@ -11,13 +11,20 @@
11
11
  // Fills page newest-first, so the walk streams: the newest points only need
12
12
  // the newest pages, and each chunk is yielded as soon as its fills are in.
13
13
 
14
- import type { Address, MarketId } from "../types/public.js";
14
+ import type { Address, MarketId, MoneyActivity } from "../types/public.js";
15
15
  import { div, mul } from "../codec/fixed.js";
16
16
  import { dec } from "../codec/units.js";
17
- import { rpc, rpcBatch, type RpcOptions } from "../transport/jsonrpc.js";
18
- import type { PodRestClient } from "../transport/rest.js";
19
-
20
- export interface PnlPoint { time: number; pnl: bigint }
17
+ import type { WireFillRow, WireSolutionRow } from "../types/wire.js";
18
+ import { rpc, type RpcOptions } from "../transport/jsonrpc.js";
19
+ import { PodHttpError, type PodRestClient } from "../transport/rest.js";
20
+
21
+ export interface PnlPoint {
22
+ time: number;
23
+ pnl: bigint;
24
+ /** Absolute account value at the point; absent when the node does not
25
+ * serve the activity feed the deposits and transfers come from. */
26
+ accountValue?: bigint;
27
+ }
21
28
  export interface PnlHistory { points: PnlPoint[]; warnings: string[] }
22
29
  export interface PnlHistoryQuery {
23
30
  /** ms */
@@ -43,12 +50,17 @@ export interface PnlLeg {
43
50
  basis?: bigint;
44
51
  }
45
52
  export interface PnlEvent {
46
- market: MarketId;
53
+ /** Absent on a cash movement. */
54
+ market?: MarketId;
47
55
  timeUs: number;
56
+ /** Signed size for a market event; the signed native amount for `cash`. */
48
57
  deltaSize: bigint;
49
- /** Fill price; unused for withdrawals. */
58
+ /** Fill price; unused for withdrawals, deposits and cash. */
50
59
  price: bigint;
51
- kind: "fill" | "withdraw" | "sweep";
60
+ /** `withdraw` and `deposit` are spot token movements: a withdrawal leaves at
61
+ * the average cost, a deposit arrives marked at the tick's clearing price.
62
+ * `cash` is native money in or out, which moves account value only. */
63
+ kind: "fill" | "withdraw" | "deposit" | "sweep" | "cash";
52
64
  }
53
65
  export interface PnlTick { mark: bigint; funding: bigint }
54
66
  export interface PnlFoldInput {
@@ -60,12 +72,15 @@ export interface PnlFoldInput {
60
72
  gridUs: number[];
61
73
  /** Newest tick at or before `timeUs`. */
62
74
  tickAt: (market: MarketId, timeUs: number) => PnlTick | undefined;
75
+ /** Account value at the reference tick; points carry `accountValue` when given. */
76
+ accountValueRef?: bigint;
63
77
  }
64
78
 
65
79
  const RESIDUAL_TOLERANCE = 10n ** 12n;
66
- // Within a tick the engine runs sweeps, then matching, then withdrawals;
67
- // walking backwards undoes them in the opposite order.
68
- const RANK = { withdraw: 0, fill: 1, sweep: 2 } as const;
80
+ // Within a tick the engine runs sweeps, then deposits, then matching, then
81
+ // transfers and withdrawals; walking backwards undoes them in the opposite
82
+ // order. Cash never touches a size, so its place only has to be consistent.
83
+ const RANK = { withdraw: 0, cash: 1, fill: 2, deposit: 3, sweep: 4 } as const;
69
84
  const newestFirst = (a: PnlEvent, b: PnlEvent) => b.timeUs - a.timeUs || RANK[a.kind] - RANK[b.kind];
70
85
 
71
86
  interface LegState { kind: "perp" | "spot"; size: bigint; basis: bigint; basisKnown: boolean }
@@ -77,12 +92,14 @@ export class PnlWalker {
77
92
  private readonly state = new Map<MarketId, LegState>();
78
93
  private pending: PnlEvent[] = [];
79
94
  private acc = 0n;
95
+ private flows = 0n;
80
96
  private refValue?: bigint;
81
97
 
82
98
  constructor(
83
99
  legs: PnlLeg[],
84
100
  private readonly tickAt: (market: MarketId, timeUs: number) => PnlTick | undefined,
85
101
  private readonly refTimeUs: number,
102
+ private readonly accountValueRef?: bigint,
86
103
  ) {
87
104
  for (const l of legs) this.state.set(l.market, { kind: l.kind, size: l.size, basis: l.basis ?? 0n, basisKnown: l.basis !== undefined });
88
105
  }
@@ -100,9 +117,15 @@ export class PnlWalker {
100
117
  this.refValue ??= this.value(this.refTimeUs);
101
118
  for (let e = this.pending[0]; e && e.timeUs > t; e = this.pending[0]) {
102
119
  this.pending.shift();
103
- this.acc += this.unapply(e);
120
+ const u = this.unapply(e);
121
+ this.acc += u.pnl;
122
+ this.flows += u.flow;
104
123
  }
105
- return { time: t / 1000, pnl: this.value(t) - this.refValue - this.acc };
124
+ const pnl = this.value(t) - this.refValue - this.acc;
125
+ const point: PnlPoint = { time: t / 1000, pnl };
126
+ // Account value is PnL plus every movement that is not PnL.
127
+ if (this.accountValueRef !== undefined) point.accountValue = this.accountValueRef + pnl - this.flows;
128
+ return point;
106
129
  }
107
130
 
108
131
  private value(t: number): bigint {
@@ -116,13 +139,16 @@ export class PnlWalker {
116
139
  return sum;
117
140
  }
118
141
 
119
- private unapply(e: PnlEvent): bigint {
142
+ /** The PnL adjustment and the non-PnL money value carried by the event. */
143
+ private unapply(e: PnlEvent): { pnl: bigint; flow: bigint } {
144
+ if (e.kind === "cash" || e.market === undefined) return { pnl: 0n, flow: e.deltaSize };
120
145
  const st = this.state.get(e.market);
121
146
  if (!st) throw new Error(`pnlHistory: event on unknown market ${e.market}`);
122
147
  const tick = this.tickAt(e.market, e.timeUs);
123
148
  const F = tick?.funding ?? 0n;
124
149
  const sizeBefore = st.size - e.deltaSize;
125
150
  let price = e.price;
151
+ let flow = 0n;
126
152
 
127
153
  if (st.kind === "spot") {
128
154
  const average = st.size !== 0n && st.basisKnown ? div(st.basis, st.size) : undefined;
@@ -132,8 +158,12 @@ export class PnlWalker {
132
158
  price = tick?.mark ?? 0n;
133
159
  } else price = average;
134
160
  }
161
+ if (e.kind === "deposit") price = tick?.mark ?? 0n;
162
+ // Tokens leave at their cost and arrive at the clearing price; either
163
+ // way the value that moved is what the event is priced at.
164
+ if (e.kind === "withdraw" || e.kind === "deposit") flow = mul(price, e.deltaSize);
135
165
  if (sizeBefore === 0n) { st.basis = 0n; st.basisKnown = true; }
136
- else if (e.deltaSize > 0n) st.basis -= mul(e.price, e.deltaSize);
166
+ else if (e.deltaSize > 0n) st.basis -= mul(price, e.deltaSize);
137
167
  else if (average !== undefined) st.basis = mul(average, sizeBefore);
138
168
  else st.basisKnown = false;
139
169
  } else {
@@ -148,34 +178,57 @@ export class PnlWalker {
148
178
  }
149
179
 
150
180
  st.size = sizeBefore;
151
- return mul(F - price, e.deltaSize);
181
+ return { pnl: mul(F - price, e.deltaSize), flow };
152
182
  }
153
183
  }
154
184
 
155
185
  export function foldPnlHistory(input: PnlFoldInput): PnlHistory {
156
- const walker = new PnlWalker(input.legs, input.tickAt, input.refTimeUs);
186
+ const walker = new PnlWalker(input.legs, input.tickAt, input.refTimeUs, input.accountValueRef);
157
187
  walker.feed(input.events);
158
188
  const points = [...input.gridUs].reverse().map((t) => walker.point(t)).reverse();
159
189
  const base = points[0]?.pnl ?? 0n;
160
- return { points: points.map((p) => ({ time: p.time, pnl: p.pnl - base })), warnings: walker.warnings };
190
+ return { points: points.map((p) => ({ ...p, pnl: p.pnl - base })), warnings: walker.warnings };
161
191
  }
162
192
 
163
193
  // --- fetch ---
164
194
 
165
- interface WireFill {
166
- orderbook_id: string;
167
- order_id: string;
168
- initial_size: string;
169
- base_amount: string;
170
- timestamp: number;
171
- price: string;
195
+ type WireFill = WireFillRow;
196
+ type WireSolution = WireSolutionRow;
197
+
198
+ /** The REST reads, with the `ob_*` JSON-RPC twins behind them for nodes that
199
+ * predate the routes. ponytail: drop the fallback once every network serves
200
+ * /clob/solutions and /clob/fills. */
201
+ function reads(rest: PodRestClient, rpcUrl: string, opts: RpcOptions) {
202
+ const missing = (e: unknown) => e instanceof PodHttpError && e.status === 404;
203
+ return {
204
+ async fills(account: Address, q: { fromUs: number; toUs: number; limit: number }): Promise<WireFill[]> {
205
+ try { return await rest.fills(account, q); }
206
+ catch (e) {
207
+ if (!missing(e)) throw e;
208
+ return (await rpc<{ fills: WireFill[] }>(rpcUrl, "ob_getFills", [account, { from_ts: q.fromUs, to_ts: q.toUs, limit: q.limit }], opts)).fills;
209
+ }
210
+ },
211
+ async solutions(q: { orderbook?: MarketId; sinceUs?: number; untilUs?: number; limit?: number }): Promise<WireSolution[]> {
212
+ try { return await rest.solutions(q); }
213
+ catch (e) {
214
+ if (!missing(e)) throw e;
215
+ // Older nodes count rows, not ticks: ask for enough rows to cover every market of one tick.
216
+ return (await rpc<{ solutions: WireSolution[] }>(rpcUrl, "ob_getSolutions", [{ orderbook_id: q.orderbook, since: q.sinceUs, until: q.untilUs, limit: q.orderbook ? q.limit : 200 }], opts)).solutions;
217
+ }
218
+ },
219
+ };
172
220
  }
173
- interface WireSolution {
174
- orderbook_id: string;
175
- timestamp: number;
176
- clearing_price?: string;
177
- mark_price: string;
178
- funding_index?: string;
221
+ type Reads = ReturnType<typeof reads>;
222
+
223
+ /** Run `fn` over `items` with at most `n` in flight, keeping order. */
224
+ async function mapLimit<A, B>(items: A[], n: number, fn: (a: A) => Promise<B>): Promise<B[]> {
225
+ const out = new Array<B>(items.length);
226
+ let next = 0;
227
+ const worker = async () => {
228
+ for (let i = next++; i < items.length; i = next++) out[i] = await fn(items[i] as A);
229
+ };
230
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker));
231
+ return out;
179
232
  }
180
233
 
181
234
  /** Immutable inputs shared by every window size and every refresh: fills as
@@ -186,6 +239,13 @@ export class PnlHistoryCache {
186
239
  readonly ticks = new Map<string, PnlTick>();
187
240
  /** Times at which every market's row was fetched at once. */
188
241
  readonly tickTimes = new Set<number>();
242
+ /** Per market, the latest time known to have no row at or before it. A
243
+ * market has no rows before its first tick, so one empty answer rules out
244
+ * every earlier point without a call. */
245
+ readonly noRowBefore = new Map<MarketId, number>();
246
+ /** Set when a range was cached from a node without the activity feed; the
247
+ * next run that does get flows drops the cache so those ranges are refetched. */
248
+ flowsMissing = false;
189
249
  private events: PnlEvent[] = [];
190
250
  private coverage: { from: number; to: number }[] = [];
191
251
 
@@ -196,10 +256,21 @@ export class PnlHistoryCache {
196
256
  clear(): void {
197
257
  this.ticks.clear();
198
258
  this.tickTimes.clear();
259
+ this.noRowBefore.clear();
260
+ this.flowsMissing = false;
199
261
  this.events = [];
200
262
  this.coverage = [];
201
263
  }
202
264
 
265
+ /** Whether a tick for the market at the time can exist at all. */
266
+ mayHaveTick(market: MarketId, timeUs: number): boolean {
267
+ return timeUs > (this.noRowBefore.get(market) ?? -1);
268
+ }
269
+
270
+ noteEmpty(market: MarketId, timeUs: number): void {
271
+ if (timeUs > (this.noRowBefore.get(market) ?? -1)) this.noRowBefore.set(market, timeUs);
272
+ }
273
+
203
274
  /** Sub-ranges of [from, to) not yet covered. */
204
275
  missing(from: number, to: number): { from: number; to: number }[] {
205
276
  const out: { from: number; to: number }[] = [];
@@ -239,15 +310,29 @@ export class PnlHistoryCache {
239
310
  slice(from: number, to: number): PnlEvent[] {
240
311
  return this.events.filter((e) => e.timeUs >= from && e.timeUs < to);
241
312
  }
313
+
314
+ /** Drop a range so it is fetched again. */
315
+ forget(from: number, to: number): void {
316
+ this.events = this.events.filter((e) => e.timeUs < from || e.timeUs >= to);
317
+ this.coverage = this.coverage.flatMap((c) => {
318
+ if (c.to <= from || c.from >= to) return [c];
319
+ const kept: { from: number; to: number }[] = [];
320
+ if (c.from < from) kept.push({ from: c.from, to: from });
321
+ if (c.to > to) kept.push({ from: to, to: c.to });
322
+ return kept;
323
+ });
324
+ }
242
325
  }
243
326
 
244
327
  export interface PnlHistoryDeps { rest: PodRestClient; rpcUrl: string; fetch?: typeof fetch; cache?: PnlHistoryCache }
245
328
 
246
329
  const MAX_POINTS = 500;
247
- const CHUNK_POINTS = 50;
330
+ // The first chunks are small so the newest points draw after one window's
331
+ // fetch; later ones grow to keep the call count down.
332
+ const CHUNK_SIZES = [10, 20, 40, 50];
248
333
  const IN_FLIGHT = 4;
249
334
  const WIDE_ACCOUNT_MARKETS = 10;
250
- const RPC_BATCH = 200;
335
+ const LOOKUPS_IN_FLIGHT = 24;
251
336
  const FILLS_PAGE = 500;
252
337
 
253
338
  /** Pages newest-first. A full page may cut a batch in half, so its oldest
@@ -258,37 +343,60 @@ class FillsCursor {
258
343
  done = false;
259
344
  readonly warnings: string[] = [];
260
345
  private to: number;
261
- constructor(private readonly rpcUrl: string, private readonly account: Address, private readonly fromUs: number, toUs: number, private readonly opts: RpcOptions) {
346
+ constructor(private readonly reads: Reads, private readonly account: Address, private readonly fromUs: number, toUs: number) {
262
347
  this.to = toUs;
263
348
  }
264
349
 
265
350
  async next(): Promise<WireFill[]> {
266
- const page = await rpc<{ fills: WireFill[] }>(this.rpcUrl, "ob_getFills", [this.account, { from_ts: this.fromUs, to_ts: this.to, limit: FILLS_PAGE }], this.opts);
267
- if (page.fills.length < FILLS_PAGE) {
351
+ const fills = await this.reads.fills(this.account, { fromUs: this.fromUs, toUs: this.to, limit: FILLS_PAGE });
352
+ if (fills.length < FILLS_PAGE) {
268
353
  this.done = true;
269
- return page.fills;
354
+ return fills;
270
355
  }
271
- const oldest = Math.min(...page.fills.map((f) => f.timestamp));
356
+ const oldest = Math.min(...fills.map((f) => f.timestamp));
272
357
  if (oldest + 1 === this.to) {
273
358
  // ponytail: a single batch with more than FILLS_PAGE fills is truncated here.
274
359
  this.warnings.push(`batch ${oldest} has more than ${FILLS_PAGE} fills; only the first page is used`);
275
360
  this.to = oldest;
276
- return page.fills;
361
+ return fills;
277
362
  }
278
363
  this.to = oldest + 1;
279
- return page.fills.filter((f) => f.timestamp !== oldest);
364
+ return fills.filter((f) => f.timestamp !== oldest);
280
365
  }
281
366
  }
282
367
 
283
- async function drainFills(rpcUrl: string, account: Address, fromUs: number, toUs: number, opts: RpcOptions): Promise<{ fills: WireFill[]; warnings: string[] }> {
284
- const cursor = new FillsCursor(rpcUrl, account, fromUs, toUs, opts);
368
+ async function drainFills(r: Reads, account: Address, fromUs: number, toUs: number): Promise<{ fills: WireFill[]; warnings: string[] }> {
369
+ const cursor = new FillsCursor(r, account, fromUs, toUs);
285
370
  const fills: WireFill[] = [];
286
371
  while (!cursor.done) fills.push(...(await cursor.next()));
287
372
  return { fills, warnings: cursor.warnings };
288
373
  }
289
374
 
290
- export async function fetchFills(rpcUrl: string, account: Address, fromUs: number, toUs: number, opts: RpcOptions): Promise<WireFill[]> {
291
- return (await drainFills(rpcUrl, account, fromUs, toUs, opts)).fills;
375
+ export async function fetchFills(rest: PodRestClient, account: Address, fromUs: number, toUs: number, rpcUrl = "", opts: RpcOptions = {}): Promise<WireFill[]> {
376
+ return (await drainFills(reads(rest, rpcUrl, opts), account, fromUs, toUs)).fills;
377
+ }
378
+
379
+ const FLOW_TYPES = ["backstop", "bridge_transfer", "transfer"] as const;
380
+
381
+ /** Money movements in [fromUs, toUs) from the activity feed, or `undefined`
382
+ * when the node does not serve it. */
383
+ async function drainFlows(rest: PodRestClient, account: Address, fromUs: number, toUs: number): Promise<MoneyActivity[] | undefined> {
384
+ const out: MoneyActivity[] = [];
385
+ let cursor: string | undefined;
386
+ try {
387
+ do {
388
+ const page = await rest.activity(account, { types: [...FLOW_TYPES], from: fromUs / 1000, to: toUs / 1000, limit: 200, cursor });
389
+ for (const e of page.activity) if (e.activityType !== "order") out.push(e);
390
+ cursor = page.nextCursor ?? undefined;
391
+ } while (cursor);
392
+ } catch (e) {
393
+ // Only a missing route means the node has no feed. Anything else (a 503,
394
+ // a timeout) must fail the run, or the range would be cached without
395
+ // its flows and every later account value would be wrong and silent.
396
+ if (e instanceof PodHttpError && e.status === 404) return undefined;
397
+ throw e;
398
+ }
399
+ return out;
292
400
  }
293
401
 
294
402
  export async function* streamPnlHistory(
@@ -297,6 +405,7 @@ export async function* streamPnlHistory(
297
405
  q: PnlHistoryQuery,
298
406
  ): AsyncGenerator<PnlHistoryChunk> {
299
407
  const rpcOpts: RpcOptions = { fetch: deps.fetch };
408
+ const io = reads(deps.rest, deps.rpcUrl, rpcOpts);
300
409
 
301
410
  // The snapshot is served from the live engine, fills and ticks from the
302
411
  // indexer. Pair them: the engine's last executed batch is read together with
@@ -315,24 +424,30 @@ export async function* streamPnlHistory(
315
424
  if (before === after || attempt >= 4) return { nowUs: before, snap, balances };
316
425
  }
317
426
  };
318
- const [markets, snapshot] = await Promise.all([deps.rest.markets(), readSnapshot()]);
427
+ const [markets, snapshot, backstop, withdrawals] = await Promise.all([
428
+ deps.rest.markets(),
429
+ readSnapshot(),
430
+ deps.rest.backstopTransfers(account),
431
+ deps.rest.bridgeWithdrawals(account, { since: q.from * 1000 - 1 }),
432
+ ]);
319
433
  const { snap, balances } = snapshot;
320
434
  let nowUs = snapshot.nowUs;
321
435
  const batchUs = Math.max(500, ...markets.map((m) => m.auctionIntervalMs)) * 1000;
322
436
  // Grid points sit on batch ticks so one range call per point serves every market.
323
437
  const fromUs = Math.ceil((q.from * 1000) / batchUs) * batchUs;
438
+ if (nowUs === 0) nowUs = (await deps.rest.status()).solutionNow * 1000;
324
439
 
325
- const indexerCaughtUp = async () => {
326
- if (nowUs === 0) { nowUs = (await deps.rest.status()).solutionNow * 1000; return; }
440
+ // The indexer is normally at the engine's tick already, so the first windows
441
+ // are fetched while this checks; if it was behind, the newest window is
442
+ // fetched again once it has caught up.
443
+ const indexerLagged = (async () => {
444
+ let lagged = false;
327
445
  for (let i = 0; i < 30 && (await deps.rest.status()).solutionNow * 1000 < nowUs; i++) {
446
+ lagged = true;
328
447
  await new Promise((r) => setTimeout(r, 100));
329
448
  }
330
- };
331
- const [backstop, withdrawals] = await Promise.all([
332
- deps.rest.backstopTransfers(account),
333
- deps.rest.bridgeWithdrawals(account, { since: fromUs - 1 }),
334
- indexerCaughtUp(),
335
- ]);
449
+ return lagged;
450
+ })();
336
451
  const toUs = Math.min(q.to === undefined ? nowUs : Math.floor((q.to * 1000) / batchUs) * batchUs, nowUs);
337
452
  if (fromUs >= toUs) throw new Error("pnlHistory: from must be before to");
338
453
 
@@ -359,7 +474,7 @@ export async function* streamPnlHistory(
359
474
  const cache = deps.cache ?? new PnlHistoryCache();
360
475
  const ticks = cache.ticks;
361
476
  const key = (m: MarketId, t: number) => `${m}:${t}`;
362
- const walker = new PnlWalker(legs, (m, t) => ticks.get(key(m, t)), nowUs);
477
+ const walker = new PnlWalker(legs, (m, t) => ticks.get(key(m, t)), nowUs, snap.accountValue);
363
478
  const known = new Set<MarketId>(legs.map((l) => l.market));
364
479
  const use = (id: MarketId) => { if (!known.has(id)) { known.add(id); walker.touch(id, kindOf(id)); } };
365
480
 
@@ -372,17 +487,19 @@ export async function* streamPnlHistory(
372
487
  funding: perp && row.funding_index !== undefined && window > 0 ? dec(row.funding_index) / BigInt(window) : 0n,
373
488
  });
374
489
  };
375
- // Every batch of 200 calls goes out at once; the node runs them concurrently.
376
- const solutions = async (params: object[]): Promise<WireSolution[][]> => {
377
- const batches: object[][] = [];
378
- for (let i = 0; i < params.length; i += RPC_BATCH) batches.push(params.slice(i, i + RPC_BATCH));
379
- const res = await Promise.all(batches.map((b) => rpcBatch<{ solutions: WireSolution[] }>(deps.rpcUrl, b.map((p) => ({ method: "ob_getSolutions", params: [p] })), rpcOpts)));
380
- return res.flat().map((r) => r.solutions);
490
+ type SolutionsQuery = Parameters<Reads["solutions"]>[0];
491
+ // A "newest row at or before t" answer is only right once the indexer has
492
+ // reached t, so no tick lookup goes out before the watermark check resolves.
493
+ // Fills do not wait: the lag retry below fetches the newest window again.
494
+ const solutions = async (queries: SolutionsQuery[]): Promise<WireSolution[][]> => {
495
+ await indexerLagged;
496
+ return mapLimit(queries, LOOKUPS_IN_FLIGHT, (q) => io.solutions(q));
381
497
  };
382
- /** All markets' rows at each tick, one call per tick. */
498
+ /** All markets' rows at each tick, one call per tick; skipped before every
499
+ * known market's first tick. */
383
500
  const fetchAllAt = async (times: number[]) => {
384
- const wanted = times.filter((t) => !cache.tickTimes.has(t));
385
- const rows = await solutions(wanted.map((t) => ({ since: t, until: t + 1, limit: RPC_BATCH })));
501
+ const wanted = times.filter((t) => !cache.tickTimes.has(t) && [...known].some((m) => cache.mayHaveTick(m, t)));
502
+ const rows = await solutions(wanted.map((t) => ({ sinceUs: t, untilUs: t + 1, limit: 1 })));
386
503
  rows.forEach((rs, i) => { const t = wanted[i] ?? 0; cache.tickTimes.add(t); for (const r of rs) store(r, t); });
387
504
  };
388
505
  /** Ticks for every known market at the grid points. One call per tick when
@@ -394,9 +511,21 @@ export async function* streamPnlHistory(
394
511
  };
395
512
  /** Newest row at or before the time, per pair; the fallback and the per-fill path. */
396
513
  const fetchPairs = async (pairs: { market: MarketId; timeUs: number }[]) => {
397
- const wanted = pairs.filter((n, i, all) => !ticks.has(key(n.market, n.timeUs)) && all.findIndex((o) => o.market === n.market && o.timeUs === n.timeUs) === i);
398
- const rows = await solutions(wanted.map((n) => ({ orderbook_id: n.market, until: n.timeUs + 1, limit: 1 })));
399
- rows.forEach((rs, i) => { const r = rs[0]; const n = wanted[i]; if (r && n) store(r, n.timeUs); });
514
+ const seen = new Set<string>();
515
+ const wanted = pairs.filter((n) => {
516
+ const k = key(n.market, n.timeUs);
517
+ if (ticks.has(k) || seen.has(k) || !cache.mayHaveTick(n.market, n.timeUs)) return false;
518
+ seen.add(k);
519
+ return true;
520
+ });
521
+ const rows = await solutions(wanted.map((n) => ({ orderbook: n.market, untilUs: n.timeUs + 1, limit: 1 })));
522
+ rows.forEach((rs, i) => {
523
+ const n = wanted[i];
524
+ if (!n) return;
525
+ const r = rs[0];
526
+ if (r) store(r, n.timeUs);
527
+ else cache.noteEmpty(n.market, n.timeUs);
528
+ });
400
529
  };
401
530
 
402
531
  const fixed: PnlEvent[] = [];
@@ -409,12 +538,38 @@ export async function* streamPnlHistory(
409
538
  for (const w of withdrawals) {
410
539
  if (w.error || w.timeUs < fromUs || w.timeUs > nowUs) continue;
411
540
  const m = spotByToken.get(w.token);
412
- if (!m) continue;
541
+ if (!m) { fixed.push({ timeUs: w.timeUs, deltaSize: -w.amount, price: 0n, kind: "cash" }); continue; }
413
542
  use(m.id);
414
543
  fixed.push({ market: m.id, timeUs: w.timeUs, deltaSize: -w.amount, price: 0n, kind: "withdraw" });
415
544
  }
416
545
  walker.feed(fixed);
417
- const preface = Promise.all([fetchTicks([nowUs]), fetchPairs(fixed.map((e) => ({ market: e.market, timeUs: e.timeUs })))]);
546
+ // Ranges cached from a node without the activity feed hold no flows. If the
547
+ // feed is there now, drop them so this run fetches everything with it.
548
+ if (cache.flowsMissing && (await drainFlows(deps.rest, account, nowUs, nowUs + 1)) !== undefined) {
549
+ cache.flowsMissing = false;
550
+ cache.forget(0, Number.MAX_SAFE_INTEGER);
551
+ }
552
+ // Deposits, transfers and backstop cash sweeps come from the activity feed;
553
+ // withdrawals and sweep legs already arrived above, so those rows are skipped.
554
+ let flowsAvailable = true;
555
+ const toFlowEvents = (rows: MoneyActivity[]): PnlEvent[] => {
556
+ const out: PnlEvent[] = [];
557
+ for (const r of rows) {
558
+ const timeUs = r.timeMs * 1000;
559
+ if (r.activityType === "backstop") {
560
+ if (!r.orderbookId) out.push({ timeUs, deltaSize: -r.cash, price: 0n, kind: "cash" });
561
+ continue;
562
+ }
563
+ if (r.error || (r.activityType === "bridge_transfer" && r.amount < 0n)) continue;
564
+ const m = spotByToken.get(r.token);
565
+ if (!m) { out.push({ timeUs, deltaSize: r.amount, price: 0n, kind: "cash" }); continue; }
566
+ use(m.id);
567
+ out.push({ market: m.id, timeUs, deltaSize: r.amount, price: 0n, kind: r.amount < 0n ? "withdraw" : "deposit" });
568
+ }
569
+ return out;
570
+ };
571
+ const pairsOf = (events: PnlEvent[]) => events.flatMap((e) => (e.market ? [{ market: e.market, timeUs: e.timeUs }] : []));
572
+ const preface = Promise.all([fetchTicks([nowUs]), fetchPairs(pairsOf(fixed))]);
418
573
 
419
574
  const toEvent = (f: WireFill): PnlEvent => {
420
575
  const base = dec(f.base_amount);
@@ -424,19 +579,24 @@ export async function* streamPnlHistory(
424
579
  // an explicit window, so the next windows load while earlier ones are folded.
425
580
  const chunks: number[][] = [];
426
581
  const newestFirstGrid = [...gridUs].reverse();
427
- for (let i = 0; i < newestFirstGrid.length; i += CHUNK_POINTS) chunks.push(newestFirstGrid.slice(i, i + CHUNK_POINTS));
582
+ for (let i = 0, n = 0; i < newestFirstGrid.length; n++) {
583
+ const size = CHUNK_SIZES[Math.min(n, CHUNK_SIZES.length - 1)] ?? 50;
584
+ chunks.push(newestFirstGrid.slice(i, i + size));
585
+ i += size;
586
+ }
428
587
  const prep = async (k: number): Promise<{ events: PnlEvent[]; warnings: string[] }> => {
429
588
  const pts = chunks[k] ?? [];
430
589
  const oldest = pts[pts.length - 1] ?? 0;
431
590
  const bound = k === 0 ? nowUs : (chunks[k - 1]?.at(-1) ?? nowUs);
432
591
  const warnings: string[] = [];
433
592
  await Promise.all(cache.missing(oldest + 1, bound + 1).map(async (r) => {
434
- const page = await drainFills(deps.rpcUrl, account, r.from, r.to, rpcOpts);
435
- cache.add(r.from, r.to, page.fills.map(toEvent));
593
+ const [page, flows] = await Promise.all([drainFills(io, account, r.from, r.to), drainFlows(deps.rest, account, r.from, r.to)]);
594
+ if (flows === undefined) { flowsAvailable = false; cache.flowsMissing = true; }
595
+ cache.add(r.from, r.to, page.fills.map(toEvent).concat(toFlowEvents(flows ?? [])));
436
596
  warnings.push(...page.warnings);
437
597
  }));
438
598
  const events = cache.slice(oldest + 1, bound + 1);
439
- await Promise.all([fetchTicks(pts), fetchPairs(events.map((e) => ({ market: e.market, timeUs: e.timeUs })))]);
599
+ await Promise.all([fetchTicks(pts), fetchPairs(pairsOf(events))]);
440
600
  return { events, warnings };
441
601
  };
442
602
  const queue: Promise<{ events: PnlEvent[]; warnings: string[] }>[] = [];
@@ -447,13 +607,22 @@ export async function* streamPnlHistory(
447
607
  p.catch(() => {});
448
608
  queue.push(p);
449
609
  }
450
- const { events, warnings } = await (queue.shift() as Promise<{ events: PnlEvent[]; warnings: string[] }>);
610
+ let first = await (queue.shift() as Promise<{ events: PnlEvent[]; warnings: string[] }>);
611
+ if (k === 0 && (await indexerLagged)) {
612
+ cache.forget((chunks[0]?.at(-1) ?? 0) + 1, nowUs + 1);
613
+ first = await prep(0);
614
+ }
615
+ const { events, warnings } = first;
451
616
  await preface;
452
- for (const e of events) use(e.market);
617
+ for (const e of events) if (e.market) use(e.market);
453
618
  walker.feed(events);
454
619
  const pts = chunks[k] ?? [];
455
620
  await fetchPairs(pts.flatMap((t) => [...known].map((market) => ({ market, timeUs: t }))));
456
621
  const out = pts.map((t) => walker.point(t)).reverse();
622
+ if (!flowsAvailable) {
623
+ for (const p of out) delete p.accountValue;
624
+ if (k === 0) warnings.push("account value unavailable: the node does not serve the activity feed");
625
+ }
457
626
  yield { points: out, done: k === chunks.length - 1, warnings: [...walker.warnings.splice(0), ...warnings] };
458
627
  }
459
628
  }
@@ -466,5 +635,5 @@ export async function fetchPnlHistory(deps: PnlHistoryDeps, account: Address, q:
466
635
  warnings.push(...c.warnings);
467
636
  }
468
637
  const base = points[0]?.pnl ?? 0n;
469
- return { points: points.map((p) => ({ time: p.time, pnl: p.pnl - base })), warnings };
638
+ return { points: points.map((p) => ({ ...p, pnl: p.pnl - base })), warnings };
470
639
  }