@pod-network/trade-sdk 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/client.d.ts +30 -11
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +37 -17
- package/dist/client.js.map +1 -1
- package/dist/codec/activity.d.ts +11 -0
- package/dist/codec/activity.d.ts.map +1 -0
- package/dist/codec/activity.js +33 -0
- package/dist/codec/activity.js.map +1 -0
- package/dist/codec/decode.d.ts +14 -6
- package/dist/codec/decode.d.ts.map +1 -1
- package/dist/codec/decode.js +58 -24
- package/dist/codec/decode.js.map +1 -1
- package/dist/codec/json.d.ts +3 -0
- package/dist/codec/json.d.ts.map +1 -0
- package/dist/codec/json.js +14 -0
- package/dist/codec/json.js.map +1 -0
- package/dist/codec/orders-v2.d.ts +9 -3
- package/dist/codec/orders-v2.d.ts.map +1 -1
- package/dist/codec/orders-v2.js +12 -9
- package/dist/codec/orders-v2.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/stores/resource.d.ts +48 -2
- package/dist/stores/resource.d.ts.map +1 -1
- package/dist/stores/resource.js +75 -6
- package/dist/stores/resource.js.map +1 -1
- package/dist/sync/activity.d.ts +58 -0
- package/dist/sync/activity.d.ts.map +1 -0
- package/dist/sync/activity.js +259 -0
- package/dist/sync/activity.js.map +1 -0
- package/dist/sync/orders.d.ts +8 -65
- package/dist/sync/orders.d.ts.map +1 -1
- package/dist/sync/orders.js +44 -192
- package/dist/sync/orders.js.map +1 -1
- package/dist/sync/pnl-history.d.ts +33 -14
- package/dist/sync/pnl-history.d.ts.map +1 -1
- package/dist/sync/pnl-history.js +255 -65
- package/dist/sync/pnl-history.js.map +1 -1
- package/dist/sync/sources.d.ts.map +1 -1
- package/dist/sync/sources.js +12 -5
- package/dist/sync/sources.js.map +1 -1
- package/dist/sync/stream.d.ts +101 -0
- package/dist/sync/stream.d.ts.map +1 -0
- package/dist/sync/stream.js +243 -0
- package/dist/sync/stream.js.map +1 -0
- package/dist/transport/jsonrpc.d.ts +0 -5
- package/dist/transport/jsonrpc.d.ts.map +1 -1
- package/dist/transport/jsonrpc.js +0 -20
- package/dist/transport/jsonrpc.js.map +1 -1
- package/dist/transport/rest.d.ts +36 -4
- package/dist/transport/rest.d.ts.map +1 -1
- package/dist/transport/rest.js +43 -10
- package/dist/transport/rest.js.map +1 -1
- package/dist/transport/ws.d.ts +5 -1
- package/dist/transport/ws.d.ts.map +1 -1
- package/dist/transport/ws.js.map +1 -1
- package/dist/types/public.d.ts +62 -23
- package/dist/types/public.d.ts.map +1 -1
- package/dist/types/wire.d.ts +98 -16
- package/dist/types/wire.d.ts.map +1 -1
- package/dist/write/index.d.ts +3 -3
- package/dist/write/index.d.ts.map +1 -1
- package/dist/write/index.js +0 -3
- package/dist/write/index.js.map +1 -1
- package/package.json +1 -1
- package/src/client.ts +49 -19
- package/src/codec/activity.ts +46 -0
- package/src/codec/decode.ts +65 -30
- package/src/codec/json.ts +18 -0
- package/src/codec/orders-v2.ts +24 -11
- package/src/index.ts +4 -2
- package/src/stores/resource.ts +111 -5
- package/src/sync/activity.ts +266 -0
- package/src/sync/orders.ts +46 -189
- package/src/sync/pnl-history.ts +244 -75
- package/src/sync/sources.ts +12 -6
- package/src/sync/stream.ts +270 -0
- package/src/transport/jsonrpc.ts +0 -23
- package/src/transport/rest.ts +62 -14
- package/src/transport/ws.ts +5 -1
- package/src/types/public.ts +62 -26
- package/src/types/wire.ts +105 -18
- package/src/write/index.ts +3 -6
package/src/sync/pnl-history.ts
CHANGED
|
@@ -11,13 +11,20 @@
|
|
|
11
11
|
// Fills page newest-first, so the walk streams: the newest points only need
|
|
12
12
|
// the newest pages, and each chunk is yielded as soon as its fills are in.
|
|
13
13
|
|
|
14
|
-
import type { Address, MarketId } from "../types/public.js";
|
|
14
|
+
import type { Address, MarketId, MoneyActivity } from "../types/public.js";
|
|
15
15
|
import { div, mul } from "../codec/fixed.js";
|
|
16
16
|
import { dec } from "../codec/units.js";
|
|
17
|
-
import {
|
|
18
|
-
import type
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
import type { WireFillRow, WireSolutionRow } from "../types/wire.js";
|
|
18
|
+
import { rpc, type RpcOptions } from "../transport/jsonrpc.js";
|
|
19
|
+
import { PodHttpError, type PodRestClient } from "../transport/rest.js";
|
|
20
|
+
|
|
21
|
+
export interface PnlPoint {
|
|
22
|
+
time: number;
|
|
23
|
+
pnl: bigint;
|
|
24
|
+
/** Absolute account value at the point; absent when the node does not
|
|
25
|
+
* serve the activity feed the deposits and transfers come from. */
|
|
26
|
+
accountValue?: bigint;
|
|
27
|
+
}
|
|
21
28
|
export interface PnlHistory { points: PnlPoint[]; warnings: string[] }
|
|
22
29
|
export interface PnlHistoryQuery {
|
|
23
30
|
/** ms */
|
|
@@ -43,12 +50,17 @@ export interface PnlLeg {
|
|
|
43
50
|
basis?: bigint;
|
|
44
51
|
}
|
|
45
52
|
export interface PnlEvent {
|
|
46
|
-
|
|
53
|
+
/** Absent on a cash movement. */
|
|
54
|
+
market?: MarketId;
|
|
47
55
|
timeUs: number;
|
|
56
|
+
/** Signed size for a market event; the signed native amount for `cash`. */
|
|
48
57
|
deltaSize: bigint;
|
|
49
|
-
/** Fill price; unused for withdrawals. */
|
|
58
|
+
/** Fill price; unused for withdrawals, deposits and cash. */
|
|
50
59
|
price: bigint;
|
|
51
|
-
|
|
60
|
+
/** `withdraw` and `deposit` are spot token movements: a withdrawal leaves at
|
|
61
|
+
* the average cost, a deposit arrives marked at the tick's clearing price.
|
|
62
|
+
* `cash` is native money in or out, which moves account value only. */
|
|
63
|
+
kind: "fill" | "withdraw" | "deposit" | "sweep" | "cash";
|
|
52
64
|
}
|
|
53
65
|
export interface PnlTick { mark: bigint; funding: bigint }
|
|
54
66
|
export interface PnlFoldInput {
|
|
@@ -60,12 +72,15 @@ export interface PnlFoldInput {
|
|
|
60
72
|
gridUs: number[];
|
|
61
73
|
/** Newest tick at or before `timeUs`. */
|
|
62
74
|
tickAt: (market: MarketId, timeUs: number) => PnlTick | undefined;
|
|
75
|
+
/** Account value at the reference tick; points carry `accountValue` when given. */
|
|
76
|
+
accountValueRef?: bigint;
|
|
63
77
|
}
|
|
64
78
|
|
|
65
79
|
const RESIDUAL_TOLERANCE = 10n ** 12n;
|
|
66
|
-
// Within a tick the engine runs sweeps, then matching, then
|
|
67
|
-
// walking backwards undoes them in the opposite
|
|
68
|
-
|
|
80
|
+
// Within a tick the engine runs sweeps, then deposits, then matching, then
|
|
81
|
+
// transfers and withdrawals; walking backwards undoes them in the opposite
|
|
82
|
+
// order. Cash never touches a size, so its place only has to be consistent.
|
|
83
|
+
const RANK = { withdraw: 0, cash: 1, fill: 2, deposit: 3, sweep: 4 } as const;
|
|
69
84
|
const newestFirst = (a: PnlEvent, b: PnlEvent) => b.timeUs - a.timeUs || RANK[a.kind] - RANK[b.kind];
|
|
70
85
|
|
|
71
86
|
interface LegState { kind: "perp" | "spot"; size: bigint; basis: bigint; basisKnown: boolean }
|
|
@@ -77,12 +92,14 @@ export class PnlWalker {
|
|
|
77
92
|
private readonly state = new Map<MarketId, LegState>();
|
|
78
93
|
private pending: PnlEvent[] = [];
|
|
79
94
|
private acc = 0n;
|
|
95
|
+
private flows = 0n;
|
|
80
96
|
private refValue?: bigint;
|
|
81
97
|
|
|
82
98
|
constructor(
|
|
83
99
|
legs: PnlLeg[],
|
|
84
100
|
private readonly tickAt: (market: MarketId, timeUs: number) => PnlTick | undefined,
|
|
85
101
|
private readonly refTimeUs: number,
|
|
102
|
+
private readonly accountValueRef?: bigint,
|
|
86
103
|
) {
|
|
87
104
|
for (const l of legs) this.state.set(l.market, { kind: l.kind, size: l.size, basis: l.basis ?? 0n, basisKnown: l.basis !== undefined });
|
|
88
105
|
}
|
|
@@ -100,9 +117,15 @@ export class PnlWalker {
|
|
|
100
117
|
this.refValue ??= this.value(this.refTimeUs);
|
|
101
118
|
for (let e = this.pending[0]; e && e.timeUs > t; e = this.pending[0]) {
|
|
102
119
|
this.pending.shift();
|
|
103
|
-
|
|
120
|
+
const u = this.unapply(e);
|
|
121
|
+
this.acc += u.pnl;
|
|
122
|
+
this.flows += u.flow;
|
|
104
123
|
}
|
|
105
|
-
|
|
124
|
+
const pnl = this.value(t) - this.refValue - this.acc;
|
|
125
|
+
const point: PnlPoint = { time: t / 1000, pnl };
|
|
126
|
+
// Account value is PnL plus every movement that is not PnL.
|
|
127
|
+
if (this.accountValueRef !== undefined) point.accountValue = this.accountValueRef + pnl - this.flows;
|
|
128
|
+
return point;
|
|
106
129
|
}
|
|
107
130
|
|
|
108
131
|
private value(t: number): bigint {
|
|
@@ -116,13 +139,16 @@ export class PnlWalker {
|
|
|
116
139
|
return sum;
|
|
117
140
|
}
|
|
118
141
|
|
|
119
|
-
|
|
142
|
+
/** The PnL adjustment and the non-PnL money value carried by the event. */
|
|
143
|
+
private unapply(e: PnlEvent): { pnl: bigint; flow: bigint } {
|
|
144
|
+
if (e.kind === "cash" || e.market === undefined) return { pnl: 0n, flow: e.deltaSize };
|
|
120
145
|
const st = this.state.get(e.market);
|
|
121
146
|
if (!st) throw new Error(`pnlHistory: event on unknown market ${e.market}`);
|
|
122
147
|
const tick = this.tickAt(e.market, e.timeUs);
|
|
123
148
|
const F = tick?.funding ?? 0n;
|
|
124
149
|
const sizeBefore = st.size - e.deltaSize;
|
|
125
150
|
let price = e.price;
|
|
151
|
+
let flow = 0n;
|
|
126
152
|
|
|
127
153
|
if (st.kind === "spot") {
|
|
128
154
|
const average = st.size !== 0n && st.basisKnown ? div(st.basis, st.size) : undefined;
|
|
@@ -132,8 +158,12 @@ export class PnlWalker {
|
|
|
132
158
|
price = tick?.mark ?? 0n;
|
|
133
159
|
} else price = average;
|
|
134
160
|
}
|
|
161
|
+
if (e.kind === "deposit") price = tick?.mark ?? 0n;
|
|
162
|
+
// Tokens leave at their cost and arrive at the clearing price; either
|
|
163
|
+
// way the value that moved is what the event is priced at.
|
|
164
|
+
if (e.kind === "withdraw" || e.kind === "deposit") flow = mul(price, e.deltaSize);
|
|
135
165
|
if (sizeBefore === 0n) { st.basis = 0n; st.basisKnown = true; }
|
|
136
|
-
else if (e.deltaSize > 0n) st.basis -= mul(
|
|
166
|
+
else if (e.deltaSize > 0n) st.basis -= mul(price, e.deltaSize);
|
|
137
167
|
else if (average !== undefined) st.basis = mul(average, sizeBefore);
|
|
138
168
|
else st.basisKnown = false;
|
|
139
169
|
} else {
|
|
@@ -148,34 +178,57 @@ export class PnlWalker {
|
|
|
148
178
|
}
|
|
149
179
|
|
|
150
180
|
st.size = sizeBefore;
|
|
151
|
-
return mul(F - price, e.deltaSize);
|
|
181
|
+
return { pnl: mul(F - price, e.deltaSize), flow };
|
|
152
182
|
}
|
|
153
183
|
}
|
|
154
184
|
|
|
155
185
|
export function foldPnlHistory(input: PnlFoldInput): PnlHistory {
|
|
156
|
-
const walker = new PnlWalker(input.legs, input.tickAt, input.refTimeUs);
|
|
186
|
+
const walker = new PnlWalker(input.legs, input.tickAt, input.refTimeUs, input.accountValueRef);
|
|
157
187
|
walker.feed(input.events);
|
|
158
188
|
const points = [...input.gridUs].reverse().map((t) => walker.point(t)).reverse();
|
|
159
189
|
const base = points[0]?.pnl ?? 0n;
|
|
160
|
-
return { points: points.map((p) => ({
|
|
190
|
+
return { points: points.map((p) => ({ ...p, pnl: p.pnl - base })), warnings: walker.warnings };
|
|
161
191
|
}
|
|
162
192
|
|
|
163
193
|
// --- fetch ---
|
|
164
194
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
195
|
+
type WireFill = WireFillRow;
|
|
196
|
+
type WireSolution = WireSolutionRow;
|
|
197
|
+
|
|
198
|
+
/** The REST reads, with the `ob_*` JSON-RPC twins behind them for nodes that
|
|
199
|
+
* predate the routes. ponytail: drop the fallback once every network serves
|
|
200
|
+
* /clob/solutions and /clob/fills. */
|
|
201
|
+
function reads(rest: PodRestClient, rpcUrl: string, opts: RpcOptions) {
|
|
202
|
+
const missing = (e: unknown) => e instanceof PodHttpError && e.status === 404;
|
|
203
|
+
return {
|
|
204
|
+
async fills(account: Address, q: { fromUs: number; toUs: number; limit: number }): Promise<WireFill[]> {
|
|
205
|
+
try { return await rest.fills(account, q); }
|
|
206
|
+
catch (e) {
|
|
207
|
+
if (!missing(e)) throw e;
|
|
208
|
+
return (await rpc<{ fills: WireFill[] }>(rpcUrl, "ob_getFills", [account, { from_ts: q.fromUs, to_ts: q.toUs, limit: q.limit }], opts)).fills;
|
|
209
|
+
}
|
|
210
|
+
},
|
|
211
|
+
async solutions(q: { orderbook?: MarketId; sinceUs?: number; untilUs?: number; limit?: number }): Promise<WireSolution[]> {
|
|
212
|
+
try { return await rest.solutions(q); }
|
|
213
|
+
catch (e) {
|
|
214
|
+
if (!missing(e)) throw e;
|
|
215
|
+
// Older nodes count rows, not ticks: ask for enough rows to cover every market of one tick.
|
|
216
|
+
return (await rpc<{ solutions: WireSolution[] }>(rpcUrl, "ob_getSolutions", [{ orderbook_id: q.orderbook, since: q.sinceUs, until: q.untilUs, limit: q.orderbook ? q.limit : 200 }], opts)).solutions;
|
|
217
|
+
}
|
|
218
|
+
},
|
|
219
|
+
};
|
|
172
220
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
221
|
+
type Reads = ReturnType<typeof reads>;
|
|
222
|
+
|
|
223
|
+
/** Run `fn` over `items` with at most `n` in flight, keeping order. */
|
|
224
|
+
async function mapLimit<A, B>(items: A[], n: number, fn: (a: A) => Promise<B>): Promise<B[]> {
|
|
225
|
+
const out = new Array<B>(items.length);
|
|
226
|
+
let next = 0;
|
|
227
|
+
const worker = async () => {
|
|
228
|
+
for (let i = next++; i < items.length; i = next++) out[i] = await fn(items[i] as A);
|
|
229
|
+
};
|
|
230
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker));
|
|
231
|
+
return out;
|
|
179
232
|
}
|
|
180
233
|
|
|
181
234
|
/** Immutable inputs shared by every window size and every refresh: fills as
|
|
@@ -186,6 +239,13 @@ export class PnlHistoryCache {
|
|
|
186
239
|
readonly ticks = new Map<string, PnlTick>();
|
|
187
240
|
/** Times at which every market's row was fetched at once. */
|
|
188
241
|
readonly tickTimes = new Set<number>();
|
|
242
|
+
/** Per market, the latest time known to have no row at or before it. A
|
|
243
|
+
* market has no rows before its first tick, so one empty answer rules out
|
|
244
|
+
* every earlier point without a call. */
|
|
245
|
+
readonly noRowBefore = new Map<MarketId, number>();
|
|
246
|
+
/** Set when a range was cached from a node without the activity feed; the
|
|
247
|
+
* next run that does get flows drops the cache so those ranges are refetched. */
|
|
248
|
+
flowsMissing = false;
|
|
189
249
|
private events: PnlEvent[] = [];
|
|
190
250
|
private coverage: { from: number; to: number }[] = [];
|
|
191
251
|
|
|
@@ -196,10 +256,21 @@ export class PnlHistoryCache {
|
|
|
196
256
|
clear(): void {
|
|
197
257
|
this.ticks.clear();
|
|
198
258
|
this.tickTimes.clear();
|
|
259
|
+
this.noRowBefore.clear();
|
|
260
|
+
this.flowsMissing = false;
|
|
199
261
|
this.events = [];
|
|
200
262
|
this.coverage = [];
|
|
201
263
|
}
|
|
202
264
|
|
|
265
|
+
/** Whether a tick for the market at the time can exist at all. */
|
|
266
|
+
mayHaveTick(market: MarketId, timeUs: number): boolean {
|
|
267
|
+
return timeUs > (this.noRowBefore.get(market) ?? -1);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
noteEmpty(market: MarketId, timeUs: number): void {
|
|
271
|
+
if (timeUs > (this.noRowBefore.get(market) ?? -1)) this.noRowBefore.set(market, timeUs);
|
|
272
|
+
}
|
|
273
|
+
|
|
203
274
|
/** Sub-ranges of [from, to) not yet covered. */
|
|
204
275
|
missing(from: number, to: number): { from: number; to: number }[] {
|
|
205
276
|
const out: { from: number; to: number }[] = [];
|
|
@@ -239,15 +310,29 @@ export class PnlHistoryCache {
|
|
|
239
310
|
slice(from: number, to: number): PnlEvent[] {
|
|
240
311
|
return this.events.filter((e) => e.timeUs >= from && e.timeUs < to);
|
|
241
312
|
}
|
|
313
|
+
|
|
314
|
+
/** Drop a range so it is fetched again. */
|
|
315
|
+
forget(from: number, to: number): void {
|
|
316
|
+
this.events = this.events.filter((e) => e.timeUs < from || e.timeUs >= to);
|
|
317
|
+
this.coverage = this.coverage.flatMap((c) => {
|
|
318
|
+
if (c.to <= from || c.from >= to) return [c];
|
|
319
|
+
const kept: { from: number; to: number }[] = [];
|
|
320
|
+
if (c.from < from) kept.push({ from: c.from, to: from });
|
|
321
|
+
if (c.to > to) kept.push({ from: to, to: c.to });
|
|
322
|
+
return kept;
|
|
323
|
+
});
|
|
324
|
+
}
|
|
242
325
|
}
|
|
243
326
|
|
|
244
327
|
export interface PnlHistoryDeps { rest: PodRestClient; rpcUrl: string; fetch?: typeof fetch; cache?: PnlHistoryCache }
|
|
245
328
|
|
|
246
329
|
const MAX_POINTS = 500;
|
|
247
|
-
|
|
330
|
+
// The first chunks are small so the newest points draw after one window's
|
|
331
|
+
// fetch; later ones grow to keep the call count down.
|
|
332
|
+
const CHUNK_SIZES = [10, 20, 40, 50];
|
|
248
333
|
const IN_FLIGHT = 4;
|
|
249
334
|
const WIDE_ACCOUNT_MARKETS = 10;
|
|
250
|
-
const
|
|
335
|
+
const LOOKUPS_IN_FLIGHT = 24;
|
|
251
336
|
const FILLS_PAGE = 500;
|
|
252
337
|
|
|
253
338
|
/** Pages newest-first. A full page may cut a batch in half, so its oldest
|
|
@@ -258,37 +343,60 @@ class FillsCursor {
|
|
|
258
343
|
done = false;
|
|
259
344
|
readonly warnings: string[] = [];
|
|
260
345
|
private to: number;
|
|
261
|
-
constructor(private readonly
|
|
346
|
+
constructor(private readonly reads: Reads, private readonly account: Address, private readonly fromUs: number, toUs: number) {
|
|
262
347
|
this.to = toUs;
|
|
263
348
|
}
|
|
264
349
|
|
|
265
350
|
async next(): Promise<WireFill[]> {
|
|
266
|
-
const
|
|
267
|
-
if (
|
|
351
|
+
const fills = await this.reads.fills(this.account, { fromUs: this.fromUs, toUs: this.to, limit: FILLS_PAGE });
|
|
352
|
+
if (fills.length < FILLS_PAGE) {
|
|
268
353
|
this.done = true;
|
|
269
|
-
return
|
|
354
|
+
return fills;
|
|
270
355
|
}
|
|
271
|
-
const oldest = Math.min(...
|
|
356
|
+
const oldest = Math.min(...fills.map((f) => f.timestamp));
|
|
272
357
|
if (oldest + 1 === this.to) {
|
|
273
358
|
// ponytail: a single batch with more than FILLS_PAGE fills is truncated here.
|
|
274
359
|
this.warnings.push(`batch ${oldest} has more than ${FILLS_PAGE} fills; only the first page is used`);
|
|
275
360
|
this.to = oldest;
|
|
276
|
-
return
|
|
361
|
+
return fills;
|
|
277
362
|
}
|
|
278
363
|
this.to = oldest + 1;
|
|
279
|
-
return
|
|
364
|
+
return fills.filter((f) => f.timestamp !== oldest);
|
|
280
365
|
}
|
|
281
366
|
}
|
|
282
367
|
|
|
283
|
-
async function drainFills(
|
|
284
|
-
const cursor = new FillsCursor(
|
|
368
|
+
async function drainFills(r: Reads, account: Address, fromUs: number, toUs: number): Promise<{ fills: WireFill[]; warnings: string[] }> {
|
|
369
|
+
const cursor = new FillsCursor(r, account, fromUs, toUs);
|
|
285
370
|
const fills: WireFill[] = [];
|
|
286
371
|
while (!cursor.done) fills.push(...(await cursor.next()));
|
|
287
372
|
return { fills, warnings: cursor.warnings };
|
|
288
373
|
}
|
|
289
374
|
|
|
290
|
-
export async function fetchFills(
|
|
291
|
-
return (await drainFills(rpcUrl, account, fromUs, toUs
|
|
375
|
+
export async function fetchFills(rest: PodRestClient, account: Address, fromUs: number, toUs: number, rpcUrl = "", opts: RpcOptions = {}): Promise<WireFill[]> {
|
|
376
|
+
return (await drainFills(reads(rest, rpcUrl, opts), account, fromUs, toUs)).fills;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
const FLOW_TYPES = ["backstop", "bridge_transfer", "transfer"] as const;
|
|
380
|
+
|
|
381
|
+
/** Money movements in [fromUs, toUs) from the activity feed, or `undefined`
|
|
382
|
+
* when the node does not serve it. */
|
|
383
|
+
async function drainFlows(rest: PodRestClient, account: Address, fromUs: number, toUs: number): Promise<MoneyActivity[] | undefined> {
|
|
384
|
+
const out: MoneyActivity[] = [];
|
|
385
|
+
let cursor: string | undefined;
|
|
386
|
+
try {
|
|
387
|
+
do {
|
|
388
|
+
const page = await rest.activity(account, { types: [...FLOW_TYPES], from: fromUs / 1000, to: toUs / 1000, limit: 200, cursor });
|
|
389
|
+
for (const e of page.activity) if (e.activityType !== "order") out.push(e);
|
|
390
|
+
cursor = page.nextCursor ?? undefined;
|
|
391
|
+
} while (cursor);
|
|
392
|
+
} catch (e) {
|
|
393
|
+
// Only a missing route means the node has no feed. Anything else (a 503,
|
|
394
|
+
// a timeout) must fail the run, or the range would be cached without
|
|
395
|
+
// its flows and every later account value would be wrong and silent.
|
|
396
|
+
if (e instanceof PodHttpError && e.status === 404) return undefined;
|
|
397
|
+
throw e;
|
|
398
|
+
}
|
|
399
|
+
return out;
|
|
292
400
|
}
|
|
293
401
|
|
|
294
402
|
export async function* streamPnlHistory(
|
|
@@ -297,6 +405,7 @@ export async function* streamPnlHistory(
|
|
|
297
405
|
q: PnlHistoryQuery,
|
|
298
406
|
): AsyncGenerator<PnlHistoryChunk> {
|
|
299
407
|
const rpcOpts: RpcOptions = { fetch: deps.fetch };
|
|
408
|
+
const io = reads(deps.rest, deps.rpcUrl, rpcOpts);
|
|
300
409
|
|
|
301
410
|
// The snapshot is served from the live engine, fills and ticks from the
|
|
302
411
|
// indexer. Pair them: the engine's last executed batch is read together with
|
|
@@ -315,24 +424,30 @@ export async function* streamPnlHistory(
|
|
|
315
424
|
if (before === after || attempt >= 4) return { nowUs: before, snap, balances };
|
|
316
425
|
}
|
|
317
426
|
};
|
|
318
|
-
const [markets, snapshot] = await Promise.all([
|
|
427
|
+
const [markets, snapshot, backstop, withdrawals] = await Promise.all([
|
|
428
|
+
deps.rest.markets(),
|
|
429
|
+
readSnapshot(),
|
|
430
|
+
deps.rest.backstopTransfers(account),
|
|
431
|
+
deps.rest.bridgeWithdrawals(account, { since: q.from * 1000 - 1 }),
|
|
432
|
+
]);
|
|
319
433
|
const { snap, balances } = snapshot;
|
|
320
434
|
let nowUs = snapshot.nowUs;
|
|
321
435
|
const batchUs = Math.max(500, ...markets.map((m) => m.auctionIntervalMs)) * 1000;
|
|
322
436
|
// Grid points sit on batch ticks so one range call per point serves every market.
|
|
323
437
|
const fromUs = Math.ceil((q.from * 1000) / batchUs) * batchUs;
|
|
438
|
+
if (nowUs === 0) nowUs = (await deps.rest.status()).solutionNow * 1000;
|
|
324
439
|
|
|
325
|
-
|
|
326
|
-
|
|
440
|
+
// The indexer is normally at the engine's tick already, so the first windows
|
|
441
|
+
// are fetched while this checks; if it was behind, the newest window is
|
|
442
|
+
// fetched again once it has caught up.
|
|
443
|
+
const indexerLagged = (async () => {
|
|
444
|
+
let lagged = false;
|
|
327
445
|
for (let i = 0; i < 30 && (await deps.rest.status()).solutionNow * 1000 < nowUs; i++) {
|
|
446
|
+
lagged = true;
|
|
328
447
|
await new Promise((r) => setTimeout(r, 100));
|
|
329
448
|
}
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
deps.rest.backstopTransfers(account),
|
|
333
|
-
deps.rest.bridgeWithdrawals(account, { since: fromUs - 1 }),
|
|
334
|
-
indexerCaughtUp(),
|
|
335
|
-
]);
|
|
449
|
+
return lagged;
|
|
450
|
+
})();
|
|
336
451
|
const toUs = Math.min(q.to === undefined ? nowUs : Math.floor((q.to * 1000) / batchUs) * batchUs, nowUs);
|
|
337
452
|
if (fromUs >= toUs) throw new Error("pnlHistory: from must be before to");
|
|
338
453
|
|
|
@@ -359,7 +474,7 @@ export async function* streamPnlHistory(
|
|
|
359
474
|
const cache = deps.cache ?? new PnlHistoryCache();
|
|
360
475
|
const ticks = cache.ticks;
|
|
361
476
|
const key = (m: MarketId, t: number) => `${m}:${t}`;
|
|
362
|
-
const walker = new PnlWalker(legs, (m, t) => ticks.get(key(m, t)), nowUs);
|
|
477
|
+
const walker = new PnlWalker(legs, (m, t) => ticks.get(key(m, t)), nowUs, snap.accountValue);
|
|
363
478
|
const known = new Set<MarketId>(legs.map((l) => l.market));
|
|
364
479
|
const use = (id: MarketId) => { if (!known.has(id)) { known.add(id); walker.touch(id, kindOf(id)); } };
|
|
365
480
|
|
|
@@ -372,17 +487,19 @@ export async function* streamPnlHistory(
|
|
|
372
487
|
funding: perp && row.funding_index !== undefined && window > 0 ? dec(row.funding_index) / BigInt(window) : 0n,
|
|
373
488
|
});
|
|
374
489
|
};
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
490
|
+
type SolutionsQuery = Parameters<Reads["solutions"]>[0];
|
|
491
|
+
// A "newest row at or before t" answer is only right once the indexer has
|
|
492
|
+
// reached t, so no tick lookup goes out before the watermark check resolves.
|
|
493
|
+
// Fills do not wait: the lag retry below fetches the newest window again.
|
|
494
|
+
const solutions = async (queries: SolutionsQuery[]): Promise<WireSolution[][]> => {
|
|
495
|
+
await indexerLagged;
|
|
496
|
+
return mapLimit(queries, LOOKUPS_IN_FLIGHT, (q) => io.solutions(q));
|
|
381
497
|
};
|
|
382
|
-
/** All markets' rows at each tick, one call per tick
|
|
498
|
+
/** All markets' rows at each tick, one call per tick; skipped before every
|
|
499
|
+
* known market's first tick. */
|
|
383
500
|
const fetchAllAt = async (times: number[]) => {
|
|
384
|
-
const wanted = times.filter((t) => !cache.tickTimes.has(t));
|
|
385
|
-
const rows = await solutions(wanted.map((t) => ({
|
|
501
|
+
const wanted = times.filter((t) => !cache.tickTimes.has(t) && [...known].some((m) => cache.mayHaveTick(m, t)));
|
|
502
|
+
const rows = await solutions(wanted.map((t) => ({ sinceUs: t, untilUs: t + 1, limit: 1 })));
|
|
386
503
|
rows.forEach((rs, i) => { const t = wanted[i] ?? 0; cache.tickTimes.add(t); for (const r of rs) store(r, t); });
|
|
387
504
|
};
|
|
388
505
|
/** Ticks for every known market at the grid points. One call per tick when
|
|
@@ -394,9 +511,21 @@ export async function* streamPnlHistory(
|
|
|
394
511
|
};
|
|
395
512
|
/** Newest row at or before the time, per pair; the fallback and the per-fill path. */
|
|
396
513
|
const fetchPairs = async (pairs: { market: MarketId; timeUs: number }[]) => {
|
|
397
|
-
const
|
|
398
|
-
const
|
|
399
|
-
|
|
514
|
+
const seen = new Set<string>();
|
|
515
|
+
const wanted = pairs.filter((n) => {
|
|
516
|
+
const k = key(n.market, n.timeUs);
|
|
517
|
+
if (ticks.has(k) || seen.has(k) || !cache.mayHaveTick(n.market, n.timeUs)) return false;
|
|
518
|
+
seen.add(k);
|
|
519
|
+
return true;
|
|
520
|
+
});
|
|
521
|
+
const rows = await solutions(wanted.map((n) => ({ orderbook: n.market, untilUs: n.timeUs + 1, limit: 1 })));
|
|
522
|
+
rows.forEach((rs, i) => {
|
|
523
|
+
const n = wanted[i];
|
|
524
|
+
if (!n) return;
|
|
525
|
+
const r = rs[0];
|
|
526
|
+
if (r) store(r, n.timeUs);
|
|
527
|
+
else cache.noteEmpty(n.market, n.timeUs);
|
|
528
|
+
});
|
|
400
529
|
};
|
|
401
530
|
|
|
402
531
|
const fixed: PnlEvent[] = [];
|
|
@@ -409,12 +538,38 @@ export async function* streamPnlHistory(
|
|
|
409
538
|
for (const w of withdrawals) {
|
|
410
539
|
if (w.error || w.timeUs < fromUs || w.timeUs > nowUs) continue;
|
|
411
540
|
const m = spotByToken.get(w.token);
|
|
412
|
-
if (!m) continue;
|
|
541
|
+
if (!m) { fixed.push({ timeUs: w.timeUs, deltaSize: -w.amount, price: 0n, kind: "cash" }); continue; }
|
|
413
542
|
use(m.id);
|
|
414
543
|
fixed.push({ market: m.id, timeUs: w.timeUs, deltaSize: -w.amount, price: 0n, kind: "withdraw" });
|
|
415
544
|
}
|
|
416
545
|
walker.feed(fixed);
|
|
417
|
-
|
|
546
|
+
// Ranges cached from a node without the activity feed hold no flows. If the
|
|
547
|
+
// feed is there now, drop them so this run fetches everything with it.
|
|
548
|
+
if (cache.flowsMissing && (await drainFlows(deps.rest, account, nowUs, nowUs + 1)) !== undefined) {
|
|
549
|
+
cache.flowsMissing = false;
|
|
550
|
+
cache.forget(0, Number.MAX_SAFE_INTEGER);
|
|
551
|
+
}
|
|
552
|
+
// Deposits, transfers and backstop cash sweeps come from the activity feed;
|
|
553
|
+
// withdrawals and sweep legs already arrived above, so those rows are skipped.
|
|
554
|
+
let flowsAvailable = true;
|
|
555
|
+
const toFlowEvents = (rows: MoneyActivity[]): PnlEvent[] => {
|
|
556
|
+
const out: PnlEvent[] = [];
|
|
557
|
+
for (const r of rows) {
|
|
558
|
+
const timeUs = r.timeMs * 1000;
|
|
559
|
+
if (r.activityType === "backstop") {
|
|
560
|
+
if (!r.orderbookId) out.push({ timeUs, deltaSize: -r.cash, price: 0n, kind: "cash" });
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
if (r.error || (r.activityType === "bridge_transfer" && r.amount < 0n)) continue;
|
|
564
|
+
const m = spotByToken.get(r.token);
|
|
565
|
+
if (!m) { out.push({ timeUs, deltaSize: r.amount, price: 0n, kind: "cash" }); continue; }
|
|
566
|
+
use(m.id);
|
|
567
|
+
out.push({ market: m.id, timeUs, deltaSize: r.amount, price: 0n, kind: r.amount < 0n ? "withdraw" : "deposit" });
|
|
568
|
+
}
|
|
569
|
+
return out;
|
|
570
|
+
};
|
|
571
|
+
const pairsOf = (events: PnlEvent[]) => events.flatMap((e) => (e.market ? [{ market: e.market, timeUs: e.timeUs }] : []));
|
|
572
|
+
const preface = Promise.all([fetchTicks([nowUs]), fetchPairs(pairsOf(fixed))]);
|
|
418
573
|
|
|
419
574
|
const toEvent = (f: WireFill): PnlEvent => {
|
|
420
575
|
const base = dec(f.base_amount);
|
|
@@ -424,19 +579,24 @@ export async function* streamPnlHistory(
|
|
|
424
579
|
// an explicit window, so the next windows load while earlier ones are folded.
|
|
425
580
|
const chunks: number[][] = [];
|
|
426
581
|
const newestFirstGrid = [...gridUs].reverse();
|
|
427
|
-
for (let i = 0; i < newestFirstGrid.length;
|
|
582
|
+
for (let i = 0, n = 0; i < newestFirstGrid.length; n++) {
|
|
583
|
+
const size = CHUNK_SIZES[Math.min(n, CHUNK_SIZES.length - 1)] ?? 50;
|
|
584
|
+
chunks.push(newestFirstGrid.slice(i, i + size));
|
|
585
|
+
i += size;
|
|
586
|
+
}
|
|
428
587
|
const prep = async (k: number): Promise<{ events: PnlEvent[]; warnings: string[] }> => {
|
|
429
588
|
const pts = chunks[k] ?? [];
|
|
430
589
|
const oldest = pts[pts.length - 1] ?? 0;
|
|
431
590
|
const bound = k === 0 ? nowUs : (chunks[k - 1]?.at(-1) ?? nowUs);
|
|
432
591
|
const warnings: string[] = [];
|
|
433
592
|
await Promise.all(cache.missing(oldest + 1, bound + 1).map(async (r) => {
|
|
434
|
-
const page = await drainFills(deps.
|
|
435
|
-
|
|
593
|
+
const [page, flows] = await Promise.all([drainFills(io, account, r.from, r.to), drainFlows(deps.rest, account, r.from, r.to)]);
|
|
594
|
+
if (flows === undefined) { flowsAvailable = false; cache.flowsMissing = true; }
|
|
595
|
+
cache.add(r.from, r.to, page.fills.map(toEvent).concat(toFlowEvents(flows ?? [])));
|
|
436
596
|
warnings.push(...page.warnings);
|
|
437
597
|
}));
|
|
438
598
|
const events = cache.slice(oldest + 1, bound + 1);
|
|
439
|
-
await Promise.all([fetchTicks(pts), fetchPairs(events
|
|
599
|
+
await Promise.all([fetchTicks(pts), fetchPairs(pairsOf(events))]);
|
|
440
600
|
return { events, warnings };
|
|
441
601
|
};
|
|
442
602
|
const queue: Promise<{ events: PnlEvent[]; warnings: string[] }>[] = [];
|
|
@@ -447,13 +607,22 @@ export async function* streamPnlHistory(
|
|
|
447
607
|
p.catch(() => {});
|
|
448
608
|
queue.push(p);
|
|
449
609
|
}
|
|
450
|
-
|
|
610
|
+
let first = await (queue.shift() as Promise<{ events: PnlEvent[]; warnings: string[] }>);
|
|
611
|
+
if (k === 0 && (await indexerLagged)) {
|
|
612
|
+
cache.forget((chunks[0]?.at(-1) ?? 0) + 1, nowUs + 1);
|
|
613
|
+
first = await prep(0);
|
|
614
|
+
}
|
|
615
|
+
const { events, warnings } = first;
|
|
451
616
|
await preface;
|
|
452
|
-
for (const e of events) use(e.market);
|
|
617
|
+
for (const e of events) if (e.market) use(e.market);
|
|
453
618
|
walker.feed(events);
|
|
454
619
|
const pts = chunks[k] ?? [];
|
|
455
620
|
await fetchPairs(pts.flatMap((t) => [...known].map((market) => ({ market, timeUs: t }))));
|
|
456
621
|
const out = pts.map((t) => walker.point(t)).reverse();
|
|
622
|
+
if (!flowsAvailable) {
|
|
623
|
+
for (const p of out) delete p.accountValue;
|
|
624
|
+
if (k === 0) warnings.push("account value unavailable: the node does not serve the activity feed");
|
|
625
|
+
}
|
|
457
626
|
yield { points: out, done: k === chunks.length - 1, warnings: [...walker.warnings.splice(0), ...warnings] };
|
|
458
627
|
}
|
|
459
628
|
}
|
|
@@ -466,5 +635,5 @@ export async function fetchPnlHistory(deps: PnlHistoryDeps, account: Address, q:
|
|
|
466
635
|
warnings.push(...c.warnings);
|
|
467
636
|
}
|
|
468
637
|
const base = points[0]?.pnl ?? 0n;
|
|
469
|
-
return { points: points.map((p) => ({
|
|
638
|
+
return { points: points.map((p) => ({ ...p, pnl: p.pnl - base })), warnings };
|
|
470
639
|
}
|
package/src/sync/sources.ts
CHANGED
|
@@ -22,6 +22,7 @@ import type {
|
|
|
22
22
|
import {
|
|
23
23
|
decodeMarketDynamics, decodeOrderbook, decodePositions, decodeTrigger,
|
|
24
24
|
} from "../codec/decode.js";
|
|
25
|
+
import { parseBig, stringifyBig } from "../codec/json.js";
|
|
25
26
|
import type { PodRestClient } from "../transport/rest.js";
|
|
26
27
|
import type { PodWsClient } from "../transport/ws.js";
|
|
27
28
|
import type { ResourceSource } from "../stores/resource.js";
|
|
@@ -51,15 +52,15 @@ export interface MarketsCache {
|
|
|
51
52
|
|
|
52
53
|
// Static-list serialization for MarketsCache. Version is embedded in the
|
|
53
54
|
// payload: a mismatch after an SDK format change just falls back to a cold
|
|
54
|
-
// start.
|
|
55
|
+
// start.
|
|
55
56
|
// 2: `status`/`minNotional` became required and the fees stopped decoding to 0.
|
|
56
|
-
|
|
57
|
+
// 3: bigints are tagged objects, not "<digits>n" strings (see codec/json).
|
|
58
|
+
const MARKETS_CACHE_VERSION = 3;
|
|
57
59
|
const serializeMarkets = (markets: Market[]): string =>
|
|
58
|
-
|
|
60
|
+
stringifyBig({ v: MARKETS_CACHE_VERSION, markets });
|
|
59
61
|
const parseCachedMarkets = (raw: string | null): Market[] | undefined => {
|
|
60
62
|
if (!raw) return undefined;
|
|
61
|
-
const data =
|
|
62
|
-
typeof v === "string" && /^-?\d+n$/.test(v) ? BigInt(v.slice(0, -1)) : v) as { v: number; markets: Market[] };
|
|
63
|
+
const data = parseBig<{ v: number; markets: Market[] }>(raw);
|
|
63
64
|
return data?.v === MARKETS_CACHE_VERSION ? data.markets : undefined;
|
|
64
65
|
};
|
|
65
66
|
|
|
@@ -109,7 +110,11 @@ export function marketsSource(
|
|
|
109
110
|
// Stable display order = the static /clob/markets order, so the list never
|
|
110
111
|
// reshuffles as dynamics arrive (markets not yet in the static list sort last).
|
|
111
112
|
const orderIndex = new Map<string, number>();
|
|
112
|
-
|
|
113
|
+
// Until REST's static list lands, anything published is built on the cached
|
|
114
|
+
// list (possibly delisted markets, partial prices) and must stay provisional,
|
|
115
|
+
// whichever path — cache, WS dynamics or stats — happens to publish it.
|
|
116
|
+
let staticLoaded = false;
|
|
117
|
+
const publish = () => (staticLoaded ? h.set : h.seed)(
|
|
113
118
|
[...byId.values()]
|
|
114
119
|
// pod_markets dynamics can arrive (and create an entry) before the static
|
|
115
120
|
// /clob/markets seed — those lack base/quote/name. Don't expose a market
|
|
@@ -147,6 +152,7 @@ export function marketsSource(
|
|
|
147
152
|
orderIndex.set(m.id, i);
|
|
148
153
|
byId.set(m.id, { ...byId.get(m.id), ...m });
|
|
149
154
|
});
|
|
155
|
+
staticLoaded = true;
|
|
150
156
|
publish();
|
|
151
157
|
try { marketsCache?.set(serializeMarkets(markets)); } catch { /* storage denied/full */ }
|
|
152
158
|
}).catch((e) => { if (!byId.size) h.fail(e); });
|