@tradejs/cli 2.0.15 → 2.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2982 -994
- package/dist/lib/aiPocketSearch.js +493 -24
- package/dist/lib/aiTrainResearch.js +19 -2
- package/dist/lib/hyperliquidWhaleBackfill.js +37 -1
- package/dist/lib/marketContextPrepare.js +112 -9
- package/dist/lib/researchAgent.js +3 -30
- package/dist/lib/runtimeDebugEvidence.js +76 -1
- package/dist/lib/runtimeEvidenceArtifacts.js +255 -0
- package/dist/lib/runtimeEvidenceSync.js +308 -0
- package/dist/lib/runtimeRedis.js +25 -0
- package/dist/lib/runtimeScorecard.js +442 -0
- package/dist/lib/runtimeTradeSync.js +9 -0
- package/dist/scripts/agentRun.js +16 -31
- package/dist/scripts/aiPocketSearch.js +717 -29
- package/dist/scripts/aiTrain.js +32 -2
- package/dist/scripts/backtest.js +136 -11
- package/dist/scripts/hyperliquidWhaleBackfill.js +27 -1
- package/dist/scripts/hyperliquidWhalesUpdate.js +1 -0
- package/dist/scripts/replay.js +122 -12
- package/dist/scripts/replayRunner.js +122 -12
- package/dist/scripts/replayRuntimeEvidence.js +68 -161
- package/dist/scripts/runtimeEvidence.js +316 -161
- package/dist/scripts/runtimeEvidenceSync.js +335 -0
- package/dist/scripts/runtimeParity.js +110 -9
- package/dist/scripts/runtimeScorecard.js +731 -0
- package/dist/scripts/signals.js +263 -145
- package/dist/scripts/signalsDaemon.js +263 -145
- package/dist/scripts/signalsSummary.js +9 -0
- package/dist/workers/testerWorker.js +2 -0
- package/package.json +11 -9
|
@@ -0,0 +1,731 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/scripts/runtimeScorecard.ts
|
|
31
|
+
var runtimeScorecard_exports = {};
|
|
32
|
+
__export(runtimeScorecard_exports, {
|
|
33
|
+
main: () => main,
|
|
34
|
+
runtimeScorecard: () => runtimeScorecard
|
|
35
|
+
});
|
|
36
|
+
module.exports = __toCommonJS(runtimeScorecard_exports);
|
|
37
|
+
var import_promises3 = __toESM(require("fs/promises"));
|
|
38
|
+
var import_node_path3 = __toESM(require("path"));
|
|
39
|
+
var import_args = __toESM(require("args"));
|
|
40
|
+
var import_chalk = __toESM(require("chalk"));
|
|
41
|
+
|
|
42
|
+
// src/lib/runtimeEvidenceArtifacts.ts
|
|
43
|
+
var import_node_crypto = require("crypto");
|
|
44
|
+
var import_promises = __toESM(require("fs/promises"));
|
|
45
|
+
var import_node_path = __toESM(require("path"));
|
|
46
|
+
var RUNTIME_EVIDENCE_MANIFEST_FILE = "manifest.json";
|
|
47
|
+
var RUNTIME_EVIDENCE_PAYLOAD_FILE = "runtime-evidence.json";
|
|
48
|
+
var RUNTIME_EVIDENCE_COMPLETE_FILE = ".complete";
|
|
49
|
+
var asRecord = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
50
|
+
var sha256 = (value) => (0, import_node_crypto.createHash)("sha256").update(value).digest("hex");
|
|
51
|
+
var assertFiniteTimestamp = (value, label) => {
|
|
52
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
53
|
+
throw new Error(`Invalid runtime evidence ${label}`);
|
|
54
|
+
}
|
|
55
|
+
return value;
|
|
56
|
+
};
|
|
57
|
+
var parseManifest = (value) => {
|
|
58
|
+
const manifest = asRecord(value);
|
|
59
|
+
const window = asRecord(manifest?.window);
|
|
60
|
+
const payload = asRecord(manifest?.payload);
|
|
61
|
+
const counts = asRecord(manifest?.counts);
|
|
62
|
+
if (manifest?.schemaVersion !== 1 || manifest.reportType !== "runtime-evidence" || typeof manifest.artifactId !== "string" || typeof manifest.deploymentId !== "string" || typeof manifest.userName !== "string" || typeof manifest.createdAt !== "number" || payload?.file !== RUNTIME_EVIDENCE_PAYLOAD_FILE || typeof payload.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(payload.sha256) || typeof payload.bytes !== "number" || !Array.isArray(manifest.lineageKeys)) {
|
|
63
|
+
throw new Error("Invalid runtime evidence manifest");
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
schemaVersion: 1,
|
|
67
|
+
artifactId: manifest.artifactId,
|
|
68
|
+
reportType: "runtime-evidence",
|
|
69
|
+
deploymentId: manifest.deploymentId,
|
|
70
|
+
userName: manifest.userName,
|
|
71
|
+
createdAt: manifest.createdAt,
|
|
72
|
+
window: {
|
|
73
|
+
startTime: assertFiniteTimestamp(window?.startTime, "window.startTime"),
|
|
74
|
+
endTime: assertFiniteTimestamp(window?.endTime, "window.endTime")
|
|
75
|
+
},
|
|
76
|
+
payload: {
|
|
77
|
+
file: RUNTIME_EVIDENCE_PAYLOAD_FILE,
|
|
78
|
+
sha256: payload.sha256,
|
|
79
|
+
bytes: payload.bytes
|
|
80
|
+
},
|
|
81
|
+
counts: Object.fromEntries(
|
|
82
|
+
Object.entries(counts ?? {}).filter(
|
|
83
|
+
(entry) => typeof entry[1] === "number" && Number.isFinite(entry[1])
|
|
84
|
+
)
|
|
85
|
+
),
|
|
86
|
+
lineageKeys: manifest.lineageKeys.filter(
|
|
87
|
+
(value2) => typeof value2 === "string" && value2.length > 0
|
|
88
|
+
)
|
|
89
|
+
};
|
|
90
|
+
};
|
|
91
|
+
var verifyRuntimeEvidenceBundle = async (bundleDir) => {
|
|
92
|
+
const manifestPath = import_node_path.default.join(bundleDir, RUNTIME_EVIDENCE_MANIFEST_FILE);
|
|
93
|
+
const payloadPath = import_node_path.default.join(bundleDir, RUNTIME_EVIDENCE_PAYLOAD_FILE);
|
|
94
|
+
const completePath = import_node_path.default.join(bundleDir, RUNTIME_EVIDENCE_COMPLETE_FILE);
|
|
95
|
+
const [manifestText, payload, completeText] = await Promise.all([
|
|
96
|
+
import_promises.default.readFile(manifestPath, "utf8"),
|
|
97
|
+
import_promises.default.readFile(payloadPath),
|
|
98
|
+
import_promises.default.readFile(completePath, "utf8")
|
|
99
|
+
]);
|
|
100
|
+
const manifest = parseManifest(JSON.parse(manifestText));
|
|
101
|
+
if (completeText.trim() !== manifest.artifactId) {
|
|
102
|
+
throw new Error(`Invalid completion marker in ${bundleDir}`);
|
|
103
|
+
}
|
|
104
|
+
if (payload.byteLength !== manifest.payload.bytes) {
|
|
105
|
+
throw new Error(`Runtime evidence size mismatch in ${bundleDir}`);
|
|
106
|
+
}
|
|
107
|
+
if (sha256(payload) !== manifest.payload.sha256) {
|
|
108
|
+
throw new Error(`Runtime evidence checksum mismatch in ${bundleDir}`);
|
|
109
|
+
}
|
|
110
|
+
const artifact = asRecord(JSON.parse(payload.toString("utf8")));
|
|
111
|
+
const artifactWindow = asRecord(artifact?.window);
|
|
112
|
+
if (artifact?.reportType !== "runtime-evidence" || artifactWindow?.startTime !== manifest.window.startTime || artifactWindow.endTime !== manifest.window.endTime) {
|
|
113
|
+
throw new Error(`Runtime evidence payload mismatch in ${bundleDir}`);
|
|
114
|
+
}
|
|
115
|
+
return {
|
|
116
|
+
bundleDir,
|
|
117
|
+
manifestPath,
|
|
118
|
+
payloadPath,
|
|
119
|
+
manifest,
|
|
120
|
+
artifact
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
var discoverRuntimeEvidenceBundles = async (rootDir) => {
|
|
124
|
+
const bundles = [];
|
|
125
|
+
const visit = async (directory, depth) => {
|
|
126
|
+
if (depth > 8) return;
|
|
127
|
+
let entries;
|
|
128
|
+
try {
|
|
129
|
+
entries = await import_promises.default.readdir(directory, { withFileTypes: true });
|
|
130
|
+
} catch (error) {
|
|
131
|
+
if (error.code === "ENOENT") return;
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
const names = new Set(entries.map((entry) => entry.name));
|
|
135
|
+
if (names.has(RUNTIME_EVIDENCE_MANIFEST_FILE) && names.has(RUNTIME_EVIDENCE_PAYLOAD_FILE) && names.has(RUNTIME_EVIDENCE_COMPLETE_FILE)) {
|
|
136
|
+
bundles.push(directory);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
await Promise.all(
|
|
140
|
+
entries.filter(
|
|
141
|
+
(entry) => entry.isDirectory() && !entry.name.startsWith(".") && !entry.name.includes(".tmp-")
|
|
142
|
+
).map((entry) => visit(import_node_path.default.join(directory, entry.name), depth + 1))
|
|
143
|
+
);
|
|
144
|
+
};
|
|
145
|
+
await visit(rootDir, 0);
|
|
146
|
+
return bundles.sort();
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
// src/lib/runtimeEvidenceSync.ts
|
|
150
|
+
var import_node_child_process = require("child_process");
|
|
151
|
+
var import_promises2 = __toESM(require("fs/promises"));
|
|
152
|
+
var import_node_path2 = __toESM(require("path"));
|
|
153
|
+
var import_node_util = require("util");
|
|
154
|
+
var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
|
|
155
|
+
var safeSegment = (value, fallback) => {
|
|
156
|
+
const normalized = value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-");
|
|
157
|
+
return normalized && normalized !== "." && normalized !== ".." ? normalized : fallback;
|
|
158
|
+
};
|
|
159
|
+
var receiptPath = (receiptsRoot, artifactId) => import_node_path2.default.join(receiptsRoot, `${safeSegment(artifactId, "artifact")}.json`);
|
|
160
|
+
var markRuntimeEvidenceBundleProcessed = async ({
|
|
161
|
+
evidenceRoot,
|
|
162
|
+
deploymentId,
|
|
163
|
+
bundleDir,
|
|
164
|
+
scorecardPath
|
|
165
|
+
}) => {
|
|
166
|
+
const safeDeploymentId = safeSegment(deploymentId, "production");
|
|
167
|
+
const verified = await verifyRuntimeEvidenceBundle(bundleDir);
|
|
168
|
+
if (verified.manifest.deploymentId !== safeDeploymentId) {
|
|
169
|
+
throw new Error(
|
|
170
|
+
`Artifact deployment mismatch: expected ${safeDeploymentId}, received ${verified.manifest.deploymentId}`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
const receiptsRoot = import_node_path2.default.join(evidenceRoot, "receipts", safeDeploymentId);
|
|
174
|
+
const finalPath = receiptPath(receiptsRoot, verified.manifest.artifactId);
|
|
175
|
+
const temporaryPath = `${finalPath}.tmp-${process.pid}`;
|
|
176
|
+
const receipt = {
|
|
177
|
+
schemaVersion: 1,
|
|
178
|
+
artifactId: verified.manifest.artifactId,
|
|
179
|
+
payloadSha256: verified.manifest.payload.sha256,
|
|
180
|
+
processedAt: Date.now(),
|
|
181
|
+
scorecardPath: scorecardPath ?? null
|
|
182
|
+
};
|
|
183
|
+
await import_promises2.default.mkdir(receiptsRoot, { recursive: true });
|
|
184
|
+
await import_promises2.default.writeFile(
|
|
185
|
+
temporaryPath,
|
|
186
|
+
`${JSON.stringify(receipt, null, 2)}
|
|
187
|
+
`,
|
|
188
|
+
"utf8"
|
|
189
|
+
);
|
|
190
|
+
await import_promises2.default.rename(temporaryPath, finalPath);
|
|
191
|
+
return finalPath;
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
// src/lib/runtimeScorecard.ts
|
|
195
|
+
var asRecord2 = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
196
|
+
var asArray = (value) => Array.isArray(value) ? value : [];
|
|
197
|
+
var finiteNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
198
|
+
var finiteString = (value) => typeof value === "string" && value.trim() ? value.trim() : null;
|
|
199
|
+
var round = (value, decimals = 6) => Number(value.toFixed(decimals));
|
|
200
|
+
var unwrapRows = (value, key) => asArray(value).map((item) => asRecord2(asRecord2(item)?.[key]) ?? asRecord2(item)).filter((item) => item != null);
|
|
201
|
+
var runtimePayload = (artifact) => {
|
|
202
|
+
const root = asRecord2(artifact) ?? {};
|
|
203
|
+
return asRecord2(root.runtime) ?? root;
|
|
204
|
+
};
|
|
205
|
+
var runtimeWindow = (artifact) => {
|
|
206
|
+
const root = asRecord2(artifact) ?? {};
|
|
207
|
+
return asRecord2(root.window) ?? asRecord2(runtimePayload(artifact).window) ?? {};
|
|
208
|
+
};
|
|
209
|
+
var extractRuntimeRows = (artifact) => {
|
|
210
|
+
const runtime = runtimePayload(artifact);
|
|
211
|
+
return {
|
|
212
|
+
evaluations: unwrapRows(runtime.evaluations, "evaluation"),
|
|
213
|
+
signals: unwrapRows(runtime.signals, "signal"),
|
|
214
|
+
trades: unwrapRows(runtime.trades, "trade"),
|
|
215
|
+
statsBuckets: asArray(runtime.evaluationStatsBuckets).map(asRecord2).filter((item) => item != null)
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
var countBy = (items, getter) => {
|
|
219
|
+
const counts = {};
|
|
220
|
+
for (const item of items) {
|
|
221
|
+
const key = getter(item);
|
|
222
|
+
if (!key) continue;
|
|
223
|
+
counts[key] = (counts[key] ?? 0) + 1;
|
|
224
|
+
}
|
|
225
|
+
return Object.fromEntries(
|
|
226
|
+
Object.entries(counts).sort(([left], [right]) => left.localeCompare(right))
|
|
227
|
+
);
|
|
228
|
+
};
|
|
229
|
+
var buildDistributions = ({
|
|
230
|
+
evaluations,
|
|
231
|
+
signals
|
|
232
|
+
}) => {
|
|
233
|
+
const candidates = evaluations.filter(
|
|
234
|
+
(evaluation) => evaluation.status === "signal"
|
|
235
|
+
);
|
|
236
|
+
return {
|
|
237
|
+
candidatesByStrategy: countBy(
|
|
238
|
+
candidates,
|
|
239
|
+
(evaluation) => finiteString(evaluation.strategy)
|
|
240
|
+
),
|
|
241
|
+
orderStatus: countBy(signals, (signal) => finiteString(signal.orderStatus)),
|
|
242
|
+
skipReasons: countBy(
|
|
243
|
+
signals,
|
|
244
|
+
(signal) => finiteString(signal.orderSkipReason)
|
|
245
|
+
),
|
|
246
|
+
aiQuality: countBy(candidates, (evaluation) => {
|
|
247
|
+
const quality = finiteNumber(asRecord2(evaluation.aiAnalysis)?.quality);
|
|
248
|
+
return quality == null ? null : String(quality);
|
|
249
|
+
})
|
|
250
|
+
};
|
|
251
|
+
};
|
|
252
|
+
var buildDistributionDelta = (current, previous) => {
|
|
253
|
+
const currentTotal = Object.values(current).reduce(
|
|
254
|
+
(total, value) => total + value,
|
|
255
|
+
0
|
|
256
|
+
);
|
|
257
|
+
const previousTotal = Object.values(previous).reduce(
|
|
258
|
+
(total, value) => total + value,
|
|
259
|
+
0
|
|
260
|
+
);
|
|
261
|
+
const keys = [
|
|
262
|
+
.../* @__PURE__ */ new Set([...Object.keys(current), ...Object.keys(previous)])
|
|
263
|
+
].sort((left, right) => left.localeCompare(right));
|
|
264
|
+
return Object.fromEntries(
|
|
265
|
+
keys.map((key) => {
|
|
266
|
+
const currentCount = current[key] ?? 0;
|
|
267
|
+
const previousCount = previous[key] ?? 0;
|
|
268
|
+
const currentShare = currentTotal > 0 ? currentCount / currentTotal : 0;
|
|
269
|
+
const previousShare = previousTotal > 0 ? previousCount / previousTotal : 0;
|
|
270
|
+
return [
|
|
271
|
+
key,
|
|
272
|
+
{
|
|
273
|
+
currentCount,
|
|
274
|
+
previousCount,
|
|
275
|
+
currentShare: round(currentShare),
|
|
276
|
+
previousShare: round(previousShare),
|
|
277
|
+
shareDelta: round(currentShare - previousShare)
|
|
278
|
+
}
|
|
279
|
+
];
|
|
280
|
+
})
|
|
281
|
+
);
|
|
282
|
+
};
|
|
283
|
+
var sum = (values) => round(values.reduce((total, value) => total + (value ?? 0), 0));
|
|
284
|
+
var statsCount = (buckets, field) => sum(buckets.map((bucket) => finiteNumber(asRecord2(bucket.stats)?.[field])));
|
|
285
|
+
var tradeTimestamp = (trade) => finiteNumber(trade.exitTimestamp) ?? finiteNumber(trade.lastSyncedAt) ?? finiteNumber(trade.entryTimestamp) ?? 0;
|
|
286
|
+
var dedupeTrades = (artifacts) => {
|
|
287
|
+
const tradesByOrderId = /* @__PURE__ */ new Map();
|
|
288
|
+
for (const artifact of artifacts) {
|
|
289
|
+
for (const trade of extractRuntimeRows(artifact).trades) {
|
|
290
|
+
const orderId = finiteString(trade.orderId);
|
|
291
|
+
if (!orderId) continue;
|
|
292
|
+
const existing = tradesByOrderId.get(orderId);
|
|
293
|
+
if (!existing || tradeTimestamp(trade) >= tradeTimestamp(existing)) {
|
|
294
|
+
tradesByOrderId.set(orderId, trade);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return [...tradesByOrderId.values()];
|
|
299
|
+
};
|
|
300
|
+
var maxDrawdown = (trades) => {
|
|
301
|
+
let cumulative = 0;
|
|
302
|
+
let peak = 0;
|
|
303
|
+
let drawdown = 0;
|
|
304
|
+
for (const trade of trades.sort(
|
|
305
|
+
(left, right) => tradeTimestamp(left) - tradeTimestamp(right)
|
|
306
|
+
)) {
|
|
307
|
+
cumulative += finiteNumber(trade.closedPnl) ?? 0;
|
|
308
|
+
peak = Math.max(peak, cumulative);
|
|
309
|
+
drawdown = Math.max(drawdown, peak - cumulative);
|
|
310
|
+
}
|
|
311
|
+
return round(drawdown);
|
|
312
|
+
};
|
|
313
|
+
var buildRollingPerformance = ({
|
|
314
|
+
trades,
|
|
315
|
+
anchorTime,
|
|
316
|
+
days
|
|
317
|
+
}) => {
|
|
318
|
+
const startTime = anchorTime - days * 24 * 60 * 60 * 1e3;
|
|
319
|
+
const closed = trades.filter((trade) => {
|
|
320
|
+
const exitTimestamp = finiteNumber(trade.exitTimestamp);
|
|
321
|
+
return trade.status === "closed" && exitTimestamp != null && exitTimestamp >= startTime && exitTimestamp < anchorTime;
|
|
322
|
+
});
|
|
323
|
+
const realizedPnl = sum(closed.map((trade) => finiteNumber(trade.closedPnl)));
|
|
324
|
+
return {
|
|
325
|
+
days,
|
|
326
|
+
closedTrades: closed.length,
|
|
327
|
+
realizedPnl,
|
|
328
|
+
expectancy: closed.length > 0 ? round(realizedPnl / closed.length) : null,
|
|
329
|
+
maxDrawdown: maxDrawdown(closed)
|
|
330
|
+
};
|
|
331
|
+
};
|
|
332
|
+
var getReplayComparison = (artifact) => {
|
|
333
|
+
const root = asRecord2(artifact) ?? {};
|
|
334
|
+
return asRecord2(asRecord2(root.replay)?.runtimeComparison) ?? null;
|
|
335
|
+
};
|
|
336
|
+
var getCalibrationSummary = (artifact) => {
|
|
337
|
+
const root = asRecord2(artifact) ?? {};
|
|
338
|
+
return asRecord2(asRecord2(root.summary)?.all) ?? null;
|
|
339
|
+
};
|
|
340
|
+
var buildRuntimeScorecard = ({
|
|
341
|
+
runtimeArtifact,
|
|
342
|
+
replayEvidenceArtifact,
|
|
343
|
+
calibrationArtifact,
|
|
344
|
+
historyRuntimeArtifacts = [],
|
|
345
|
+
thresholds = {
|
|
346
|
+
minimumParityRatio: 0.95,
|
|
347
|
+
maximumSlippageResidualBps: 3,
|
|
348
|
+
minimumClosedTrades: 20,
|
|
349
|
+
minimumExpectancy: 0
|
|
350
|
+
},
|
|
351
|
+
generatedAt = Date.now()
|
|
352
|
+
}) => {
|
|
353
|
+
const rows = extractRuntimeRows(runtimeArtifact);
|
|
354
|
+
const window = runtimeWindow(runtimeArtifact);
|
|
355
|
+
const startTime = finiteNumber(window.startTime) ?? generatedAt - 864e5;
|
|
356
|
+
const endTime = finiteNumber(window.endTime) ?? generatedAt;
|
|
357
|
+
const evaluationsFromStats = statsCount(rows.statsBuckets, "evaluated");
|
|
358
|
+
const candidatesFromStats = statsCount(rows.statsBuckets, "signals");
|
|
359
|
+
const signalEvaluations = rows.evaluations.filter(
|
|
360
|
+
(evaluation) => evaluation.status === "signal"
|
|
361
|
+
);
|
|
362
|
+
const gateDecisions = signalEvaluations.map((evaluation) => ({
|
|
363
|
+
evaluation,
|
|
364
|
+
decision: finiteString(asRecord2(evaluation.aiAnalysis)?.gateDecision)
|
|
365
|
+
})).filter(
|
|
366
|
+
({ decision }) => decision === "approved" || decision === "rejected"
|
|
367
|
+
);
|
|
368
|
+
const allocatorDecisions = signalEvaluations.map((evaluation) => asRecord2(evaluation.allocatorDecision)).filter((item) => item != null);
|
|
369
|
+
const riskDecisions = signalEvaluations.map((evaluation) => asRecord2(evaluation.riskDecision)).filter((item) => item != null);
|
|
370
|
+
const orderAttempts = rows.signals.filter(
|
|
371
|
+
(signal) => signal.orderStatus === "completed" || signal.orderStatus === "failed"
|
|
372
|
+
);
|
|
373
|
+
const currentClosedTrades = rows.trades.filter((trade) => {
|
|
374
|
+
const exitTimestamp = finiteNumber(trade.exitTimestamp);
|
|
375
|
+
return trade.status === "closed" && exitTimestamp != null && exitTimestamp >= startTime && exitTimestamp < endTime;
|
|
376
|
+
});
|
|
377
|
+
const replayComparison = getReplayComparison(replayEvidenceArtifact);
|
|
378
|
+
const replayCounts = asRecord2(replayComparison?.counts);
|
|
379
|
+
const matched = finiteNumber(replayCounts?.matched) ?? 0;
|
|
380
|
+
const backtestOnly = finiteNumber(replayCounts?.backtestOnly) ?? 0;
|
|
381
|
+
const runtimeOnly = finiteNumber(replayCounts?.runtimeOnly) ?? 0;
|
|
382
|
+
const comparisonTotal = matched + backtestOnly + runtimeOnly;
|
|
383
|
+
const parityRatio = comparisonTotal > 0 ? matched / comparisonTotal : null;
|
|
384
|
+
const lineage = asRecord2(replayComparison?.lineage);
|
|
385
|
+
const lineageReason = finiteString(lineage?.reason);
|
|
386
|
+
const calibration = getCalibrationSummary(calibrationArtifact);
|
|
387
|
+
const actualSlippageBps = finiteNumber(
|
|
388
|
+
asRecord2(calibration?.signalToFillAdverseBps)?.avg
|
|
389
|
+
);
|
|
390
|
+
const residualVsModelBps = finiteNumber(
|
|
391
|
+
asRecord2(calibration?.residualVsCurrentModelBps)?.avg
|
|
392
|
+
);
|
|
393
|
+
const historyTrades = dedupeTrades([
|
|
394
|
+
...historyRuntimeArtifacts,
|
|
395
|
+
runtimeArtifact
|
|
396
|
+
]);
|
|
397
|
+
const currentDistributions = buildDistributions(rows);
|
|
398
|
+
const previousArtifact = historyRuntimeArtifacts.map((artifact) => ({ artifact, window: runtimeWindow(artifact) })).filter(({ window: candidateWindow }) => {
|
|
399
|
+
const candidateEndTime = finiteNumber(candidateWindow.endTime);
|
|
400
|
+
return candidateEndTime != null && candidateEndTime <= startTime;
|
|
401
|
+
}).sort(
|
|
402
|
+
(left, right) => (finiteNumber(right.window.endTime) ?? 0) - (finiteNumber(left.window.endTime) ?? 0)
|
|
403
|
+
)[0];
|
|
404
|
+
const previousDistributions = previousArtifact ? buildDistributions(extractRuntimeRows(previousArtifact.artifact)) : null;
|
|
405
|
+
const rolling = [7, 30, 90].map(
|
|
406
|
+
(days) => buildRollingPerformance({
|
|
407
|
+
trades: [...historyTrades],
|
|
408
|
+
anchorTime: endTime,
|
|
409
|
+
days
|
|
410
|
+
})
|
|
411
|
+
);
|
|
412
|
+
const sevenDay = rolling[0];
|
|
413
|
+
const reactions = [];
|
|
414
|
+
if (lineageReason) {
|
|
415
|
+
reactions.push({
|
|
416
|
+
code: "LINEAGE_NOT_COMPARABLE",
|
|
417
|
+
severity: "blocking",
|
|
418
|
+
message: lineageReason
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
if (parityRatio != null && parityRatio < thresholds.minimumParityRatio) {
|
|
422
|
+
reactions.push({
|
|
423
|
+
code: "PARITY_REGRESSION",
|
|
424
|
+
severity: "blocking",
|
|
425
|
+
message: `Parity ${round(parityRatio * 100, 2)}% is below ${round(thresholds.minimumParityRatio * 100, 2)}%.`
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
if (residualVsModelBps != null && residualVsModelBps > thresholds.maximumSlippageResidualBps) {
|
|
429
|
+
reactions.push({
|
|
430
|
+
code: "SLIPPAGE_DRIFT",
|
|
431
|
+
severity: "warning",
|
|
432
|
+
message: `Execution residual ${round(residualVsModelBps, 2)} bps exceeds ${thresholds.maximumSlippageResidualBps} bps.`
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
if (sevenDay.closedTrades >= thresholds.minimumClosedTrades && sevenDay.expectancy != null && sevenDay.expectancy < thresholds.minimumExpectancy) {
|
|
436
|
+
reactions.push({
|
|
437
|
+
code: "EXPECTANCY_DEGRADATION",
|
|
438
|
+
severity: "blocking",
|
|
439
|
+
message: `7d expectancy ${sevenDay.expectancy} is below ${thresholds.minimumExpectancy}.`
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
if (sevenDay.closedTrades < thresholds.minimumClosedTrades) {
|
|
443
|
+
reactions.push({
|
|
444
|
+
code: "INSUFFICIENT_RECENT_SAMPLE",
|
|
445
|
+
severity: "info",
|
|
446
|
+
message: `Only ${sevenDay.closedTrades} closed trades in 7d; ${thresholds.minimumClosedTrades} required for expectancy reaction.`
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
const promotionStatus = reactions.some(
|
|
450
|
+
(reaction) => reaction.severity === "blocking"
|
|
451
|
+
) ? "PROMOTION_BLOCKED" : reactions.some((reaction) => reaction.severity === "warning") ? "REVIEW_REQUIRED" : "OBSERVE";
|
|
452
|
+
return {
|
|
453
|
+
reportType: "runtime-scorecard",
|
|
454
|
+
schemaVersion: 1,
|
|
455
|
+
generatedAt,
|
|
456
|
+
window: { startTime, endTime },
|
|
457
|
+
promotionStatus,
|
|
458
|
+
thresholds,
|
|
459
|
+
funnel: {
|
|
460
|
+
evaluations: evaluationsFromStats > 0 ? evaluationsFromStats : rows.evaluations.length,
|
|
461
|
+
coreCandidates: candidatesFromStats > 0 ? candidatesFromStats : signalEvaluations.length,
|
|
462
|
+
gate: {
|
|
463
|
+
available: gateDecisions.length > 0,
|
|
464
|
+
approved: gateDecisions.filter(
|
|
465
|
+
({ decision }) => decision === "approved"
|
|
466
|
+
).length,
|
|
467
|
+
rejected: gateDecisions.filter(
|
|
468
|
+
({ decision }) => decision === "rejected"
|
|
469
|
+
).length,
|
|
470
|
+
coverage: signalEvaluations.length ? round(gateDecisions.length / signalEvaluations.length) : null
|
|
471
|
+
},
|
|
472
|
+
allocator: {
|
|
473
|
+
available: allocatorDecisions.length > 0,
|
|
474
|
+
approved: allocatorDecisions.filter(
|
|
475
|
+
(decision) => decision.decision === "approved"
|
|
476
|
+
).length,
|
|
477
|
+
rejected: allocatorDecisions.filter(
|
|
478
|
+
(decision) => decision.decision === "rejected"
|
|
479
|
+
).length
|
|
480
|
+
},
|
|
481
|
+
risk: {
|
|
482
|
+
available: riskDecisions.length > 0,
|
|
483
|
+
approved: riskDecisions.filter(
|
|
484
|
+
(decision) => decision.decision === "approved"
|
|
485
|
+
).length,
|
|
486
|
+
rejected: riskDecisions.filter(
|
|
487
|
+
(decision) => decision.decision === "rejected"
|
|
488
|
+
).length
|
|
489
|
+
},
|
|
490
|
+
orderAttempts: orderAttempts.length,
|
|
491
|
+
orderFailures: orderAttempts.filter(
|
|
492
|
+
(signal) => signal.orderStatus === "failed"
|
|
493
|
+
).length,
|
|
494
|
+
balanceRejects: rows.signals.filter(
|
|
495
|
+
(signal) => /BALANCE|MARGIN/i.test(
|
|
496
|
+
`${finiteString(signal.orderSkipReason) ?? ""} ${finiteString(signal.orderFailureReason) ?? ""}`
|
|
497
|
+
)
|
|
498
|
+
).length,
|
|
499
|
+
fills: rows.trades.length,
|
|
500
|
+
closedTrades: currentClosedTrades.length
|
|
501
|
+
},
|
|
502
|
+
realized: {
|
|
503
|
+
pnl: sum(
|
|
504
|
+
currentClosedTrades.map((trade) => finiteNumber(trade.closedPnl))
|
|
505
|
+
),
|
|
506
|
+
fees: sum(
|
|
507
|
+
currentClosedTrades.map((trade) => finiteNumber(trade.totalFee))
|
|
508
|
+
),
|
|
509
|
+
funding: sum(
|
|
510
|
+
currentClosedTrades.map((trade) => finiteNumber(trade.fundingFee))
|
|
511
|
+
)
|
|
512
|
+
},
|
|
513
|
+
parity: {
|
|
514
|
+
available: comparisonTotal > 0,
|
|
515
|
+
matched,
|
|
516
|
+
backtestOnly,
|
|
517
|
+
runtimeOnly,
|
|
518
|
+
ratio: parityRatio == null ? null : round(parityRatio),
|
|
519
|
+
lineageReason
|
|
520
|
+
},
|
|
521
|
+
execution: {
|
|
522
|
+
available: calibration != null,
|
|
523
|
+
actualSignalToFillSlippageBps: actualSlippageBps,
|
|
524
|
+
residualVsCurrentModelBps: residualVsModelBps
|
|
525
|
+
},
|
|
526
|
+
rolling,
|
|
527
|
+
distributions: currentDistributions,
|
|
528
|
+
distributionChanges: {
|
|
529
|
+
available: previousDistributions != null,
|
|
530
|
+
baselineWindow: previousArtifact ? {
|
|
531
|
+
startTime: finiteNumber(previousArtifact.window.startTime),
|
|
532
|
+
endTime: finiteNumber(previousArtifact.window.endTime)
|
|
533
|
+
} : null,
|
|
534
|
+
metrics: previousDistributions ? Object.fromEntries(
|
|
535
|
+
Object.entries(currentDistributions).map(([name, current]) => [
|
|
536
|
+
name,
|
|
537
|
+
buildDistributionDelta(
|
|
538
|
+
current,
|
|
539
|
+
previousDistributions[name]
|
|
540
|
+
)
|
|
541
|
+
])
|
|
542
|
+
) : {}
|
|
543
|
+
},
|
|
544
|
+
reactions
|
|
545
|
+
};
|
|
546
|
+
};
|
|
547
|
+
var formatRuntimeScorecardMarkdown = (scorecard) => {
|
|
548
|
+
const rollingRows = scorecard.rolling.map(
|
|
549
|
+
(row) => `| ${row.days}d | ${row.closedTrades} | ${row.realizedPnl} | ${row.expectancy ?? "n/a"} | ${row.maxDrawdown} |`
|
|
550
|
+
).join("\n");
|
|
551
|
+
const reactions = scorecard.reactions.length ? scorecard.reactions.map(
|
|
552
|
+
(reaction) => `- ${reaction.severity.toUpperCase()} \`${reaction.code}\`: ${reaction.message}`
|
|
553
|
+
).join("\n") : "- No automatic reactions.";
|
|
554
|
+
const distributionChanges = Object.entries(
|
|
555
|
+
scorecard.distributionChanges.metrics
|
|
556
|
+
).flatMap(
|
|
557
|
+
([metric, values]) => Object.entries(values).map(([value, change]) => ({
|
|
558
|
+
metric,
|
|
559
|
+
value,
|
|
560
|
+
shareDelta: change.shareDelta
|
|
561
|
+
}))
|
|
562
|
+
).sort(
|
|
563
|
+
(left, right) => Math.abs(right.shareDelta) - Math.abs(left.shareDelta)
|
|
564
|
+
).slice(0, 10).map(
|
|
565
|
+
(change) => `- ${change.metric}.${change.value}: ${change.shareDelta >= 0 ? "+" : ""}${round(change.shareDelta * 100, 2)} pp`
|
|
566
|
+
).join("\n");
|
|
567
|
+
return `# Runtime scorecard
|
|
568
|
+
|
|
569
|
+
Status: **${scorecard.promotionStatus}**
|
|
570
|
+
|
|
571
|
+
Window: ${new Date(scorecard.window.startTime).toISOString()} \u2014 ${new Date(scorecard.window.endTime).toISOString()}
|
|
572
|
+
|
|
573
|
+
## Funnel
|
|
574
|
+
|
|
575
|
+
- Evaluations: ${scorecard.funnel.evaluations}
|
|
576
|
+
- Core candidates: ${scorecard.funnel.coreCandidates}
|
|
577
|
+
- Gate approvals/rejects: ${scorecard.funnel.gate.approved}/${scorecard.funnel.gate.rejected}
|
|
578
|
+
- Allocator approvals/rejects: ${scorecard.funnel.allocator.available ? `${scorecard.funnel.allocator.approved}/${scorecard.funnel.allocator.rejected}` : "unavailable"}
|
|
579
|
+
- Risk approvals/rejects: ${scorecard.funnel.risk.available ? `${scorecard.funnel.risk.approved}/${scorecard.funnel.risk.rejected}` : "unavailable"}
|
|
580
|
+
- Order attempts/failures: ${scorecard.funnel.orderAttempts}/${scorecard.funnel.orderFailures}
|
|
581
|
+
- Fills: ${scorecard.funnel.fills}
|
|
582
|
+
- Closed trades: ${scorecard.funnel.closedTrades}
|
|
583
|
+
|
|
584
|
+
## Replay and execution
|
|
585
|
+
|
|
586
|
+
- Parity: ${scorecard.parity.ratio == null ? "n/a" : `${round(scorecard.parity.ratio * 100, 2)}%`}
|
|
587
|
+
- Runtime/replay mismatches: ${scorecard.parity.backtestOnly + scorecard.parity.runtimeOnly}
|
|
588
|
+
- Actual signal-to-fill slippage: ${scorecard.execution.actualSignalToFillSlippageBps ?? "n/a"} bps
|
|
589
|
+
- Residual vs current model: ${scorecard.execution.residualVsCurrentModelBps ?? "n/a"} bps
|
|
590
|
+
|
|
591
|
+
## Rolling performance
|
|
592
|
+
|
|
593
|
+
| Window | Closed | Realized PnL | Expectancy | Max drawdown |
|
|
594
|
+
| --- | ---: | ---: | ---: | ---: |
|
|
595
|
+
${rollingRows}
|
|
596
|
+
|
|
597
|
+
## Distribution changes
|
|
598
|
+
|
|
599
|
+
${scorecard.distributionChanges.available ? distributionChanges || "- No distribution changes." : "- Previous comparable artifact is unavailable."}
|
|
600
|
+
|
|
601
|
+
## Reactions
|
|
602
|
+
|
|
603
|
+
${reactions}
|
|
604
|
+
`;
|
|
605
|
+
};
|
|
606
|
+
|
|
607
|
+
// src/scripts/runtimeScorecard.ts
|
|
608
|
+
import_args.default.option("runtimeEvidence", "Verified runtime evidence JSON");
|
|
609
|
+
import_args.default.option("replayEvidence", "Replay runtime evidence JSON");
|
|
610
|
+
import_args.default.option("calibration", "Execution calibration JSON");
|
|
611
|
+
import_args.default.option("historyDir", "Directory containing verified evidence bundles");
|
|
612
|
+
import_args.default.option(
|
|
613
|
+
"evidenceDir",
|
|
614
|
+
"Local evidence storage root used for processing receipts",
|
|
615
|
+
process.env.RUNTIME_EVIDENCE_LOCAL_DIR || "data/runtime-evidence"
|
|
616
|
+
);
|
|
617
|
+
import_args.default.option(
|
|
618
|
+
"deployment",
|
|
619
|
+
"Deployment id used for processing receipts",
|
|
620
|
+
process.env.RUNTIME_EVIDENCE_DEPLOYMENT_ID || "production"
|
|
621
|
+
);
|
|
622
|
+
import_args.default.option("markProcessed", "Write a processing receipt on success", false);
|
|
623
|
+
import_args.default.option(
|
|
624
|
+
["P", "minimumParityRatio"],
|
|
625
|
+
"Minimum acceptable replay parity",
|
|
626
|
+
0.95
|
|
627
|
+
);
|
|
628
|
+
import_args.default.option(
|
|
629
|
+
["S", "maximumSlippageResidualBps"],
|
|
630
|
+
"Maximum acceptable execution residual in bps",
|
|
631
|
+
3
|
|
632
|
+
);
|
|
633
|
+
import_args.default.option(
|
|
634
|
+
["T", "minimumClosedTrades"],
|
|
635
|
+
"Minimum 7d closed trades for expectancy reactions",
|
|
636
|
+
20
|
|
637
|
+
);
|
|
638
|
+
import_args.default.option(["X", "minimumExpectancy"], "Minimum acceptable 7d expectancy", 0);
|
|
639
|
+
import_args.default.option(["o", "out"], "Output JSON path", "output/runtime-scorecard.json");
|
|
640
|
+
var flags = import_args.default.parse(process.argv);
|
|
641
|
+
var projectRoot = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
642
|
+
var resolveOptionalPath = (value) => {
|
|
643
|
+
const raw = String(value ?? "").trim();
|
|
644
|
+
return raw ? import_node_path3.default.resolve(projectRoot, raw) : null;
|
|
645
|
+
};
|
|
646
|
+
var readJson = async (filePath) => filePath ? JSON.parse(await import_promises3.default.readFile(filePath, "utf8")) : void 0;
|
|
647
|
+
var loadHistoryArtifacts = async (historyDir) => {
|
|
648
|
+
if (!historyDir) return [];
|
|
649
|
+
const bundleDirs = await discoverRuntimeEvidenceBundles(historyDir);
|
|
650
|
+
const bundles = [];
|
|
651
|
+
for (const bundleDir of bundleDirs) {
|
|
652
|
+
bundles.push(await verifyRuntimeEvidenceBundle(bundleDir));
|
|
653
|
+
}
|
|
654
|
+
return bundles.map((bundle) => bundle.artifact);
|
|
655
|
+
};
|
|
656
|
+
var runtimeScorecard = async () => {
|
|
657
|
+
const runtimeEvidencePath = resolveOptionalPath(flags.runtimeEvidence);
|
|
658
|
+
if (!runtimeEvidencePath) {
|
|
659
|
+
throw new Error("Provide --runtimeEvidence for the runtime scorecard.");
|
|
660
|
+
}
|
|
661
|
+
const replayEvidencePath = resolveOptionalPath(flags.replayEvidence);
|
|
662
|
+
const calibrationPath = resolveOptionalPath(flags.calibration);
|
|
663
|
+
const historyDir = resolveOptionalPath(flags.historyDir);
|
|
664
|
+
const [
|
|
665
|
+
runtimeArtifact,
|
|
666
|
+
replayEvidenceArtifact,
|
|
667
|
+
calibrationArtifact,
|
|
668
|
+
history
|
|
669
|
+
] = await Promise.all([
|
|
670
|
+
readJson(runtimeEvidencePath),
|
|
671
|
+
readJson(replayEvidencePath),
|
|
672
|
+
readJson(calibrationPath),
|
|
673
|
+
loadHistoryArtifacts(historyDir)
|
|
674
|
+
]);
|
|
675
|
+
const scorecard = buildRuntimeScorecard({
|
|
676
|
+
runtimeArtifact,
|
|
677
|
+
replayEvidenceArtifact,
|
|
678
|
+
calibrationArtifact,
|
|
679
|
+
historyRuntimeArtifacts: history,
|
|
680
|
+
thresholds: {
|
|
681
|
+
minimumParityRatio: Number(flags.minimumParityRatio),
|
|
682
|
+
maximumSlippageResidualBps: Number(flags.maximumSlippageResidualBps),
|
|
683
|
+
minimumClosedTrades: Number(flags.minimumClosedTrades),
|
|
684
|
+
minimumExpectancy: Number(flags.minimumExpectancy)
|
|
685
|
+
}
|
|
686
|
+
});
|
|
687
|
+
const outPath = import_node_path3.default.resolve(projectRoot, String(flags.out));
|
|
688
|
+
const markdownPath = outPath.replace(/\.json$/i, "") + ".md";
|
|
689
|
+
await import_promises3.default.mkdir(import_node_path3.default.dirname(outPath), { recursive: true });
|
|
690
|
+
await Promise.all([
|
|
691
|
+
import_promises3.default.writeFile(outPath, `${JSON.stringify(scorecard, null, 2)}
|
|
692
|
+
`, "utf8"),
|
|
693
|
+
import_promises3.default.writeFile(
|
|
694
|
+
markdownPath,
|
|
695
|
+
formatRuntimeScorecardMarkdown(scorecard),
|
|
696
|
+
"utf8"
|
|
697
|
+
)
|
|
698
|
+
]);
|
|
699
|
+
let receiptPath2 = null;
|
|
700
|
+
if (flags.markProcessed) {
|
|
701
|
+
receiptPath2 = await markRuntimeEvidenceBundleProcessed({
|
|
702
|
+
evidenceRoot: import_node_path3.default.resolve(projectRoot, String(flags.evidenceDir)),
|
|
703
|
+
deploymentId: String(flags.deployment),
|
|
704
|
+
bundleDir: import_node_path3.default.dirname(runtimeEvidencePath),
|
|
705
|
+
scorecardPath: outPath
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
console.log(import_chalk.default.green(`Wrote ${outPath}`));
|
|
709
|
+
console.log(import_chalk.default.green(`Wrote ${markdownPath}`));
|
|
710
|
+
console.log(
|
|
711
|
+
JSON.stringify(
|
|
712
|
+
{
|
|
713
|
+
promotionStatus: scorecard.promotionStatus,
|
|
714
|
+
funnel: scorecard.funnel,
|
|
715
|
+
parity: scorecard.parity,
|
|
716
|
+
execution: scorecard.execution,
|
|
717
|
+
rolling: scorecard.rolling,
|
|
718
|
+
reactions: scorecard.reactions,
|
|
719
|
+
receiptPath: receiptPath2
|
|
720
|
+
},
|
|
721
|
+
null,
|
|
722
|
+
2
|
|
723
|
+
)
|
|
724
|
+
);
|
|
725
|
+
};
|
|
726
|
+
var main = runtimeScorecard;
|
|
727
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
728
|
+
0 && (module.exports = {
|
|
729
|
+
main,
|
|
730
|
+
runtimeScorecard
|
|
731
|
+
});
|