backtest-kit 3.0.18 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -1
- package/build/index.cjs +327 -126
- package/build/index.mjs +327 -127
- package/package.json +1 -1
- package/types.d.ts +59 -1
package/build/index.mjs
CHANGED
|
@@ -422,6 +422,13 @@ const GLOBAL_CONFIG = {
|
|
|
422
422
|
* Default: 50 signals
|
|
423
423
|
*/
|
|
424
424
|
CC_MAX_SIGNALS: 50,
|
|
425
|
+
/**
|
|
426
|
+
* Enables mutex locking for candle fetching to prevent concurrent fetches of the same candles.
|
|
427
|
+
* This can help avoid redundant API calls and ensure data consistency when multiple processes/threads attempt to fetch candles simultaneously.
|
|
428
|
+
*
|
|
429
|
+
* Default: true (mutex locking enabled for candle fetching)
|
|
430
|
+
*/
|
|
431
|
+
CC_ENABLE_CANDLE_FETCH_MUTEX: true,
|
|
425
432
|
};
|
|
426
433
|
const DEFAULT_CONFIG = Object.freeze({ ...GLOBAL_CONFIG });
|
|
427
434
|
|
|
@@ -692,7 +699,7 @@ async function writeFileAtomic(file, data, options = {}) {
|
|
|
692
699
|
}
|
|
693
700
|
}
|
|
694
701
|
|
|
695
|
-
var _a$
|
|
702
|
+
var _a$3;
|
|
696
703
|
const BASE_WAIT_FOR_INIT_SYMBOL = Symbol("wait-for-init");
|
|
697
704
|
// Calculate step in milliseconds for candle close time validation
|
|
698
705
|
const INTERVAL_MINUTES$8 = {
|
|
@@ -810,7 +817,7 @@ class PersistBase {
|
|
|
810
817
|
constructor(entityName, baseDir = join(process.cwd(), "logs/data")) {
|
|
811
818
|
this.entityName = entityName;
|
|
812
819
|
this.baseDir = baseDir;
|
|
813
|
-
this[_a$
|
|
820
|
+
this[_a$3] = singleshot(async () => await BASE_WAIT_FOR_INIT_FN(this));
|
|
814
821
|
bt.loggerService.debug(PERSIST_BASE_METHOD_NAME_CTOR, {
|
|
815
822
|
entityName: this.entityName,
|
|
816
823
|
baseDir,
|
|
@@ -913,7 +920,7 @@ class PersistBase {
|
|
|
913
920
|
}
|
|
914
921
|
}
|
|
915
922
|
}
|
|
916
|
-
_a$
|
|
923
|
+
_a$3 = BASE_WAIT_FOR_INIT_SYMBOL;
|
|
917
924
|
// @ts-ignore
|
|
918
925
|
PersistBase = makeExtendable(PersistBase);
|
|
919
926
|
/**
|
|
@@ -1920,6 +1927,69 @@ class PersistNotificationUtils {
|
|
|
1920
1927
|
*/
|
|
1921
1928
|
const PersistNotificationAdapter = new PersistNotificationUtils();
|
|
1922
1929
|
|
|
1930
|
+
var _a$2, _b$2;
|
|
1931
|
+
const BUSY_DELAY = 100;
|
|
1932
|
+
const SET_BUSY_SYMBOL = Symbol("setBusy");
|
|
1933
|
+
const GET_BUSY_SYMBOL = Symbol("getBusy");
|
|
1934
|
+
const ACQUIRE_LOCK_SYMBOL = Symbol("acquireLock");
|
|
1935
|
+
const RELEASE_LOCK_SYMBOL = Symbol("releaseLock");
|
|
1936
|
+
const ACQUIRE_LOCK_FN = async (self) => {
|
|
1937
|
+
while (self[GET_BUSY_SYMBOL]()) {
|
|
1938
|
+
await sleep(BUSY_DELAY);
|
|
1939
|
+
}
|
|
1940
|
+
self[SET_BUSY_SYMBOL](true);
|
|
1941
|
+
};
|
|
1942
|
+
class Lock {
|
|
1943
|
+
constructor() {
|
|
1944
|
+
this._isBusy = 0;
|
|
1945
|
+
this[_a$2] = queued(ACQUIRE_LOCK_FN);
|
|
1946
|
+
this[_b$2] = () => this[SET_BUSY_SYMBOL](false);
|
|
1947
|
+
this.acquireLock = async () => {
|
|
1948
|
+
await this[ACQUIRE_LOCK_SYMBOL](this);
|
|
1949
|
+
};
|
|
1950
|
+
this.releaseLock = async () => {
|
|
1951
|
+
await this[RELEASE_LOCK_SYMBOL]();
|
|
1952
|
+
};
|
|
1953
|
+
}
|
|
1954
|
+
[SET_BUSY_SYMBOL](isBusy) {
|
|
1955
|
+
this._isBusy += isBusy ? 1 : -1;
|
|
1956
|
+
if (this._isBusy < 0) {
|
|
1957
|
+
throw new Error("Extra release in finally block");
|
|
1958
|
+
}
|
|
1959
|
+
}
|
|
1960
|
+
[GET_BUSY_SYMBOL]() {
|
|
1961
|
+
return !!this._isBusy;
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
_a$2 = ACQUIRE_LOCK_SYMBOL, _b$2 = RELEASE_LOCK_SYMBOL;
|
|
1965
|
+
|
|
1966
|
+
const METHOD_NAME_ACQUIRE_LOCK = "CandleUtils.acquireLock";
|
|
1967
|
+
const METHOD_NAME_RELEASE_LOCK = "CandleUtils.releaseLock";
|
|
1968
|
+
class CandleUtils {
|
|
1969
|
+
constructor() {
|
|
1970
|
+
this._lock = new Lock();
|
|
1971
|
+
this.acquireLock = async (source) => {
|
|
1972
|
+
bt.loggerService.info(METHOD_NAME_ACQUIRE_LOCK, {
|
|
1973
|
+
source,
|
|
1974
|
+
});
|
|
1975
|
+
if (!GLOBAL_CONFIG.CC_ENABLE_CANDLE_FETCH_MUTEX) {
|
|
1976
|
+
return;
|
|
1977
|
+
}
|
|
1978
|
+
return await this._lock.acquireLock();
|
|
1979
|
+
};
|
|
1980
|
+
this.releaseLock = async (source) => {
|
|
1981
|
+
bt.loggerService.info(METHOD_NAME_RELEASE_LOCK, {
|
|
1982
|
+
source,
|
|
1983
|
+
});
|
|
1984
|
+
if (!GLOBAL_CONFIG.CC_ENABLE_CANDLE_FETCH_MUTEX) {
|
|
1985
|
+
return;
|
|
1986
|
+
}
|
|
1987
|
+
return await this._lock.releaseLock();
|
|
1988
|
+
};
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
const Candle = new CandleUtils();
|
|
1992
|
+
|
|
1923
1993
|
const MS_PER_MINUTE$5 = 60000;
|
|
1924
1994
|
const INTERVAL_MINUTES$7 = {
|
|
1925
1995
|
"1m": 1,
|
|
@@ -2099,34 +2169,40 @@ const GET_CANDLES_FN = async (dto, since, self) => {
|
|
|
2099
2169
|
const step = INTERVAL_MINUTES$7[dto.interval];
|
|
2100
2170
|
const sinceTimestamp = since.getTime();
|
|
2101
2171
|
const untilTimestamp = sinceTimestamp + dto.limit * step * MS_PER_MINUTE$5;
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
let lastError;
|
|
2109
|
-
for (let i = 0; i !== GLOBAL_CONFIG.CC_GET_CANDLES_RETRY_COUNT; i++) {
|
|
2110
|
-
try {
|
|
2111
|
-
const result = await self.params.getCandles(dto.symbol, dto.interval, since, dto.limit, self.params.execution.context.backtest);
|
|
2112
|
-
VALIDATE_NO_INCOMPLETE_CANDLES_FN(result);
|
|
2113
|
-
// Write to cache after successful fetch
|
|
2114
|
-
await WRITE_CANDLES_CACHE_FN$1(result, dto, self);
|
|
2115
|
-
return result;
|
|
2172
|
+
await Candle.acquireLock(`ClientExchange GET_CANDLES_FN symbol=${dto.symbol} interval=${dto.interval} limit=${dto.limit}`);
|
|
2173
|
+
try {
|
|
2174
|
+
// Try to read from cache first
|
|
2175
|
+
const cachedCandles = await READ_CANDLES_CACHE_FN$1(dto, sinceTimestamp, untilTimestamp, self);
|
|
2176
|
+
if (cachedCandles !== null) {
|
|
2177
|
+
return cachedCandles;
|
|
2116
2178
|
}
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2179
|
+
// Cache miss or error - fetch from API
|
|
2180
|
+
let lastError;
|
|
2181
|
+
for (let i = 0; i !== GLOBAL_CONFIG.CC_GET_CANDLES_RETRY_COUNT; i++) {
|
|
2182
|
+
try {
|
|
2183
|
+
const result = await self.params.getCandles(dto.symbol, dto.interval, since, dto.limit, self.params.execution.context.backtest);
|
|
2184
|
+
VALIDATE_NO_INCOMPLETE_CANDLES_FN(result);
|
|
2185
|
+
// Write to cache after successful fetch
|
|
2186
|
+
await WRITE_CANDLES_CACHE_FN$1(result, dto, self);
|
|
2187
|
+
return result;
|
|
2188
|
+
}
|
|
2189
|
+
catch (err) {
|
|
2190
|
+
const message = `ClientExchange GET_CANDLES_FN: attempt ${i + 1} failed for symbol=${dto.symbol}, interval=${dto.interval}, since=${since.toISOString()}, limit=${dto.limit}}`;
|
|
2191
|
+
const payload = {
|
|
2192
|
+
error: errorData(err),
|
|
2193
|
+
message: getErrorMessage(err),
|
|
2194
|
+
};
|
|
2195
|
+
self.params.logger.warn(message, payload);
|
|
2196
|
+
console.warn(message, payload);
|
|
2197
|
+
lastError = err;
|
|
2198
|
+
await sleep(GLOBAL_CONFIG.CC_GET_CANDLES_RETRY_DELAY_MS);
|
|
2199
|
+
}
|
|
2127
2200
|
}
|
|
2201
|
+
throw lastError;
|
|
2202
|
+
}
|
|
2203
|
+
finally {
|
|
2204
|
+
Candle.releaseLock(`ClientExchange GET_CANDLES_FN symbol=${dto.symbol} interval=${dto.interval} limit=${dto.limit}`);
|
|
2128
2205
|
}
|
|
2129
|
-
throw lastError;
|
|
2130
2206
|
};
|
|
2131
2207
|
/**
|
|
2132
2208
|
* Wrapper to call onCandleData callback with error handling.
|
|
@@ -2809,8 +2885,9 @@ class ExchangeConnectionService {
|
|
|
2809
2885
|
*
|
|
2810
2886
|
* For signals with partial closes:
|
|
2811
2887
|
* - Calculates weighted PNL: Σ(percent_i × pnl_i) for each partial + (remaining% × final_pnl)
|
|
2812
|
-
* - Each partial close has its own
|
|
2813
|
-
* -
|
|
2888
|
+
* - Each partial close has its own slippage
|
|
2889
|
+
* - Open fee is charged once; close fees are proportional to each partial's size
|
|
2890
|
+
* - Total fees = CC_PERCENT_FEE (open) + CC_PERCENT_FEE × 1 (closes sum to 100%) = 2 × CC_PERCENT_FEE
|
|
2814
2891
|
*
|
|
2815
2892
|
* Formula breakdown:
|
|
2816
2893
|
* 1. Apply slippage to open/close prices (worse execution)
|
|
@@ -2857,7 +2934,8 @@ const toProfitLossDto = (signal, priceClose) => {
|
|
|
2857
2934
|
// Calculate weighted PNL with partial closes
|
|
2858
2935
|
if (signal._partial && signal._partial.length > 0) {
|
|
2859
2936
|
let totalWeightedPnl = 0;
|
|
2860
|
-
|
|
2937
|
+
// Open fee is paid once for the whole position
|
|
2938
|
+
let totalFees = GLOBAL_CONFIG.CC_PERCENT_FEE;
|
|
2861
2939
|
// Calculate PNL for each partial close
|
|
2862
2940
|
for (const partial of signal._partial) {
|
|
2863
2941
|
const partialPercent = partial.percent;
|
|
@@ -2884,8 +2962,8 @@ const toProfitLossDto = (signal, priceClose) => {
|
|
|
2884
2962
|
// Weight by percentage of position closed
|
|
2885
2963
|
const weightedPnl = (partialPercent / 100) * partialPnl;
|
|
2886
2964
|
totalWeightedPnl += weightedPnl;
|
|
2887
|
-
//
|
|
2888
|
-
totalFees += GLOBAL_CONFIG.CC_PERCENT_FEE *
|
|
2965
|
+
// Close fee is proportional to the size of this partial
|
|
2966
|
+
totalFees += GLOBAL_CONFIG.CC_PERCENT_FEE * (partialPercent / 100);
|
|
2889
2967
|
}
|
|
2890
2968
|
// Calculate PNL for remaining position (if any)
|
|
2891
2969
|
// Compute totalClosed from _partial array
|
|
@@ -2914,10 +2992,11 @@ const toProfitLossDto = (signal, priceClose) => {
|
|
|
2914
2992
|
// Weight by remaining percentage
|
|
2915
2993
|
const weightedRemainingPnl = (remainingPercent / 100) * remainingPnl;
|
|
2916
2994
|
totalWeightedPnl += weightedRemainingPnl;
|
|
2917
|
-
//
|
|
2918
|
-
totalFees += GLOBAL_CONFIG.CC_PERCENT_FEE *
|
|
2995
|
+
// Close fee is proportional to the remaining size
|
|
2996
|
+
totalFees += GLOBAL_CONFIG.CC_PERCENT_FEE * (remainingPercent / 100);
|
|
2919
2997
|
}
|
|
2920
2998
|
// Subtract total fees from weighted PNL
|
|
2999
|
+
// totalFees = CC_PERCENT_FEE (open) + CC_PERCENT_FEE × 1 (all closes sum to 100%) = 2 × CC_PERCENT_FEE
|
|
2921
3000
|
const pnlPercentage = totalWeightedPnl - totalFees;
|
|
2922
3001
|
return {
|
|
2923
3002
|
pnlPercentage,
|
|
@@ -26307,55 +26386,61 @@ class ExchangeInstance {
|
|
|
26307
26386
|
const sinceTimestamp = alignedWhen - limit * stepMs;
|
|
26308
26387
|
const since = new Date(sinceTimestamp);
|
|
26309
26388
|
const untilTimestamp = alignedWhen;
|
|
26310
|
-
|
|
26311
|
-
|
|
26312
|
-
|
|
26313
|
-
|
|
26314
|
-
|
|
26315
|
-
|
|
26316
|
-
|
|
26317
|
-
|
|
26318
|
-
|
|
26319
|
-
|
|
26320
|
-
|
|
26321
|
-
|
|
26322
|
-
const
|
|
26323
|
-
|
|
26324
|
-
|
|
26325
|
-
|
|
26326
|
-
|
|
26327
|
-
|
|
26328
|
-
|
|
26389
|
+
await Candle.acquireLock(`ExchangeInstance.getCandles symbol=${symbol} interval=${interval} limit=${limit}`);
|
|
26390
|
+
try {
|
|
26391
|
+
// Try to read from cache first
|
|
26392
|
+
const cachedCandles = await READ_CANDLES_CACHE_FN({ symbol, interval, limit }, sinceTimestamp, untilTimestamp, this.exchangeName);
|
|
26393
|
+
if (cachedCandles !== null) {
|
|
26394
|
+
return cachedCandles;
|
|
26395
|
+
}
|
|
26396
|
+
let allData = [];
|
|
26397
|
+
// If limit exceeds CC_MAX_CANDLES_PER_REQUEST, fetch data in chunks
|
|
26398
|
+
if (limit > GLOBAL_CONFIG.CC_MAX_CANDLES_PER_REQUEST) {
|
|
26399
|
+
let remaining = limit;
|
|
26400
|
+
let currentSince = new Date(since.getTime());
|
|
26401
|
+
const isBacktest = await GET_BACKTEST_FN();
|
|
26402
|
+
while (remaining > 0) {
|
|
26403
|
+
const chunkLimit = Math.min(remaining, GLOBAL_CONFIG.CC_MAX_CANDLES_PER_REQUEST);
|
|
26404
|
+
const chunkData = await getCandles(symbol, interval, currentSince, chunkLimit, isBacktest);
|
|
26405
|
+
allData.push(...chunkData);
|
|
26406
|
+
remaining -= chunkLimit;
|
|
26407
|
+
if (remaining > 0) {
|
|
26408
|
+
// Move currentSince forward by the number of candles fetched
|
|
26409
|
+
currentSince = new Date(currentSince.getTime() + chunkLimit * stepMs);
|
|
26410
|
+
}
|
|
26329
26411
|
}
|
|
26330
26412
|
}
|
|
26413
|
+
else {
|
|
26414
|
+
const isBacktest = await GET_BACKTEST_FN();
|
|
26415
|
+
allData = await getCandles(symbol, interval, since, limit, isBacktest);
|
|
26416
|
+
}
|
|
26417
|
+
// Apply distinct by timestamp to remove duplicates
|
|
26418
|
+
const uniqueData = Array.from(new Map(allData.map((candle) => [candle.timestamp, candle])).values());
|
|
26419
|
+
if (allData.length !== uniqueData.length) {
|
|
26420
|
+
bt.loggerService.warn(`ExchangeInstance getCandles: Removed ${allData.length - uniqueData.length} duplicate candles by timestamp`);
|
|
26421
|
+
}
|
|
26422
|
+
// Validate adapter returned data
|
|
26423
|
+
if (uniqueData.length === 0) {
|
|
26424
|
+
throw new Error(`ExchangeInstance getCandles: adapter returned empty array. ` +
|
|
26425
|
+
`Expected ${limit} candles starting from openTime=${sinceTimestamp}.`);
|
|
26426
|
+
}
|
|
26427
|
+
if (uniqueData[0].timestamp !== sinceTimestamp) {
|
|
26428
|
+
throw new Error(`ExchangeInstance getCandles: first candle timestamp mismatch. ` +
|
|
26429
|
+
`Expected openTime=${sinceTimestamp}, got=${uniqueData[0].timestamp}. ` +
|
|
26430
|
+
`Adapter must return candles with timestamp=openTime, starting from aligned since.`);
|
|
26431
|
+
}
|
|
26432
|
+
if (uniqueData.length !== limit) {
|
|
26433
|
+
throw new Error(`ExchangeInstance getCandles: candle count mismatch. ` +
|
|
26434
|
+
`Expected ${limit} candles, got ${uniqueData.length}. ` +
|
|
26435
|
+
`Adapter must return exact number of candles requested.`);
|
|
26436
|
+
}
|
|
26437
|
+
// Write to cache after successful fetch
|
|
26438
|
+
await WRITE_CANDLES_CACHE_FN(uniqueData, { symbol, interval, limit }, this.exchangeName);
|
|
26439
|
+
return uniqueData;
|
|
26331
26440
|
}
|
|
26332
|
-
|
|
26333
|
-
|
|
26334
|
-
allData = await getCandles(symbol, interval, since, limit, isBacktest);
|
|
26335
|
-
}
|
|
26336
|
-
// Apply distinct by timestamp to remove duplicates
|
|
26337
|
-
const uniqueData = Array.from(new Map(allData.map((candle) => [candle.timestamp, candle])).values());
|
|
26338
|
-
if (allData.length !== uniqueData.length) {
|
|
26339
|
-
bt.loggerService.warn(`ExchangeInstance getCandles: Removed ${allData.length - uniqueData.length} duplicate candles by timestamp`);
|
|
26340
|
-
}
|
|
26341
|
-
// Validate adapter returned data
|
|
26342
|
-
if (uniqueData.length === 0) {
|
|
26343
|
-
throw new Error(`ExchangeInstance getCandles: adapter returned empty array. ` +
|
|
26344
|
-
`Expected ${limit} candles starting from openTime=${sinceTimestamp}.`);
|
|
26345
|
-
}
|
|
26346
|
-
if (uniqueData[0].timestamp !== sinceTimestamp) {
|
|
26347
|
-
throw new Error(`ExchangeInstance getCandles: first candle timestamp mismatch. ` +
|
|
26348
|
-
`Expected openTime=${sinceTimestamp}, got=${uniqueData[0].timestamp}. ` +
|
|
26349
|
-
`Adapter must return candles with timestamp=openTime, starting from aligned since.`);
|
|
26350
|
-
}
|
|
26351
|
-
if (uniqueData.length !== limit) {
|
|
26352
|
-
throw new Error(`ExchangeInstance getCandles: candle count mismatch. ` +
|
|
26353
|
-
`Expected ${limit} candles, got ${uniqueData.length}. ` +
|
|
26354
|
-
`Adapter must return exact number of candles requested.`);
|
|
26441
|
+
finally {
|
|
26442
|
+
Candle.releaseLock(`ExchangeInstance.getCandles symbol=${symbol} interval=${interval} limit=${limit}`);
|
|
26355
26443
|
}
|
|
26356
|
-
// Write to cache after successful fetch
|
|
26357
|
-
await WRITE_CANDLES_CACHE_FN(uniqueData, { symbol, interval, limit }, this.exchangeName);
|
|
26358
|
-
return uniqueData;
|
|
26359
26444
|
};
|
|
26360
26445
|
/**
|
|
26361
26446
|
* Calculates VWAP (Volume Weighted Average Price) from last N 1m candles.
|
|
@@ -26589,56 +26674,62 @@ class ExchangeInstance {
|
|
|
26589
26674
|
`Provide one of: (sDate+eDate+limit), (sDate+eDate), (eDate+limit), (sDate+limit), or (limit only). ` +
|
|
26590
26675
|
`Got: sDate=${sDate}, eDate=${eDate}, limit=${limit}`);
|
|
26591
26676
|
}
|
|
26592
|
-
|
|
26593
|
-
|
|
26594
|
-
|
|
26595
|
-
|
|
26596
|
-
|
|
26597
|
-
|
|
26598
|
-
|
|
26599
|
-
|
|
26600
|
-
|
|
26601
|
-
|
|
26602
|
-
|
|
26603
|
-
|
|
26604
|
-
|
|
26605
|
-
|
|
26606
|
-
|
|
26607
|
-
|
|
26608
|
-
|
|
26609
|
-
|
|
26610
|
-
|
|
26611
|
-
|
|
26612
|
-
|
|
26677
|
+
await Candle.acquireLock(`ExchangeInstance.getRawCandles symbol=${symbol} interval=${interval} limit=${calculatedLimit}`);
|
|
26678
|
+
try {
|
|
26679
|
+
// Try to read from cache first
|
|
26680
|
+
const untilTimestamp = sinceTimestamp + calculatedLimit * stepMs;
|
|
26681
|
+
const cachedCandles = await READ_CANDLES_CACHE_FN({ symbol, interval, limit: calculatedLimit }, sinceTimestamp, untilTimestamp, this.exchangeName);
|
|
26682
|
+
if (cachedCandles !== null) {
|
|
26683
|
+
return cachedCandles;
|
|
26684
|
+
}
|
|
26685
|
+
// Fetch candles
|
|
26686
|
+
const since = new Date(sinceTimestamp);
|
|
26687
|
+
let allData = [];
|
|
26688
|
+
const isBacktest = await GET_BACKTEST_FN();
|
|
26689
|
+
const getCandles = this._methods.getCandles;
|
|
26690
|
+
if (calculatedLimit > GLOBAL_CONFIG.CC_MAX_CANDLES_PER_REQUEST) {
|
|
26691
|
+
let remaining = calculatedLimit;
|
|
26692
|
+
let currentSince = new Date(since.getTime());
|
|
26693
|
+
while (remaining > 0) {
|
|
26694
|
+
const chunkLimit = Math.min(remaining, GLOBAL_CONFIG.CC_MAX_CANDLES_PER_REQUEST);
|
|
26695
|
+
const chunkData = await getCandles(symbol, interval, currentSince, chunkLimit, isBacktest);
|
|
26696
|
+
allData.push(...chunkData);
|
|
26697
|
+
remaining -= chunkLimit;
|
|
26698
|
+
if (remaining > 0) {
|
|
26699
|
+
currentSince = new Date(currentSince.getTime() + chunkLimit * stepMs);
|
|
26700
|
+
}
|
|
26613
26701
|
}
|
|
26614
26702
|
}
|
|
26703
|
+
else {
|
|
26704
|
+
allData = await getCandles(symbol, interval, since, calculatedLimit, isBacktest);
|
|
26705
|
+
}
|
|
26706
|
+
// Apply distinct by timestamp to remove duplicates
|
|
26707
|
+
const uniqueData = Array.from(new Map(allData.map((candle) => [candle.timestamp, candle])).values());
|
|
26708
|
+
if (allData.length !== uniqueData.length) {
|
|
26709
|
+
bt.loggerService.warn(`ExchangeInstance getRawCandles: Removed ${allData.length - uniqueData.length} duplicate candles by timestamp`);
|
|
26710
|
+
}
|
|
26711
|
+
// Validate adapter returned data
|
|
26712
|
+
if (uniqueData.length === 0) {
|
|
26713
|
+
throw new Error(`ExchangeInstance getRawCandles: adapter returned empty array. ` +
|
|
26714
|
+
`Expected ${calculatedLimit} candles starting from openTime=${sinceTimestamp}.`);
|
|
26715
|
+
}
|
|
26716
|
+
if (uniqueData[0].timestamp !== sinceTimestamp) {
|
|
26717
|
+
throw new Error(`ExchangeInstance getRawCandles: first candle timestamp mismatch. ` +
|
|
26718
|
+
`Expected openTime=${sinceTimestamp}, got=${uniqueData[0].timestamp}. ` +
|
|
26719
|
+
`Adapter must return candles with timestamp=openTime, starting from aligned since.`);
|
|
26720
|
+
}
|
|
26721
|
+
if (uniqueData.length !== calculatedLimit) {
|
|
26722
|
+
throw new Error(`ExchangeInstance getRawCandles: candle count mismatch. ` +
|
|
26723
|
+
`Expected ${calculatedLimit} candles, got ${uniqueData.length}. ` +
|
|
26724
|
+
`Adapter must return exact number of candles requested.`);
|
|
26725
|
+
}
|
|
26726
|
+
// Write to cache after successful fetch
|
|
26727
|
+
await WRITE_CANDLES_CACHE_FN(uniqueData, { symbol, interval, limit: calculatedLimit }, this.exchangeName);
|
|
26728
|
+
return uniqueData;
|
|
26615
26729
|
}
|
|
26616
|
-
|
|
26617
|
-
|
|
26618
|
-
}
|
|
26619
|
-
// Apply distinct by timestamp to remove duplicates
|
|
26620
|
-
const uniqueData = Array.from(new Map(allData.map((candle) => [candle.timestamp, candle])).values());
|
|
26621
|
-
if (allData.length !== uniqueData.length) {
|
|
26622
|
-
bt.loggerService.warn(`ExchangeInstance getRawCandles: Removed ${allData.length - uniqueData.length} duplicate candles by timestamp`);
|
|
26623
|
-
}
|
|
26624
|
-
// Validate adapter returned data
|
|
26625
|
-
if (uniqueData.length === 0) {
|
|
26626
|
-
throw new Error(`ExchangeInstance getRawCandles: adapter returned empty array. ` +
|
|
26627
|
-
`Expected ${calculatedLimit} candles starting from openTime=${sinceTimestamp}.`);
|
|
26628
|
-
}
|
|
26629
|
-
if (uniqueData[0].timestamp !== sinceTimestamp) {
|
|
26630
|
-
throw new Error(`ExchangeInstance getRawCandles: first candle timestamp mismatch. ` +
|
|
26631
|
-
`Expected openTime=${sinceTimestamp}, got=${uniqueData[0].timestamp}. ` +
|
|
26632
|
-
`Adapter must return candles with timestamp=openTime, starting from aligned since.`);
|
|
26633
|
-
}
|
|
26634
|
-
if (uniqueData.length !== calculatedLimit) {
|
|
26635
|
-
throw new Error(`ExchangeInstance getRawCandles: candle count mismatch. ` +
|
|
26636
|
-
`Expected ${calculatedLimit} candles, got ${uniqueData.length}. ` +
|
|
26637
|
-
`Adapter must return exact number of candles requested.`);
|
|
26730
|
+
finally {
|
|
26731
|
+
Candle.releaseLock(`ExchangeInstance.getRawCandles symbol=${symbol} interval=${interval} limit=${calculatedLimit}`);
|
|
26638
26732
|
}
|
|
26639
|
-
// Write to cache after successful fetch
|
|
26640
|
-
await WRITE_CANDLES_CACHE_FN(uniqueData, { symbol, interval, limit: calculatedLimit }, this.exchangeName);
|
|
26641
|
-
return uniqueData;
|
|
26642
26733
|
};
|
|
26643
26734
|
const schema = bt.exchangeSchemaService.get(this.exchangeName);
|
|
26644
26735
|
this._methods = CREATE_EXCHANGE_INSTANCE_FN(schema);
|
|
@@ -30125,6 +30216,115 @@ function listenStrategyCommitOnce(filterFn, fn) {
|
|
|
30125
30216
|
return strategyCommitSubject.filter(filterFn).once(fn);
|
|
30126
30217
|
}
|
|
30127
30218
|
|
|
30219
|
+
const WARN_KB = 30;
|
|
30220
|
+
const DUMP_MESSAGES_METHOD_NAME = "dump.dumpMessages";
|
|
30221
|
+
/**
|
|
30222
|
+
* Dumps chat history and result data to markdown files in a structured directory.
|
|
30223
|
+
*
|
|
30224
|
+
* Creates a subfolder named after `resultId` inside `outputDir`.
|
|
30225
|
+
* If the subfolder already exists, the function returns early without overwriting.
|
|
30226
|
+
* Writes:
|
|
30227
|
+
* - `00_system_prompt.md` — system messages and output data summary
|
|
30228
|
+
* - `NN_user_message.md` — each user message as a separate file
|
|
30229
|
+
* - `NN_llm_output.md` — final LLM output data
|
|
30230
|
+
*
|
|
30231
|
+
* Warns via logger if any user message exceeds 30 KB.
|
|
30232
|
+
*
|
|
30233
|
+
* @param resultId - Unique identifier for the result (used as subfolder name)
|
|
30234
|
+
* @param history - Full chat history containing system, user, and assistant messages
|
|
30235
|
+
* @param result - Structured output data to include in the dump
|
|
30236
|
+
* @param outputDir - Base directory for output files (default: `./dump/strategy`)
|
|
30237
|
+
* @returns Promise that resolves when all files are written
|
|
30238
|
+
*
|
|
30239
|
+
* @example
|
|
30240
|
+
* ```typescript
|
|
30241
|
+
* import { dumpMessages } from "backtest-kit";
|
|
30242
|
+
*
|
|
30243
|
+
* await dumpMessages("result-123", history, { profit: 42 });
|
|
30244
|
+
* ```
|
|
30245
|
+
*/
|
|
30246
|
+
async function dumpMessages(resultId, history, result, outputDir = "./dump/strategy") {
|
|
30247
|
+
bt.loggerService.info(DUMP_MESSAGES_METHOD_NAME, {
|
|
30248
|
+
resultId,
|
|
30249
|
+
outputDir,
|
|
30250
|
+
});
|
|
30251
|
+
// Extract system messages and system reminders from existing data
|
|
30252
|
+
const systemMessages = history.filter((m) => m.role === "system");
|
|
30253
|
+
const userMessages = history.filter((m) => m.role === "user");
|
|
30254
|
+
const subfolderPath = path.join(outputDir, String(resultId));
|
|
30255
|
+
try {
|
|
30256
|
+
await fs__default.access(subfolderPath);
|
|
30257
|
+
return;
|
|
30258
|
+
}
|
|
30259
|
+
catch {
|
|
30260
|
+
await fs__default.mkdir(subfolderPath, { recursive: true });
|
|
30261
|
+
}
|
|
30262
|
+
{
|
|
30263
|
+
let summary = "# Outline Result Summary\n";
|
|
30264
|
+
{
|
|
30265
|
+
summary += "\n";
|
|
30266
|
+
summary += `**ResultId**: ${resultId}\n`;
|
|
30267
|
+
summary += "\n";
|
|
30268
|
+
}
|
|
30269
|
+
if (result) {
|
|
30270
|
+
summary += "## Output Data\n\n";
|
|
30271
|
+
summary += "```json\n";
|
|
30272
|
+
summary += JSON.stringify(result, null, 2);
|
|
30273
|
+
summary += "\n```\n\n";
|
|
30274
|
+
}
|
|
30275
|
+
// Add system messages to summary
|
|
30276
|
+
if (systemMessages.length > 0) {
|
|
30277
|
+
summary += "## System Messages\n\n";
|
|
30278
|
+
systemMessages.forEach((msg, idx) => {
|
|
30279
|
+
summary += `### System Message ${idx + 1}\n\n`;
|
|
30280
|
+
summary += msg.content;
|
|
30281
|
+
summary += "\n";
|
|
30282
|
+
});
|
|
30283
|
+
}
|
|
30284
|
+
const summaryFile = path.join(subfolderPath, "00_system_prompt.md");
|
|
30285
|
+
await fs__default.writeFile(summaryFile, summary, "utf8");
|
|
30286
|
+
}
|
|
30287
|
+
{
|
|
30288
|
+
await Promise.all(Array.from(userMessages.entries()).map(async ([idx, message]) => {
|
|
30289
|
+
const messageNum = String(idx + 1).padStart(2, "0");
|
|
30290
|
+
const contentFileName = `${messageNum}_user_message.md`;
|
|
30291
|
+
const contentFilePath = path.join(subfolderPath, contentFileName);
|
|
30292
|
+
{
|
|
30293
|
+
const messageSizeBytes = Buffer.byteLength(message.content, "utf8");
|
|
30294
|
+
const messageSizeKb = Math.floor(messageSizeBytes / 1024);
|
|
30295
|
+
if (messageSizeKb > WARN_KB) {
|
|
30296
|
+
console.warn(`User message ${idx + 1} is ${messageSizeBytes} bytes (${messageSizeKb}kb), which exceeds warning limit`);
|
|
30297
|
+
bt.loggerService.warn(DUMP_MESSAGES_METHOD_NAME, {
|
|
30298
|
+
resultId,
|
|
30299
|
+
messageIndex: idx + 1,
|
|
30300
|
+
messageSizeBytes,
|
|
30301
|
+
messageSizeKb,
|
|
30302
|
+
});
|
|
30303
|
+
}
|
|
30304
|
+
}
|
|
30305
|
+
let content = `# User Input ${idx + 1}\n\n`;
|
|
30306
|
+
content += `**ResultId**: ${resultId}\n\n`;
|
|
30307
|
+
content += message.content;
|
|
30308
|
+
content += "\n";
|
|
30309
|
+
await fs__default.writeFile(contentFilePath, content, "utf8");
|
|
30310
|
+
}));
|
|
30311
|
+
}
|
|
30312
|
+
{
|
|
30313
|
+
const messageNum = String(userMessages.length + 1).padStart(2, "0");
|
|
30314
|
+
const contentFileName = `${messageNum}_llm_output.md`;
|
|
30315
|
+
const contentFilePath = path.join(subfolderPath, contentFileName);
|
|
30316
|
+
let content = "# Full Outline Result\n\n";
|
|
30317
|
+
content += `**ResultId**: ${resultId}\n\n`;
|
|
30318
|
+
if (result) {
|
|
30319
|
+
content += "## Output Data\n\n";
|
|
30320
|
+
content += "```json\n";
|
|
30321
|
+
content += JSON.stringify(result, null, 2);
|
|
30322
|
+
content += "\n```\n";
|
|
30323
|
+
}
|
|
30324
|
+
await fs__default.writeFile(contentFilePath, content, "utf8");
|
|
30325
|
+
}
|
|
30326
|
+
}
|
|
30327
|
+
|
|
30128
30328
|
const BACKTEST_METHOD_NAME_RUN = "BacktestUtils.run";
|
|
30129
30329
|
const BACKTEST_METHOD_NAME_BACKGROUND = "BacktestUtils.background";
|
|
30130
30330
|
const BACKTEST_METHOD_NAME_STOP = "BacktestUtils.stop";
|
|
@@ -37384,4 +37584,4 @@ const set = (object, path, value) => {
|
|
|
37384
37584
|
}
|
|
37385
37585
|
};
|
|
37386
37586
|
|
|
37387
|
-
export { ActionBase, Backtest, Breakeven, Cache, Constant, Exchange, ExecutionContextService, Heat, Live, Markdown, MarkdownFileBase, MarkdownFolderBase, MethodContextService, Notification, NotificationBacktest, NotificationLive, Partial, Performance, PersistBase, PersistBreakevenAdapter, PersistCandleAdapter, PersistNotificationAdapter, PersistPartialAdapter, PersistRiskAdapter, PersistScheduleAdapter, PersistSignalAdapter, PersistStorageAdapter, PositionSize, Report, ReportBase, Risk, Schedule, Storage, StorageBacktest, StorageLive, Strategy, Walker, addActionSchema, addExchangeSchema, addFrameSchema, addRiskSchema, addSizingSchema, addStrategySchema, addWalkerSchema, alignToInterval, checkCandles, commitActivateScheduled, commitBreakeven, commitCancelScheduled, commitClosePending, commitPartialLoss, commitPartialProfit, commitTrailingStop, commitTrailingTake, emitters, formatPrice, formatQuantity, get, getActionSchema, getAveragePrice, getBacktestTimeframe, getCandles, getColumns, getConfig, getContext, getDate, getDefaultColumns, getDefaultConfig, getExchangeSchema, getFrameSchema, getMode, getNextCandles, getOrderBook, getRawCandles, getRiskSchema, getSizingSchema, getStrategySchema, getSymbol, getWalkerSchema, hasTradeContext, backtest as lib, listExchangeSchema, listFrameSchema, listRiskSchema, listSizingSchema, listStrategySchema, listWalkerSchema, listenActivePing, listenActivePingOnce, listenBacktestProgress, listenBreakevenAvailable, listenBreakevenAvailableOnce, listenDoneBacktest, listenDoneBacktestOnce, listenDoneLive, listenDoneLiveOnce, listenDoneWalker, listenDoneWalkerOnce, listenError, listenExit, listenPartialLossAvailable, listenPartialLossAvailableOnce, listenPartialProfitAvailable, listenPartialProfitAvailableOnce, listenPerformance, listenRisk, listenRiskOnce, listenSchedulePing, listenSchedulePingOnce, listenSignal, listenSignalBacktest, listenSignalBacktestOnce, listenSignalLive, listenSignalLiveOnce, listenSignalOnce, listenStrategyCommit, listenStrategyCommitOnce, listenValidation, listenWalker, listenWalkerComplete, listenWalkerOnce, listenWalkerProgress, overrideActionSchema, overrideExchangeSchema, overrideFrameSchema, overrideRiskSchema, overrideSizingSchema, overrideStrategySchema, overrideWalkerSchema, parseArgs, roundTicks, set, setColumns, setConfig, setLogger, stopStrategy, validate, waitForCandle, warmCandles };
|
|
37587
|
+
export { ActionBase, Backtest, Breakeven, Cache, Constant, Exchange, ExecutionContextService, Heat, Live, Markdown, MarkdownFileBase, MarkdownFolderBase, MethodContextService, Notification, NotificationBacktest, NotificationLive, Partial, Performance, PersistBase, PersistBreakevenAdapter, PersistCandleAdapter, PersistNotificationAdapter, PersistPartialAdapter, PersistRiskAdapter, PersistScheduleAdapter, PersistSignalAdapter, PersistStorageAdapter, PositionSize, Report, ReportBase, Risk, Schedule, Storage, StorageBacktest, StorageLive, Strategy, Walker, addActionSchema, addExchangeSchema, addFrameSchema, addRiskSchema, addSizingSchema, addStrategySchema, addWalkerSchema, alignToInterval, checkCandles, commitActivateScheduled, commitBreakeven, commitCancelScheduled, commitClosePending, commitPartialLoss, commitPartialProfit, commitTrailingStop, commitTrailingTake, dumpMessages, emitters, formatPrice, formatQuantity, get, getActionSchema, getAveragePrice, getBacktestTimeframe, getCandles, getColumns, getConfig, getContext, getDate, getDefaultColumns, getDefaultConfig, getExchangeSchema, getFrameSchema, getMode, getNextCandles, getOrderBook, getRawCandles, getRiskSchema, getSizingSchema, getStrategySchema, getSymbol, getWalkerSchema, hasTradeContext, backtest as lib, listExchangeSchema, listFrameSchema, listRiskSchema, listSizingSchema, listStrategySchema, listWalkerSchema, listenActivePing, listenActivePingOnce, listenBacktestProgress, listenBreakevenAvailable, listenBreakevenAvailableOnce, listenDoneBacktest, listenDoneBacktestOnce, listenDoneLive, listenDoneLiveOnce, listenDoneWalker, listenDoneWalkerOnce, listenError, listenExit, listenPartialLossAvailable, listenPartialLossAvailableOnce, listenPartialProfitAvailable, listenPartialProfitAvailableOnce, listenPerformance, listenRisk, listenRiskOnce, listenSchedulePing, listenSchedulePingOnce, listenSignal, listenSignalBacktest, listenSignalBacktestOnce, listenSignalLive, listenSignalLiveOnce, listenSignalOnce, listenStrategyCommit, listenStrategyCommitOnce, listenValidation, listenWalker, listenWalkerComplete, listenWalkerOnce, listenWalkerProgress, overrideActionSchema, overrideExchangeSchema, overrideFrameSchema, overrideRiskSchema, overrideSizingSchema, overrideStrategySchema, overrideWalkerSchema, parseArgs, roundTicks, set, setColumns, setConfig, setLogger, stopStrategy, validate, waitForCandle, warmCandles };
|