@tradejs/cli 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/cli.js +1910 -813
- package/dist/lib/aiTrainMetrics.js +150 -0
- package/dist/scripts/aiExport.js +192 -0
- package/dist/scripts/aiExportSelect.js +135 -0
- package/dist/scripts/aiTrain.js +2239 -0
- package/dist/scripts/backtest.js +218 -32
- package/dist/scripts/derivativesIngest.js +5 -0
- package/dist/scripts/derivativesIngestCoinalyzeAll.js +21 -12
- package/dist/scripts/mlExport.js +112 -13
- package/dist/scripts/mlExportSelect.js +43 -9
- package/dist/scripts/mlTrainLatestSelect.js +4 -3
- package/dist/scripts/resolveExportStrategy.js +133 -0
- package/dist/scripts/selectStrategy.js +4 -3
- package/dist/scripts/signals.js +109 -25
- package/dist/scripts/spawnSiblingScript.js +71 -0
- package/dist/workers/testerWorker.js +13 -2
- package/package.json +11 -9
package/dist/cli.js
CHANGED
|
@@ -33,6 +33,321 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
33
33
|
mod
|
|
34
34
|
));
|
|
35
35
|
|
|
36
|
+
// src/scripts/selectStrategy.ts
|
|
37
|
+
var import_readline, import_chalk, import_strategies, defaultStrategy, projectRoot, getStrategyChoices, selectStrategy;
|
|
38
|
+
var init_selectStrategy = __esm({
|
|
39
|
+
"src/scripts/selectStrategy.ts"() {
|
|
40
|
+
"use strict";
|
|
41
|
+
import_readline = __toESM(require("readline"));
|
|
42
|
+
import_chalk = __toESM(require("chalk"));
|
|
43
|
+
import_strategies = require("@tradejs/node/strategies");
|
|
44
|
+
defaultStrategy = "TrendLine";
|
|
45
|
+
projectRoot = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
46
|
+
getStrategyChoices = async () => {
|
|
47
|
+
try {
|
|
48
|
+
const loaded = await (0, import_strategies.getAvailableStrategyNames)(projectRoot);
|
|
49
|
+
if (loaded.length) {
|
|
50
|
+
return loaded;
|
|
51
|
+
}
|
|
52
|
+
} catch (error) {
|
|
53
|
+
console.warn(`Failed to load strategy list: ${String(error)}`);
|
|
54
|
+
}
|
|
55
|
+
return [
|
|
56
|
+
"Breakout",
|
|
57
|
+
"MaStrategy",
|
|
58
|
+
"AdaptiveMomentumRibbon",
|
|
59
|
+
"TrendLine",
|
|
60
|
+
"VolumeDivergence"
|
|
61
|
+
];
|
|
62
|
+
};
|
|
63
|
+
selectStrategy = async (promptLabel = "Select strategy", options) => {
|
|
64
|
+
const strategies = options?.strategies && options.strategies.length ? options.strategies : await getStrategyChoices();
|
|
65
|
+
const preferredDefault = options?.defaultStrategy || defaultStrategy;
|
|
66
|
+
const fallbackStrategy = strategies.includes(preferredDefault) ? preferredDefault : strategies[0];
|
|
67
|
+
if (!process.stdin.isTTY) {
|
|
68
|
+
return fallbackStrategy;
|
|
69
|
+
}
|
|
70
|
+
console.log(import_chalk.default.cyan("Available strategies:"));
|
|
71
|
+
strategies.forEach((name, index) => {
|
|
72
|
+
const isDefault = name === defaultStrategy;
|
|
73
|
+
const label = isDefault ? import_chalk.default.green(name) : name;
|
|
74
|
+
const suffix = isDefault ? import_chalk.default.gray(" (default)") : "";
|
|
75
|
+
console.log(` ${import_chalk.default.yellow(String(index + 1))}) ${label}${suffix}`);
|
|
76
|
+
});
|
|
77
|
+
const rl = import_readline.default.createInterface({
|
|
78
|
+
input: process.stdin,
|
|
79
|
+
output: process.stdout
|
|
80
|
+
});
|
|
81
|
+
const question = (text) => new Promise((resolve) => rl.question(text, resolve));
|
|
82
|
+
const answer = await question(
|
|
83
|
+
`${promptLabel} [${import_chalk.default.green(fallbackStrategy)}]: `
|
|
84
|
+
);
|
|
85
|
+
rl.close();
|
|
86
|
+
const trimmed = answer.trim();
|
|
87
|
+
if (!trimmed) {
|
|
88
|
+
return fallbackStrategy;
|
|
89
|
+
}
|
|
90
|
+
const asNumber = Number(trimmed);
|
|
91
|
+
if (Number.isFinite(asNumber) && asNumber >= 1 && asNumber <= strategies.length) {
|
|
92
|
+
return strategies[asNumber - 1];
|
|
93
|
+
}
|
|
94
|
+
const byName = strategies.find(
|
|
95
|
+
(name) => name.toLowerCase() === trimmed.toLowerCase()
|
|
96
|
+
);
|
|
97
|
+
if (byName) {
|
|
98
|
+
return byName;
|
|
99
|
+
}
|
|
100
|
+
console.warn(`Unknown strategy "${trimmed}", using ${fallbackStrategy}.`);
|
|
101
|
+
return fallbackStrategy;
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// src/scripts/resolveExportStrategy.ts
|
|
107
|
+
var resolveExportStrategy;
|
|
108
|
+
var init_resolveExportStrategy = __esm({
|
|
109
|
+
"src/scripts/resolveExportStrategy.ts"() {
|
|
110
|
+
"use strict";
|
|
111
|
+
init_selectStrategy();
|
|
112
|
+
resolveExportStrategy = async ({
|
|
113
|
+
explicitStrategy,
|
|
114
|
+
outDir,
|
|
115
|
+
datasetLabel,
|
|
116
|
+
promptLabel,
|
|
117
|
+
listStrategies
|
|
118
|
+
}) => {
|
|
119
|
+
const raw = String(explicitStrategy || "").trim();
|
|
120
|
+
if (raw) {
|
|
121
|
+
return raw;
|
|
122
|
+
}
|
|
123
|
+
const availableStrategies = await listStrategies({ outDir });
|
|
124
|
+
if (!availableStrategies.length) {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
if (availableStrategies.length === 1) {
|
|
128
|
+
return availableStrategies[0];
|
|
129
|
+
}
|
|
130
|
+
if (!process.stdin.isTTY) {
|
|
131
|
+
throw new Error(
|
|
132
|
+
`Multiple ${datasetLabel} chunk strategies found in ${outDir}: ${availableStrategies.join(", ")}. Pass --strategy.`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
return selectStrategy(promptLabel, {
|
|
136
|
+
strategies: availableStrategies,
|
|
137
|
+
defaultStrategy: availableStrategies[0]
|
|
138
|
+
});
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// src/scripts/aiExport.ts
|
|
144
|
+
var aiExport_exports = {};
|
|
145
|
+
var import_args, import_chalk2, import_promises, import_path, import_ai, flags, main;
|
|
146
|
+
var init_aiExport = __esm({
|
|
147
|
+
"src/scripts/aiExport.ts"() {
|
|
148
|
+
"use strict";
|
|
149
|
+
import_args = __toESM(require("args"));
|
|
150
|
+
import_chalk2 = __toESM(require("chalk"));
|
|
151
|
+
import_promises = __toESM(require("fs/promises"));
|
|
152
|
+
import_path = __toESM(require("path"));
|
|
153
|
+
import_ai = require("@tradejs/infra/ai");
|
|
154
|
+
init_resolveExportStrategy();
|
|
155
|
+
import_args.default.example(
|
|
156
|
+
"yarn ts-node ./src/scripts/aiExport --strategy trendline",
|
|
157
|
+
"Merge per-worker AI prompt chunks into one dataset JSONL"
|
|
158
|
+
);
|
|
159
|
+
import_args.default.option(["o", "outDir"], "Dataset directory", "data/ai/export");
|
|
160
|
+
import_args.default.option(["s", "strategy"], "Strategy name", "");
|
|
161
|
+
import_args.default.option(
|
|
162
|
+
["k", "keepChunks"],
|
|
163
|
+
"Keep source chunk files after successful merge",
|
|
164
|
+
false
|
|
165
|
+
);
|
|
166
|
+
flags = import_args.default.parse(process.argv);
|
|
167
|
+
main = async () => {
|
|
168
|
+
const outDir = String(flags.outDir || "data/ai/export");
|
|
169
|
+
await import_promises.default.mkdir(outDir, { recursive: true });
|
|
170
|
+
const strategyName = await resolveExportStrategy({
|
|
171
|
+
explicitStrategy: String(flags.strategy || ""),
|
|
172
|
+
outDir,
|
|
173
|
+
datasetLabel: "AI",
|
|
174
|
+
promptLabel: "Select AI export strategy",
|
|
175
|
+
listStrategies: import_ai.listAiChunkStrategies
|
|
176
|
+
});
|
|
177
|
+
if (!strategyName) {
|
|
178
|
+
console.log(import_chalk2.default.yellow(`No AI chunk files found in ${outDir}`));
|
|
179
|
+
process.exit(0);
|
|
180
|
+
}
|
|
181
|
+
const chunkFiles = await (0, import_ai.listAiChunkFiles)({
|
|
182
|
+
strategyName,
|
|
183
|
+
outDir
|
|
184
|
+
});
|
|
185
|
+
if (!chunkFiles.length) {
|
|
186
|
+
console.log(
|
|
187
|
+
import_chalk2.default.yellow(
|
|
188
|
+
`No chunk files found for strategy "${strategyName}" in ${outDir}`
|
|
189
|
+
)
|
|
190
|
+
);
|
|
191
|
+
process.exit(0);
|
|
192
|
+
}
|
|
193
|
+
const mergedPath = import_path.default.join(
|
|
194
|
+
outDir,
|
|
195
|
+
`ai-dataset-${(0, import_ai.toFileToken)(strategyName)}-merged-${Date.now()}.jsonl`
|
|
196
|
+
);
|
|
197
|
+
await (0, import_ai.mergeAiJsonlFiles)({
|
|
198
|
+
filePaths: chunkFiles,
|
|
199
|
+
outPath: mergedPath
|
|
200
|
+
});
|
|
201
|
+
const shouldDeleteChunks = !Boolean(flags.keepChunks);
|
|
202
|
+
if (shouldDeleteChunks) {
|
|
203
|
+
for (const filePath of chunkFiles) {
|
|
204
|
+
await import_promises.default.rm(filePath, { force: true });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
console.log(import_chalk2.default.green(`Merged AI dataset saved: ${mergedPath}`));
|
|
208
|
+
console.log(
|
|
209
|
+
import_chalk2.default.gray(
|
|
210
|
+
`strategy=${strategyName}, source_chunks=${chunkFiles.length}, deleteChunks=${Boolean(
|
|
211
|
+
shouldDeleteChunks
|
|
212
|
+
)}`
|
|
213
|
+
)
|
|
214
|
+
);
|
|
215
|
+
process.exit(0);
|
|
216
|
+
};
|
|
217
|
+
main().catch((error) => {
|
|
218
|
+
console.error(import_chalk2.default.red(error?.message || String(error)));
|
|
219
|
+
process.exit(1);
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
// src/lib/aiTrainMetrics.ts
|
|
225
|
+
var divideOrNull, qualitySortKey, summarizeAiTrainEvaluations, getDirectionSortKey, summarizeAiTrainEvaluationsByDirection;
|
|
226
|
+
var init_aiTrainMetrics = __esm({
|
|
227
|
+
"src/lib/aiTrainMetrics.ts"() {
|
|
228
|
+
"use strict";
|
|
229
|
+
divideOrNull = (num, denom) => {
|
|
230
|
+
if (denom <= 0) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
return num / denom;
|
|
234
|
+
};
|
|
235
|
+
qualitySortKey = (quality) => quality == null ? Number.POSITIVE_INFINITY : quality;
|
|
236
|
+
summarizeAiTrainEvaluations = (evaluations) => {
|
|
237
|
+
let approved = 0;
|
|
238
|
+
let rejected = 0;
|
|
239
|
+
let correct = 0;
|
|
240
|
+
let incorrect = 0;
|
|
241
|
+
let profitable = 0;
|
|
242
|
+
let unprofitable = 0;
|
|
243
|
+
let flat = 0;
|
|
244
|
+
let truePositive = 0;
|
|
245
|
+
let falsePositive = 0;
|
|
246
|
+
let trueNegative = 0;
|
|
247
|
+
let falseNegative = 0;
|
|
248
|
+
let totalProfitAll = 0;
|
|
249
|
+
let totalProfitApproved = 0;
|
|
250
|
+
const bucketMap = /* @__PURE__ */ new Map();
|
|
251
|
+
for (const evaluation of evaluations) {
|
|
252
|
+
const { profit, profitableTrade, aiApproved, quality } = evaluation;
|
|
253
|
+
totalProfitAll += profit;
|
|
254
|
+
if (profitableTrade) {
|
|
255
|
+
profitable += 1;
|
|
256
|
+
} else if (profit < 0) {
|
|
257
|
+
unprofitable += 1;
|
|
258
|
+
} else {
|
|
259
|
+
flat += 1;
|
|
260
|
+
}
|
|
261
|
+
if (aiApproved) {
|
|
262
|
+
approved += 1;
|
|
263
|
+
totalProfitApproved += profit;
|
|
264
|
+
} else {
|
|
265
|
+
rejected += 1;
|
|
266
|
+
}
|
|
267
|
+
const isCorrect = aiApproved === profitableTrade;
|
|
268
|
+
if (isCorrect) {
|
|
269
|
+
correct += 1;
|
|
270
|
+
} else {
|
|
271
|
+
incorrect += 1;
|
|
272
|
+
}
|
|
273
|
+
if (aiApproved && profitableTrade) {
|
|
274
|
+
truePositive += 1;
|
|
275
|
+
} else if (aiApproved && !profitableTrade) {
|
|
276
|
+
falsePositive += 1;
|
|
277
|
+
} else if (!aiApproved && profitableTrade) {
|
|
278
|
+
falseNegative += 1;
|
|
279
|
+
} else {
|
|
280
|
+
trueNegative += 1;
|
|
281
|
+
}
|
|
282
|
+
const bucket = bucketMap.get(quality) ?? {
|
|
283
|
+
quality,
|
|
284
|
+
count: 0,
|
|
285
|
+
approved: 0,
|
|
286
|
+
profitable: 0,
|
|
287
|
+
totalProfit: 0
|
|
288
|
+
};
|
|
289
|
+
bucket.count += 1;
|
|
290
|
+
bucket.totalProfit += profit;
|
|
291
|
+
if (aiApproved) {
|
|
292
|
+
bucket.approved += 1;
|
|
293
|
+
}
|
|
294
|
+
if (profitableTrade) {
|
|
295
|
+
bucket.profitable += 1;
|
|
296
|
+
}
|
|
297
|
+
bucketMap.set(quality, bucket);
|
|
298
|
+
}
|
|
299
|
+
const avgProfitAll = divideOrNull(totalProfitAll, evaluations.length);
|
|
300
|
+
const avgProfitApproved = divideOrNull(totalProfitApproved, approved);
|
|
301
|
+
const expectancyDelta = avgProfitAll == null || avgProfitApproved == null ? null : avgProfitApproved - avgProfitAll;
|
|
302
|
+
return {
|
|
303
|
+
approved,
|
|
304
|
+
rejected,
|
|
305
|
+
correct,
|
|
306
|
+
incorrect,
|
|
307
|
+
truePositive,
|
|
308
|
+
falsePositive,
|
|
309
|
+
trueNegative,
|
|
310
|
+
falseNegative,
|
|
311
|
+
profitable,
|
|
312
|
+
unprofitable,
|
|
313
|
+
flat,
|
|
314
|
+
precisionApproved: divideOrNull(truePositive, approved),
|
|
315
|
+
recallWinners: divideOrNull(truePositive, profitable),
|
|
316
|
+
avgProfitAll,
|
|
317
|
+
avgProfitApproved,
|
|
318
|
+
expectancyDelta,
|
|
319
|
+
qualityBuckets: [...bucketMap.values()].sort(
|
|
320
|
+
(a, b) => qualitySortKey(a.quality) - qualitySortKey(b.quality)
|
|
321
|
+
)
|
|
322
|
+
};
|
|
323
|
+
};
|
|
324
|
+
getDirectionSortKey = (direction) => {
|
|
325
|
+
if (direction === "LONG") {
|
|
326
|
+
return 0;
|
|
327
|
+
}
|
|
328
|
+
if (direction === "SHORT") {
|
|
329
|
+
return 1;
|
|
330
|
+
}
|
|
331
|
+
return 2;
|
|
332
|
+
};
|
|
333
|
+
summarizeAiTrainEvaluationsByDirection = (evaluations) => {
|
|
334
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
335
|
+
for (const evaluation of evaluations) {
|
|
336
|
+
const direction = typeof evaluation.direction === "string" && evaluation.direction.trim() ? evaluation.direction : "UNKNOWN";
|
|
337
|
+
const bucket = grouped.get(direction) ?? [];
|
|
338
|
+
bucket.push(evaluation);
|
|
339
|
+
grouped.set(direction, bucket);
|
|
340
|
+
}
|
|
341
|
+
return [...grouped.entries()].sort(
|
|
342
|
+
([leftDirection], [rightDirection]) => getDirectionSortKey(leftDirection) - getDirectionSortKey(rightDirection) || leftDirection.localeCompare(rightDirection)
|
|
343
|
+
).map(([direction, directionEvaluations]) => ({
|
|
344
|
+
direction,
|
|
345
|
+
summary: summarizeAiTrainEvaluations(directionEvaluations)
|
|
346
|
+
}));
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
|
|
36
351
|
// ../../node_modules/debug/node_modules/ms/index.js
|
|
37
352
|
var require_ms = __commonJS({
|
|
38
353
|
"../../node_modules/debug/node_modules/ms/index.js"(exports2, module2) {
|
|
@@ -183,7 +498,7 @@ var require_common = __commonJS({
|
|
|
183
498
|
let enableOverride = null;
|
|
184
499
|
let namespacesCache;
|
|
185
500
|
let enabledCache;
|
|
186
|
-
function debug(...
|
|
501
|
+
function debug(...args17) {
|
|
187
502
|
if (!debug.enabled) {
|
|
188
503
|
return;
|
|
189
504
|
}
|
|
@@ -194,28 +509,28 @@ var require_common = __commonJS({
|
|
|
194
509
|
self.prev = prevTime;
|
|
195
510
|
self.curr = curr;
|
|
196
511
|
prevTime = curr;
|
|
197
|
-
|
|
198
|
-
if (typeof
|
|
199
|
-
|
|
512
|
+
args17[0] = createDebug.coerce(args17[0]);
|
|
513
|
+
if (typeof args17[0] !== "string") {
|
|
514
|
+
args17.unshift("%O");
|
|
200
515
|
}
|
|
201
516
|
let index = 0;
|
|
202
|
-
|
|
517
|
+
args17[0] = args17[0].replace(/%([a-zA-Z%])/g, (match, format2) => {
|
|
203
518
|
if (match === "%%") {
|
|
204
519
|
return "%";
|
|
205
520
|
}
|
|
206
521
|
index++;
|
|
207
522
|
const formatter = createDebug.formatters[format2];
|
|
208
523
|
if (typeof formatter === "function") {
|
|
209
|
-
const val =
|
|
524
|
+
const val = args17[index];
|
|
210
525
|
match = formatter.call(self, val);
|
|
211
|
-
|
|
526
|
+
args17.splice(index, 1);
|
|
212
527
|
index--;
|
|
213
528
|
}
|
|
214
529
|
return match;
|
|
215
530
|
});
|
|
216
|
-
createDebug.formatArgs.call(self,
|
|
531
|
+
createDebug.formatArgs.call(self, args17);
|
|
217
532
|
const logFn = self.log || createDebug.log;
|
|
218
|
-
logFn.apply(self,
|
|
533
|
+
logFn.apply(self, args17);
|
|
219
534
|
}
|
|
220
535
|
debug.namespace = namespace;
|
|
221
536
|
debug.useColors = createDebug.useColors();
|
|
@@ -423,16 +738,16 @@ var require_browser = __commonJS({
|
|
|
423
738
|
typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/) && parseInt(RegExp.$1, 10) >= 31 || // Double check webkit in userAgent just in case we are in a worker
|
|
424
739
|
typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/applewebkit\/(\d+)/);
|
|
425
740
|
}
|
|
426
|
-
function formatArgs(
|
|
427
|
-
|
|
741
|
+
function formatArgs(args17) {
|
|
742
|
+
args17[0] = (this.useColors ? "%c" : "") + this.namespace + (this.useColors ? " %c" : " ") + args17[0] + (this.useColors ? "%c " : " ") + "+" + module2.exports.humanize(this.diff);
|
|
428
743
|
if (!this.useColors) {
|
|
429
744
|
return;
|
|
430
745
|
}
|
|
431
746
|
const c = "color: " + this.color;
|
|
432
|
-
|
|
747
|
+
args17.splice(1, 0, c, "color: inherit");
|
|
433
748
|
let index = 0;
|
|
434
749
|
let lastC = 0;
|
|
435
|
-
|
|
750
|
+
args17[0].replace(/%[a-zA-Z%]/g, (match) => {
|
|
436
751
|
if (match === "%%") {
|
|
437
752
|
return;
|
|
438
753
|
}
|
|
@@ -441,7 +756,7 @@ var require_browser = __commonJS({
|
|
|
441
756
|
lastC = index;
|
|
442
757
|
}
|
|
443
758
|
});
|
|
444
|
-
|
|
759
|
+
args17.splice(lastC, 0, c);
|
|
445
760
|
}
|
|
446
761
|
exports2.log = console.debug || console.log || (() => {
|
|
447
762
|
});
|
|
@@ -723,16 +1038,16 @@ var require_node = __commonJS({
|
|
|
723
1038
|
function useColors() {
|
|
724
1039
|
return "colors" in exports2.inspectOpts ? Boolean(exports2.inspectOpts.colors) : tty.isatty(process.stderr.fd);
|
|
725
1040
|
}
|
|
726
|
-
function formatArgs(
|
|
1041
|
+
function formatArgs(args17) {
|
|
727
1042
|
const { namespace: name, useColors: useColors2 } = this;
|
|
728
1043
|
if (useColors2) {
|
|
729
1044
|
const c = this.color;
|
|
730
1045
|
const colorCode = "\x1B[3" + (c < 8 ? c : "8;5;" + c);
|
|
731
1046
|
const prefix = ` ${colorCode};1m${name} \x1B[0m`;
|
|
732
|
-
|
|
733
|
-
|
|
1047
|
+
args17[0] = prefix + args17[0].split("\n").join("\n" + prefix);
|
|
1048
|
+
args17.push(colorCode + "m+" + module2.exports.humanize(this.diff) + "\x1B[0m");
|
|
734
1049
|
} else {
|
|
735
|
-
|
|
1050
|
+
args17[0] = getDate() + name + " " + args17[0];
|
|
736
1051
|
}
|
|
737
1052
|
}
|
|
738
1053
|
function getDate() {
|
|
@@ -741,8 +1056,8 @@ var require_node = __commonJS({
|
|
|
741
1056
|
}
|
|
742
1057
|
return (/* @__PURE__ */ new Date()).toISOString() + " ";
|
|
743
1058
|
}
|
|
744
|
-
function log(...
|
|
745
|
-
return process.stderr.write(util.format(...
|
|
1059
|
+
function log(...args17) {
|
|
1060
|
+
return process.stderr.write(util.format(...args17) + "\n");
|
|
746
1061
|
}
|
|
747
1062
|
function save(namespaces) {
|
|
748
1063
|
if (namespaces) {
|
|
@@ -1602,34 +1917,548 @@ var require_list_it = __commonJS({
|
|
|
1602
1917
|
"../../node_modules/list-it/index.js"(exports2, module2) {
|
|
1603
1918
|
"use strict";
|
|
1604
1919
|
var ListItBuffer = require_list_it_buffer();
|
|
1605
|
-
function
|
|
1920
|
+
function ListIt4(opt) {
|
|
1606
1921
|
ListItBuffer.call(this, opt);
|
|
1607
1922
|
}
|
|
1608
|
-
|
|
1609
|
-
|
|
1923
|
+
ListIt4.prototype = new ListItBuffer();
|
|
1924
|
+
ListIt4.buffer = function(opt) {
|
|
1610
1925
|
opt = opt || {};
|
|
1611
1926
|
if (!("autoAlign" in opt)) {
|
|
1612
1927
|
opt.autoAlign = false;
|
|
1613
1928
|
}
|
|
1614
|
-
return new
|
|
1929
|
+
return new ListIt4(opt);
|
|
1930
|
+
};
|
|
1931
|
+
module2.exports = ListIt4;
|
|
1932
|
+
}
|
|
1933
|
+
});
|
|
1934
|
+
|
|
1935
|
+
// src/scripts/aiTrain.ts
|
|
1936
|
+
var aiTrain_exports = {};
|
|
1937
|
+
var import_args2, import_chalk3, import_promises2, import_path2, import_progress, import_async, import_ai2, import_ai3, import_constants, ListIt, flags2, createListIt, createTable, formatRatio, formatProfit, colorizeRatio, colorizePercent, colorizeProfit, colorizeQuality, normalizeInt, normalizePositiveInt, normalizeQuality, isProfitableTrade, isAiApproval, listMergedFiles, deriveStrategyNameFromFile, extractSignalFromDatasetRow, resolvePromptRunContext, getDeterministicGateContext, resolveDeterministicGateEvaluation, summarizeDeterministicGateEvaluations, printSection, resolveDatasetFile, main2;
|
|
1938
|
+
var init_aiTrain = __esm({
|
|
1939
|
+
"src/scripts/aiTrain.ts"() {
|
|
1940
|
+
"use strict";
|
|
1941
|
+
import_args2 = __toESM(require("args"));
|
|
1942
|
+
import_chalk3 = __toESM(require("chalk"));
|
|
1943
|
+
import_promises2 = __toESM(require("fs/promises"));
|
|
1944
|
+
import_path2 = __toESM(require("path"));
|
|
1945
|
+
import_progress = __toESM(require("progress"));
|
|
1946
|
+
import_async = require("@tradejs/core/async");
|
|
1947
|
+
import_ai2 = require("@tradejs/infra/ai");
|
|
1948
|
+
import_ai3 = require("@tradejs/node/ai");
|
|
1949
|
+
import_constants = require("@tradejs/node/constants");
|
|
1950
|
+
init_aiTrainMetrics();
|
|
1951
|
+
ListIt = require_list_it();
|
|
1952
|
+
import_args2.default.example(
|
|
1953
|
+
"yarn ai-train -n 50 --minQuality 4",
|
|
1954
|
+
"Replay latest AI prompt dataset and measure approval accuracy"
|
|
1955
|
+
);
|
|
1956
|
+
import_args2.default.option(["o", "outDir"], "Dataset directory", "data/ai/export");
|
|
1957
|
+
import_args2.default.option(["s", "strategy"], "Strategy name filter for merged file", "");
|
|
1958
|
+
import_args2.default.option(["f", "file"], "Explicit merged dataset file path", "");
|
|
1959
|
+
import_args2.default.option(
|
|
1960
|
+
["n", "recent"],
|
|
1961
|
+
"How many recent trades to evaluate from the end (0 = all)",
|
|
1962
|
+
50
|
|
1963
|
+
);
|
|
1964
|
+
import_args2.default.option(
|
|
1965
|
+
"skip",
|
|
1966
|
+
"How many recent trades to skip from the end before selecting replay rows",
|
|
1967
|
+
0
|
|
1968
|
+
);
|
|
1969
|
+
import_args2.default.option(
|
|
1970
|
+
["p", "parallel"],
|
|
1971
|
+
"Concurrent AI requests during replay",
|
|
1972
|
+
import_constants.AI_CONCURRENCY_LIMIT
|
|
1973
|
+
);
|
|
1974
|
+
import_args2.default.option(
|
|
1975
|
+
"model",
|
|
1976
|
+
"OpenRouter model id override for this replay run",
|
|
1977
|
+
import_ai3.DEFAULT_AI_MODEL
|
|
1978
|
+
);
|
|
1979
|
+
import_args2.default.option("minQuality", "Minimum AI quality required to approve entry", 4);
|
|
1980
|
+
import_args2.default.option(
|
|
1981
|
+
"localOnly",
|
|
1982
|
+
"Replay deterministic adapter gate without AI provider calls",
|
|
1983
|
+
false
|
|
1984
|
+
);
|
|
1985
|
+
flags2 = import_args2.default.parse(process.argv);
|
|
1986
|
+
createListIt = () => new ListIt({
|
|
1987
|
+
autoAlign: true,
|
|
1988
|
+
headerUnderline: true
|
|
1989
|
+
});
|
|
1990
|
+
createTable = (headers, rows) => createListIt().setHeaderRow(headers).d(rows).toString();
|
|
1991
|
+
formatRatio = (value) => value == null ? "n/a" : `${(value * 100).toFixed(1)}%`;
|
|
1992
|
+
formatProfit = (value) => value == null ? "n/a" : value.toFixed(2);
|
|
1993
|
+
colorizeRatio = (value) => {
|
|
1994
|
+
const text = formatRatio(value);
|
|
1995
|
+
if (value == null) {
|
|
1996
|
+
return import_chalk3.default.gray(text);
|
|
1997
|
+
}
|
|
1998
|
+
if (value >= 0.6) {
|
|
1999
|
+
return import_chalk3.default.green(text);
|
|
2000
|
+
}
|
|
2001
|
+
if (value >= 0.3) {
|
|
2002
|
+
return import_chalk3.default.yellow(text);
|
|
2003
|
+
}
|
|
2004
|
+
return import_chalk3.default.red(text);
|
|
2005
|
+
};
|
|
2006
|
+
colorizePercent = (value, total) => {
|
|
2007
|
+
const ratio = total > 0 ? value / total : null;
|
|
2008
|
+
return colorizeRatio(ratio);
|
|
2009
|
+
};
|
|
2010
|
+
colorizeProfit = (value) => {
|
|
2011
|
+
const text = formatProfit(value);
|
|
2012
|
+
if (value == null) {
|
|
2013
|
+
return import_chalk3.default.gray(text);
|
|
2014
|
+
}
|
|
2015
|
+
if (value > 0) {
|
|
2016
|
+
return import_chalk3.default.green(text);
|
|
2017
|
+
}
|
|
2018
|
+
if (value < 0) {
|
|
2019
|
+
return import_chalk3.default.red(text);
|
|
2020
|
+
}
|
|
2021
|
+
return import_chalk3.default.yellow(text);
|
|
2022
|
+
};
|
|
2023
|
+
colorizeQuality = (quality) => {
|
|
2024
|
+
const text = quality == null ? "n/a" : String(quality);
|
|
2025
|
+
if (quality == null) {
|
|
2026
|
+
return import_chalk3.default.gray(text);
|
|
2027
|
+
}
|
|
2028
|
+
if (quality >= 4) {
|
|
2029
|
+
return import_chalk3.default.green(text);
|
|
2030
|
+
}
|
|
2031
|
+
if (quality === 3) {
|
|
2032
|
+
return import_chalk3.default.yellow(text);
|
|
2033
|
+
}
|
|
2034
|
+
return import_chalk3.default.red(text);
|
|
2035
|
+
};
|
|
2036
|
+
normalizeInt = (value, fallback) => {
|
|
2037
|
+
const parsed = Number(value);
|
|
2038
|
+
return Number.isFinite(parsed) ? Math.max(0, Math.trunc(parsed)) : fallback;
|
|
2039
|
+
};
|
|
2040
|
+
normalizePositiveInt = (value, fallback) => {
|
|
2041
|
+
const parsed = Number(value);
|
|
2042
|
+
return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : fallback;
|
|
2043
|
+
};
|
|
2044
|
+
normalizeQuality = (analysis) => {
|
|
2045
|
+
const quality = Number(analysis?.quality);
|
|
2046
|
+
return Number.isFinite(quality) ? Math.round(quality) : null;
|
|
2047
|
+
};
|
|
2048
|
+
isProfitableTrade = (row) => Number(row.profit) > 0;
|
|
2049
|
+
isAiApproval = (row, analysis, minQuality) => {
|
|
2050
|
+
const quality = normalizeQuality(analysis);
|
|
2051
|
+
return analysis.direction === row.direction && quality != null && quality >= minQuality;
|
|
2052
|
+
};
|
|
2053
|
+
listMergedFiles = async (params) => {
|
|
2054
|
+
const { outDir, strategyName } = params;
|
|
2055
|
+
let entries = [];
|
|
2056
|
+
try {
|
|
2057
|
+
entries = await import_promises2.default.readdir(outDir);
|
|
2058
|
+
} catch {
|
|
2059
|
+
return [];
|
|
2060
|
+
}
|
|
2061
|
+
const prefix = strategyName ? `ai-dataset-${(0, import_ai2.toFileToken)(strategyName)}-merged-` : "ai-dataset-";
|
|
2062
|
+
return entries.filter((name) => name.startsWith(prefix)).filter((name) => name.includes("-merged-") && name.endsWith(".jsonl")).sort().map((name) => import_path2.default.join(outDir, name));
|
|
2063
|
+
};
|
|
2064
|
+
deriveStrategyNameFromFile = (filePath) => {
|
|
2065
|
+
const match = import_path2.default.basename(filePath).match(/^ai-dataset-(.+)-merged-\d+\.jsonl$/);
|
|
2066
|
+
return match?.[1] ? match[1] : "unknown";
|
|
2067
|
+
};
|
|
2068
|
+
extractSignalFromDatasetRow = (row) => {
|
|
2069
|
+
const { payload } = row;
|
|
2070
|
+
return {
|
|
2071
|
+
...payload.signal,
|
|
2072
|
+
strategy: payload.signal.strategy,
|
|
2073
|
+
figures: payload.figures ?? {},
|
|
2074
|
+
indicators: payload.indicators ?? {},
|
|
2075
|
+
additionalIndicators: payload.additionalIndicators ?? {},
|
|
2076
|
+
prices: payload.signal.prices
|
|
2077
|
+
};
|
|
2078
|
+
};
|
|
2079
|
+
resolvePromptRunContext = (row) => {
|
|
2080
|
+
const signal = extractSignalFromDatasetRow(row);
|
|
2081
|
+
return {
|
|
2082
|
+
signal,
|
|
2083
|
+
promptPair: (0, import_ai3.buildAiPrompts)(signal)
|
|
2084
|
+
};
|
|
2085
|
+
};
|
|
2086
|
+
getDeterministicGateContext = (signal) => {
|
|
2087
|
+
const payload = (0, import_ai3.buildAiPayload)(signal);
|
|
2088
|
+
return (0, import_ai3.getDeterministicAiGateContext)(payload);
|
|
2089
|
+
};
|
|
2090
|
+
resolveDeterministicGateEvaluation = (signal, fallbackDirection) => {
|
|
2091
|
+
if (!signal) {
|
|
2092
|
+
return {
|
|
2093
|
+
direction: fallbackDirection || "UNKNOWN",
|
|
2094
|
+
signalAvailable: false,
|
|
2095
|
+
coreBlocked: false,
|
|
2096
|
+
adapterBlocked: false,
|
|
2097
|
+
modelCandidate: false
|
|
2098
|
+
};
|
|
2099
|
+
}
|
|
2100
|
+
const gateContext = getDeterministicGateContext(signal);
|
|
2101
|
+
const structuralHardBlockReasons = Array.isArray(
|
|
2102
|
+
gateContext?.structuralHardBlockReasons
|
|
2103
|
+
) ? gateContext.structuralHardBlockReasons.filter(
|
|
2104
|
+
(reason) => typeof reason === "string" && reason.trim().length > 0
|
|
2105
|
+
) : [];
|
|
2106
|
+
const coreBlocked = structuralHardBlockReasons.length > 0;
|
|
2107
|
+
const approvalAllowedNow = typeof gateContext?.approvalAllowedNow === "boolean" ? gateContext.approvalAllowedNow : null;
|
|
2108
|
+
const adapterBlocked = !coreBlocked && approvalAllowedNow === false;
|
|
2109
|
+
return {
|
|
2110
|
+
direction: typeof signal.direction === "string" && signal.direction.trim() ? signal.direction : fallbackDirection || "UNKNOWN",
|
|
2111
|
+
signalAvailable: true,
|
|
2112
|
+
coreBlocked,
|
|
2113
|
+
adapterBlocked,
|
|
2114
|
+
modelCandidate: !coreBlocked && !adapterBlocked
|
|
2115
|
+
};
|
|
2116
|
+
};
|
|
2117
|
+
summarizeDeterministicGateEvaluations = (deterministicEvaluations, evaluations) => {
|
|
2118
|
+
const summary = {
|
|
2119
|
+
selected: deterministicEvaluations.length,
|
|
2120
|
+
signalAvailable: 0,
|
|
2121
|
+
signalMissing: 0,
|
|
2122
|
+
coreBlocked: 0,
|
|
2123
|
+
adapterBlocked: 0,
|
|
2124
|
+
modelCandidate: 0,
|
|
2125
|
+
modelApproved: 0,
|
|
2126
|
+
modelRejected: 0,
|
|
2127
|
+
modelFailed: 0
|
|
2128
|
+
};
|
|
2129
|
+
for (const evaluation of deterministicEvaluations) {
|
|
2130
|
+
if (evaluation.signalAvailable) {
|
|
2131
|
+
summary.signalAvailable += 1;
|
|
2132
|
+
} else {
|
|
2133
|
+
summary.signalMissing += 1;
|
|
2134
|
+
}
|
|
2135
|
+
if (evaluation.coreBlocked) {
|
|
2136
|
+
summary.coreBlocked += 1;
|
|
2137
|
+
} else if (evaluation.adapterBlocked) {
|
|
2138
|
+
summary.adapterBlocked += 1;
|
|
2139
|
+
} else if (evaluation.modelCandidate) {
|
|
2140
|
+
summary.modelCandidate += 1;
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
for (const evaluation of evaluations) {
|
|
2144
|
+
if (!evaluation.modelCandidate) {
|
|
2145
|
+
continue;
|
|
2146
|
+
}
|
|
2147
|
+
if (evaluation.aiApproved) {
|
|
2148
|
+
summary.modelApproved += 1;
|
|
2149
|
+
} else {
|
|
2150
|
+
summary.modelRejected += 1;
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
summary.modelFailed = summary.modelCandidate - summary.modelApproved - summary.modelRejected;
|
|
2154
|
+
return summary;
|
|
1615
2155
|
};
|
|
1616
|
-
|
|
2156
|
+
printSection = (title, table) => {
|
|
2157
|
+
console.log(import_chalk3.default.gray(`${title}:`));
|
|
2158
|
+
console.log(table);
|
|
2159
|
+
console.log("");
|
|
2160
|
+
};
|
|
2161
|
+
resolveDatasetFile = async () => {
|
|
2162
|
+
const explicitFile = String(flags2.file || "").trim();
|
|
2163
|
+
if (explicitFile) {
|
|
2164
|
+
return import_path2.default.resolve(explicitFile);
|
|
2165
|
+
}
|
|
2166
|
+
const outDir = String(flags2.outDir || "data/ai/export");
|
|
2167
|
+
const strategyName = String(flags2.strategy || "").trim() || void 0;
|
|
2168
|
+
const mergedFiles = await listMergedFiles({
|
|
2169
|
+
outDir,
|
|
2170
|
+
strategyName
|
|
2171
|
+
});
|
|
2172
|
+
if (!mergedFiles.length) {
|
|
2173
|
+
throw new Error(
|
|
2174
|
+
strategyName ? `No merged AI dataset found for strategy "${strategyName}" in ${outDir}. Run yarn ai-export first.` : `No merged AI dataset found in ${outDir}. Run yarn ai-export first.`
|
|
2175
|
+
);
|
|
2176
|
+
}
|
|
2177
|
+
return mergedFiles[mergedFiles.length - 1];
|
|
2178
|
+
};
|
|
2179
|
+
main2 = async () => {
|
|
2180
|
+
const recent = normalizeInt(flags2.recent, 50);
|
|
2181
|
+
const skip = normalizeInt(flags2.skip, 0);
|
|
2182
|
+
const minQuality = normalizeInt(flags2.minQuality, 4);
|
|
2183
|
+
const localOnly = Boolean(flags2.localOnly);
|
|
2184
|
+
const model = String(flags2.model || import_ai3.DEFAULT_AI_MODEL).trim() || import_ai3.DEFAULT_AI_MODEL;
|
|
2185
|
+
const parallel = normalizePositiveInt(flags2.parallel, import_constants.AI_CONCURRENCY_LIMIT);
|
|
2186
|
+
await (0, import_ai3.ensureAiStrategyPluginsLoaded)();
|
|
2187
|
+
const filePath = await resolveDatasetFile();
|
|
2188
|
+
const { rows, totalRows } = await (0, import_ai2.readAiDatasetRows)({
|
|
2189
|
+
filePath,
|
|
2190
|
+
limitFromEnd: recent,
|
|
2191
|
+
skipFromEnd: skip
|
|
2192
|
+
});
|
|
2193
|
+
if (!rows.length) {
|
|
2194
|
+
console.log(
|
|
2195
|
+
import_chalk3.default.yellow(
|
|
2196
|
+
`No AI prompt rows selected in ${filePath} (recent=${recent || "all"}, skip=${skip})`
|
|
2197
|
+
)
|
|
2198
|
+
);
|
|
2199
|
+
process.exit(0);
|
|
2200
|
+
}
|
|
2201
|
+
const strategyName = rows[0]?.strategyName || deriveStrategyNameFromFile(filePath);
|
|
2202
|
+
const preparedRows = rows.map((row) => {
|
|
2203
|
+
const { promptPair, signal } = resolvePromptRunContext(row);
|
|
2204
|
+
const payload = (0, import_ai3.buildAiPayload)(signal);
|
|
2205
|
+
return {
|
|
2206
|
+
row,
|
|
2207
|
+
promptPair,
|
|
2208
|
+
payload,
|
|
2209
|
+
signal,
|
|
2210
|
+
deterministic: resolveDeterministicGateEvaluation(signal, row.direction)
|
|
2211
|
+
};
|
|
2212
|
+
});
|
|
2213
|
+
const concurrency = Math.max(1, Math.min(parallel, rows.length));
|
|
2214
|
+
const bar = new import_progress.default(
|
|
2215
|
+
":current/:total [:bar][:percent] :symbol :status",
|
|
2216
|
+
{
|
|
2217
|
+
total: rows.length,
|
|
2218
|
+
width: 20
|
|
2219
|
+
}
|
|
2220
|
+
);
|
|
2221
|
+
let failed = 0;
|
|
2222
|
+
const errorMessages2 = [];
|
|
2223
|
+
const evaluations = [];
|
|
2224
|
+
await (0, import_async.runWithConcurrency)(preparedRows, concurrency, async (preparedRow) => {
|
|
2225
|
+
const { row, promptPair, payload, signal, deterministic } = preparedRow;
|
|
2226
|
+
const profit = Number(row.profit);
|
|
2227
|
+
const profitableTrade = isProfitableTrade(row);
|
|
2228
|
+
try {
|
|
2229
|
+
const analysis = localOnly ? await (0, import_ai3.runAiPromptLocal)(signal, { payload }) : await (0, import_ai3.runAiPrompt)(
|
|
2230
|
+
promptPair,
|
|
2231
|
+
signal ? {
|
|
2232
|
+
model,
|
|
2233
|
+
signal,
|
|
2234
|
+
payload
|
|
2235
|
+
} : { model }
|
|
2236
|
+
);
|
|
2237
|
+
const aiApproved = isAiApproval(row, analysis, minQuality);
|
|
2238
|
+
const quality = normalizeQuality(analysis);
|
|
2239
|
+
const isCorrect = aiApproved === profitableTrade;
|
|
2240
|
+
evaluations.push({
|
|
2241
|
+
profit,
|
|
2242
|
+
profitableTrade,
|
|
2243
|
+
aiApproved,
|
|
2244
|
+
quality,
|
|
2245
|
+
direction: row.direction,
|
|
2246
|
+
modelCandidate: deterministic.modelCandidate
|
|
2247
|
+
});
|
|
2248
|
+
bar.tick(1, {
|
|
2249
|
+
symbol: import_chalk3.default.gray(row.symbol),
|
|
2250
|
+
status: isCorrect ? import_chalk3.default.green("ok") : import_chalk3.default.red("miss")
|
|
2251
|
+
});
|
|
2252
|
+
} catch (error) {
|
|
2253
|
+
failed += 1;
|
|
2254
|
+
if (errorMessages2.length < 5) {
|
|
2255
|
+
errorMessages2.push(
|
|
2256
|
+
`[${row.symbol}/${row.signalId}] ${error instanceof Error ? error.message : String(error)}`
|
|
2257
|
+
);
|
|
2258
|
+
}
|
|
2259
|
+
bar.tick(1, {
|
|
2260
|
+
symbol: import_chalk3.default.gray(row.symbol),
|
|
2261
|
+
status: import_chalk3.default.yellow("error")
|
|
2262
|
+
});
|
|
2263
|
+
}
|
|
2264
|
+
});
|
|
2265
|
+
const summary = summarizeAiTrainEvaluations(evaluations);
|
|
2266
|
+
const directionSummaries = summarizeAiTrainEvaluationsByDirection(evaluations);
|
|
2267
|
+
const deterministicSummary = summarizeDeterministicGateEvaluations(
|
|
2268
|
+
preparedRows.map((preparedRow) => preparedRow.deterministic),
|
|
2269
|
+
evaluations
|
|
2270
|
+
);
|
|
2271
|
+
const evaluated = summary.correct + summary.incorrect;
|
|
2272
|
+
console.log("");
|
|
2273
|
+
console.log(import_chalk3.default.green("AI train finished"));
|
|
2274
|
+
console.log(import_chalk3.default.gray(filePath));
|
|
2275
|
+
console.log("");
|
|
2276
|
+
printSection(
|
|
2277
|
+
"RUN",
|
|
2278
|
+
createTable(
|
|
2279
|
+
[import_chalk3.default.gray("FIELD"), import_chalk3.default.gray("VALUE")],
|
|
2280
|
+
[
|
|
2281
|
+
["strategy", import_chalk3.default.yellow(strategyName)],
|
|
2282
|
+
["selected", import_chalk3.default.blue(String(rows.length))],
|
|
2283
|
+
["source_rows", import_chalk3.default.blue(String(totalRows))],
|
|
2284
|
+
["recent", import_chalk3.default.blue(recent === 0 ? "all" : String(recent))],
|
|
2285
|
+
["skip", import_chalk3.default.blue(String(skip))],
|
|
2286
|
+
["min_quality", import_chalk3.default.magenta(String(minQuality))],
|
|
2287
|
+
[
|
|
2288
|
+
"mode",
|
|
2289
|
+
localOnly ? import_chalk3.default.green("local-deterministic") : import_chalk3.default.yellow("llm")
|
|
2290
|
+
],
|
|
2291
|
+
["model", import_chalk3.default.yellow(localOnly ? "local-deterministic" : model)],
|
|
2292
|
+
["parallel", import_chalk3.default.magenta(String(concurrency))]
|
|
2293
|
+
]
|
|
2294
|
+
)
|
|
2295
|
+
);
|
|
2296
|
+
printSection(
|
|
2297
|
+
"OUTCOME",
|
|
2298
|
+
createTable(
|
|
2299
|
+
[import_chalk3.default.gray("METRIC"), import_chalk3.default.gray("VALUE")],
|
|
2300
|
+
[
|
|
2301
|
+
["accuracy", colorizePercent(summary.correct, evaluated)],
|
|
2302
|
+
["evaluated", import_chalk3.default.blue(String(evaluated))],
|
|
2303
|
+
["correct", import_chalk3.default.green(String(summary.correct))],
|
|
2304
|
+
["incorrect", import_chalk3.default.red(String(summary.incorrect))],
|
|
2305
|
+
[
|
|
2306
|
+
"failed",
|
|
2307
|
+
failed > 0 ? import_chalk3.default.yellow(String(failed)) : import_chalk3.default.green("0")
|
|
2308
|
+
],
|
|
2309
|
+
["approved", import_chalk3.default.cyan(String(summary.approved))],
|
|
2310
|
+
["rejected", import_chalk3.default.cyan(String(summary.rejected))],
|
|
2311
|
+
["profitable", import_chalk3.default.green(String(summary.profitable))],
|
|
2312
|
+
["unprofitable", import_chalk3.default.red(String(summary.unprofitable))],
|
|
2313
|
+
["flat", import_chalk3.default.yellow(String(summary.flat))],
|
|
2314
|
+
["precision_approved", colorizeRatio(summary.precisionApproved)],
|
|
2315
|
+
["recall_winners", colorizeRatio(summary.recallWinners)],
|
|
2316
|
+
["avg_profit_all", colorizeProfit(summary.avgProfitAll)],
|
|
2317
|
+
["avg_profit_approved", colorizeProfit(summary.avgProfitApproved)],
|
|
2318
|
+
["expectancy_delta", colorizeProfit(summary.expectancyDelta)]
|
|
2319
|
+
]
|
|
2320
|
+
)
|
|
2321
|
+
);
|
|
2322
|
+
printSection(
|
|
2323
|
+
"CONFUSION",
|
|
2324
|
+
createTable(
|
|
2325
|
+
[import_chalk3.default.green("TP"), import_chalk3.default.red("FP"), import_chalk3.default.green("TN"), import_chalk3.default.red("FN")],
|
|
2326
|
+
[
|
|
2327
|
+
[
|
|
2328
|
+
import_chalk3.default.green(String(summary.truePositive)),
|
|
2329
|
+
import_chalk3.default.red(String(summary.falsePositive)),
|
|
2330
|
+
import_chalk3.default.green(String(summary.trueNegative)),
|
|
2331
|
+
import_chalk3.default.red(String(summary.falseNegative))
|
|
2332
|
+
]
|
|
2333
|
+
]
|
|
2334
|
+
)
|
|
2335
|
+
);
|
|
2336
|
+
if (directionSummaries.length) {
|
|
2337
|
+
printSection(
|
|
2338
|
+
"BY DIRECTION",
|
|
2339
|
+
createTable(
|
|
2340
|
+
[
|
|
2341
|
+
import_chalk3.default.gray("DIR"),
|
|
2342
|
+
import_chalk3.default.gray("EVAL"),
|
|
2343
|
+
import_chalk3.default.gray("ACCURACY"),
|
|
2344
|
+
import_chalk3.default.gray("APPROVED"),
|
|
2345
|
+
import_chalk3.default.green("TP"),
|
|
2346
|
+
import_chalk3.default.red("FP"),
|
|
2347
|
+
import_chalk3.default.green("TN"),
|
|
2348
|
+
import_chalk3.default.red("FN"),
|
|
2349
|
+
import_chalk3.default.gray("PRECISION"),
|
|
2350
|
+
import_chalk3.default.gray("RECALL")
|
|
2351
|
+
],
|
|
2352
|
+
directionSummaries.map(({ direction, summary: directionSummary }) => {
|
|
2353
|
+
const directionEvaluated = directionSummary.correct + directionSummary.incorrect;
|
|
2354
|
+
return [
|
|
2355
|
+
import_chalk3.default.yellow(direction),
|
|
2356
|
+
import_chalk3.default.blue(String(directionEvaluated)),
|
|
2357
|
+
colorizePercent(directionSummary.correct, directionEvaluated),
|
|
2358
|
+
import_chalk3.default.cyan(String(directionSummary.approved)),
|
|
2359
|
+
import_chalk3.default.green(String(directionSummary.truePositive)),
|
|
2360
|
+
import_chalk3.default.red(String(directionSummary.falsePositive)),
|
|
2361
|
+
import_chalk3.default.green(String(directionSummary.trueNegative)),
|
|
2362
|
+
import_chalk3.default.red(String(directionSummary.falseNegative)),
|
|
2363
|
+
colorizeRatio(directionSummary.precisionApproved),
|
|
2364
|
+
colorizeRatio(directionSummary.recallWinners)
|
|
2365
|
+
];
|
|
2366
|
+
})
|
|
2367
|
+
)
|
|
2368
|
+
);
|
|
2369
|
+
}
|
|
2370
|
+
printSection(
|
|
2371
|
+
"DETERMINISTIC FLOW",
|
|
2372
|
+
createTable(
|
|
2373
|
+
[import_chalk3.default.gray("METRIC"), import_chalk3.default.gray("VALUE")],
|
|
2374
|
+
[
|
|
2375
|
+
["selected", import_chalk3.default.blue(String(deterministicSummary.selected))],
|
|
2376
|
+
[
|
|
2377
|
+
"signal_available",
|
|
2378
|
+
import_chalk3.default.blue(String(deterministicSummary.signalAvailable))
|
|
2379
|
+
],
|
|
2380
|
+
[
|
|
2381
|
+
"signal_missing",
|
|
2382
|
+
deterministicSummary.signalMissing > 0 ? import_chalk3.default.yellow(String(deterministicSummary.signalMissing)) : import_chalk3.default.green("0")
|
|
2383
|
+
],
|
|
2384
|
+
[
|
|
2385
|
+
"core_blocked_now",
|
|
2386
|
+
deterministicSummary.coreBlocked > 0 ? import_chalk3.default.yellow(String(deterministicSummary.coreBlocked)) : import_chalk3.default.green("0")
|
|
2387
|
+
],
|
|
2388
|
+
[
|
|
2389
|
+
"adapter_blocked_now",
|
|
2390
|
+
deterministicSummary.adapterBlocked > 0 ? import_chalk3.default.yellow(String(deterministicSummary.adapterBlocked)) : import_chalk3.default.green("0")
|
|
2391
|
+
],
|
|
2392
|
+
[
|
|
2393
|
+
"left_to_model_now",
|
|
2394
|
+
import_chalk3.default.cyan(String(deterministicSummary.modelCandidate))
|
|
2395
|
+
],
|
|
2396
|
+
[
|
|
2397
|
+
"model_approved",
|
|
2398
|
+
import_chalk3.default.green(String(deterministicSummary.modelApproved))
|
|
2399
|
+
],
|
|
2400
|
+
[
|
|
2401
|
+
"model_rejected",
|
|
2402
|
+
import_chalk3.default.red(String(deterministicSummary.modelRejected))
|
|
2403
|
+
],
|
|
2404
|
+
[
|
|
2405
|
+
"model_failed",
|
|
2406
|
+
deterministicSummary.modelFailed > 0 ? import_chalk3.default.yellow(String(deterministicSummary.modelFailed)) : import_chalk3.default.green("0")
|
|
2407
|
+
]
|
|
2408
|
+
]
|
|
2409
|
+
)
|
|
2410
|
+
);
|
|
2411
|
+
if (summary.qualityBuckets.length) {
|
|
2412
|
+
printSection(
|
|
2413
|
+
"QUALITY BREAKDOWN",
|
|
2414
|
+
createTable(
|
|
2415
|
+
[
|
|
2416
|
+
import_chalk3.default.gray("QUALITY"),
|
|
2417
|
+
import_chalk3.default.gray("COUNT"),
|
|
2418
|
+
import_chalk3.default.gray("APPROVED"),
|
|
2419
|
+
import_chalk3.default.gray("APPROVAL_RATE"),
|
|
2420
|
+
import_chalk3.default.gray("WINRATE"),
|
|
2421
|
+
import_chalk3.default.gray("AVG_PROFIT")
|
|
2422
|
+
],
|
|
2423
|
+
summary.qualityBuckets.map((bucket) => [
|
|
2424
|
+
colorizeQuality(bucket.quality),
|
|
2425
|
+
import_chalk3.default.blue(String(bucket.count)),
|
|
2426
|
+
import_chalk3.default.cyan(String(bucket.approved)),
|
|
2427
|
+
colorizeRatio(bucket.approved / bucket.count),
|
|
2428
|
+
colorizeRatio(bucket.profitable / bucket.count),
|
|
2429
|
+
colorizeProfit(bucket.totalProfit / bucket.count)
|
|
2430
|
+
])
|
|
2431
|
+
)
|
|
2432
|
+
);
|
|
2433
|
+
}
|
|
2434
|
+
if (errorMessages2.length) {
|
|
2435
|
+
console.log(import_chalk3.default.yellow("AI provider errors:"));
|
|
2436
|
+
errorMessages2.forEach((message) => {
|
|
2437
|
+
console.log(import_chalk3.default.yellow(`- ${message}`));
|
|
2438
|
+
});
|
|
2439
|
+
}
|
|
2440
|
+
process.exit(failed === rows.length ? 1 : 0);
|
|
2441
|
+
};
|
|
2442
|
+
main2().catch((error) => {
|
|
2443
|
+
console.error(import_chalk3.default.red(error?.message || String(error)));
|
|
2444
|
+
process.exit(1);
|
|
2445
|
+
});
|
|
1617
2446
|
}
|
|
1618
2447
|
});
|
|
1619
2448
|
|
|
1620
2449
|
// src/scripts/backtest.ts
|
|
1621
2450
|
var backtest_exports = {};
|
|
1622
|
-
var
|
|
2451
|
+
var import_args3, import_progress2, import_child_process, import_fs, import_path3, import_os, import_chalk4, import_lodash, import_date_fns, import_uuid, import_connectors, import_connectors2, import_cli, import_backtest, import_data, import_constants2, import_redis, ListIt2, MAX_PARALLEL, normalizedArgv, flags3, interval, progressStep, testsLimit, testsSkip, testItemTimeoutMs, uuid, projectRoot2, testerWorkerPathCandidates, testerWorkerPath, testerNeedsTsRuntime, HEADERS_RESULTS, HEADERS_RESULTS_BY_TICKERS, successTests, errorTests, errorMessages, results, resultsByTickers, resultArtifactsByTestName, artifactRefCountsByTestName, bestTickerTestNameBySymbol, topResultNames, userName, runStartedAt, createListIt2, createTable2, createTimestamp, isGoodTest, getResultAmount, recordError, stripInlineLogs, getInlineArtifacts, retainArtifacts, releaseArtifacts, insertTopResult, updateTopResults, updateBestTickerResult, isStrategyConfigGrid, resolveBacktestConnectorName, getLogsById, cacheArtifactsById, resolveResultArtifacts, setTestData, backtest, saveAndPrintResults, saveAndPrintResultsByTickers, finish;
|
|
1623
2452
|
var init_backtest = __esm({
|
|
1624
2453
|
"src/scripts/backtest.ts"() {
|
|
1625
2454
|
"use strict";
|
|
1626
|
-
|
|
1627
|
-
|
|
2455
|
+
import_args3 = __toESM(require("args"));
|
|
2456
|
+
import_progress2 = __toESM(require("progress"));
|
|
1628
2457
|
import_child_process = require("child_process");
|
|
1629
2458
|
import_fs = __toESM(require("fs"));
|
|
1630
|
-
|
|
2459
|
+
import_path3 = __toESM(require("path"));
|
|
1631
2460
|
import_os = __toESM(require("os"));
|
|
1632
|
-
|
|
2461
|
+
import_chalk4 = __toESM(require("chalk"));
|
|
1633
2462
|
import_lodash = __toESM(require("lodash"));
|
|
1634
2463
|
import_date_fns = require("date-fns");
|
|
1635
2464
|
import_uuid = require("uuid");
|
|
@@ -1638,49 +2467,67 @@ var init_backtest = __esm({
|
|
|
1638
2467
|
import_cli = require("@tradejs/node/cli");
|
|
1639
2468
|
import_backtest = require("@tradejs/core/backtest");
|
|
1640
2469
|
import_data = require("@tradejs/core/data");
|
|
1641
|
-
|
|
2470
|
+
import_constants2 = require("@tradejs/core/constants");
|
|
1642
2471
|
import_redis = require("@tradejs/infra/redis");
|
|
1643
|
-
|
|
2472
|
+
ListIt2 = require_list_it();
|
|
1644
2473
|
MAX_PARALLEL = Math.min(import_os.default.cpus().length, 6);
|
|
1645
|
-
|
|
2474
|
+
import_args3.default.example(
|
|
1646
2475
|
" yarn backtest -t 400 --cacheOnly",
|
|
1647
2476
|
"Run tests on uploaded data for 400 tickers"
|
|
1648
2477
|
);
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
2478
|
+
import_args3.default.option(["t", "tickers"], "Selected tickers");
|
|
2479
|
+
import_args3.default.option(["e", "exclude"], "Exclude tickers from tests");
|
|
2480
|
+
import_args3.default.option(["l", "tickersLimit"], "Tickers limit");
|
|
2481
|
+
import_args3.default.option(["n", "tests"], "Tests limit", import_constants2.TESTS_LIMIT);
|
|
2482
|
+
import_args3.default.option("skip", "Skip first N tests", 0);
|
|
2483
|
+
import_args3.default.option(["p", "parallel"], "Parallel tasks", MAX_PARALLEL);
|
|
2484
|
+
import_args3.default.option(["f", "timeframe"], "Timeframe", 15);
|
|
2485
|
+
import_args3.default.option(["T", "top"], "Return N best tests", import_constants2.TESTS_TOP_LIMIT);
|
|
2486
|
+
import_args3.default.option(["u", "updateOnly"], "Only update tickers history", false);
|
|
2487
|
+
import_args3.default.option(["C", "cacheOnly"], "Do not update tickers history", false);
|
|
2488
|
+
import_args3.default.option(["c", "config"], "Backtest config", "breakout");
|
|
2489
|
+
import_args3.default.option(["L", "showTickersList"], "Just show only ticker list", false);
|
|
2490
|
+
import_args3.default.option(["S", "progressStep"], "Progress step", 100);
|
|
2491
|
+
import_args3.default.option(["U", "user"], "Use user config", "root");
|
|
2492
|
+
import_args3.default.option(
|
|
1663
2493
|
"connector",
|
|
1664
2494
|
"Connector provider or name for backtest (e.g. bybit, binance, coinbase, custom)",
|
|
1665
2495
|
"bybit"
|
|
1666
2496
|
);
|
|
1667
|
-
|
|
2497
|
+
import_args3.default.option(
|
|
1668
2498
|
["m", "ml"],
|
|
1669
2499
|
"Write ML dataset rows to per-worker JSONL chunks",
|
|
1670
2500
|
false
|
|
1671
2501
|
);
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
2502
|
+
import_args3.default.option(
|
|
2503
|
+
["A", "ai"],
|
|
2504
|
+
"Write AI prompt rows to per-worker JSONL chunks",
|
|
2505
|
+
false
|
|
2506
|
+
);
|
|
2507
|
+
normalizedArgv = process.argv.map((arg) => {
|
|
2508
|
+
if (arg === "--ML") {
|
|
2509
|
+
return "--ml";
|
|
2510
|
+
}
|
|
2511
|
+
if (arg === "--AI") {
|
|
2512
|
+
return "--ai";
|
|
2513
|
+
}
|
|
2514
|
+
return arg;
|
|
2515
|
+
});
|
|
2516
|
+
flags3 = import_args3.default.parse(normalizedArgv);
|
|
2517
|
+
interval = flags3.timeframe.toString();
|
|
2518
|
+
progressStep = Math.max(1, parseInt(String(flags3.progressStep), 10));
|
|
2519
|
+
testsLimit = Math.max(0, parseInt(String(flags3.tests), 10));
|
|
2520
|
+
testsSkip = Math.max(0, parseInt(String(flags3.skip ?? 0), 10));
|
|
2521
|
+
testItemTimeoutMs = 12e4;
|
|
1675
2522
|
uuid = (len = 12) => (0, import_uuid.v4)().slice(-len);
|
|
1676
|
-
|
|
2523
|
+
projectRoot2 = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
1677
2524
|
testerWorkerPathCandidates = [
|
|
1678
2525
|
// `tradejs` bin runs from dist/cli.js, so bundled commands resolve from `dist`.
|
|
1679
|
-
|
|
2526
|
+
import_path3.default.resolve(__dirname, "./workers/testerWorker.js"),
|
|
1680
2527
|
// Direct execution of dist/scripts/backtest.js resolves from `dist/scripts`.
|
|
1681
|
-
|
|
2528
|
+
import_path3.default.resolve(__dirname, "../workers/testerWorker.js"),
|
|
1682
2529
|
// ts-node execution resolves from `src/scripts`.
|
|
1683
|
-
|
|
2530
|
+
import_path3.default.resolve(__dirname, "../workers/testerWorker.ts")
|
|
1684
2531
|
];
|
|
1685
2532
|
testerWorkerPath = testerWorkerPathCandidates.find(
|
|
1686
2533
|
(candidate) => import_fs.default.existsSync(candidate)
|
|
@@ -1692,40 +2539,142 @@ var init_backtest = __esm({
|
|
|
1692
2539
|
}
|
|
1693
2540
|
testerNeedsTsRuntime = testerWorkerPath.endsWith(".ts");
|
|
1694
2541
|
HEADERS_RESULTS = [
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
2542
|
+
import_chalk4.default.blue("ID"),
|
|
2543
|
+
import_chalk4.default.yellow("SYMBOL"),
|
|
2544
|
+
import_chalk4.default.cyan("PROFIT"),
|
|
2545
|
+
import_chalk4.default.cyan("ORDERS"),
|
|
2546
|
+
import_chalk4.default.cyan("WIN/LOSS (%)"),
|
|
2547
|
+
import_chalk4.default.cyan("RISK"),
|
|
2548
|
+
import_chalk4.default.cyan("MAX DRAWDOWN (%)")
|
|
1702
2549
|
];
|
|
1703
2550
|
HEADERS_RESULTS_BY_TICKERS = [
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
2551
|
+
import_chalk4.default.blue("ID"),
|
|
2552
|
+
import_chalk4.default.yellow("SYMBOL"),
|
|
2553
|
+
import_chalk4.default.cyan("PROFIT"),
|
|
2554
|
+
import_chalk4.default.cyan("ORDERS"),
|
|
2555
|
+
import_chalk4.default.cyan("WIN/LOSS (%)"),
|
|
2556
|
+
import_chalk4.default.cyan("RISK"),
|
|
2557
|
+
import_chalk4.default.cyan("MAX DRAWDOWN (%)")
|
|
1711
2558
|
];
|
|
1712
2559
|
successTests = 0;
|
|
1713
2560
|
errorTests = 0;
|
|
1714
2561
|
errorMessages = [];
|
|
1715
2562
|
results = [];
|
|
1716
2563
|
resultsByTickers = /* @__PURE__ */ new Map();
|
|
1717
|
-
|
|
2564
|
+
resultArtifactsByTestName = /* @__PURE__ */ new Map();
|
|
2565
|
+
artifactRefCountsByTestName = /* @__PURE__ */ new Map();
|
|
2566
|
+
bestTickerTestNameBySymbol = /* @__PURE__ */ new Map();
|
|
2567
|
+
topResultNames = /* @__PURE__ */ new Set();
|
|
2568
|
+
userName = flags3.user;
|
|
1718
2569
|
runStartedAt = Date.now();
|
|
1719
|
-
|
|
2570
|
+
createListIt2 = () => new ListIt2({
|
|
1720
2571
|
autoAlign: true,
|
|
1721
2572
|
headerUnderline: true
|
|
1722
2573
|
});
|
|
1723
|
-
|
|
2574
|
+
createTable2 = (headers, rows) => createListIt2().setHeaderRow(headers).d(rows).toString();
|
|
1724
2575
|
createTimestamp = (date) => (0, import_date_fns.format)(date, "yyyyMMddHHmm");
|
|
1725
|
-
|
|
2576
|
+
isGoodTest = (result) => result.stat?.orders > 5 && result.stat?.profit > 10;
|
|
2577
|
+
getResultAmount = (result) => result.stat.amount ?? 0;
|
|
1726
2578
|
recordError = (error) => {
|
|
1727
2579
|
errorMessages.push(error);
|
|
1728
2580
|
};
|
|
2581
|
+
stripInlineLogs = (result) => {
|
|
2582
|
+
const { inlineOrderLog, inlinePositionLog, ...resultWithoutLogs } = result;
|
|
2583
|
+
return resultWithoutLogs;
|
|
2584
|
+
};
|
|
2585
|
+
getInlineArtifacts = (result) => {
|
|
2586
|
+
if (!result.inlineOrderLog && !result.inlinePositionLog) {
|
|
2587
|
+
return null;
|
|
2588
|
+
}
|
|
2589
|
+
return {
|
|
2590
|
+
orderLog: result.inlineOrderLog,
|
|
2591
|
+
positionLog: result.inlinePositionLog
|
|
2592
|
+
};
|
|
2593
|
+
};
|
|
2594
|
+
retainArtifacts = (testName, artifacts) => {
|
|
2595
|
+
if (artifacts) {
|
|
2596
|
+
resultArtifactsByTestName.set(testName, artifacts);
|
|
2597
|
+
}
|
|
2598
|
+
artifactRefCountsByTestName.set(
|
|
2599
|
+
testName,
|
|
2600
|
+
(artifactRefCountsByTestName.get(testName) ?? 0) + 1
|
|
2601
|
+
);
|
|
2602
|
+
};
|
|
2603
|
+
releaseArtifacts = (testName) => {
|
|
2604
|
+
const nextCount = (artifactRefCountsByTestName.get(testName) ?? 0) - 1;
|
|
2605
|
+
if (nextCount <= 0) {
|
|
2606
|
+
artifactRefCountsByTestName.delete(testName);
|
|
2607
|
+
resultArtifactsByTestName.delete(testName);
|
|
2608
|
+
return;
|
|
2609
|
+
}
|
|
2610
|
+
artifactRefCountsByTestName.set(testName, nextCount);
|
|
2611
|
+
};
|
|
2612
|
+
insertTopResult = (currentResults, nextResult, limit) => {
|
|
2613
|
+
if (limit <= 0) {
|
|
2614
|
+
return {
|
|
2615
|
+
results: [],
|
|
2616
|
+
added: false
|
|
2617
|
+
};
|
|
2618
|
+
}
|
|
2619
|
+
const nextResults = [...currentResults];
|
|
2620
|
+
const nextAmount = getResultAmount(nextResult);
|
|
2621
|
+
let insertIndex = nextResults.length;
|
|
2622
|
+
for (let index = 0; index < nextResults.length; index += 1) {
|
|
2623
|
+
if (nextAmount > getResultAmount(nextResults[index])) {
|
|
2624
|
+
insertIndex = index;
|
|
2625
|
+
break;
|
|
2626
|
+
}
|
|
2627
|
+
}
|
|
2628
|
+
if (insertIndex === nextResults.length && nextResults.length >= limit) {
|
|
2629
|
+
return {
|
|
2630
|
+
results: currentResults,
|
|
2631
|
+
added: false
|
|
2632
|
+
};
|
|
2633
|
+
}
|
|
2634
|
+
nextResults.splice(insertIndex, 0, nextResult);
|
|
2635
|
+
if (nextResults.length > limit) {
|
|
2636
|
+
nextResults.length = limit;
|
|
2637
|
+
}
|
|
2638
|
+
return {
|
|
2639
|
+
results: nextResults,
|
|
2640
|
+
added: nextResults.some(
|
|
2641
|
+
(candidate) => candidate.test.name === nextResult.test.name
|
|
2642
|
+
)
|
|
2643
|
+
};
|
|
2644
|
+
};
|
|
2645
|
+
updateTopResults = (nextResults, nextResult, artifacts) => {
|
|
2646
|
+
const previousTopNames = topResultNames;
|
|
2647
|
+
const nextTopNames = new Set(nextResults.map((result) => result.test.name));
|
|
2648
|
+
if (artifacts && !previousTopNames.has(nextResult.test.name) && nextTopNames.has(nextResult.test.name)) {
|
|
2649
|
+
retainArtifacts(nextResult.test.name, artifacts);
|
|
2650
|
+
}
|
|
2651
|
+
for (const testName of previousTopNames) {
|
|
2652
|
+
if (!nextTopNames.has(testName)) {
|
|
2653
|
+
releaseArtifacts(testName);
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
results = nextResults;
|
|
2657
|
+
topResultNames = nextTopNames;
|
|
2658
|
+
};
|
|
2659
|
+
updateBestTickerResult = (result, artifacts) => {
|
|
2660
|
+
if (!isGoodTest(result)) {
|
|
2661
|
+
return false;
|
|
2662
|
+
}
|
|
2663
|
+
const previousResult = resultsByTickers.get(result.test.symbol);
|
|
2664
|
+
if (previousResult && previousResult.stat.profit >= result.stat.profit) {
|
|
2665
|
+
return false;
|
|
2666
|
+
}
|
|
2667
|
+
const previousTestName = bestTickerTestNameBySymbol.get(result.test.symbol);
|
|
2668
|
+
if (previousTestName) {
|
|
2669
|
+
releaseArtifacts(previousTestName);
|
|
2670
|
+
}
|
|
2671
|
+
resultsByTickers.set(result.test.symbol, result);
|
|
2672
|
+
bestTickerTestNameBySymbol.set(result.test.symbol, result.test.name);
|
|
2673
|
+
if (artifacts) {
|
|
2674
|
+
retainArtifacts(result.test.name, artifacts);
|
|
2675
|
+
}
|
|
2676
|
+
return true;
|
|
2677
|
+
};
|
|
1729
2678
|
isStrategyConfigGrid = (value) => {
|
|
1730
2679
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1731
2680
|
return false;
|
|
@@ -1735,12 +2684,12 @@ var init_backtest = __esm({
|
|
|
1735
2684
|
);
|
|
1736
2685
|
};
|
|
1737
2686
|
resolveBacktestConnectorName = async (value) => {
|
|
1738
|
-
const connectorName = await (0, import_connectors2.resolveConnectorName)(value,
|
|
2687
|
+
const connectorName = await (0, import_connectors2.resolveConnectorName)(value, projectRoot2);
|
|
1739
2688
|
if (connectorName) {
|
|
1740
2689
|
return connectorName;
|
|
1741
2690
|
}
|
|
1742
2691
|
console.log(
|
|
1743
|
-
|
|
2692
|
+
import_chalk4.default.yellow(
|
|
1744
2693
|
`Unknown connector "${String(value || "").trim() || String(value)}". Fallback to ${import_connectors2.DEFAULT_CONNECTOR_NAME}.`
|
|
1745
2694
|
)
|
|
1746
2695
|
);
|
|
@@ -1748,101 +2697,127 @@ var init_backtest = __esm({
|
|
|
1748
2697
|
};
|
|
1749
2698
|
getLogsById = async (orderLogId) => {
|
|
1750
2699
|
const orderLog = await (0, import_redis.getData)(
|
|
1751
|
-
import_redis.redisKeys.cacheOrders(userName, orderLogId)
|
|
2700
|
+
import_redis.redisKeys.cacheOrders(userName, orderLogId),
|
|
2701
|
+
null
|
|
1752
2702
|
);
|
|
1753
2703
|
const positionLog = await (0, import_redis.getData)(
|
|
1754
|
-
import_redis.redisKeys.cachePositions(userName, orderLogId)
|
|
2704
|
+
import_redis.redisKeys.cachePositions(userName, orderLogId),
|
|
2705
|
+
null
|
|
1755
2706
|
);
|
|
1756
2707
|
return { orderLog, positionLog };
|
|
1757
2708
|
};
|
|
2709
|
+
cacheArtifactsById = async (orderLogId, artifacts) => {
|
|
2710
|
+
if (!artifacts || !Array.isArray(artifacts.orderLog) || !Array.isArray(artifacts.positionLog)) {
|
|
2711
|
+
return;
|
|
2712
|
+
}
|
|
2713
|
+
await Promise.all([
|
|
2714
|
+
(0, import_redis.setData)(import_redis.redisKeys.cacheOrders(userName, orderLogId), artifacts.orderLog, {
|
|
2715
|
+
expire: import_constants2.TTL_1D
|
|
2716
|
+
}),
|
|
2717
|
+
(0, import_redis.setData)(
|
|
2718
|
+
import_redis.redisKeys.cachePositions(userName, orderLogId),
|
|
2719
|
+
artifacts.positionLog,
|
|
2720
|
+
{
|
|
2721
|
+
expire: import_constants2.TTL_1D
|
|
2722
|
+
}
|
|
2723
|
+
)
|
|
2724
|
+
]);
|
|
2725
|
+
};
|
|
2726
|
+
resolveResultArtifacts = async (result) => {
|
|
2727
|
+
const inlineArtifacts = resultArtifactsByTestName.get(result.test.name);
|
|
2728
|
+
if (inlineArtifacts?.orderLog && inlineArtifacts?.positionLog) {
|
|
2729
|
+
return inlineArtifacts;
|
|
2730
|
+
}
|
|
2731
|
+
return getLogsById(result.orderLogId);
|
|
2732
|
+
};
|
|
1758
2733
|
setTestData = async (test, stat, orderLog) => {
|
|
1759
2734
|
await (0, import_redis.setData)(
|
|
1760
2735
|
import_redis.redisKeys.testOrders(test.userName, test.strategyName, test.name),
|
|
1761
2736
|
orderLog,
|
|
1762
2737
|
{
|
|
1763
|
-
expire:
|
|
2738
|
+
expire: import_constants2.TTL_1M
|
|
1764
2739
|
}
|
|
1765
2740
|
);
|
|
1766
2741
|
await (0, import_redis.setData)(
|
|
1767
2742
|
import_redis.redisKeys.testConfig(test.userName, test.strategyName, test.name),
|
|
1768
2743
|
test,
|
|
1769
2744
|
{
|
|
1770
|
-
expire:
|
|
2745
|
+
expire: import_constants2.TTL_1M
|
|
1771
2746
|
}
|
|
1772
2747
|
);
|
|
1773
2748
|
await (0, import_redis.setData)(
|
|
1774
2749
|
import_redis.redisKeys.testStat(test.userName, test.strategyName, test.name),
|
|
1775
2750
|
stat,
|
|
1776
2751
|
{
|
|
1777
|
-
expire:
|
|
2752
|
+
expire: import_constants2.TTL_1M
|
|
1778
2753
|
}
|
|
1779
2754
|
);
|
|
1780
2755
|
};
|
|
1781
2756
|
backtest = async () => {
|
|
1782
|
-
if (!
|
|
2757
|
+
if (!flags3.config) {
|
|
1783
2758
|
throw new Error("Backtest config not send");
|
|
1784
2759
|
}
|
|
1785
|
-
const strategyName =
|
|
1786
|
-
if (!
|
|
2760
|
+
const strategyName = flags3.config.split(":")[0];
|
|
2761
|
+
if (!flags3.config) {
|
|
1787
2762
|
throw new Error("Strategy name not found");
|
|
1788
2763
|
}
|
|
1789
2764
|
const backtestConfig = await (0, import_redis.getData)(
|
|
1790
|
-
import_redis.redisKeys.backtestConfig(userName,
|
|
2765
|
+
import_redis.redisKeys.backtestConfig(userName, flags3.config),
|
|
1791
2766
|
null
|
|
1792
2767
|
);
|
|
1793
2768
|
if (!backtestConfig) {
|
|
1794
|
-
throw new Error(`Backtest config "${
|
|
2769
|
+
throw new Error(`Backtest config "${flags3.config}" not found`);
|
|
1795
2770
|
}
|
|
1796
2771
|
const typedBacktestConfig = backtestConfig;
|
|
1797
2772
|
if (!isStrategyConfigGrid(typedBacktestConfig)) {
|
|
1798
2773
|
throw new Error(
|
|
1799
|
-
`Backtest config "${
|
|
2774
|
+
`Backtest config "${flags3.config}" must include strategyName and strategyConfig grid`
|
|
1800
2775
|
);
|
|
1801
2776
|
}
|
|
1802
|
-
const connectorName = await resolveBacktestConnectorName(
|
|
2777
|
+
const connectorName = await resolveBacktestConnectorName(flags3.connector);
|
|
1803
2778
|
const connectorFactory = await (0, import_connectors2.getConnectorCreatorByName)(
|
|
1804
2779
|
connectorName,
|
|
1805
|
-
|
|
2780
|
+
projectRoot2
|
|
1806
2781
|
);
|
|
1807
2782
|
if (!connectorFactory) {
|
|
1808
2783
|
throw new Error(`Connector "${connectorName}" is not registered`);
|
|
1809
2784
|
}
|
|
1810
2785
|
const marketConnector = await connectorFactory({
|
|
1811
|
-
userName:
|
|
2786
|
+
userName: flags3.user
|
|
1812
2787
|
});
|
|
1813
2788
|
const tickers = await (0, import_cli.getTickers)(
|
|
1814
2789
|
marketConnector,
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
2790
|
+
flags3.tickers,
|
|
2791
|
+
flags3.exclude,
|
|
2792
|
+
flags3.tickersLimit
|
|
1818
2793
|
);
|
|
1819
|
-
if (
|
|
1820
|
-
console.log(
|
|
2794
|
+
if (flags3.showTickersList) {
|
|
2795
|
+
console.log(import_chalk4.default.gray(JSON.stringify(tickers.sort(), null, 2)));
|
|
1821
2796
|
return;
|
|
1822
2797
|
}
|
|
1823
|
-
if (!
|
|
2798
|
+
if (!flags3.cacheOnly) {
|
|
1824
2799
|
await (0, import_cli.update)(marketConnector, interval, tickers);
|
|
1825
2800
|
const binanceConnectorCreator = await (0, import_connectors2.getConnectorCreatorByName)(
|
|
1826
2801
|
import_connectors.ConnectorNames.Binance,
|
|
1827
|
-
|
|
2802
|
+
projectRoot2
|
|
1828
2803
|
);
|
|
1829
2804
|
const coinbaseConnectorCreator = await (0, import_connectors2.getConnectorCreatorByName)(
|
|
1830
2805
|
import_connectors.ConnectorNames.Coinbase,
|
|
1831
|
-
|
|
2806
|
+
projectRoot2
|
|
1832
2807
|
);
|
|
1833
2808
|
if (!binanceConnectorCreator || !coinbaseConnectorCreator) {
|
|
1834
2809
|
throw new Error("Binance/Coinbase connectors are required");
|
|
1835
2810
|
}
|
|
1836
2811
|
const binanceConnector = await binanceConnectorCreator({
|
|
1837
|
-
userName:
|
|
2812
|
+
userName: flags3.user
|
|
1838
2813
|
});
|
|
1839
2814
|
const coinbaseConnector = await coinbaseConnectorCreator({
|
|
1840
|
-
userName:
|
|
2815
|
+
userName: flags3.user
|
|
1841
2816
|
});
|
|
1842
2817
|
await (0, import_cli.update)(binanceConnector, interval, ["BTCUSDT"]);
|
|
1843
2818
|
await (0, import_cli.update)(coinbaseConnector, interval, ["BTCUSDT"]);
|
|
1844
2819
|
}
|
|
1845
|
-
if (
|
|
2820
|
+
if (flags3.updateOnly) {
|
|
1846
2821
|
return;
|
|
1847
2822
|
}
|
|
1848
2823
|
let testSuite = (0, import_backtest.createTestSuite)(
|
|
@@ -1851,15 +2826,38 @@ var init_backtest = __esm({
|
|
|
1851
2826
|
strategyName,
|
|
1852
2827
|
typedBacktestConfig,
|
|
1853
2828
|
connectorName
|
|
1854
|
-
)
|
|
1855
|
-
const mlEnabled = Boolean(
|
|
1856
|
-
|
|
1857
|
-
|
|
2829
|
+
);
|
|
2830
|
+
const mlEnabled = Boolean(flags3.ml);
|
|
2831
|
+
const aiEnabled = Boolean(flags3.ai);
|
|
2832
|
+
testSuite = testSuite.map((test) => ({
|
|
2833
|
+
...test,
|
|
2834
|
+
ml: mlEnabled,
|
|
2835
|
+
ai: aiEnabled,
|
|
2836
|
+
timeoutMs: testItemTimeoutMs
|
|
2837
|
+
})).slice(testsSkip, testsSkip + testsLimit);
|
|
2838
|
+
if (!testSuite.length) {
|
|
2839
|
+
console.log(
|
|
2840
|
+
import_chalk4.default.yellow(
|
|
2841
|
+
`No tests selected (skip=${testsSkip}, limit=${testsLimit}).`
|
|
2842
|
+
)
|
|
2843
|
+
);
|
|
2844
|
+
return;
|
|
2845
|
+
}
|
|
2846
|
+
const chunkSize = Math.ceil(testSuite.length / parseInt(flags3.parallel));
|
|
1858
2847
|
const chunks = import_lodash.default.chunk(testSuite, chunkSize);
|
|
1859
|
-
let completedWorkers = 0;
|
|
1860
2848
|
let completedTests = 0;
|
|
1861
2849
|
let isFinishing = false;
|
|
1862
2850
|
const workers = /* @__PURE__ */ new Set();
|
|
2851
|
+
const maybeFinish = async () => {
|
|
2852
|
+
if (isFinishing) {
|
|
2853
|
+
return;
|
|
2854
|
+
}
|
|
2855
|
+
if (completedTests !== testSuite.length || workers.size > 0) {
|
|
2856
|
+
return;
|
|
2857
|
+
}
|
|
2858
|
+
isFinishing = true;
|
|
2859
|
+
await finish();
|
|
2860
|
+
};
|
|
1863
2861
|
const stopWorkers = () => {
|
|
1864
2862
|
for (const worker of workers) {
|
|
1865
2863
|
if (!worker.killed) {
|
|
@@ -1875,9 +2873,9 @@ var init_backtest = __esm({
|
|
|
1875
2873
|
stopWorkers();
|
|
1876
2874
|
process.exit(143);
|
|
1877
2875
|
});
|
|
1878
|
-
console.log(
|
|
2876
|
+
console.log(import_chalk4.default.yellow(`tests: ${testSuite.length}`));
|
|
1879
2877
|
console.log("");
|
|
1880
|
-
const bar = new
|
|
2878
|
+
const bar = new import_progress2.default(
|
|
1881
2879
|
":current/:total [:bar][:percent] :symbol :amount :eta(s)",
|
|
1882
2880
|
{
|
|
1883
2881
|
total: testSuite.length,
|
|
@@ -1899,13 +2897,8 @@ var init_backtest = __esm({
|
|
|
1899
2897
|
workers.add(tester);
|
|
1900
2898
|
tester.on("message", async (msg) => {
|
|
1901
2899
|
if (msg.done) {
|
|
1902
|
-
completedWorkers++;
|
|
1903
2900
|
workers.delete(tester);
|
|
1904
|
-
|
|
1905
|
-
isFinishing = true;
|
|
1906
|
-
results = (0, import_backtest.sortBestTests)(results, flags.top);
|
|
1907
|
-
await finish();
|
|
1908
|
-
}
|
|
2901
|
+
await maybeFinish();
|
|
1909
2902
|
return;
|
|
1910
2903
|
}
|
|
1911
2904
|
completedTests++;
|
|
@@ -1917,49 +2910,55 @@ var init_backtest = __esm({
|
|
|
1917
2910
|
payload: msg
|
|
1918
2911
|
});
|
|
1919
2912
|
console.error(
|
|
1920
|
-
|
|
2913
|
+
import_chalk4.default.red(`Error in test #${msg.id}: ${JSON.stringify(msg)}`)
|
|
1921
2914
|
);
|
|
1922
2915
|
return;
|
|
1923
2916
|
} else {
|
|
1924
2917
|
successTests++;
|
|
1925
2918
|
}
|
|
1926
|
-
|
|
1927
|
-
const
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
}
|
|
2919
|
+
const fullResult = msg;
|
|
2920
|
+
const result = stripInlineLogs(fullResult);
|
|
2921
|
+
const inlineArtifacts = getInlineArtifacts(fullResult);
|
|
2922
|
+
const nextTopResults = insertTopResult(results, result, flags3.top);
|
|
2923
|
+
const shouldCacheTopArtifacts = nextTopResults.added;
|
|
2924
|
+
if (nextTopResults.added || results !== nextTopResults.results) {
|
|
2925
|
+
updateTopResults(nextTopResults.results, result, inlineArtifacts);
|
|
2926
|
+
}
|
|
2927
|
+
const updatedBestTickerResult = updateBestTickerResult(
|
|
2928
|
+
result,
|
|
2929
|
+
inlineArtifacts
|
|
2930
|
+
);
|
|
2931
|
+
if ((shouldCacheTopArtifacts || updatedBestTickerResult) && inlineArtifacts) {
|
|
2932
|
+
await cacheArtifactsById(result.orderLogId, inlineArtifacts);
|
|
2933
|
+
}
|
|
1934
2934
|
if (completedTests % progressStep === 0 || completedTests === testSuite.length) {
|
|
1935
|
-
|
|
1936
|
-
const
|
|
1937
|
-
|
|
1938
|
-
stat: { profit }
|
|
1939
|
-
} = results[0];
|
|
2935
|
+
const bestResult = results[0];
|
|
2936
|
+
const symbol = bestResult?.test.symbol || "-";
|
|
2937
|
+
const profit = bestResult?.stat.profit || 0;
|
|
1940
2938
|
const profitStr = `${(profit || 0).toFixed(2)}$`;
|
|
1941
2939
|
bar.tick(
|
|
1942
2940
|
completedTests === testSuite.length ? completedTests % progressStep : progressStep,
|
|
1943
2941
|
{
|
|
1944
|
-
symbol:
|
|
1945
|
-
amount: profit > 0 ?
|
|
2942
|
+
symbol: import_chalk4.default.yellow(symbol),
|
|
2943
|
+
amount: profit > 0 ? import_chalk4.default.green(profitStr) : import_chalk4.default.red(profitStr)
|
|
1946
2944
|
}
|
|
1947
2945
|
);
|
|
1948
2946
|
}
|
|
1949
2947
|
});
|
|
1950
2948
|
tester.on("error", (err) => {
|
|
1951
2949
|
recordError({ error: err?.message ?? err });
|
|
1952
|
-
console.error(
|
|
2950
|
+
console.error(import_chalk4.default.red(`Worker error: ${err.message}`));
|
|
1953
2951
|
});
|
|
1954
2952
|
tester.on("exit", (code) => {
|
|
1955
2953
|
workers.delete(tester);
|
|
1956
2954
|
if (code !== 0) {
|
|
1957
2955
|
recordError({ error: `Worker exited with code ${code}` });
|
|
1958
|
-
console.error(
|
|
2956
|
+
console.error(import_chalk4.default.red(`Worker exited with code ${code}`));
|
|
1959
2957
|
}
|
|
2958
|
+
void maybeFinish();
|
|
1960
2959
|
});
|
|
1961
2960
|
await (0, import_redis.setData)(import_redis.redisKeys.cacheChunk(userName, chunkId), chunkWithId, {
|
|
1962
|
-
expire:
|
|
2961
|
+
expire: import_constants2.TTL_1D
|
|
1963
2962
|
});
|
|
1964
2963
|
tester.send({ chunkId, userName });
|
|
1965
2964
|
}
|
|
@@ -1967,14 +2966,22 @@ var init_backtest = __esm({
|
|
|
1967
2966
|
saveAndPrintResults = async () => {
|
|
1968
2967
|
const colorizedResults = [];
|
|
1969
2968
|
for await (const result of results) {
|
|
1970
|
-
const { test
|
|
2969
|
+
const { test } = result;
|
|
1971
2970
|
const { symbol, name } = test;
|
|
1972
|
-
const { orderLog, positionLog } = await
|
|
2971
|
+
const { orderLog, positionLog } = await resolveResultArtifacts(result);
|
|
2972
|
+
if (!orderLog || !positionLog) {
|
|
2973
|
+
throw new Error(`Logs not found for test ${name}`);
|
|
2974
|
+
}
|
|
1973
2975
|
const stat = (0, import_backtest.calculateStatsFull)(positionLog);
|
|
2976
|
+
if (!stat && result.stat.orders > 0) {
|
|
2977
|
+
throw new Error(
|
|
2978
|
+
`Position log is empty for test ${name} despite ${result.stat.orders} closed orders`
|
|
2979
|
+
);
|
|
2980
|
+
}
|
|
1974
2981
|
await setTestData(test, stat, orderLog);
|
|
1975
2982
|
const statRow = [
|
|
1976
|
-
|
|
1977
|
-
|
|
2983
|
+
import_chalk4.default.blue(name),
|
|
2984
|
+
import_chalk4.default.yellow(symbol),
|
|
1978
2985
|
...(0, import_cli.drawStatInCLI)(stat, [
|
|
1979
2986
|
"netProfit",
|
|
1980
2987
|
"orders",
|
|
@@ -1987,20 +2994,28 @@ var init_backtest = __esm({
|
|
|
1987
2994
|
}
|
|
1988
2995
|
console.log("");
|
|
1989
2996
|
console.log("RESULTS:");
|
|
1990
|
-
console.log(
|
|
2997
|
+
console.log(createTable2(HEADERS_RESULTS, colorizedResults));
|
|
1991
2998
|
console.log("");
|
|
1992
2999
|
};
|
|
1993
3000
|
saveAndPrintResultsByTickers = async () => {
|
|
1994
3001
|
const colorizedResultsByTickers = [];
|
|
1995
3002
|
for await (const result of resultsByTickers.values()) {
|
|
1996
|
-
const { test
|
|
3003
|
+
const { test } = result;
|
|
1997
3004
|
const { symbol, name } = test;
|
|
1998
|
-
const { orderLog, positionLog } = await
|
|
3005
|
+
const { orderLog, positionLog } = await resolveResultArtifacts(result);
|
|
3006
|
+
if (!orderLog || !positionLog) {
|
|
3007
|
+
throw new Error(`Logs not found for ticker result ${name}`);
|
|
3008
|
+
}
|
|
1999
3009
|
const stat = (0, import_backtest.calculateStatsFull)(positionLog);
|
|
3010
|
+
if (!stat && result.stat.orders > 0) {
|
|
3011
|
+
throw new Error(
|
|
3012
|
+
`Position log is empty for ticker result ${name} despite ${result.stat.orders} closed orders`
|
|
3013
|
+
);
|
|
3014
|
+
}
|
|
2000
3015
|
await setTestData(test, stat, orderLog);
|
|
2001
3016
|
const statRow = [
|
|
2002
|
-
|
|
2003
|
-
|
|
3017
|
+
import_chalk4.default.blue(name),
|
|
3018
|
+
import_chalk4.default.yellow(symbol),
|
|
2004
3019
|
...(0, import_cli.drawStatInCLI)(stat, [
|
|
2005
3020
|
"netProfit",
|
|
2006
3021
|
"orders",
|
|
@@ -2014,28 +3029,28 @@ var init_backtest = __esm({
|
|
|
2014
3029
|
console.log("");
|
|
2015
3030
|
console.log("RESULTS BY TICKERS:");
|
|
2016
3031
|
console.log(
|
|
2017
|
-
|
|
3032
|
+
createTable2(HEADERS_RESULTS_BY_TICKERS, colorizedResultsByTickers)
|
|
2018
3033
|
);
|
|
2019
3034
|
console.log("");
|
|
2020
3035
|
};
|
|
2021
3036
|
finish = async () => {
|
|
2022
|
-
if (
|
|
3037
|
+
if (flags3.tickers) {
|
|
2023
3038
|
await saveAndPrintResults();
|
|
2024
3039
|
} else {
|
|
2025
3040
|
await saveAndPrintResultsByTickers();
|
|
2026
3041
|
}
|
|
2027
3042
|
const bestConfig = results[0]?.test.strategyConfig;
|
|
2028
|
-
console.log(
|
|
2029
|
-
console.log(
|
|
3043
|
+
console.log(import_chalk4.default.gray("BEST CONFIG:"));
|
|
3044
|
+
console.log(import_chalk4.default.green((0, import_data.toJson)(bestConfig, true)));
|
|
2030
3045
|
console.log("");
|
|
2031
3046
|
const mergedConfig = (0, import_backtest.mergeConfigs)(
|
|
2032
3047
|
results.map(({ test: { strategyConfig } }) => strategyConfig)
|
|
2033
3048
|
);
|
|
2034
|
-
console.log(
|
|
2035
|
-
console.log(
|
|
3049
|
+
console.log(import_chalk4.default.gray("MERGED CONFIG:"));
|
|
3050
|
+
console.log(import_chalk4.default.blue((0, import_data.toJson)(mergedConfig, true)));
|
|
2036
3051
|
console.log("");
|
|
2037
|
-
console.log(`${
|
|
2038
|
-
console.log(`${
|
|
3052
|
+
console.log(`${import_chalk4.default.green("SUCCESS TESTS")}: ${successTests}`);
|
|
3053
|
+
console.log(`${import_chalk4.default.red("ERRORS")}: ${errorTests}`);
|
|
2039
3054
|
console.log("");
|
|
2040
3055
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
2041
3056
|
const durationSeconds = Number(
|
|
@@ -2043,15 +3058,15 @@ var init_backtest = __esm({
|
|
|
2043
3058
|
);
|
|
2044
3059
|
const timestamp = createTimestamp(finishedAt);
|
|
2045
3060
|
await (0, import_redis.setData)(
|
|
2046
|
-
import_redis.redisKeys.backtestResults(userName,
|
|
3061
|
+
import_redis.redisKeys.backtestResults(userName, flags3.config, timestamp),
|
|
2047
3062
|
{
|
|
2048
|
-
config:
|
|
3063
|
+
config: flags3.config,
|
|
2049
3064
|
user: userName,
|
|
2050
3065
|
startedAt: new Date(runStartedAt).toISOString(),
|
|
2051
3066
|
finishedAt: finishedAt.toISOString(),
|
|
2052
3067
|
durationSeconds,
|
|
2053
3068
|
results,
|
|
2054
|
-
resultsByTickers: resultsByTickers.values(),
|
|
3069
|
+
resultsByTickers: Array.from(resultsByTickers.values()),
|
|
2055
3070
|
bestConfig,
|
|
2056
3071
|
mergedConfig,
|
|
2057
3072
|
successTests,
|
|
@@ -2069,27 +3084,27 @@ var init_backtest = __esm({
|
|
|
2069
3084
|
});
|
|
2070
3085
|
|
|
2071
3086
|
// src/lib/runBot.ts
|
|
2072
|
-
var import_lodash2, import_connectors3, import_data2,
|
|
3087
|
+
var import_lodash2, import_connectors3, import_data2, import_strategies2, import_async2, import_constants3, import_time, import_logger, import_redis2, projectRoot3, runBot;
|
|
2073
3088
|
var init_runBot = __esm({
|
|
2074
3089
|
"src/lib/runBot.ts"() {
|
|
2075
3090
|
"use strict";
|
|
2076
3091
|
import_lodash2 = __toESM(require("lodash"));
|
|
2077
3092
|
import_connectors3 = require("@tradejs/node/connectors");
|
|
2078
3093
|
import_data2 = require("@tradejs/core/data");
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
3094
|
+
import_strategies2 = require("@tradejs/node/strategies");
|
|
3095
|
+
import_async2 = require("@tradejs/core/async");
|
|
3096
|
+
import_constants3 = require("@tradejs/core/constants");
|
|
2082
3097
|
import_time = require("@tradejs/core/time");
|
|
2083
3098
|
import_logger = require("@tradejs/infra/logger");
|
|
2084
3099
|
import_redis2 = require("@tradejs/infra/redis");
|
|
2085
|
-
|
|
3100
|
+
projectRoot3 = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
2086
3101
|
runBot = async () => {
|
|
2087
3102
|
const botResults = [];
|
|
2088
|
-
const preloadStart = (0, import_time.getTimestamp)(
|
|
3103
|
+
const preloadStart = (0, import_time.getTimestamp)(import_constants3.BOT_PRELOAD_DAYS);
|
|
2089
3104
|
const end = (0, import_time.getTimestamp)();
|
|
2090
3105
|
const keys = await (0, import_redis2.getKeys)(import_redis2.redisKeys.botsPrefix());
|
|
2091
3106
|
const botKeys = keys.filter((key) => key.endsWith(":bots"));
|
|
2092
|
-
await (0,
|
|
3107
|
+
await (0, import_async2.delay)(5e3);
|
|
2093
3108
|
import_logger.logger.log("info", "files count: %s", botKeys.length);
|
|
2094
3109
|
for await (const key of botKeys) {
|
|
2095
3110
|
const userName2 = key.split(":")[1];
|
|
@@ -2108,16 +3123,16 @@ var init_runBot = __esm({
|
|
|
2108
3123
|
continue;
|
|
2109
3124
|
}
|
|
2110
3125
|
try {
|
|
2111
|
-
const strategyCreator = await (0,
|
|
3126
|
+
const strategyCreator = await (0, import_strategies2.getStrategyCreator)(
|
|
2112
3127
|
strategyName,
|
|
2113
|
-
|
|
3128
|
+
projectRoot3
|
|
2114
3129
|
);
|
|
2115
3130
|
if (!strategyCreator) {
|
|
2116
3131
|
throw new Error(`Unknown strategy: ${strategyName}`);
|
|
2117
3132
|
}
|
|
2118
3133
|
const connectorCreator = await (0, import_connectors3.getConnectorCreatorByName)(
|
|
2119
3134
|
connectorName,
|
|
2120
|
-
|
|
3135
|
+
projectRoot3
|
|
2121
3136
|
);
|
|
2122
3137
|
if (!connectorCreator) {
|
|
2123
3138
|
throw new Error(`Unknown connector: ${connectorName}`);
|
|
@@ -2127,11 +3142,11 @@ var init_runBot = __esm({
|
|
|
2127
3142
|
});
|
|
2128
3143
|
const binanceConnectorCreator = await (0, import_connectors3.getConnectorCreatorByName)(
|
|
2129
3144
|
import_connectors3.BUILTIN_CONNECTOR_NAMES.Binance,
|
|
2130
|
-
|
|
3145
|
+
projectRoot3
|
|
2131
3146
|
);
|
|
2132
3147
|
const coinbaseConnectorCreator = await (0, import_connectors3.getConnectorCreatorByName)(
|
|
2133
3148
|
import_connectors3.BUILTIN_CONNECTOR_NAMES.Coinbase,
|
|
2134
|
-
|
|
3149
|
+
projectRoot3
|
|
2135
3150
|
);
|
|
2136
3151
|
if (!binanceConnectorCreator || !coinbaseConnectorCreator) {
|
|
2137
3152
|
throw new Error("Binance/Coinbase connectors are required");
|
|
@@ -2221,17 +3236,17 @@ var init_bot = __esm({
|
|
|
2221
3236
|
|
|
2222
3237
|
// src/scripts/cleanDir.ts
|
|
2223
3238
|
var cleanDir_exports = {};
|
|
2224
|
-
var
|
|
3239
|
+
var import_args4, import_cli2, flags4, getDir, run2;
|
|
2225
3240
|
var init_cleanDir = __esm({
|
|
2226
3241
|
"src/scripts/cleanDir.ts"() {
|
|
2227
3242
|
"use strict";
|
|
2228
|
-
|
|
3243
|
+
import_args4 = __toESM(require("args"));
|
|
2229
3244
|
import_cli2 = require("@tradejs/node/cli");
|
|
2230
|
-
|
|
2231
|
-
|
|
3245
|
+
import_args4.default.option("dir", "Directory clean up", "cache");
|
|
3246
|
+
flags4 = import_args4.default.parse(process.argv);
|
|
2232
3247
|
getDir = () => {
|
|
2233
|
-
if (
|
|
2234
|
-
return `data/${
|
|
3248
|
+
if (flags4.dir) {
|
|
3249
|
+
return `data/${flags4.dir}`;
|
|
2235
3250
|
}
|
|
2236
3251
|
return "data/cache";
|
|
2237
3252
|
};
|
|
@@ -2245,17 +3260,17 @@ var init_cleanDir = __esm({
|
|
|
2245
3260
|
|
|
2246
3261
|
// src/scripts/cleanRedis.ts
|
|
2247
3262
|
var cleanRedis_exports = {};
|
|
2248
|
-
var
|
|
3263
|
+
var import_args5, import_cli3, flags5, getArea, run3;
|
|
2249
3264
|
var init_cleanRedis = __esm({
|
|
2250
3265
|
"src/scripts/cleanRedis.ts"() {
|
|
2251
3266
|
"use strict";
|
|
2252
|
-
|
|
3267
|
+
import_args5 = __toESM(require("args"));
|
|
2253
3268
|
import_cli3 = require("@tradejs/node/cli");
|
|
2254
|
-
|
|
2255
|
-
|
|
3269
|
+
import_args5.default.option("area", "Area clean up", "cache");
|
|
3270
|
+
flags5 = import_args5.default.parse(process.argv);
|
|
2256
3271
|
getArea = () => {
|
|
2257
|
-
if (
|
|
2258
|
-
return
|
|
3272
|
+
if (flags5.area) {
|
|
3273
|
+
return flags5.area;
|
|
2259
3274
|
}
|
|
2260
3275
|
return "cache";
|
|
2261
3276
|
};
|
|
@@ -2269,17 +3284,17 @@ var init_cleanRedis = __esm({
|
|
|
2269
3284
|
|
|
2270
3285
|
// src/scripts/cleanTests.ts
|
|
2271
3286
|
var cleanTests_exports = {};
|
|
2272
|
-
var
|
|
3287
|
+
var import_args6, import_chalk5, import_cli4, import_redis3, flags6, extractUsers, getUsersToClean, cleanUserTests, run4;
|
|
2273
3288
|
var init_cleanTests = __esm({
|
|
2274
3289
|
"src/scripts/cleanTests.ts"() {
|
|
2275
3290
|
"use strict";
|
|
2276
|
-
|
|
2277
|
-
|
|
3291
|
+
import_args6 = __toESM(require("args"));
|
|
3292
|
+
import_chalk5 = __toESM(require("chalk"));
|
|
2278
3293
|
import_cli4 = require("@tradejs/node/cli");
|
|
2279
3294
|
import_redis3 = require("@tradejs/infra/redis");
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
3295
|
+
import_args6.default.option(["U", "user"], "Clean tests for user", "");
|
|
3296
|
+
import_args6.default.option(["C", "cache"], "Clean only cache keys", false);
|
|
3297
|
+
flags6 = import_args6.default.parse(process.argv);
|
|
2283
3298
|
extractUsers = (keys) => {
|
|
2284
3299
|
const users = /* @__PURE__ */ new Set();
|
|
2285
3300
|
for (const key of keys) {
|
|
@@ -2298,7 +3313,7 @@ var init_cleanTests = __esm({
|
|
|
2298
3313
|
return Array.from(users).sort();
|
|
2299
3314
|
};
|
|
2300
3315
|
getUsersToClean = async () => {
|
|
2301
|
-
const userName2 = String(
|
|
3316
|
+
const userName2 = String(flags6.user || "").trim();
|
|
2302
3317
|
if (userName2) {
|
|
2303
3318
|
return [userName2];
|
|
2304
3319
|
}
|
|
@@ -2308,19 +3323,19 @@ var init_cleanTests = __esm({
|
|
|
2308
3323
|
cleanUserTests = async (userName2) => {
|
|
2309
3324
|
const testsPrefix = `users:${userName2}:tests:`;
|
|
2310
3325
|
const testsCachePrefix = `users:${userName2}:cache:tests:`;
|
|
2311
|
-
if (
|
|
2312
|
-
console.log(
|
|
3326
|
+
if (flags6.cache) {
|
|
3327
|
+
console.log(import_chalk5.default.yellow(`clean user cache: ${userName2}`));
|
|
2313
3328
|
await (0, import_cli4.cleanRedis)(testsCachePrefix);
|
|
2314
3329
|
return;
|
|
2315
3330
|
}
|
|
2316
|
-
console.log(
|
|
3331
|
+
console.log(import_chalk5.default.yellow(`clean user tests: ${userName2}`));
|
|
2317
3332
|
await (0, import_cli4.cleanRedis)(testsPrefix);
|
|
2318
3333
|
await (0, import_cli4.cleanRedis)(testsCachePrefix);
|
|
2319
3334
|
};
|
|
2320
3335
|
run4 = async () => {
|
|
2321
3336
|
const users = await getUsersToClean();
|
|
2322
3337
|
if (users.length === 0) {
|
|
2323
|
-
console.log(
|
|
3338
|
+
console.log(import_chalk5.default.yellow("No users found to clean tests."));
|
|
2324
3339
|
process.exit();
|
|
2325
3340
|
}
|
|
2326
3341
|
for await (const userName2 of users) {
|
|
@@ -2334,33 +3349,33 @@ var init_cleanTests = __esm({
|
|
|
2334
3349
|
|
|
2335
3350
|
// src/scripts/continuity.ts
|
|
2336
3351
|
var continuity_exports = {};
|
|
2337
|
-
var
|
|
3352
|
+
var import_args7, import_progress3, import_chalk6, import_connectors4, import_cli5, import_constants4, import_time2, import_logger2, import_timescale, flags7, interval2, intervalKey, projectRoot4, parseProviders, findGapInData, continuity;
|
|
2338
3353
|
var init_continuity = __esm({
|
|
2339
3354
|
"src/scripts/continuity.ts"() {
|
|
2340
3355
|
"use strict";
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
3356
|
+
import_args7 = __toESM(require("args"));
|
|
3357
|
+
import_progress3 = __toESM(require("progress"));
|
|
3358
|
+
import_chalk6 = __toESM(require("chalk"));
|
|
2344
3359
|
import_connectors4 = require("@tradejs/node/connectors");
|
|
2345
3360
|
import_cli5 = require("@tradejs/node/cli");
|
|
2346
|
-
|
|
3361
|
+
import_constants4 = require("@tradejs/core/constants");
|
|
2347
3362
|
import_time2 = require("@tradejs/core/time");
|
|
2348
3363
|
import_logger2 = require("@tradejs/infra/logger");
|
|
2349
3364
|
import_timescale = require("@tradejs/infra/timescale");
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
3365
|
+
import_args7.default.option(["t", "tickers"], "Selected tickers");
|
|
3366
|
+
import_args7.default.option(["f", "timeframe"], "Timeframe", 15);
|
|
3367
|
+
import_args7.default.option(
|
|
2353
3368
|
["p", "provider"],
|
|
2354
3369
|
"Data provider: all|bybit|binance|coinbase or comma list",
|
|
2355
3370
|
"all"
|
|
2356
3371
|
);
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
interval2 = Number(
|
|
2360
|
-
intervalKey =
|
|
2361
|
-
|
|
3372
|
+
import_args7.default.option(["U", "user"], "Use user confg", "root");
|
|
3373
|
+
flags7 = import_args7.default.parse(process.argv);
|
|
3374
|
+
interval2 = Number(flags7.timeframe);
|
|
3375
|
+
intervalKey = flags7.timeframe.toString();
|
|
3376
|
+
projectRoot4 = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
2362
3377
|
parseProviders = async (value) => {
|
|
2363
|
-
const allProviders = await (0, import_connectors4.getAvailableConnectorProviders)(
|
|
3378
|
+
const allProviders = await (0, import_connectors4.getAvailableConnectorProviders)(projectRoot4);
|
|
2364
3379
|
const raw = String(value || "").trim().toLowerCase();
|
|
2365
3380
|
if (!raw || raw === "all") {
|
|
2366
3381
|
return allProviders;
|
|
@@ -2395,18 +3410,18 @@ var init_continuity = __esm({
|
|
|
2395
3410
|
};
|
|
2396
3411
|
continuity = async () => {
|
|
2397
3412
|
if (!Number.isFinite(interval2) || interval2 <= 0) {
|
|
2398
|
-
import_logger2.logger.error("Invalid timeframe: %s",
|
|
3413
|
+
import_logger2.logger.error("Invalid timeframe: %s", flags7.timeframe);
|
|
2399
3414
|
process.exit(1);
|
|
2400
3415
|
}
|
|
2401
3416
|
await (0, import_timescale.waitForDbReady)();
|
|
2402
|
-
const reloadStart = (0, import_time2.getTimestamp)(
|
|
3417
|
+
const reloadStart = (0, import_time2.getTimestamp)(import_constants4.PRELOAD_DAYS);
|
|
2403
3418
|
const reloadEnd = (0, import_time2.getTimestamp)();
|
|
2404
|
-
const providerIds = await parseProviders(
|
|
3419
|
+
const providerIds = await parseProviders(flags7.provider);
|
|
2405
3420
|
const providers = await Promise.all(
|
|
2406
3421
|
providerIds.map(async (providerId) => {
|
|
2407
3422
|
const connectorName = await (0, import_connectors4.getConnectorNameByProvider)(
|
|
2408
3423
|
providerId,
|
|
2409
|
-
|
|
3424
|
+
projectRoot4
|
|
2410
3425
|
);
|
|
2411
3426
|
if (!connectorName) {
|
|
2412
3427
|
import_logger2.logger.warn(
|
|
@@ -2417,7 +3432,7 @@ var init_continuity = __esm({
|
|
|
2417
3432
|
}
|
|
2418
3433
|
const creator = await (0, import_connectors4.getConnectorCreatorByName)(
|
|
2419
3434
|
connectorName,
|
|
2420
|
-
|
|
3435
|
+
projectRoot4
|
|
2421
3436
|
);
|
|
2422
3437
|
if (!creator) {
|
|
2423
3438
|
import_logger2.logger.warn(
|
|
@@ -2441,10 +3456,10 @@ var init_continuity = __esm({
|
|
|
2441
3456
|
}
|
|
2442
3457
|
for await (const provider of activeProviders) {
|
|
2443
3458
|
const connector = await provider.create({
|
|
2444
|
-
userName:
|
|
3459
|
+
userName: flags7.user
|
|
2445
3460
|
});
|
|
2446
|
-
const tickers = await (0, import_cli5.getTickers)(connector,
|
|
2447
|
-
const bar = new
|
|
3461
|
+
const tickers = await (0, import_cli5.getTickers)(connector, flags7.tickers);
|
|
3462
|
+
const bar = new import_progress3.default(
|
|
2448
3463
|
":current/:total [:bar][:percent] broken::broken fixed::fixed :eta(s) :symbol",
|
|
2449
3464
|
{
|
|
2450
3465
|
total: tickers.length,
|
|
@@ -2454,7 +3469,7 @@ var init_continuity = __esm({
|
|
|
2454
3469
|
let broken = 0;
|
|
2455
3470
|
let fixed = 0;
|
|
2456
3471
|
const expectedMs = interval2 * 60 * 1e3;
|
|
2457
|
-
import_logger2.logger.info(
|
|
3472
|
+
import_logger2.logger.info(import_chalk6.default.yellow(`continuity ${provider.name}: ${tickers.length}`));
|
|
2458
3473
|
for await (const symbol of tickers) {
|
|
2459
3474
|
const data = await connector.kline({
|
|
2460
3475
|
symbol,
|
|
@@ -2489,13 +3504,13 @@ var init_continuity = __esm({
|
|
|
2489
3504
|
}
|
|
2490
3505
|
}
|
|
2491
3506
|
bar.tick(1, {
|
|
2492
|
-
broken:
|
|
2493
|
-
fixed:
|
|
2494
|
-
symbol:
|
|
3507
|
+
broken: import_chalk6.default.yellow(broken),
|
|
3508
|
+
fixed: import_chalk6.default.cyan(fixed),
|
|
3509
|
+
symbol: import_chalk6.default.gray(symbol)
|
|
2495
3510
|
});
|
|
2496
3511
|
}
|
|
2497
3512
|
import_logger2.logger.info(
|
|
2498
|
-
|
|
3513
|
+
import_chalk6.default.yellow(
|
|
2499
3514
|
`[${provider.id}] broken: ${broken}/${tickers.length}, fixed: ${fixed}`
|
|
2500
3515
|
)
|
|
2501
3516
|
);
|
|
@@ -2508,46 +3523,50 @@ var init_continuity = __esm({
|
|
|
2508
3523
|
|
|
2509
3524
|
// src/scripts/derivativesIngest.ts
|
|
2510
3525
|
var derivativesIngest_exports = {};
|
|
2511
|
-
var import_config2,
|
|
3526
|
+
var import_config2, import_args8, import_chalk7, import_indicators, import_timescale2, import_userSettings, import_connectors5, flags8, asInt, run5;
|
|
2512
3527
|
var init_derivativesIngest = __esm({
|
|
2513
3528
|
"src/scripts/derivativesIngest.ts"() {
|
|
2514
3529
|
"use strict";
|
|
2515
3530
|
import_config2 = require("dotenv/config");
|
|
2516
|
-
|
|
2517
|
-
|
|
3531
|
+
import_args8 = __toESM(require("args"));
|
|
3532
|
+
import_chalk7 = __toESM(require("chalk"));
|
|
2518
3533
|
import_indicators = require("@tradejs/core/indicators");
|
|
2519
3534
|
import_timescale2 = require("@tradejs/infra/timescale");
|
|
3535
|
+
import_userSettings = require("@tradejs/infra/userSettings");
|
|
2520
3536
|
import_connectors5 = require("@tradejs/connectors");
|
|
2521
|
-
|
|
3537
|
+
import_args8.default.example(
|
|
2522
3538
|
"yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m,1h --days 120",
|
|
2523
3539
|
"Ingest market features (derivatives/spread) into Timescale by provider"
|
|
2524
3540
|
);
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
3541
|
+
import_args8.default.option(["s", "symbols"], "Comma-separated symbols", "BTCUSDT,ETHUSDT");
|
|
3542
|
+
import_args8.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m,1h");
|
|
3543
|
+
import_args8.default.option(["d", "days"], "Lookback in days", 120);
|
|
3544
|
+
import_args8.default.option(
|
|
2529
3545
|
["p", "provider"],
|
|
2530
3546
|
"Provider name: coinalyze | binance_coinbase_spread",
|
|
2531
3547
|
"coinalyze"
|
|
2532
3548
|
);
|
|
2533
|
-
|
|
2534
|
-
|
|
3549
|
+
import_args8.default.option(["U", "user"], "User settings profile name from Redis", "root");
|
|
3550
|
+
import_args8.default.option(["b", "batchDays"], "Request chunk size in days", 7);
|
|
3551
|
+
flags8 = import_args8.default.parse(process.argv);
|
|
2535
3552
|
asInt = (value, fallback) => {
|
|
2536
3553
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
2537
3554
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
2538
3555
|
};
|
|
2539
3556
|
run5 = async () => {
|
|
2540
|
-
const providerName = String(
|
|
3557
|
+
const providerName = String(flags8.provider || "coinalyze").trim().toLowerCase();
|
|
2541
3558
|
const provider = import_connectors5.marketDataProviders[providerName];
|
|
2542
3559
|
if (!provider) {
|
|
2543
3560
|
throw new Error(`Unsupported provider: ${providerName}`);
|
|
2544
3561
|
}
|
|
2545
|
-
const symbols = (0, import_indicators.normalizeCoinalyzeSymbols)(
|
|
3562
|
+
const symbols = (0, import_indicators.normalizeCoinalyzeSymbols)(flags8.symbols);
|
|
2546
3563
|
const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
|
|
2547
|
-
|
|
3564
|
+
flags8.intervals
|
|
2548
3565
|
);
|
|
2549
|
-
const days = asInt(
|
|
2550
|
-
const batchDays = asInt(
|
|
3566
|
+
const days = asInt(flags8.days, 120);
|
|
3567
|
+
const batchDays = asInt(flags8.batchDays, 7);
|
|
3568
|
+
const userName2 = String(flags8.user || "root").trim() || "root";
|
|
3569
|
+
const coinalyzeApiKey = providerName === "coinalyze" ? (await (0, import_userSettings.getUserSettings)(userName2)).COINALYZE_API_KEY : void 0;
|
|
2551
3570
|
if (!symbols.length) throw new Error("No symbols provided");
|
|
2552
3571
|
if (!intervals.length) throw new Error("No intervals provided");
|
|
2553
3572
|
await (0, import_timescale2.waitForDbReady)();
|
|
@@ -2561,10 +3580,11 @@ var init_derivativesIngest = __esm({
|
|
|
2561
3580
|
while (cursor < now2) {
|
|
2562
3581
|
const toMs = Math.min(now2, cursor + batchDays * 24 * 60 * 60 * 1e3);
|
|
2563
3582
|
process.stdout.write(
|
|
2564
|
-
`\r${
|
|
3583
|
+
`\r${import_chalk7.default.cyan(providerName)} ${import_chalk7.default.yellow(symbol)} ${interval4} ${new Date(cursor).toISOString()} .. ${new Date(toMs).toISOString()} `
|
|
2565
3584
|
);
|
|
2566
3585
|
const window2 = await provider.fetchWindow({
|
|
2567
3586
|
symbol,
|
|
3587
|
+
apiKey: coinalyzeApiKey,
|
|
2568
3588
|
interval: interval4,
|
|
2569
3589
|
fromMs: cursor,
|
|
2570
3590
|
toMs
|
|
@@ -2585,13 +3605,13 @@ var init_derivativesIngest = __esm({
|
|
|
2585
3605
|
}
|
|
2586
3606
|
}
|
|
2587
3607
|
console.log(
|
|
2588
|
-
|
|
3608
|
+
import_chalk7.default.green(
|
|
2589
3609
|
`Done. provider=${providerName} derivatives_rows=${totalDerivativesRows} spread_rows=${totalSpreadRows}`
|
|
2590
3610
|
)
|
|
2591
3611
|
);
|
|
2592
3612
|
};
|
|
2593
3613
|
run5().catch((error) => {
|
|
2594
|
-
console.error(
|
|
3614
|
+
console.error(import_chalk7.default.red(`derivativesIngest failed: ${error}`));
|
|
2595
3615
|
process.exit(1);
|
|
2596
3616
|
});
|
|
2597
3617
|
}
|
|
@@ -2599,80 +3619,77 @@ var init_derivativesIngest = __esm({
|
|
|
2599
3619
|
|
|
2600
3620
|
// src/scripts/derivativesIngestCoinalyzeAll.ts
|
|
2601
3621
|
var derivativesIngestCoinalyzeAll_exports = {};
|
|
2602
|
-
var import_config3,
|
|
3622
|
+
var import_config3, import_args9, import_chalk8, import_progress4, import_lodash3, import_connectors6, import_indicators2, import_async3, import_userSettings2, import_cli6, import_timescale3, coinalyzeIntervalMap, flags9, asInt2, parseList, coinalyzeBaseUrl, coinalyzeMaxRetries, lastRequestTs, fetchJsonWithRateLimit, fetchCoinalyzeMarkets, selectBestMarket, buildMatches, normalizeMetricPoint, toSeriesMap, fetchMetricBatch, run6;
|
|
2603
3623
|
var init_derivativesIngestCoinalyzeAll = __esm({
|
|
2604
3624
|
"src/scripts/derivativesIngestCoinalyzeAll.ts"() {
|
|
2605
3625
|
"use strict";
|
|
2606
3626
|
import_config3 = require("dotenv/config");
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
3627
|
+
import_args9 = __toESM(require("args"));
|
|
3628
|
+
import_chalk8 = __toESM(require("chalk"));
|
|
3629
|
+
import_progress4 = __toESM(require("progress"));
|
|
2610
3630
|
import_lodash3 = __toESM(require("lodash"));
|
|
2611
3631
|
import_connectors6 = require("@tradejs/connectors");
|
|
2612
3632
|
import_indicators2 = require("@tradejs/core/indicators");
|
|
2613
|
-
|
|
3633
|
+
import_async3 = require("@tradejs/core/async");
|
|
3634
|
+
import_userSettings2 = require("@tradejs/infra/userSettings");
|
|
2614
3635
|
import_cli6 = require("@tradejs/node/cli");
|
|
2615
3636
|
import_timescale3 = require("@tradejs/infra/timescale");
|
|
2616
3637
|
coinalyzeIntervalMap = {
|
|
2617
3638
|
"15m": "15min",
|
|
2618
3639
|
"1h": "1hour"
|
|
2619
3640
|
};
|
|
2620
|
-
|
|
3641
|
+
import_args9.default.example(
|
|
2621
3642
|
"yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m,1h",
|
|
2622
3643
|
"Fetch derivatives for all getTickers symbols matched to Coinalyze markets"
|
|
2623
3644
|
);
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
3645
|
+
import_args9.default.option(["U", "user"], "User settings profile name from Redis", "root");
|
|
3646
|
+
import_args9.default.option(["t", "tickers"], "Comma-separated include symbols");
|
|
3647
|
+
import_args9.default.option(["e", "exclude"], "Comma-separated exclude symbols");
|
|
3648
|
+
import_args9.default.option(["l", "tickersLimit"], "Tickers limit");
|
|
3649
|
+
import_args9.default.option(["c", "chunk"], "Chunk selector, e.g. 1/4");
|
|
3650
|
+
import_args9.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m,1h");
|
|
3651
|
+
import_args9.default.option(["d", "days"], "Lookback in days", 120);
|
|
3652
|
+
import_args9.default.option(["b", "batchDays"], "Request chunk size in days", 120);
|
|
3653
|
+
import_args9.default.option(
|
|
2633
3654
|
["E", "exchangePriority"],
|
|
2634
3655
|
"Coinalyze exchange priority, comma-separated",
|
|
2635
3656
|
"A,6,0"
|
|
2636
3657
|
);
|
|
2637
|
-
|
|
3658
|
+
import_args9.default.option(
|
|
2638
3659
|
["S", "symbolBatchSize"],
|
|
2639
3660
|
"How many symbols to request in one Coinalyze call",
|
|
2640
3661
|
8
|
|
2641
3662
|
);
|
|
2642
|
-
|
|
3663
|
+
import_args9.default.option(
|
|
2643
3664
|
["w", "requestDelayMs"],
|
|
2644
3665
|
"Delay between API requests to smooth rate limit",
|
|
2645
3666
|
200
|
|
2646
3667
|
);
|
|
2647
|
-
|
|
3668
|
+
import_args9.default.option(
|
|
2648
3669
|
["T", "requestTimeoutMs"],
|
|
2649
3670
|
"HTTP request timeout in milliseconds",
|
|
2650
3671
|
45e3
|
|
2651
3672
|
);
|
|
2652
|
-
|
|
2653
|
-
|
|
3673
|
+
import_args9.default.option(["L", "showTickersList"], "Print matched symbols only", false);
|
|
3674
|
+
flags9 = import_args9.default.parse(process.argv);
|
|
2654
3675
|
asInt2 = (value, fallback) => {
|
|
2655
3676
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
2656
3677
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
2657
3678
|
};
|
|
2658
3679
|
parseList = (value) => String(value ?? "").split(",").map((item) => item.trim().toUpperCase()).filter(Boolean);
|
|
2659
|
-
apiKey = process.env.COINALYZE_API_KEY?.trim();
|
|
2660
|
-
if (!apiKey) {
|
|
2661
|
-
throw new Error("Missing COINALYZE_API_KEY");
|
|
2662
|
-
}
|
|
2663
3680
|
coinalyzeBaseUrl = process.env.COINALYZE_BASE_URL?.trim() || "https://api.coinalyze.net/v1";
|
|
2664
3681
|
coinalyzeMaxRetries = asInt2(process.env.COINALYZE_MAX_RETRIES, 4);
|
|
2665
3682
|
lastRequestTs = 0;
|
|
2666
|
-
fetchJsonWithRateLimit = async (url) => {
|
|
2667
|
-
const requestDelayMs = Math.max(100, asInt2(
|
|
3683
|
+
fetchJsonWithRateLimit = async (url, apiKey) => {
|
|
3684
|
+
const requestDelayMs = Math.max(100, asInt2(flags9.requestDelayMs, 200));
|
|
2668
3685
|
const requestTimeoutMs = Math.max(
|
|
2669
3686
|
5e3,
|
|
2670
|
-
asInt2(
|
|
3687
|
+
asInt2(flags9.requestTimeoutMs, 45e3)
|
|
2671
3688
|
);
|
|
2672
3689
|
for (let attempt = 0; attempt <= coinalyzeMaxRetries; attempt += 1) {
|
|
2673
3690
|
const waitMs = Math.max(0, lastRequestTs + requestDelayMs - Date.now());
|
|
2674
3691
|
if (waitMs > 0) {
|
|
2675
|
-
await (0,
|
|
3692
|
+
await (0, import_async3.delay)(waitMs);
|
|
2676
3693
|
}
|
|
2677
3694
|
lastRequestTs = Date.now();
|
|
2678
3695
|
const controller = new AbortController();
|
|
@@ -2692,7 +3709,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2692
3709
|
const abortError = error instanceof Error && (error.name === "AbortError" || String(error.message).toLowerCase().includes("aborted"));
|
|
2693
3710
|
if (attempt < coinalyzeMaxRetries && abortError) {
|
|
2694
3711
|
const backoffMs = Math.min(12e3, 800 * 2 ** attempt);
|
|
2695
|
-
await (0,
|
|
3712
|
+
await (0, import_async3.delay)(backoffMs);
|
|
2696
3713
|
continue;
|
|
2697
3714
|
}
|
|
2698
3715
|
throw error;
|
|
@@ -2708,16 +3725,17 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2708
3725
|
const transient = response.status === 429 || response.status >= 500;
|
|
2709
3726
|
if (attempt < coinalyzeMaxRetries && transient) {
|
|
2710
3727
|
const backoffMs = Math.min(12e3, 800 * 2 ** attempt);
|
|
2711
|
-
await (0,
|
|
3728
|
+
await (0, import_async3.delay)(retryAfterMs ?? backoffMs);
|
|
2712
3729
|
continue;
|
|
2713
3730
|
}
|
|
2714
3731
|
throw new Error(`Coinalyze ${response.status}: ${text}`);
|
|
2715
3732
|
}
|
|
2716
3733
|
throw new Error("Coinalyze request failed after retries");
|
|
2717
3734
|
};
|
|
2718
|
-
fetchCoinalyzeMarkets = async () => {
|
|
3735
|
+
fetchCoinalyzeMarkets = async (apiKey) => {
|
|
2719
3736
|
const raw = await fetchJsonWithRateLimit(
|
|
2720
|
-
`${coinalyzeBaseUrl}/future-markets
|
|
3737
|
+
`${coinalyzeBaseUrl}/future-markets`,
|
|
3738
|
+
apiKey
|
|
2721
3739
|
);
|
|
2722
3740
|
return Array.isArray(raw) ? raw : [];
|
|
2723
3741
|
};
|
|
@@ -2800,40 +3818,48 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2800
3818
|
return out;
|
|
2801
3819
|
};
|
|
2802
3820
|
fetchMetricBatch = async (params) => {
|
|
2803
|
-
const { endpoint, metric, marketSymbols, interval: interval4, fromMs, toMs } = params;
|
|
3821
|
+
const { endpoint, metric, marketSymbols, apiKey, interval: interval4, fromMs, toMs } = params;
|
|
2804
3822
|
const url = new URL(`${coinalyzeBaseUrl}${endpoint}`);
|
|
2805
3823
|
url.searchParams.set("symbols", marketSymbols.join(","));
|
|
2806
3824
|
url.searchParams.set("interval", coinalyzeIntervalMap[interval4] || interval4);
|
|
2807
3825
|
url.searchParams.set("from", String(Math.floor(fromMs / 1e3)));
|
|
2808
3826
|
url.searchParams.set("to", String(Math.floor(toMs / 1e3)));
|
|
2809
|
-
const raw = await fetchJsonWithRateLimit(url.toString());
|
|
3827
|
+
const raw = await fetchJsonWithRateLimit(url.toString(), apiKey);
|
|
2810
3828
|
return toSeriesMap(raw, metric);
|
|
2811
3829
|
};
|
|
2812
3830
|
run6 = async () => {
|
|
3831
|
+
const userName2 = String(flags9.user || "root").trim() || "root";
|
|
2813
3832
|
const intervals = (0, import_indicators2.normalizeDerivativesIntervals)(
|
|
2814
|
-
|
|
3833
|
+
flags9.intervals
|
|
2815
3834
|
);
|
|
2816
|
-
const days = asInt2(
|
|
2817
|
-
const batchDays = asInt2(
|
|
2818
|
-
const symbolBatchSize = asInt2(
|
|
2819
|
-
const tickersLimit =
|
|
2820
|
-
const exchangePriority = parseList(
|
|
3835
|
+
const days = asInt2(flags9.days, 120);
|
|
3836
|
+
const batchDays = asInt2(flags9.batchDays, 120);
|
|
3837
|
+
const symbolBatchSize = asInt2(flags9.symbolBatchSize, 25);
|
|
3838
|
+
const tickersLimit = flags9.tickersLimit !== void 0 ? asInt2(flags9.tickersLimit, 0) || void 0 : void 0;
|
|
3839
|
+
const exchangePriority = parseList(flags9.exchangePriority || "A,6,0");
|
|
2821
3840
|
if (!intervals.length) throw new Error("No intervals provided");
|
|
3841
|
+
const settings = await (0, import_userSettings2.getUserSettings)(userName2);
|
|
3842
|
+
const coinalyzeApiKey = settings.COINALYZE_API_KEY.trim();
|
|
3843
|
+
if (!coinalyzeApiKey) {
|
|
3844
|
+
throw new Error(
|
|
3845
|
+
`Missing COINALYZE_API_KEY in user settings for user ${userName2}`
|
|
3846
|
+
);
|
|
3847
|
+
}
|
|
2822
3848
|
const connectorFactory = import_connectors6.connectors[import_connectors6.ConnectorNames.ByBit];
|
|
2823
3849
|
const bybit = await connectorFactory({
|
|
2824
|
-
userName:
|
|
3850
|
+
userName: userName2
|
|
2825
3851
|
});
|
|
2826
3852
|
const tickers = await (0, import_cli6.getTickers)(
|
|
2827
3853
|
bybit,
|
|
2828
|
-
String(
|
|
2829
|
-
String(
|
|
3854
|
+
String(flags9.tickers || ""),
|
|
3855
|
+
String(flags9.exclude || ""),
|
|
2830
3856
|
tickersLimit,
|
|
2831
|
-
String(
|
|
3857
|
+
String(flags9.chunk || "")
|
|
2832
3858
|
);
|
|
2833
3859
|
if (!tickers.length) {
|
|
2834
3860
|
throw new Error("No tickers loaded via getTickers");
|
|
2835
3861
|
}
|
|
2836
|
-
const markets = await fetchCoinalyzeMarkets();
|
|
3862
|
+
const markets = await fetchCoinalyzeMarkets(coinalyzeApiKey);
|
|
2837
3863
|
if (!markets.length) {
|
|
2838
3864
|
throw new Error("No markets returned by Coinalyze /future-markets");
|
|
2839
3865
|
}
|
|
@@ -2846,18 +3872,18 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2846
3872
|
throw new Error("No matched symbols between getTickers and Coinalyze");
|
|
2847
3873
|
}
|
|
2848
3874
|
console.log(
|
|
2849
|
-
|
|
3875
|
+
import_chalk8.default.cyan(
|
|
2850
3876
|
`Tickers=${tickers.length}, matched=${matches.length}, unmatched=${unmatched.length}, intervals=${intervals.join(",")}, symbolBatch=${symbolBatchSize}`
|
|
2851
3877
|
)
|
|
2852
3878
|
);
|
|
2853
3879
|
if (unmatched.length) {
|
|
2854
3880
|
console.log(
|
|
2855
|
-
|
|
3881
|
+
import_chalk8.default.yellow(
|
|
2856
3882
|
`Unmatched sample (${Math.min(20, unmatched.length)}): ${unmatched.slice(0, 20).join(", ")}`
|
|
2857
3883
|
)
|
|
2858
3884
|
);
|
|
2859
3885
|
}
|
|
2860
|
-
if (
|
|
3886
|
+
if (flags9.showTickersList) {
|
|
2861
3887
|
console.log(
|
|
2862
3888
|
JSON.stringify(
|
|
2863
3889
|
matches.map((item) => ({
|
|
@@ -2877,7 +3903,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2877
3903
|
const symbolBatches = import_lodash3.default.chunk(matches, symbolBatchSize);
|
|
2878
3904
|
const windowsPerPair = Math.ceil(days / batchDays);
|
|
2879
3905
|
const totalWindows = symbolBatches.length * intervals.length * windowsPerPair;
|
|
2880
|
-
const bar = new
|
|
3906
|
+
const bar = new import_progress4.default(
|
|
2881
3907
|
":current/:total [:bar][:percent] :eta(s) :batch rows=:rows fail=:fail",
|
|
2882
3908
|
{
|
|
2883
3909
|
total: Math.max(1, totalWindows),
|
|
@@ -2901,6 +3927,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2901
3927
|
endpoint: oiPath,
|
|
2902
3928
|
metric: "oi",
|
|
2903
3929
|
marketSymbols,
|
|
3930
|
+
apiKey: coinalyzeApiKey,
|
|
2904
3931
|
interval: interval4,
|
|
2905
3932
|
fromMs: cursor,
|
|
2906
3933
|
toMs
|
|
@@ -2909,6 +3936,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2909
3936
|
endpoint: fundingPath,
|
|
2910
3937
|
metric: "funding",
|
|
2911
3938
|
marketSymbols,
|
|
3939
|
+
apiKey: coinalyzeApiKey,
|
|
2912
3940
|
interval: interval4,
|
|
2913
3941
|
fromMs: cursor,
|
|
2914
3942
|
toMs
|
|
@@ -2917,6 +3945,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2917
3945
|
endpoint: liqPath,
|
|
2918
3946
|
metric: "liq",
|
|
2919
3947
|
marketSymbols,
|
|
3948
|
+
apiKey: coinalyzeApiKey,
|
|
2920
3949
|
interval: interval4,
|
|
2921
3950
|
fromMs: cursor,
|
|
2922
3951
|
toMs
|
|
@@ -2938,13 +3967,13 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2938
3967
|
} catch (error) {
|
|
2939
3968
|
failedWindows += 1;
|
|
2940
3969
|
console.error(
|
|
2941
|
-
|
|
3970
|
+
import_chalk8.default.red(
|
|
2942
3971
|
`batch window failed batch=${batchIdx + 1}/${symbolBatches.length} interval=${interval4} ${new Date(cursor).toISOString()}..${new Date(toMs).toISOString()}: ${error}`
|
|
2943
3972
|
)
|
|
2944
3973
|
);
|
|
2945
3974
|
} finally {
|
|
2946
3975
|
bar.tick(1, {
|
|
2947
|
-
batch:
|
|
3976
|
+
batch: import_chalk8.default.gray(
|
|
2948
3977
|
`${batchIdx + 1}/${symbolBatches.length} ${interval4} (${batch.length} symbols)`
|
|
2949
3978
|
),
|
|
2950
3979
|
rows: totalRows,
|
|
@@ -2957,7 +3986,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2957
3986
|
}
|
|
2958
3987
|
console.log("");
|
|
2959
3988
|
console.log(
|
|
2960
|
-
|
|
3989
|
+
import_chalk8.default.green(
|
|
2961
3990
|
`Done. symbols=${matches.length} intervals=${intervals.join(",")} rows=${totalRows} failed_windows=${failedWindows}`
|
|
2962
3991
|
)
|
|
2963
3992
|
);
|
|
@@ -2966,7 +3995,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
|
|
|
2966
3995
|
}
|
|
2967
3996
|
};
|
|
2968
3997
|
run6().catch((error) => {
|
|
2969
|
-
console.error(
|
|
3998
|
+
console.error(import_chalk8.default.red(`derivativesIngestCoinalyzeAll failed: ${error}`));
|
|
2970
3999
|
process.exit(1);
|
|
2971
4000
|
});
|
|
2972
4001
|
}
|
|
@@ -3112,14 +4141,14 @@ var require_postgres_date = __commonJS({
|
|
|
3112
4141
|
var ms = matches[7];
|
|
3113
4142
|
ms = ms ? 1e3 * parseFloat(ms) : 0;
|
|
3114
4143
|
var date;
|
|
3115
|
-
var
|
|
3116
|
-
if (
|
|
4144
|
+
var offset = timeZoneOffset(isoDate);
|
|
4145
|
+
if (offset != null) {
|
|
3117
4146
|
date = new Date(Date.UTC(year, month, day, hour, minute, second, ms));
|
|
3118
4147
|
if (is0To99(year)) {
|
|
3119
4148
|
date.setUTCFullYear(year);
|
|
3120
4149
|
}
|
|
3121
|
-
if (
|
|
3122
|
-
date.setTime(date.getTime() -
|
|
4150
|
+
if (offset !== 0) {
|
|
4151
|
+
date.setTime(date.getTime() - offset);
|
|
3123
4152
|
}
|
|
3124
4153
|
} else {
|
|
3125
4154
|
date = new Date(year, month, day, hour, minute, second, ms);
|
|
@@ -3158,8 +4187,8 @@ var require_postgres_date = __commonJS({
|
|
|
3158
4187
|
return 0;
|
|
3159
4188
|
}
|
|
3160
4189
|
var sign = type === "-" ? -1 : 1;
|
|
3161
|
-
var
|
|
3162
|
-
return
|
|
4190
|
+
var offset = parseInt(zone[2], 10) * 3600 + parseInt(zone[3] || 0, 10) * 60 + parseInt(zone[4] || 0, 10);
|
|
4191
|
+
return offset * sign * 1e3;
|
|
3163
4192
|
}
|
|
3164
4193
|
function bcYearToNegativeYear(year) {
|
|
3165
4194
|
return -(year - 1);
|
|
@@ -3602,13 +4631,13 @@ var require_binaryParsers = __commonJS({
|
|
|
3602
4631
|
"../../node_modules/pg-types/lib/binaryParsers.js"(exports2, module2) {
|
|
3603
4632
|
"use strict";
|
|
3604
4633
|
var parseInt64 = require_pg_int8();
|
|
3605
|
-
var parseBits = function(data, bits,
|
|
3606
|
-
|
|
4634
|
+
var parseBits = function(data, bits, offset, invert, callback) {
|
|
4635
|
+
offset = offset || 0;
|
|
3607
4636
|
invert = invert || false;
|
|
3608
4637
|
callback = callback || function(lastValue, newValue, bits2) {
|
|
3609
4638
|
return lastValue * Math.pow(2, bits2) + newValue;
|
|
3610
4639
|
};
|
|
3611
|
-
var offsetBytes =
|
|
4640
|
+
var offsetBytes = offset >> 3;
|
|
3612
4641
|
var inv = function(value) {
|
|
3613
4642
|
if (invert) {
|
|
3614
4643
|
return ~value & 255;
|
|
@@ -3616,23 +4645,23 @@ var require_binaryParsers = __commonJS({
|
|
|
3616
4645
|
return value;
|
|
3617
4646
|
};
|
|
3618
4647
|
var mask = 255;
|
|
3619
|
-
var firstBits = 8 -
|
|
4648
|
+
var firstBits = 8 - offset % 8;
|
|
3620
4649
|
if (bits < firstBits) {
|
|
3621
4650
|
mask = 255 << 8 - bits & 255;
|
|
3622
4651
|
firstBits = bits;
|
|
3623
4652
|
}
|
|
3624
|
-
if (
|
|
3625
|
-
mask = mask >>
|
|
4653
|
+
if (offset) {
|
|
4654
|
+
mask = mask >> offset % 8;
|
|
3626
4655
|
}
|
|
3627
4656
|
var result = 0;
|
|
3628
|
-
if (
|
|
4657
|
+
if (offset % 8 + bits >= 8) {
|
|
3629
4658
|
result = callback(0, inv(data[offsetBytes]) & mask, firstBits);
|
|
3630
4659
|
}
|
|
3631
|
-
var bytes = bits +
|
|
4660
|
+
var bytes = bits + offset >> 3;
|
|
3632
4661
|
for (var i = offsetBytes + 1; i < bytes; i++) {
|
|
3633
4662
|
result = callback(result, inv(data[i]), 8);
|
|
3634
4663
|
}
|
|
3635
|
-
var lastBits = (bits +
|
|
4664
|
+
var lastBits = (bits + offset) % 8;
|
|
3636
4665
|
if (lastBits > 0) {
|
|
3637
4666
|
result = callback(result, inv(data[bytes]) >> 8 - lastBits, lastBits);
|
|
3638
4667
|
}
|
|
@@ -3722,28 +4751,28 @@ var require_binaryParsers = __commonJS({
|
|
|
3722
4751
|
};
|
|
3723
4752
|
var parseArray = function(value) {
|
|
3724
4753
|
var dim = parseBits(value, 32);
|
|
3725
|
-
var
|
|
4754
|
+
var flags17 = parseBits(value, 32, 32);
|
|
3726
4755
|
var elementType = parseBits(value, 32, 64);
|
|
3727
|
-
var
|
|
4756
|
+
var offset = 96;
|
|
3728
4757
|
var dims = [];
|
|
3729
4758
|
for (var i = 0; i < dim; i++) {
|
|
3730
|
-
dims[i] = parseBits(value, 32,
|
|
3731
|
-
|
|
3732
|
-
|
|
4759
|
+
dims[i] = parseBits(value, 32, offset);
|
|
4760
|
+
offset += 32;
|
|
4761
|
+
offset += 32;
|
|
3733
4762
|
}
|
|
3734
4763
|
var parseElement = function(elementType2) {
|
|
3735
|
-
var length = parseBits(value, 32,
|
|
3736
|
-
|
|
4764
|
+
var length = parseBits(value, 32, offset);
|
|
4765
|
+
offset += 32;
|
|
3737
4766
|
if (length == 4294967295) {
|
|
3738
4767
|
return null;
|
|
3739
4768
|
}
|
|
3740
4769
|
var result;
|
|
3741
4770
|
if (elementType2 == 23 || elementType2 == 20) {
|
|
3742
|
-
result = parseBits(value, length * 8,
|
|
3743
|
-
|
|
4771
|
+
result = parseBits(value, length * 8, offset);
|
|
4772
|
+
offset += length * 8;
|
|
3744
4773
|
return result;
|
|
3745
4774
|
} else if (elementType2 == 25) {
|
|
3746
|
-
result = value.toString(this.encoding,
|
|
4775
|
+
result = value.toString(this.encoding, offset >> 3, (offset += length << 3) >> 3);
|
|
3747
4776
|
return result;
|
|
3748
4777
|
} else {
|
|
3749
4778
|
console.log("ERROR: ElementType not implemented: " + elementType2);
|
|
@@ -4051,18 +5080,18 @@ var require_utils = __commonJS({
|
|
|
4051
5080
|
return JSON.stringify(val);
|
|
4052
5081
|
}
|
|
4053
5082
|
function dateToString(date) {
|
|
4054
|
-
let
|
|
5083
|
+
let offset = -date.getTimezoneOffset();
|
|
4055
5084
|
let year = date.getFullYear();
|
|
4056
5085
|
const isBCYear = year < 1;
|
|
4057
5086
|
if (isBCYear) year = Math.abs(year) + 1;
|
|
4058
5087
|
let ret = String(year).padStart(4, "0") + "-" + String(date.getMonth() + 1).padStart(2, "0") + "-" + String(date.getDate()).padStart(2, "0") + "T" + String(date.getHours()).padStart(2, "0") + ":" + String(date.getMinutes()).padStart(2, "0") + ":" + String(date.getSeconds()).padStart(2, "0") + "." + String(date.getMilliseconds()).padStart(3, "0");
|
|
4059
|
-
if (
|
|
5088
|
+
if (offset < 0) {
|
|
4060
5089
|
ret += "-";
|
|
4061
|
-
|
|
5090
|
+
offset *= -1;
|
|
4062
5091
|
} else {
|
|
4063
5092
|
ret += "+";
|
|
4064
5093
|
}
|
|
4065
|
-
ret += String(Math.floor(
|
|
5094
|
+
ret += String(Math.floor(offset / 60)).padStart(2, "0") + ":" + String(offset % 60).padStart(2, "0");
|
|
4066
5095
|
if (isBCYear) ret += " BC";
|
|
4067
5096
|
return ret;
|
|
4068
5097
|
}
|
|
@@ -4604,15 +5633,15 @@ var require_pg_connection_string = __commonJS({
|
|
|
4604
5633
|
if (config.sslcert || config.sslkey || config.sslrootcert || config.sslmode) {
|
|
4605
5634
|
config.ssl = {};
|
|
4606
5635
|
}
|
|
4607
|
-
const
|
|
5636
|
+
const fs7 = config.sslcert || config.sslkey || config.sslrootcert ? require("fs") : null;
|
|
4608
5637
|
if (config.sslcert) {
|
|
4609
|
-
config.ssl.cert =
|
|
5638
|
+
config.ssl.cert = fs7.readFileSync(config.sslcert).toString();
|
|
4610
5639
|
}
|
|
4611
5640
|
if (config.sslkey) {
|
|
4612
|
-
config.ssl.key =
|
|
5641
|
+
config.ssl.key = fs7.readFileSync(config.sslkey).toString();
|
|
4613
5642
|
}
|
|
4614
5643
|
if (config.sslrootcert) {
|
|
4615
|
-
config.ssl.ca =
|
|
5644
|
+
config.ssl.ca = fs7.readFileSync(config.sslrootcert).toString();
|
|
4616
5645
|
}
|
|
4617
5646
|
if (options.useLibpqCompat && config.uselibpqcompat) {
|
|
4618
5647
|
throw new Error("Both useLibpqCompat and uselibpqcompat are set. Please use only one of them.");
|
|
@@ -5613,13 +6642,13 @@ var require_buffer_reader = __commonJS({
|
|
|
5613
6642
|
exports2.BufferReader = void 0;
|
|
5614
6643
|
var emptyBuffer = Buffer.allocUnsafe(0);
|
|
5615
6644
|
var BufferReader = class {
|
|
5616
|
-
constructor(
|
|
5617
|
-
this.offset =
|
|
6645
|
+
constructor(offset = 0) {
|
|
6646
|
+
this.offset = offset;
|
|
5618
6647
|
this.buffer = emptyBuffer;
|
|
5619
6648
|
this.encoding = "utf-8";
|
|
5620
6649
|
}
|
|
5621
|
-
setBuffer(
|
|
5622
|
-
this.offset =
|
|
6650
|
+
setBuffer(offset, buffer) {
|
|
6651
|
+
this.offset = offset;
|
|
5623
6652
|
this.buffer = buffer;
|
|
5624
6653
|
}
|
|
5625
6654
|
int16() {
|
|
@@ -5691,26 +6720,26 @@ var require_parser = __commonJS({
|
|
|
5691
6720
|
parse(buffer, callback) {
|
|
5692
6721
|
this.mergeBuffer(buffer);
|
|
5693
6722
|
const bufferFullLength = this.bufferOffset + this.bufferLength;
|
|
5694
|
-
let
|
|
5695
|
-
while (
|
|
5696
|
-
const code = this.buffer[
|
|
5697
|
-
const length = this.buffer.readUInt32BE(
|
|
6723
|
+
let offset = this.bufferOffset;
|
|
6724
|
+
while (offset + HEADER_LENGTH <= bufferFullLength) {
|
|
6725
|
+
const code = this.buffer[offset];
|
|
6726
|
+
const length = this.buffer.readUInt32BE(offset + CODE_LENGTH);
|
|
5698
6727
|
const fullMessageLength = CODE_LENGTH + length;
|
|
5699
|
-
if (fullMessageLength +
|
|
5700
|
-
const message = this.handlePacket(
|
|
6728
|
+
if (fullMessageLength + offset <= bufferFullLength) {
|
|
6729
|
+
const message = this.handlePacket(offset + HEADER_LENGTH, code, length, this.buffer);
|
|
5701
6730
|
callback(message);
|
|
5702
|
-
|
|
6731
|
+
offset += fullMessageLength;
|
|
5703
6732
|
} else {
|
|
5704
6733
|
break;
|
|
5705
6734
|
}
|
|
5706
6735
|
}
|
|
5707
|
-
if (
|
|
6736
|
+
if (offset === bufferFullLength) {
|
|
5708
6737
|
this.buffer = emptyBuffer;
|
|
5709
6738
|
this.bufferLength = 0;
|
|
5710
6739
|
this.bufferOffset = 0;
|
|
5711
6740
|
} else {
|
|
5712
|
-
this.bufferLength = bufferFullLength -
|
|
5713
|
-
this.bufferOffset =
|
|
6741
|
+
this.bufferLength = bufferFullLength - offset;
|
|
6742
|
+
this.bufferOffset = offset;
|
|
5714
6743
|
}
|
|
5715
6744
|
}
|
|
5716
6745
|
mergeBuffer(buffer) {
|
|
@@ -5740,7 +6769,7 @@ var require_parser = __commonJS({
|
|
|
5740
6769
|
this.bufferLength = buffer.byteLength;
|
|
5741
6770
|
}
|
|
5742
6771
|
}
|
|
5743
|
-
handlePacket(
|
|
6772
|
+
handlePacket(offset, code, length, bytes) {
|
|
5744
6773
|
switch (code) {
|
|
5745
6774
|
case 50:
|
|
5746
6775
|
return messages_1.bindComplete;
|
|
@@ -5759,59 +6788,59 @@ var require_parser = __commonJS({
|
|
|
5759
6788
|
case 73:
|
|
5760
6789
|
return messages_1.emptyQuery;
|
|
5761
6790
|
case 68:
|
|
5762
|
-
return this.parseDataRowMessage(
|
|
6791
|
+
return this.parseDataRowMessage(offset, length, bytes);
|
|
5763
6792
|
case 67:
|
|
5764
|
-
return this.parseCommandCompleteMessage(
|
|
6793
|
+
return this.parseCommandCompleteMessage(offset, length, bytes);
|
|
5765
6794
|
case 90:
|
|
5766
|
-
return this.parseReadyForQueryMessage(
|
|
6795
|
+
return this.parseReadyForQueryMessage(offset, length, bytes);
|
|
5767
6796
|
case 65:
|
|
5768
|
-
return this.parseNotificationMessage(
|
|
6797
|
+
return this.parseNotificationMessage(offset, length, bytes);
|
|
5769
6798
|
case 82:
|
|
5770
|
-
return this.parseAuthenticationResponse(
|
|
6799
|
+
return this.parseAuthenticationResponse(offset, length, bytes);
|
|
5771
6800
|
case 83:
|
|
5772
|
-
return this.parseParameterStatusMessage(
|
|
6801
|
+
return this.parseParameterStatusMessage(offset, length, bytes);
|
|
5773
6802
|
case 75:
|
|
5774
|
-
return this.parseBackendKeyData(
|
|
6803
|
+
return this.parseBackendKeyData(offset, length, bytes);
|
|
5775
6804
|
case 69:
|
|
5776
|
-
return this.parseErrorMessage(
|
|
6805
|
+
return this.parseErrorMessage(offset, length, bytes, "error");
|
|
5777
6806
|
case 78:
|
|
5778
|
-
return this.parseErrorMessage(
|
|
6807
|
+
return this.parseErrorMessage(offset, length, bytes, "notice");
|
|
5779
6808
|
case 84:
|
|
5780
|
-
return this.parseRowDescriptionMessage(
|
|
6809
|
+
return this.parseRowDescriptionMessage(offset, length, bytes);
|
|
5781
6810
|
case 116:
|
|
5782
|
-
return this.parseParameterDescriptionMessage(
|
|
6811
|
+
return this.parseParameterDescriptionMessage(offset, length, bytes);
|
|
5783
6812
|
case 71:
|
|
5784
|
-
return this.parseCopyInMessage(
|
|
6813
|
+
return this.parseCopyInMessage(offset, length, bytes);
|
|
5785
6814
|
case 72:
|
|
5786
|
-
return this.parseCopyOutMessage(
|
|
6815
|
+
return this.parseCopyOutMessage(offset, length, bytes);
|
|
5787
6816
|
case 100:
|
|
5788
|
-
return this.parseCopyData(
|
|
6817
|
+
return this.parseCopyData(offset, length, bytes);
|
|
5789
6818
|
default:
|
|
5790
6819
|
return new messages_1.DatabaseError("received invalid response: " + code.toString(16), length, "error");
|
|
5791
6820
|
}
|
|
5792
6821
|
}
|
|
5793
|
-
parseReadyForQueryMessage(
|
|
5794
|
-
this.reader.setBuffer(
|
|
6822
|
+
parseReadyForQueryMessage(offset, length, bytes) {
|
|
6823
|
+
this.reader.setBuffer(offset, bytes);
|
|
5795
6824
|
const status = this.reader.string(1);
|
|
5796
6825
|
return new messages_1.ReadyForQueryMessage(length, status);
|
|
5797
6826
|
}
|
|
5798
|
-
parseCommandCompleteMessage(
|
|
5799
|
-
this.reader.setBuffer(
|
|
6827
|
+
parseCommandCompleteMessage(offset, length, bytes) {
|
|
6828
|
+
this.reader.setBuffer(offset, bytes);
|
|
5800
6829
|
const text = this.reader.cstring();
|
|
5801
6830
|
return new messages_1.CommandCompleteMessage(length, text);
|
|
5802
6831
|
}
|
|
5803
|
-
parseCopyData(
|
|
5804
|
-
const chunk = bytes.slice(
|
|
6832
|
+
parseCopyData(offset, length, bytes) {
|
|
6833
|
+
const chunk = bytes.slice(offset, offset + (length - 4));
|
|
5805
6834
|
return new messages_1.CopyDataMessage(length, chunk);
|
|
5806
6835
|
}
|
|
5807
|
-
parseCopyInMessage(
|
|
5808
|
-
return this.parseCopyMessage(
|
|
6836
|
+
parseCopyInMessage(offset, length, bytes) {
|
|
6837
|
+
return this.parseCopyMessage(offset, length, bytes, "copyInResponse");
|
|
5809
6838
|
}
|
|
5810
|
-
parseCopyOutMessage(
|
|
5811
|
-
return this.parseCopyMessage(
|
|
6839
|
+
parseCopyOutMessage(offset, length, bytes) {
|
|
6840
|
+
return this.parseCopyMessage(offset, length, bytes, "copyOutResponse");
|
|
5812
6841
|
}
|
|
5813
|
-
parseCopyMessage(
|
|
5814
|
-
this.reader.setBuffer(
|
|
6842
|
+
parseCopyMessage(offset, length, bytes, messageName) {
|
|
6843
|
+
this.reader.setBuffer(offset, bytes);
|
|
5815
6844
|
const isBinary = this.reader.byte() !== 0;
|
|
5816
6845
|
const columnCount = this.reader.int16();
|
|
5817
6846
|
const message = new messages_1.CopyResponse(length, messageName, isBinary, columnCount);
|
|
@@ -5820,15 +6849,15 @@ var require_parser = __commonJS({
|
|
|
5820
6849
|
}
|
|
5821
6850
|
return message;
|
|
5822
6851
|
}
|
|
5823
|
-
parseNotificationMessage(
|
|
5824
|
-
this.reader.setBuffer(
|
|
6852
|
+
parseNotificationMessage(offset, length, bytes) {
|
|
6853
|
+
this.reader.setBuffer(offset, bytes);
|
|
5825
6854
|
const processId = this.reader.int32();
|
|
5826
6855
|
const channel = this.reader.cstring();
|
|
5827
6856
|
const payload = this.reader.cstring();
|
|
5828
6857
|
return new messages_1.NotificationResponseMessage(length, processId, channel, payload);
|
|
5829
6858
|
}
|
|
5830
|
-
parseRowDescriptionMessage(
|
|
5831
|
-
this.reader.setBuffer(
|
|
6859
|
+
parseRowDescriptionMessage(offset, length, bytes) {
|
|
6860
|
+
this.reader.setBuffer(offset, bytes);
|
|
5832
6861
|
const fieldCount = this.reader.int16();
|
|
5833
6862
|
const message = new messages_1.RowDescriptionMessage(length, fieldCount);
|
|
5834
6863
|
for (let i = 0; i < fieldCount; i++) {
|
|
@@ -5846,8 +6875,8 @@ var require_parser = __commonJS({
|
|
|
5846
6875
|
const mode = this.reader.int16() === 0 ? "text" : "binary";
|
|
5847
6876
|
return new messages_1.Field(name, tableID, columnID, dataTypeID, dataTypeSize, dataTypeModifier, mode);
|
|
5848
6877
|
}
|
|
5849
|
-
parseParameterDescriptionMessage(
|
|
5850
|
-
this.reader.setBuffer(
|
|
6878
|
+
parseParameterDescriptionMessage(offset, length, bytes) {
|
|
6879
|
+
this.reader.setBuffer(offset, bytes);
|
|
5851
6880
|
const parameterCount = this.reader.int16();
|
|
5852
6881
|
const message = new messages_1.ParameterDescriptionMessage(length, parameterCount);
|
|
5853
6882
|
for (let i = 0; i < parameterCount; i++) {
|
|
@@ -5855,8 +6884,8 @@ var require_parser = __commonJS({
|
|
|
5855
6884
|
}
|
|
5856
6885
|
return message;
|
|
5857
6886
|
}
|
|
5858
|
-
parseDataRowMessage(
|
|
5859
|
-
this.reader.setBuffer(
|
|
6887
|
+
parseDataRowMessage(offset, length, bytes) {
|
|
6888
|
+
this.reader.setBuffer(offset, bytes);
|
|
5860
6889
|
const fieldCount = this.reader.int16();
|
|
5861
6890
|
const fields = new Array(fieldCount);
|
|
5862
6891
|
for (let i = 0; i < fieldCount; i++) {
|
|
@@ -5865,20 +6894,20 @@ var require_parser = __commonJS({
|
|
|
5865
6894
|
}
|
|
5866
6895
|
return new messages_1.DataRowMessage(length, fields);
|
|
5867
6896
|
}
|
|
5868
|
-
parseParameterStatusMessage(
|
|
5869
|
-
this.reader.setBuffer(
|
|
6897
|
+
parseParameterStatusMessage(offset, length, bytes) {
|
|
6898
|
+
this.reader.setBuffer(offset, bytes);
|
|
5870
6899
|
const name = this.reader.cstring();
|
|
5871
6900
|
const value = this.reader.cstring();
|
|
5872
6901
|
return new messages_1.ParameterStatusMessage(length, name, value);
|
|
5873
6902
|
}
|
|
5874
|
-
parseBackendKeyData(
|
|
5875
|
-
this.reader.setBuffer(
|
|
6903
|
+
parseBackendKeyData(offset, length, bytes) {
|
|
6904
|
+
this.reader.setBuffer(offset, bytes);
|
|
5876
6905
|
const processID = this.reader.int32();
|
|
5877
6906
|
const secretKey = this.reader.int32();
|
|
5878
6907
|
return new messages_1.BackendKeyDataMessage(length, processID, secretKey);
|
|
5879
6908
|
}
|
|
5880
|
-
parseAuthenticationResponse(
|
|
5881
|
-
this.reader.setBuffer(
|
|
6909
|
+
parseAuthenticationResponse(offset, length, bytes) {
|
|
6910
|
+
this.reader.setBuffer(offset, bytes);
|
|
5882
6911
|
const code = this.reader.int32();
|
|
5883
6912
|
const message = {
|
|
5884
6913
|
name: "authenticationOk",
|
|
@@ -5925,8 +6954,8 @@ var require_parser = __commonJS({
|
|
|
5925
6954
|
}
|
|
5926
6955
|
return message;
|
|
5927
6956
|
}
|
|
5928
|
-
parseErrorMessage(
|
|
5929
|
-
this.reader.setBuffer(
|
|
6957
|
+
parseErrorMessage(offset, length, bytes, name) {
|
|
6958
|
+
this.reader.setBuffer(offset, bytes);
|
|
5930
6959
|
const fields = {};
|
|
5931
6960
|
let fieldType = this.reader.string(1);
|
|
5932
6961
|
while (fieldType !== "\0") {
|
|
@@ -6347,7 +7376,7 @@ var require_split2 = __commonJS({
|
|
|
6347
7376
|
var require_helper = __commonJS({
|
|
6348
7377
|
"../../node_modules/pgpass/lib/helper.js"(exports2, module2) {
|
|
6349
7378
|
"use strict";
|
|
6350
|
-
var
|
|
7379
|
+
var path8 = require("path");
|
|
6351
7380
|
var Stream = require("stream").Stream;
|
|
6352
7381
|
var split = require_split2();
|
|
6353
7382
|
var util = require("util");
|
|
@@ -6367,8 +7396,8 @@ var require_helper = __commonJS({
|
|
|
6367
7396
|
function warn() {
|
|
6368
7397
|
var isWritable = warnStream instanceof Stream && true === warnStream.writable;
|
|
6369
7398
|
if (isWritable) {
|
|
6370
|
-
var
|
|
6371
|
-
warnStream.write(util.format.apply(util,
|
|
7399
|
+
var args17 = Array.prototype.slice.call(arguments).concat("\n");
|
|
7400
|
+
warnStream.write(util.format.apply(util, args17));
|
|
6372
7401
|
}
|
|
6373
7402
|
}
|
|
6374
7403
|
Object.defineProperty(module2.exports, "isWin", {
|
|
@@ -6386,7 +7415,7 @@ var require_helper = __commonJS({
|
|
|
6386
7415
|
};
|
|
6387
7416
|
module2.exports.getFileName = function(rawEnv) {
|
|
6388
7417
|
var env = rawEnv || process.env;
|
|
6389
|
-
var file = env.PGPASSFILE || (isWin ?
|
|
7418
|
+
var file = env.PGPASSFILE || (isWin ? path8.join(env.APPDATA || "./", "postgresql", "pgpass.conf") : path8.join(env.HOME || "./", ".pgpass"));
|
|
6390
7419
|
return file;
|
|
6391
7420
|
};
|
|
6392
7421
|
module2.exports.usePgPass = function(stats, fname) {
|
|
@@ -6518,16 +7547,16 @@ var require_helper = __commonJS({
|
|
|
6518
7547
|
var require_lib = __commonJS({
|
|
6519
7548
|
"../../node_modules/pgpass/lib/index.js"(exports2, module2) {
|
|
6520
7549
|
"use strict";
|
|
6521
|
-
var
|
|
6522
|
-
var
|
|
7550
|
+
var path8 = require("path");
|
|
7551
|
+
var fs7 = require("fs");
|
|
6523
7552
|
var helper = require_helper();
|
|
6524
7553
|
module2.exports = function(connInfo, cb) {
|
|
6525
7554
|
var file = helper.getFileName();
|
|
6526
|
-
|
|
7555
|
+
fs7.stat(file, function(err, stat) {
|
|
6527
7556
|
if (err || !helper.usePgPass(stat, file)) {
|
|
6528
7557
|
return cb(void 0);
|
|
6529
7558
|
}
|
|
6530
|
-
var st =
|
|
7559
|
+
var st = fs7.createReadStream(file);
|
|
6531
7560
|
helper.getPassword(connInfo, st, cb);
|
|
6532
7561
|
});
|
|
6533
7562
|
};
|
|
@@ -7925,19 +8954,19 @@ var init_esm = __esm({
|
|
|
7925
8954
|
|
|
7926
8955
|
// src/scripts/doctor.ts
|
|
7927
8956
|
var doctor_exports = {};
|
|
7928
|
-
var import_config4,
|
|
8957
|
+
var import_config4, import_args10, import_chalk9, import_ioredis, import_node_net, okLabel, failLabel, infoLabel, toPort, parseAddress, checkRedis, checkPostgres, checkMlGrpc, flags10, run7;
|
|
7929
8958
|
var init_doctor = __esm({
|
|
7930
8959
|
"src/scripts/doctor.ts"() {
|
|
7931
8960
|
"use strict";
|
|
7932
8961
|
import_config4 = require("dotenv/config");
|
|
7933
|
-
|
|
7934
|
-
|
|
8962
|
+
import_args10 = __toESM(require("args"));
|
|
8963
|
+
import_chalk9 = __toESM(require("chalk"));
|
|
7935
8964
|
import_ioredis = __toESM(require("ioredis"));
|
|
7936
8965
|
init_esm();
|
|
7937
8966
|
import_node_net = __toESM(require("net"));
|
|
7938
|
-
okLabel = (text) => `${
|
|
7939
|
-
failLabel = (text) => `${
|
|
7940
|
-
infoLabel = (text) => `${
|
|
8967
|
+
okLabel = (text) => `${import_chalk9.default.green("OK")} ${text}`;
|
|
8968
|
+
failLabel = (text) => `${import_chalk9.default.red("FAIL")} ${text}`;
|
|
8969
|
+
infoLabel = (text) => `${import_chalk9.default.yellow("INFO")} ${text}`;
|
|
7941
8970
|
toPort = (value, fallback) => {
|
|
7942
8971
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
7943
8972
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
@@ -8048,20 +9077,20 @@ var init_doctor = __esm({
|
|
|
8048
9077
|
);
|
|
8049
9078
|
});
|
|
8050
9079
|
};
|
|
8051
|
-
|
|
8052
|
-
|
|
9080
|
+
import_args10.default.option("require-ml", "Treat ML gRPC connectivity as required", false).option("skip-ml", "Skip ML gRPC check", false);
|
|
9081
|
+
flags10 = import_args10.default.parse(process.argv);
|
|
8053
9082
|
run7 = async () => {
|
|
8054
9083
|
const checks = [];
|
|
8055
9084
|
checks.push(await checkRedis());
|
|
8056
9085
|
checks.push(await checkPostgres());
|
|
8057
|
-
if (!
|
|
9086
|
+
if (!flags10.skipMl) {
|
|
8058
9087
|
const ml = await checkMlGrpc();
|
|
8059
|
-
if (
|
|
9088
|
+
if (flags10.requireMl) {
|
|
8060
9089
|
ml.required = true;
|
|
8061
9090
|
}
|
|
8062
9091
|
checks.push(ml);
|
|
8063
9092
|
}
|
|
8064
|
-
console.log(
|
|
9093
|
+
console.log(import_chalk9.default.cyan("TradeJS doctor"));
|
|
8065
9094
|
for (const item of checks) {
|
|
8066
9095
|
const label = item.ok ? okLabel(item.name) : failLabel(item.name);
|
|
8067
9096
|
console.log(`${label} (${item.required ? "required" : "optional"})`);
|
|
@@ -8072,7 +9101,7 @@ var init_doctor = __esm({
|
|
|
8072
9101
|
if (failedRequired.length) {
|
|
8073
9102
|
console.log("");
|
|
8074
9103
|
console.log(
|
|
8075
|
-
|
|
9104
|
+
import_chalk9.default.red(
|
|
8076
9105
|
`Doctor failed: ${failedRequired.length} required check(s) failed.`
|
|
8077
9106
|
)
|
|
8078
9107
|
);
|
|
@@ -8092,21 +9121,21 @@ var init_doctor = __esm({
|
|
|
8092
9121
|
);
|
|
8093
9122
|
}
|
|
8094
9123
|
console.log("");
|
|
8095
|
-
console.log(
|
|
9124
|
+
console.log(import_chalk9.default.green("Doctor passed."));
|
|
8096
9125
|
};
|
|
8097
9126
|
run7().catch((error) => {
|
|
8098
|
-
console.error(
|
|
9127
|
+
console.error(import_chalk9.default.red(`doctor failed: ${String(error)}`));
|
|
8099
9128
|
process.exit(1);
|
|
8100
9129
|
});
|
|
8101
9130
|
}
|
|
8102
9131
|
});
|
|
8103
9132
|
|
|
8104
9133
|
// src/scripts/infraCommon.ts
|
|
8105
|
-
var
|
|
9134
|
+
var import_chalk10, import_node_child_process, import_node_fs, import_node_path, DEV_COMPOSE_FILE, resolveProjectRoot, DEV_COMPOSE_TEMPLATE, getDevComposePath, initDevComposeFile, requireDevComposeFile, runDockerCompose;
|
|
8106
9135
|
var init_infraCommon = __esm({
|
|
8107
9136
|
"src/scripts/infraCommon.ts"() {
|
|
8108
9137
|
"use strict";
|
|
8109
|
-
|
|
9138
|
+
import_chalk10 = __toESM(require("chalk"));
|
|
8110
9139
|
import_node_child_process = require("child_process");
|
|
8111
9140
|
import_node_fs = require("fs");
|
|
8112
9141
|
import_node_path = __toESM(require("path"));
|
|
@@ -8165,28 +9194,28 @@ volumes:
|
|
|
8165
9194
|
const composePath = getDevComposePath();
|
|
8166
9195
|
if ((0, import_node_fs.existsSync)(composePath)) {
|
|
8167
9196
|
console.log(
|
|
8168
|
-
|
|
9197
|
+
import_chalk10.default.yellow(
|
|
8169
9198
|
`${DEV_COMPOSE_FILE} already exists. Keeping user-defined file as is.`
|
|
8170
9199
|
)
|
|
8171
9200
|
);
|
|
8172
9201
|
return composePath;
|
|
8173
9202
|
}
|
|
8174
9203
|
(0, import_node_fs.writeFileSync)(composePath, DEV_COMPOSE_TEMPLATE, "utf8");
|
|
8175
|
-
console.log(
|
|
9204
|
+
console.log(import_chalk10.default.green(`Created ${DEV_COMPOSE_FILE} in project root`));
|
|
8176
9205
|
return composePath;
|
|
8177
9206
|
};
|
|
8178
9207
|
requireDevComposeFile = () => {
|
|
8179
9208
|
const composePath = import_node_path.default.resolve(resolveProjectRoot(), DEV_COMPOSE_FILE);
|
|
8180
9209
|
if ((0, import_node_fs.existsSync)(composePath)) {
|
|
8181
|
-
console.log(
|
|
9210
|
+
console.log(import_chalk10.default.gray(`Using existing ${DEV_COMPOSE_FILE}`));
|
|
8182
9211
|
return composePath;
|
|
8183
9212
|
}
|
|
8184
9213
|
throw new Error(
|
|
8185
9214
|
`${DEV_COMPOSE_FILE} not found in project root. Run "npx @tradejs/cli infra-init" first.`
|
|
8186
9215
|
);
|
|
8187
9216
|
};
|
|
8188
|
-
runDockerCompose = (composePath,
|
|
8189
|
-
const result = (0, import_node_child_process.spawnSync)("docker", ["compose", "-f", composePath, ...
|
|
9217
|
+
runDockerCompose = (composePath, args17) => {
|
|
9218
|
+
const result = (0, import_node_child_process.spawnSync)("docker", ["compose", "-f", composePath, ...args17], {
|
|
8190
9219
|
stdio: "inherit"
|
|
8191
9220
|
});
|
|
8192
9221
|
if (result.error) {
|
|
@@ -8201,20 +9230,20 @@ volumes:
|
|
|
8201
9230
|
|
|
8202
9231
|
// src/scripts/infraDown.ts
|
|
8203
9232
|
var infraDown_exports = {};
|
|
8204
|
-
var
|
|
9233
|
+
var import_chalk11, run8;
|
|
8205
9234
|
var init_infraDown = __esm({
|
|
8206
9235
|
"src/scripts/infraDown.ts"() {
|
|
8207
9236
|
"use strict";
|
|
8208
|
-
|
|
9237
|
+
import_chalk11 = __toESM(require("chalk"));
|
|
8209
9238
|
init_infraCommon();
|
|
8210
9239
|
run8 = async () => {
|
|
8211
9240
|
const composePath = requireDevComposeFile();
|
|
8212
|
-
console.log(
|
|
9241
|
+
console.log(import_chalk11.default.cyan("Stopping dev infra (timescale, redis)..."));
|
|
8213
9242
|
runDockerCompose(composePath, ["stop", "timescale", "redis"]);
|
|
8214
|
-
console.log(
|
|
9243
|
+
console.log(import_chalk11.default.green("Dev infra is stopped."));
|
|
8215
9244
|
};
|
|
8216
9245
|
run8().catch((error) => {
|
|
8217
|
-
console.error(
|
|
9246
|
+
console.error(import_chalk11.default.red(`infra-down failed: ${String(error)}`));
|
|
8218
9247
|
process.exit(1);
|
|
8219
9248
|
});
|
|
8220
9249
|
}
|
|
@@ -8222,18 +9251,18 @@ var init_infraDown = __esm({
|
|
|
8222
9251
|
|
|
8223
9252
|
// src/scripts/infraInit.ts
|
|
8224
9253
|
var infraInit_exports = {};
|
|
8225
|
-
var
|
|
9254
|
+
var import_chalk12, run9;
|
|
8226
9255
|
var init_infraInit = __esm({
|
|
8227
9256
|
"src/scripts/infraInit.ts"() {
|
|
8228
9257
|
"use strict";
|
|
8229
|
-
|
|
9258
|
+
import_chalk12 = __toESM(require("chalk"));
|
|
8230
9259
|
init_infraCommon();
|
|
8231
9260
|
run9 = async () => {
|
|
8232
9261
|
initDevComposeFile();
|
|
8233
|
-
console.log(
|
|
9262
|
+
console.log(import_chalk12.default.green("Dev infra files are initialized."));
|
|
8234
9263
|
};
|
|
8235
9264
|
run9().catch((error) => {
|
|
8236
|
-
console.error(
|
|
9265
|
+
console.error(import_chalk12.default.red(`infra-init failed: ${String(error)}`));
|
|
8237
9266
|
process.exit(1);
|
|
8238
9267
|
});
|
|
8239
9268
|
}
|
|
@@ -8241,20 +9270,20 @@ var init_infraInit = __esm({
|
|
|
8241
9270
|
|
|
8242
9271
|
// src/scripts/infraUp.ts
|
|
8243
9272
|
var infraUp_exports = {};
|
|
8244
|
-
var
|
|
9273
|
+
var import_chalk13, run10;
|
|
8245
9274
|
var init_infraUp = __esm({
|
|
8246
9275
|
"src/scripts/infraUp.ts"() {
|
|
8247
9276
|
"use strict";
|
|
8248
|
-
|
|
9277
|
+
import_chalk13 = __toESM(require("chalk"));
|
|
8249
9278
|
init_infraCommon();
|
|
8250
9279
|
run10 = async () => {
|
|
8251
9280
|
const composePath = requireDevComposeFile();
|
|
8252
|
-
console.log(
|
|
9281
|
+
console.log(import_chalk13.default.cyan("Starting dev infra (timescale, redis)..."));
|
|
8253
9282
|
runDockerCompose(composePath, ["up", "-d", "timescale", "redis"]);
|
|
8254
|
-
console.log(
|
|
9283
|
+
console.log(import_chalk13.default.green("Dev infra is up."));
|
|
8255
9284
|
};
|
|
8256
9285
|
run10().catch((error) => {
|
|
8257
|
-
console.error(
|
|
9286
|
+
console.error(import_chalk13.default.red(`infra-up failed: ${String(error)}`));
|
|
8258
9287
|
process.exit(1);
|
|
8259
9288
|
});
|
|
8260
9289
|
}
|
|
@@ -8262,19 +9291,19 @@ var init_infraUp = __esm({
|
|
|
8262
9291
|
|
|
8263
9292
|
// src/scripts/migration.ts
|
|
8264
9293
|
var migration_exports = {};
|
|
8265
|
-
var
|
|
9294
|
+
var import_progress5, import_chalk14, import_logger3, import_files, import_timescale4, DIR, re, BATCH, projectRoot5, migrateFile, migration;
|
|
8266
9295
|
var init_migration = __esm({
|
|
8267
9296
|
"src/scripts/migration.ts"() {
|
|
8268
9297
|
"use strict";
|
|
8269
|
-
|
|
8270
|
-
|
|
9298
|
+
import_progress5 = __toESM(require("progress"));
|
|
9299
|
+
import_chalk14 = __toESM(require("chalk"));
|
|
8271
9300
|
import_logger3 = require("@tradejs/infra/logger");
|
|
8272
9301
|
import_files = require("@tradejs/infra/files");
|
|
8273
9302
|
import_timescale4 = require("@tradejs/infra/timescale");
|
|
8274
9303
|
DIR = "data/history";
|
|
8275
9304
|
re = /^(.+?)_(\d+)\.json$/;
|
|
8276
9305
|
BATCH = 2e3;
|
|
8277
|
-
|
|
9306
|
+
projectRoot5 = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
8278
9307
|
migrateFile = async (file) => {
|
|
8279
9308
|
const m = re.exec(file);
|
|
8280
9309
|
if (!m) return;
|
|
@@ -8284,7 +9313,7 @@ var init_migration = __esm({
|
|
|
8284
9313
|
DIR,
|
|
8285
9314
|
`${symbol}_${interval4}`,
|
|
8286
9315
|
[],
|
|
8287
|
-
|
|
9316
|
+
projectRoot5
|
|
8288
9317
|
);
|
|
8289
9318
|
const rows = (0, import_timescale4.toRows)(symbol, interval4, data);
|
|
8290
9319
|
for (let i = 0; i < rows.length; i += BATCH) {
|
|
@@ -8292,8 +9321,8 @@ var init_migration = __esm({
|
|
|
8292
9321
|
}
|
|
8293
9322
|
};
|
|
8294
9323
|
migration = async () => {
|
|
8295
|
-
const files = await (0, import_files.getFiles)(DIR,
|
|
8296
|
-
const bar = new
|
|
9324
|
+
const files = await (0, import_files.getFiles)(DIR, projectRoot5);
|
|
9325
|
+
const bar = new import_progress5.default(
|
|
8297
9326
|
":current/:total [:bar][:percent] :eta(s) :file",
|
|
8298
9327
|
{
|
|
8299
9328
|
total: files.length,
|
|
@@ -8307,7 +9336,7 @@ var init_migration = __esm({
|
|
|
8307
9336
|
import_logger3.logger.error("Failed: %s %s", file, e);
|
|
8308
9337
|
} finally {
|
|
8309
9338
|
bar.tick(1, {
|
|
8310
|
-
file:
|
|
9339
|
+
file: import_chalk14.default.gray(file)
|
|
8311
9340
|
});
|
|
8312
9341
|
}
|
|
8313
9342
|
}
|
|
@@ -8317,91 +9346,84 @@ var init_migration = __esm({
|
|
|
8317
9346
|
}
|
|
8318
9347
|
});
|
|
8319
9348
|
|
|
8320
|
-
// src/scripts/
|
|
8321
|
-
var
|
|
8322
|
-
var
|
|
8323
|
-
|
|
9349
|
+
// src/scripts/mlExport.ts
|
|
9350
|
+
var mlExport_exports = {};
|
|
9351
|
+
var import_args11, import_chalk15, import_promises3, import_path4, import_ml, flags11, listChunkFiles, main3;
|
|
9352
|
+
var init_mlExport = __esm({
|
|
9353
|
+
"src/scripts/mlExport.ts"() {
|
|
8324
9354
|
"use strict";
|
|
8325
|
-
|
|
8326
|
-
|
|
8327
|
-
|
|
8328
|
-
|
|
8329
|
-
|
|
8330
|
-
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
8334
|
-
|
|
8335
|
-
|
|
8336
|
-
|
|
8337
|
-
|
|
8338
|
-
|
|
8339
|
-
|
|
8340
|
-
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
|
|
8344
|
-
|
|
8345
|
-
|
|
9355
|
+
import_args11 = __toESM(require("args"));
|
|
9356
|
+
import_chalk15 = __toESM(require("chalk"));
|
|
9357
|
+
import_promises3 = __toESM(require("fs/promises"));
|
|
9358
|
+
import_path4 = __toESM(require("path"));
|
|
9359
|
+
import_ml = require("@tradejs/infra/ml");
|
|
9360
|
+
init_resolveExportStrategy();
|
|
9361
|
+
import_args11.default.example(
|
|
9362
|
+
"yarn ts-node ./src/scripts/mlExport --strategy trendline",
|
|
9363
|
+
"Merge per-worker ML chunks into one dataset JSONL"
|
|
9364
|
+
);
|
|
9365
|
+
import_args11.default.option(["o", "outDir"], "Dataset directory", "data/ml/export");
|
|
9366
|
+
import_args11.default.option(["s", "strategy"], "Strategy name", "");
|
|
9367
|
+
import_args11.default.option(
|
|
9368
|
+
["k", "keepChunks"],
|
|
9369
|
+
"Keep source chunk files after successful merge",
|
|
9370
|
+
false
|
|
9371
|
+
);
|
|
9372
|
+
flags11 = import_args11.default.parse(process.argv);
|
|
9373
|
+
listChunkFiles = async (outDir, strategyName) => {
|
|
9374
|
+
const prefix = `ml-dataset-${(0, import_ml.toFileToken)(strategyName)}-chunk-`;
|
|
9375
|
+
const entries = await import_promises3.default.readdir(outDir, { withFileTypes: true });
|
|
9376
|
+
return entries.filter((entry) => entry.isFile()).map((entry) => entry.name).filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => import_path4.default.join(outDir, name)).sort();
|
|
8346
9377
|
};
|
|
8347
|
-
|
|
8348
|
-
const
|
|
8349
|
-
|
|
8350
|
-
|
|
8351
|
-
|
|
8352
|
-
|
|
8353
|
-
|
|
8354
|
-
|
|
8355
|
-
|
|
8356
|
-
const label = isDefault ? import_chalk12.default.green(name) : name;
|
|
8357
|
-
const suffix = isDefault ? import_chalk12.default.gray(" (default)") : "";
|
|
8358
|
-
console.log(` ${import_chalk12.default.yellow(String(index + 1))}) ${label}${suffix}`);
|
|
8359
|
-
});
|
|
8360
|
-
const rl = import_readline.default.createInterface({
|
|
8361
|
-
input: process.stdin,
|
|
8362
|
-
output: process.stdout
|
|
9378
|
+
main3 = async () => {
|
|
9379
|
+
const outDir = String(flags11.outDir || "data/ml/export");
|
|
9380
|
+
await import_promises3.default.mkdir(outDir, { recursive: true });
|
|
9381
|
+
const strategyName = await resolveExportStrategy({
|
|
9382
|
+
explicitStrategy: String(flags11.strategy || ""),
|
|
9383
|
+
outDir,
|
|
9384
|
+
datasetLabel: "ML",
|
|
9385
|
+
promptLabel: "Select ML export strategy",
|
|
9386
|
+
listStrategies: import_ml.listMlChunkStrategies
|
|
8363
9387
|
});
|
|
8364
|
-
|
|
8365
|
-
|
|
8366
|
-
|
|
8367
|
-
);
|
|
8368
|
-
rl.close();
|
|
8369
|
-
const trimmed = answer.trim();
|
|
8370
|
-
if (!trimmed) {
|
|
8371
|
-
return fallbackStrategy;
|
|
9388
|
+
if (!strategyName) {
|
|
9389
|
+
console.log(import_chalk15.default.yellow(`No ML chunk files found in ${outDir}`));
|
|
9390
|
+
process.exit(0);
|
|
8372
9391
|
}
|
|
8373
|
-
const
|
|
8374
|
-
if (
|
|
8375
|
-
|
|
9392
|
+
const chunkFiles = await listChunkFiles(outDir, strategyName);
|
|
9393
|
+
if (!chunkFiles.length) {
|
|
9394
|
+
console.log(
|
|
9395
|
+
import_chalk15.default.yellow(
|
|
9396
|
+
`No chunk files found for strategy "${strategyName}" in ${outDir}`
|
|
9397
|
+
)
|
|
9398
|
+
);
|
|
9399
|
+
process.exit(0);
|
|
8376
9400
|
}
|
|
8377
|
-
const
|
|
8378
|
-
|
|
9401
|
+
const mergedPath = import_path4.default.join(
|
|
9402
|
+
outDir,
|
|
9403
|
+
`ml-dataset-${(0, import_ml.toFileToken)(strategyName)}-merged-${Date.now()}.jsonl`
|
|
8379
9404
|
);
|
|
8380
|
-
|
|
8381
|
-
|
|
9405
|
+
await (0, import_ml.mergeJsonlFiles)({
|
|
9406
|
+
filePaths: chunkFiles,
|
|
9407
|
+
outPath: mergedPath
|
|
9408
|
+
});
|
|
9409
|
+
const shouldDeleteChunks = !Boolean(flags11.keepChunks);
|
|
9410
|
+
if (shouldDeleteChunks) {
|
|
9411
|
+
for (const filePath of chunkFiles) {
|
|
9412
|
+
await import_promises3.default.rm(filePath, { force: true });
|
|
9413
|
+
}
|
|
8382
9414
|
}
|
|
8383
|
-
console.
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8388
|
-
|
|
8389
|
-
|
|
8390
|
-
|
|
8391
|
-
|
|
8392
|
-
var init_mlExportSelect = __esm({
|
|
8393
|
-
"src/scripts/mlExportSelect.ts"() {
|
|
8394
|
-
"use strict";
|
|
8395
|
-
import_child_process2 = require("child_process");
|
|
8396
|
-
init_selectStrategy();
|
|
8397
|
-
run11 = async () => {
|
|
8398
|
-
const selected = await selectStrategy();
|
|
8399
|
-
const args14 = ["ts-node", "./src/scripts/mlExport", "--strategy", selected];
|
|
8400
|
-
const result = (0, import_child_process2.spawnSync)("yarn", args14, { stdio: "inherit" });
|
|
8401
|
-
process.exit(result.status ?? 1);
|
|
9415
|
+
console.log(import_chalk15.default.green(`Merged dataset saved: ${mergedPath}`));
|
|
9416
|
+
console.log(
|
|
9417
|
+
import_chalk15.default.gray(
|
|
9418
|
+
`strategy=${strategyName}, source_chunks=${chunkFiles.length}, deleteChunks=${Boolean(
|
|
9419
|
+
shouldDeleteChunks
|
|
9420
|
+
)}`
|
|
9421
|
+
)
|
|
9422
|
+
);
|
|
9423
|
+
process.exit(0);
|
|
8402
9424
|
};
|
|
8403
|
-
|
|
8404
|
-
console.error(
|
|
9425
|
+
main3().catch((error) => {
|
|
9426
|
+
console.error(import_chalk15.default.red(error?.message || String(error)));
|
|
8405
9427
|
process.exit(1);
|
|
8406
9428
|
});
|
|
8407
9429
|
}
|
|
@@ -8409,33 +9431,33 @@ var init_mlExportSelect = __esm({
|
|
|
8409
9431
|
|
|
8410
9432
|
// src/scripts/mlInspect.ts
|
|
8411
9433
|
var mlInspect_exports = {};
|
|
8412
|
-
var
|
|
9434
|
+
var import_args12, import_chalk16, import_promises4, import_fs2, import_path5, import_readline2, import_child_process2, flags12, toMode, toInspectTool, selectInspectTool, asPositiveInt, readRowsHead, readRowsTail, readRowsSample, quantileSorted, mean, std, findLatestDataset, buildNumericStats, formatNumber, runQuickInspect, runYDataInspect, main4;
|
|
8413
9435
|
var init_mlInspect = __esm({
|
|
8414
9436
|
"src/scripts/mlInspect.ts"() {
|
|
8415
9437
|
"use strict";
|
|
8416
|
-
|
|
8417
|
-
|
|
8418
|
-
|
|
9438
|
+
import_args12 = __toESM(require("args"));
|
|
9439
|
+
import_chalk16 = __toESM(require("chalk"));
|
|
9440
|
+
import_promises4 = __toESM(require("fs/promises"));
|
|
8419
9441
|
import_fs2 = require("fs");
|
|
8420
|
-
|
|
9442
|
+
import_path5 = __toESM(require("path"));
|
|
8421
9443
|
import_readline2 = __toESM(require("readline"));
|
|
8422
|
-
|
|
8423
|
-
|
|
9444
|
+
import_child_process2 = require("child_process");
|
|
9445
|
+
import_args12.default.example(
|
|
8424
9446
|
"yarn ml-inspect --rows 10000 --mode sample",
|
|
8425
9447
|
"Inspect the latest ML dataset chunk and highlight problematic features"
|
|
8426
9448
|
);
|
|
8427
|
-
|
|
8428
|
-
|
|
8429
|
-
|
|
8430
|
-
|
|
8431
|
-
|
|
9449
|
+
import_args12.default.option(["d", "dir"], "Dataset directory", "data/ml/export");
|
|
9450
|
+
import_args12.default.option(["r", "rows"], "Rows to inspect", 1e4);
|
|
9451
|
+
import_args12.default.option(["m", "mode"], "head | tail | sample", "sample");
|
|
9452
|
+
import_args12.default.option(["S", "strategy"], "Strategy token in dataset filename", "");
|
|
9453
|
+
import_args12.default.option(
|
|
8432
9454
|
["f", "file"],
|
|
8433
9455
|
"Explicit dataset file path (overrides auto-select)"
|
|
8434
9456
|
);
|
|
8435
|
-
|
|
8436
|
-
|
|
8437
|
-
|
|
8438
|
-
|
|
9457
|
+
import_args12.default.option(["L", "limitIssues"], "How many fields to print in report", 25);
|
|
9458
|
+
import_args12.default.option(["M", "minFieldValues"], "Min valid values per numeric field", 50);
|
|
9459
|
+
import_args12.default.option(["T", "tool"], "quick | ydata", "");
|
|
9460
|
+
flags12 = import_args12.default.parse(process.argv);
|
|
8439
9461
|
toMode = (value) => {
|
|
8440
9462
|
const mode = String(value || "sample").toLowerCase();
|
|
8441
9463
|
if (mode === "head" || mode === "tail" || mode === "sample") {
|
|
@@ -8454,12 +9476,12 @@ var init_mlInspect = __esm({
|
|
|
8454
9476
|
return defaultTool;
|
|
8455
9477
|
}
|
|
8456
9478
|
const options = ["quick", "ydata"];
|
|
8457
|
-
console.log(
|
|
9479
|
+
console.log(import_chalk16.default.cyan("Available inspect tools:"));
|
|
8458
9480
|
options.forEach((name, index) => {
|
|
8459
9481
|
const isDefault = name === defaultTool;
|
|
8460
|
-
const label = isDefault ?
|
|
8461
|
-
const suffix = isDefault ?
|
|
8462
|
-
console.log(` ${
|
|
9482
|
+
const label = isDefault ? import_chalk16.default.green(name) : name;
|
|
9483
|
+
const suffix = isDefault ? import_chalk16.default.gray(" (default)") : "";
|
|
9484
|
+
console.log(` ${import_chalk16.default.yellow(String(index + 1))}) ${label}${suffix}`);
|
|
8463
9485
|
});
|
|
8464
9486
|
const rl = import_readline2.default.createInterface({
|
|
8465
9487
|
input: process.stdin,
|
|
@@ -8467,7 +9489,7 @@ var init_mlInspect = __esm({
|
|
|
8467
9489
|
});
|
|
8468
9490
|
const question = (text) => new Promise((resolve) => rl.question(text, resolve));
|
|
8469
9491
|
const answer = await question(
|
|
8470
|
-
`Select inspect tool [${
|
|
9492
|
+
`Select inspect tool [${import_chalk16.default.green(defaultTool)}]: `
|
|
8471
9493
|
);
|
|
8472
9494
|
rl.close();
|
|
8473
9495
|
const trimmed = answer.trim().toLowerCase();
|
|
@@ -8592,7 +9614,7 @@ var init_mlInspect = __esm({
|
|
|
8592
9614
|
};
|
|
8593
9615
|
findLatestDataset = async (params) => {
|
|
8594
9616
|
const { dir, strategy } = params;
|
|
8595
|
-
const entries = await
|
|
9617
|
+
const entries = await import_promises4.default.readdir(dir, { withFileTypes: true });
|
|
8596
9618
|
const strategyToken = strategy.trim().toLowerCase();
|
|
8597
9619
|
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name).filter((name) => name.endsWith(".jsonl")).filter(
|
|
8598
9620
|
(name) => !name.includes(".train.") && !name.includes(".test.")
|
|
@@ -8604,14 +9626,14 @@ var init_mlInspect = __esm({
|
|
|
8604
9626
|
}
|
|
8605
9627
|
const withMtime = await Promise.all(
|
|
8606
9628
|
files.map(async (name) => {
|
|
8607
|
-
const stat = await
|
|
9629
|
+
const stat = await import_promises4.default.stat(import_path5.default.join(dir, name));
|
|
8608
9630
|
return { name, mtime: stat.mtimeMs };
|
|
8609
9631
|
})
|
|
8610
9632
|
);
|
|
8611
9633
|
withMtime.sort(
|
|
8612
9634
|
(a, b) => b.mtime - a.mtime
|
|
8613
9635
|
);
|
|
8614
|
-
return
|
|
9636
|
+
return import_path5.default.join(dir, withMtime[0].name);
|
|
8615
9637
|
};
|
|
8616
9638
|
buildNumericStats = (rows, minFieldValues) => {
|
|
8617
9639
|
const allFields = /* @__PURE__ */ new Set();
|
|
@@ -8773,35 +9795,35 @@ var init_mlInspect = __esm({
|
|
|
8773
9795
|
rows = await readRowsSample(datasetPath, rowsToInspect);
|
|
8774
9796
|
}
|
|
8775
9797
|
if (!rows.length) {
|
|
8776
|
-
console.error(
|
|
9798
|
+
console.error(import_chalk16.default.red("No rows could be read from dataset."));
|
|
8777
9799
|
process.exit(1);
|
|
8778
9800
|
}
|
|
8779
9801
|
const numericStats = buildNumericStats(rows, minFieldValues);
|
|
8780
9802
|
if (!numericStats.length) {
|
|
8781
9803
|
console.error(
|
|
8782
|
-
|
|
9804
|
+
import_chalk16.default.red(
|
|
8783
9805
|
"No numeric fields with enough values were found in sampled rows."
|
|
8784
9806
|
)
|
|
8785
9807
|
);
|
|
8786
9808
|
process.exit(1);
|
|
8787
9809
|
}
|
|
8788
9810
|
const problematic = numericStats.filter((item) => item.score > 0);
|
|
8789
|
-
console.log(
|
|
8790
|
-
console.log(
|
|
8791
|
-
console.log(
|
|
8792
|
-
console.log(
|
|
8793
|
-
console.log(
|
|
8794
|
-
console.log(
|
|
9811
|
+
console.log(import_chalk16.default.gray("ML dataset inspection (quick)"));
|
|
9812
|
+
console.log(import_chalk16.default.gray(`file: ${datasetPath}`));
|
|
9813
|
+
console.log(import_chalk16.default.gray(`mode: ${mode}`));
|
|
9814
|
+
console.log(import_chalk16.default.gray(`rows inspected: ${rows.length}`));
|
|
9815
|
+
console.log(import_chalk16.default.gray(`numeric fields analyzed: ${numericStats.length}`));
|
|
9816
|
+
console.log(import_chalk16.default.gray(`fields with recommendations: ${problematic.length}`));
|
|
8795
9817
|
console.log("");
|
|
8796
9818
|
if (!problematic.length) {
|
|
8797
9819
|
console.log(
|
|
8798
|
-
|
|
9820
|
+
import_chalk16.default.green("No problematic fields detected by current rules.")
|
|
8799
9821
|
);
|
|
8800
9822
|
process.exit(0);
|
|
8801
9823
|
}
|
|
8802
9824
|
const top = problematic.slice(0, limitIssues);
|
|
8803
9825
|
for (const item of top) {
|
|
8804
|
-
console.log(
|
|
9826
|
+
console.log(import_chalk16.default.yellow(item.field));
|
|
8805
9827
|
console.log(
|
|
8806
9828
|
` score=${item.score} issues=${item.issues.join("; ") || "none"}`
|
|
8807
9829
|
);
|
|
@@ -8815,13 +9837,13 @@ var init_mlInspect = __esm({
|
|
|
8815
9837
|
if (problematic.length > limitIssues) {
|
|
8816
9838
|
console.log("");
|
|
8817
9839
|
console.log(
|
|
8818
|
-
|
|
9840
|
+
import_chalk16.default.gray(
|
|
8819
9841
|
`... ${problematic.length - limitIssues} more fields hidden (raise --limitIssues)`
|
|
8820
9842
|
)
|
|
8821
9843
|
);
|
|
8822
9844
|
}
|
|
8823
9845
|
console.log("");
|
|
8824
|
-
console.log(
|
|
9846
|
+
console.log(import_chalk16.default.cyan("How to fix common issues:"));
|
|
8825
9847
|
console.log(
|
|
8826
9848
|
" - Large scale spread: normalize feature (log1p / relative to price / robust scaling)."
|
|
8827
9849
|
);
|
|
@@ -8836,32 +9858,32 @@ var init_mlInspect = __esm({
|
|
|
8836
9858
|
runYDataInspect = async (params) => {
|
|
8837
9859
|
const { datasetPath, mode, rowsToInspect } = params;
|
|
8838
9860
|
const cwd = process.cwd();
|
|
8839
|
-
const absDatasetPath =
|
|
8840
|
-
const dataRoot =
|
|
9861
|
+
const absDatasetPath = import_path5.default.resolve(datasetPath);
|
|
9862
|
+
const dataRoot = import_path5.default.resolve(cwd, "data");
|
|
8841
9863
|
if (!absDatasetPath.startsWith(dataRoot)) {
|
|
8842
9864
|
console.error(
|
|
8843
|
-
|
|
9865
|
+
import_chalk16.default.red(
|
|
8844
9866
|
`ydata mode supports only files under ${dataRoot}. Use --file inside data/`
|
|
8845
9867
|
)
|
|
8846
9868
|
);
|
|
8847
9869
|
process.exit(1);
|
|
8848
9870
|
}
|
|
8849
|
-
const relToData =
|
|
9871
|
+
const relToData = import_path5.default.relative(dataRoot, absDatasetPath);
|
|
8850
9872
|
const inputInContainer = `/app/data/${relToData}`;
|
|
8851
|
-
const reportDir =
|
|
8852
|
-
await
|
|
8853
|
-
const reportPath =
|
|
9873
|
+
const reportDir = import_path5.default.dirname(absDatasetPath);
|
|
9874
|
+
await import_promises4.default.mkdir(reportDir, { recursive: true });
|
|
9875
|
+
const reportPath = import_path5.default.join(
|
|
8854
9876
|
reportDir,
|
|
8855
|
-
`${
|
|
9877
|
+
`${import_path5.default.basename(datasetPath, import_path5.default.extname(datasetPath))}.profile.html`
|
|
8856
9878
|
);
|
|
8857
|
-
const reportRelToData =
|
|
9879
|
+
const reportRelToData = import_path5.default.relative(dataRoot, reportPath);
|
|
8858
9880
|
const reportInContainer = `/app/data/${reportRelToData}`;
|
|
8859
9881
|
console.log(
|
|
8860
|
-
|
|
9882
|
+
import_chalk16.default.gray(
|
|
8861
9883
|
"Running ydata-profiling report via docker ml-profile service..."
|
|
8862
9884
|
)
|
|
8863
9885
|
);
|
|
8864
|
-
const result = (0,
|
|
9886
|
+
const result = (0, import_child_process2.spawnSync)(
|
|
8865
9887
|
"docker",
|
|
8866
9888
|
[
|
|
8867
9889
|
"compose",
|
|
@@ -8881,7 +9903,7 @@ var init_mlInspect = __esm({
|
|
|
8881
9903
|
"--output",
|
|
8882
9904
|
reportInContainer,
|
|
8883
9905
|
"--title",
|
|
8884
|
-
`ML Profile: ${
|
|
9906
|
+
`ML Profile: ${import_path5.default.basename(datasetPath)}`
|
|
8885
9907
|
],
|
|
8886
9908
|
{
|
|
8887
9909
|
stdio: "inherit",
|
|
@@ -8893,30 +9915,30 @@ var init_mlInspect = __esm({
|
|
|
8893
9915
|
);
|
|
8894
9916
|
if ((result.status ?? 1) !== 0) {
|
|
8895
9917
|
console.error(
|
|
8896
|
-
|
|
9918
|
+
import_chalk16.default.red(
|
|
8897
9919
|
"ydata-profiling failed. Build profile image first: docker compose -f docker-compose.ml.yml build ml-profile"
|
|
8898
9920
|
)
|
|
8899
9921
|
);
|
|
8900
9922
|
process.exit(result.status ?? 1);
|
|
8901
9923
|
}
|
|
8902
|
-
console.log(
|
|
8903
|
-
};
|
|
8904
|
-
|
|
8905
|
-
const dir = String(
|
|
8906
|
-
const rowsToInspect = asPositiveInt(
|
|
8907
|
-
const mode = toMode(
|
|
8908
|
-
const strategy = String(
|
|
8909
|
-
const limitIssues = asPositiveInt(
|
|
8910
|
-
const minFieldValues = asPositiveInt(
|
|
8911
|
-
const toolFromFlags = toInspectTool(
|
|
8912
|
-
const explicitFile =
|
|
9924
|
+
console.log(import_chalk16.default.green(`Profile report saved: ${reportPath}`));
|
|
9925
|
+
};
|
|
9926
|
+
main4 = async () => {
|
|
9927
|
+
const dir = String(flags12.dir || "data/ml/export");
|
|
9928
|
+
const rowsToInspect = asPositiveInt(flags12.rows, 1e4);
|
|
9929
|
+
const mode = toMode(flags12.mode);
|
|
9930
|
+
const strategy = String(flags12.strategy || "");
|
|
9931
|
+
const limitIssues = asPositiveInt(flags12.limitIssues, 25);
|
|
9932
|
+
const minFieldValues = asPositiveInt(flags12.minFieldValues, 50);
|
|
9933
|
+
const toolFromFlags = toInspectTool(flags12.tool);
|
|
9934
|
+
const explicitFile = flags12.file ? String(flags12.file) : "";
|
|
8913
9935
|
const datasetPath = explicitFile || await findLatestDataset({
|
|
8914
9936
|
dir,
|
|
8915
9937
|
strategy
|
|
8916
9938
|
});
|
|
8917
9939
|
if (!datasetPath) {
|
|
8918
9940
|
console.error(
|
|
8919
|
-
|
|
9941
|
+
import_chalk16.default.red(`No dataset found. Expected ml-dataset-*.jsonl in ${dir}`)
|
|
8920
9942
|
);
|
|
8921
9943
|
process.exit(1);
|
|
8922
9944
|
}
|
|
@@ -8937,7 +9959,7 @@ var init_mlInspect = __esm({
|
|
|
8937
9959
|
minFieldValues
|
|
8938
9960
|
});
|
|
8939
9961
|
};
|
|
8940
|
-
|
|
9962
|
+
main4().catch((err) => {
|
|
8941
9963
|
console.error("ml-inspect failed:", err);
|
|
8942
9964
|
process.exit(1);
|
|
8943
9965
|
});
|
|
@@ -8946,22 +9968,22 @@ var init_mlInspect = __esm({
|
|
|
8946
9968
|
|
|
8947
9969
|
// src/scripts/mlTrainLatestSelect.ts
|
|
8948
9970
|
var mlTrainLatestSelect_exports = {};
|
|
8949
|
-
var
|
|
9971
|
+
var import_path6, import_promises5, import_fs3, import_crypto, import_child_process3, import_events, import_readline3, import_args13, import_chalk17, import_ml2, toFileToken4, asBool, asInt3, asNonNegativeInt, formatBytes, getMlContainerMemUsage, MODEL_TYPES, STRATEGIES, flags13, parseStrategy, parseModelType, selectModelType, listDatasetFiles, concatFiles, pipeNormalized, DAY_MS, SPLIT_PROGRESS_EVERY, parseTimestampMs, scanMaxLabeledTimestampMs, scanMaxTrainTimestampMs, hashFileNameSha1, prepareTrainWindowFiles, run11;
|
|
8950
9972
|
var init_mlTrainLatestSelect = __esm({
|
|
8951
9973
|
"src/scripts/mlTrainLatestSelect.ts"() {
|
|
8952
9974
|
"use strict";
|
|
8953
|
-
|
|
8954
|
-
|
|
9975
|
+
import_path6 = __toESM(require("path"));
|
|
9976
|
+
import_promises5 = __toESM(require("fs/promises"));
|
|
8955
9977
|
import_fs3 = require("fs");
|
|
8956
9978
|
import_crypto = require("crypto");
|
|
8957
|
-
|
|
9979
|
+
import_child_process3 = require("child_process");
|
|
8958
9980
|
import_events = require("events");
|
|
8959
9981
|
import_readline3 = __toESM(require("readline"));
|
|
8960
|
-
|
|
8961
|
-
|
|
9982
|
+
import_args13 = __toESM(require("args"));
|
|
9983
|
+
import_chalk17 = __toESM(require("chalk"));
|
|
8962
9984
|
init_selectStrategy();
|
|
8963
|
-
|
|
8964
|
-
|
|
9985
|
+
import_ml2 = require("@tradejs/infra/ml");
|
|
9986
|
+
toFileToken4 = (value) => value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "") || "any";
|
|
8965
9987
|
asBool = (value) => ["1", "true", "yes", "on"].includes((value ?? "").trim().toLowerCase());
|
|
8966
9988
|
asInt3 = (value, fallback) => {
|
|
8967
9989
|
const parsed = Number.parseInt((value ?? "").trim(), 10);
|
|
@@ -8984,7 +10006,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
8984
10006
|
};
|
|
8985
10007
|
getMlContainerMemUsage = () => {
|
|
8986
10008
|
try {
|
|
8987
|
-
const psRaw = (0,
|
|
10009
|
+
const psRaw = (0, import_child_process3.execSync)(
|
|
8988
10010
|
'docker ps --filter status=running --filter label=com.docker.compose.service=ml --format "{{.ID}}\\t{{.Names}}\\t{{.Label \\"com.docker.compose.oneoff\\"}}"',
|
|
8989
10011
|
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }
|
|
8990
10012
|
).trim();
|
|
@@ -8996,7 +10018,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
8996
10018
|
if (!rows.length) return "n/a";
|
|
8997
10019
|
const chosen = rows.find((row) => row.oneoff === "True") ?? rows[0];
|
|
8998
10020
|
if (!chosen?.id) return "n/a";
|
|
8999
|
-
const memUsage = (0,
|
|
10021
|
+
const memUsage = (0, import_child_process3.execSync)(
|
|
9000
10022
|
`docker stats --no-stream --format "{{.MemUsage}}" ${chosen.id}`,
|
|
9001
10023
|
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }
|
|
9002
10024
|
).trim();
|
|
@@ -9021,17 +10043,17 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9021
10043
|
"VolumeDivergence",
|
|
9022
10044
|
"any"
|
|
9023
10045
|
];
|
|
9024
|
-
|
|
9025
|
-
|
|
10046
|
+
import_args13.default.option(["s", "strategy"], "Strategy name (e.g. TrendLine)");
|
|
10047
|
+
import_args13.default.option(
|
|
9026
10048
|
["m", "model"],
|
|
9027
10049
|
"Model type: catboost | random_forest | extra_trees | xgboost | lightgbm"
|
|
9028
10050
|
);
|
|
9029
|
-
|
|
10051
|
+
import_args13.default.option(
|
|
9030
10052
|
["L", "latestOnly"],
|
|
9031
10053
|
"Use only latest dataset file (overrides ML_TRAIN_USE_LATEST_ONLY)",
|
|
9032
10054
|
false
|
|
9033
10055
|
);
|
|
9034
|
-
|
|
10056
|
+
flags13 = import_args13.default.parse(process.argv);
|
|
9035
10057
|
parseStrategy = (value) => {
|
|
9036
10058
|
const raw = String(value ?? "").trim();
|
|
9037
10059
|
if (!raw) return null;
|
|
@@ -9049,12 +10071,12 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9049
10071
|
if (!process.stdin.isTTY) {
|
|
9050
10072
|
return defaultModel;
|
|
9051
10073
|
}
|
|
9052
|
-
console.log(
|
|
10074
|
+
console.log(import_chalk17.default.cyan("Available models:"));
|
|
9053
10075
|
MODEL_TYPES.forEach((name, index) => {
|
|
9054
10076
|
const isDefault = name === defaultModel;
|
|
9055
|
-
const label = isDefault ?
|
|
9056
|
-
const suffix = isDefault ?
|
|
9057
|
-
console.log(` ${
|
|
10077
|
+
const label = isDefault ? import_chalk17.default.green(name) : name;
|
|
10078
|
+
const suffix = isDefault ? import_chalk17.default.gray(" (default)") : "";
|
|
10079
|
+
console.log(` ${import_chalk17.default.yellow(String(index + 1))}) ${label}${suffix}`);
|
|
9058
10080
|
});
|
|
9059
10081
|
const rl = import_readline3.default.createInterface({
|
|
9060
10082
|
input: process.stdin,
|
|
@@ -9062,7 +10084,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9062
10084
|
});
|
|
9063
10085
|
const question = (text) => new Promise((resolve) => rl.question(text, resolve));
|
|
9064
10086
|
const answer = await question(
|
|
9065
|
-
`Select model [${
|
|
10087
|
+
`Select model [${import_chalk17.default.green(defaultModel)}]: `
|
|
9066
10088
|
);
|
|
9067
10089
|
rl.close();
|
|
9068
10090
|
const trimmed = answer.trim().toLowerCase();
|
|
@@ -9080,8 +10102,8 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9080
10102
|
return defaultModel;
|
|
9081
10103
|
};
|
|
9082
10104
|
listDatasetFiles = async (dir, strategyName) => {
|
|
9083
|
-
const entries = await
|
|
9084
|
-
const prefix = `ml-dataset-${
|
|
10105
|
+
const entries = await import_promises5.default.readdir(dir, { withFileTypes: true });
|
|
10106
|
+
const prefix = `ml-dataset-${toFileToken4(strategyName)}-`;
|
|
9085
10107
|
const legacyPrefix = "ml-dataset-";
|
|
9086
10108
|
const fileNames = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
|
9087
10109
|
const hasScopedFiles = fileNames.some((name) => name.startsWith(prefix));
|
|
@@ -9091,7 +10113,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9091
10113
|
).filter((name) => !name.includes(".train.") && !name.includes(".test.")).filter((name) => !name.includes(".holdout-train.")).filter((name) => !name.includes(".holdout-test.")).filter((name) => !name.includes(".walk-forward.")).filter((name) => !name.includes(".prod."));
|
|
9092
10114
|
const withMtime = await Promise.all(
|
|
9093
10115
|
jsonlFiles.map(async (name) => {
|
|
9094
|
-
const stat = await
|
|
10116
|
+
const stat = await import_promises5.default.stat(import_path6.default.join(dir, name));
|
|
9095
10117
|
return { name, mtime: stat.mtimeMs };
|
|
9096
10118
|
})
|
|
9097
10119
|
);
|
|
@@ -9212,7 +10234,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9212
10234
|
}
|
|
9213
10235
|
return maxTrainTs;
|
|
9214
10236
|
};
|
|
9215
|
-
hashFileNameSha1 = (filePath) => (0, import_crypto.createHash)("sha1").update(
|
|
10237
|
+
hashFileNameSha1 = (filePath) => (0, import_crypto.createHash)("sha1").update(import_path6.default.basename(filePath)).digest("hex");
|
|
9216
10238
|
prepareTrainWindowFiles = async ({
|
|
9217
10239
|
inputPath,
|
|
9218
10240
|
testDays,
|
|
@@ -9220,8 +10242,8 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9220
10242
|
walkForwardFolds,
|
|
9221
10243
|
enforceCausalityGuard
|
|
9222
10244
|
}) => {
|
|
9223
|
-
const inputBaseName =
|
|
9224
|
-
if ((0,
|
|
10245
|
+
const inputBaseName = import_path6.default.basename(inputPath);
|
|
10246
|
+
if ((0, import_ml2.isDerivedDatasetFileName)(inputBaseName)) {
|
|
9225
10247
|
throw new Error(
|
|
9226
10248
|
`Refusing to split derived dataset file: ${inputBaseName}. Expected base export file.`
|
|
9227
10249
|
);
|
|
@@ -9235,17 +10257,17 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9235
10257
|
causalityGuardVersion: enforceCausalityGuard ? 1 : 0
|
|
9236
10258
|
});
|
|
9237
10259
|
const key = (0, import_crypto.createHash)("sha1").update(keyPayload).digest("hex").slice(0, 12);
|
|
9238
|
-
const dir =
|
|
9239
|
-
const parsed =
|
|
9240
|
-
const holdoutTrainPath =
|
|
10260
|
+
const dir = import_path6.default.dirname(inputPath);
|
|
10261
|
+
const parsed = import_path6.default.parse(inputPath);
|
|
10262
|
+
const holdoutTrainPath = import_path6.default.join(
|
|
9241
10263
|
dir,
|
|
9242
10264
|
`${parsed.name}.holdout-train.${key}.jsonl`
|
|
9243
10265
|
);
|
|
9244
|
-
const holdoutTestPath =
|
|
10266
|
+
const holdoutTestPath = import_path6.default.join(
|
|
9245
10267
|
dir,
|
|
9246
10268
|
`${parsed.name}.holdout-test.${key}.jsonl`
|
|
9247
10269
|
);
|
|
9248
|
-
const prodPath =
|
|
10270
|
+
const prodPath = import_path6.default.join(dir, `${parsed.name}.prod.${key}.jsonl`);
|
|
9249
10271
|
const walkForwardFoldDefs = Array.from(
|
|
9250
10272
|
{ length: Math.max(walkForwardFolds, 0) },
|
|
9251
10273
|
(_4, idx) => {
|
|
@@ -9253,27 +10275,27 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9253
10275
|
const foldToken = `fold-${fold}`;
|
|
9254
10276
|
return {
|
|
9255
10277
|
fold,
|
|
9256
|
-
trainPath:
|
|
10278
|
+
trainPath: import_path6.default.join(
|
|
9257
10279
|
dir,
|
|
9258
10280
|
`${parsed.name}.walk-forward-${foldToken}.train.${key}.jsonl`
|
|
9259
10281
|
),
|
|
9260
|
-
testPath:
|
|
10282
|
+
testPath: import_path6.default.join(
|
|
9261
10283
|
dir,
|
|
9262
10284
|
`${parsed.name}.walk-forward-${foldToken}.test.${key}.jsonl`
|
|
9263
10285
|
)
|
|
9264
10286
|
};
|
|
9265
10287
|
}
|
|
9266
10288
|
);
|
|
9267
|
-
const metaPath =
|
|
10289
|
+
const metaPath = import_path6.default.join(dir, `${parsed.name}.windows.${key}.meta.json`);
|
|
9268
10290
|
try {
|
|
9269
10291
|
const [metaRaw] = await Promise.all([
|
|
9270
|
-
|
|
9271
|
-
|
|
9272
|
-
|
|
9273
|
-
|
|
10292
|
+
import_promises5.default.readFile(metaPath, "utf8"),
|
|
10293
|
+
import_promises5.default.access(holdoutTrainPath),
|
|
10294
|
+
import_promises5.default.access(holdoutTestPath),
|
|
10295
|
+
import_promises5.default.access(prodPath),
|
|
9274
10296
|
...walkForwardFoldDefs.flatMap((entry) => [
|
|
9275
|
-
|
|
9276
|
-
|
|
10297
|
+
import_promises5.default.access(entry.trainPath),
|
|
10298
|
+
import_promises5.default.access(entry.testPath)
|
|
9277
10299
|
])
|
|
9278
10300
|
]);
|
|
9279
10301
|
const meta2 = JSON.parse(metaRaw);
|
|
@@ -9363,7 +10385,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9363
10385
|
wfStartMs,
|
|
9364
10386
|
prodStartMs,
|
|
9365
10387
|
folds
|
|
9366
|
-
} = (0,
|
|
10388
|
+
} = (0, import_ml2.computeWindowBoundaries)({
|
|
9367
10389
|
maxLabeledTs,
|
|
9368
10390
|
maxTrainTs,
|
|
9369
10391
|
testDays,
|
|
@@ -9444,7 +10466,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9444
10466
|
const ts = parseTimestampMs(parsedRow.entryTimestamp);
|
|
9445
10467
|
if (!ts) continue;
|
|
9446
10468
|
if (enforceCausalityGuard) {
|
|
9447
|
-
const violations = (0,
|
|
10469
|
+
const violations = (0, import_ml2.findLookaheadViolations)(
|
|
9448
10470
|
parsedRow
|
|
9449
10471
|
);
|
|
9450
10472
|
if (violations.length) {
|
|
@@ -9562,7 +10584,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9562
10584
|
}
|
|
9563
10585
|
}
|
|
9564
10586
|
const meta = {
|
|
9565
|
-
exportFile:
|
|
10587
|
+
exportFile: import_path6.default.basename(inputPath),
|
|
9566
10588
|
exportHash,
|
|
9567
10589
|
params: {
|
|
9568
10590
|
testDays,
|
|
@@ -9575,72 +10597,72 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9575
10597
|
holdoutTrainRows,
|
|
9576
10598
|
holdoutTestRows,
|
|
9577
10599
|
holdoutCutoffMs,
|
|
9578
|
-
holdoutCutoffDt: (0,
|
|
10600
|
+
holdoutCutoffDt: (0, import_ml2.toIsoUtcOrNull)(holdoutCutoffMs),
|
|
9579
10601
|
holdoutTrainStartMs,
|
|
9580
|
-
holdoutTrainStartDt: (0,
|
|
10602
|
+
holdoutTrainStartDt: (0, import_ml2.toIsoUtcOrNull)(holdoutTrainStartMs),
|
|
9581
10603
|
holdoutTrainMinTs: holdoutTrainRows > 0 ? holdoutTrainMinTs : null,
|
|
9582
|
-
holdoutTrainMinDt: (0,
|
|
10604
|
+
holdoutTrainMinDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9583
10605
|
holdoutTrainRows > 0 ? holdoutTrainMinTs : null
|
|
9584
10606
|
),
|
|
9585
10607
|
holdoutTrainMaxTs: holdoutTrainRows > 0 ? holdoutTrainMaxTs : null,
|
|
9586
|
-
holdoutTrainMaxDt: (0,
|
|
10608
|
+
holdoutTrainMaxDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9587
10609
|
holdoutTrainRows > 0 ? holdoutTrainMaxTs : null
|
|
9588
10610
|
),
|
|
9589
10611
|
holdoutTestMinTs: holdoutTestRows > 0 ? holdoutTestMinTs : null,
|
|
9590
|
-
holdoutTestMinDt: (0,
|
|
10612
|
+
holdoutTestMinDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9591
10613
|
holdoutTestRows > 0 ? holdoutTestMinTs : null
|
|
9592
10614
|
),
|
|
9593
10615
|
holdoutTestMaxTs: holdoutTestRows > 0 ? holdoutTestMaxTs : null,
|
|
9594
|
-
holdoutTestMaxDt: (0,
|
|
10616
|
+
holdoutTestMaxDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9595
10617
|
holdoutTestRows > 0 ? holdoutTestMaxTs : null
|
|
9596
10618
|
),
|
|
9597
|
-
holdoutTrain:
|
|
9598
|
-
holdoutTest:
|
|
10619
|
+
holdoutTrain: import_path6.default.basename(holdoutTrainPath),
|
|
10620
|
+
holdoutTest: import_path6.default.basename(holdoutTestPath)
|
|
9599
10621
|
},
|
|
9600
10622
|
prod: {
|
|
9601
10623
|
prodRows,
|
|
9602
10624
|
prodStartMs: Number.isFinite(prodStartMs) ? prodStartMs : null,
|
|
9603
|
-
prodStartDt: (0,
|
|
10625
|
+
prodStartDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9604
10626
|
Number.isFinite(prodStartMs) ? prodStartMs : null
|
|
9605
10627
|
),
|
|
9606
10628
|
prodMinTs: prodRows > 0 ? prodMinTs : null,
|
|
9607
|
-
prodMinDt: (0,
|
|
10629
|
+
prodMinDt: (0, import_ml2.toIsoUtcOrNull)(prodRows > 0 ? prodMinTs : null),
|
|
9608
10630
|
prodMaxTs: prodRows > 0 ? prodMaxTs : null,
|
|
9609
|
-
prodMaxDt: (0,
|
|
9610
|
-
prod:
|
|
10631
|
+
prodMaxDt: (0, import_ml2.toIsoUtcOrNull)(prodRows > 0 ? prodMaxTs : null),
|
|
10632
|
+
prod: import_path6.default.basename(prodPath)
|
|
9611
10633
|
},
|
|
9612
10634
|
walkForwardFolds: walkForwardWriters.map((entry) => ({
|
|
9613
10635
|
fold: entry.fold,
|
|
9614
10636
|
walkForwardSourceRows,
|
|
9615
10637
|
startTs: entry.startTs,
|
|
9616
|
-
startDt: (0,
|
|
10638
|
+
startDt: (0, import_ml2.toIsoUtcOrNull)(entry.startTs),
|
|
9617
10639
|
endTs: entry.endTs,
|
|
9618
|
-
endDt: (0,
|
|
10640
|
+
endDt: (0, import_ml2.toIsoUtcOrNull)(entry.endTs),
|
|
9619
10641
|
trainStartMs: Number.isFinite(wfStartMs) ? wfStartMs : null,
|
|
9620
|
-
trainStartDt: (0,
|
|
10642
|
+
trainStartDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9621
10643
|
Number.isFinite(wfStartMs) ? wfStartMs : null
|
|
9622
10644
|
),
|
|
9623
10645
|
trainMinTs: entry.trainRows > 0 ? entry.trainMinTs : null,
|
|
9624
|
-
trainMinDt: (0,
|
|
10646
|
+
trainMinDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9625
10647
|
entry.trainRows > 0 ? entry.trainMinTs : null
|
|
9626
10648
|
),
|
|
9627
10649
|
trainMaxTs: entry.trainRows > 0 ? entry.trainMaxTs : null,
|
|
9628
|
-
trainMaxDt: (0,
|
|
10650
|
+
trainMaxDt: (0, import_ml2.toIsoUtcOrNull)(
|
|
9629
10651
|
entry.trainRows > 0 ? entry.trainMaxTs : null
|
|
9630
10652
|
),
|
|
9631
10653
|
testMinTs: entry.testRows > 0 ? entry.testMinTs : null,
|
|
9632
|
-
testMinDt: (0,
|
|
10654
|
+
testMinDt: (0, import_ml2.toIsoUtcOrNull)(entry.testRows > 0 ? entry.testMinTs : null),
|
|
9633
10655
|
testMaxTs: entry.testRows > 0 ? entry.testMaxTs : null,
|
|
9634
|
-
testMaxDt: (0,
|
|
10656
|
+
testMaxDt: (0, import_ml2.toIsoUtcOrNull)(entry.testRows > 0 ? entry.testMaxTs : null),
|
|
9635
10657
|
trainRows: entry.trainRows,
|
|
9636
10658
|
testRows: entry.testRows,
|
|
9637
|
-
train:
|
|
9638
|
-
test:
|
|
10659
|
+
train: import_path6.default.basename(entry.trainPath),
|
|
10660
|
+
test: import_path6.default.basename(entry.testPath)
|
|
9639
10661
|
}))
|
|
9640
10662
|
},
|
|
9641
10663
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9642
10664
|
};
|
|
9643
|
-
await
|
|
10665
|
+
await import_promises5.default.writeFile(metaPath, `${JSON.stringify(meta, null, 2)}
|
|
9644
10666
|
`, "utf8");
|
|
9645
10667
|
return {
|
|
9646
10668
|
holdoutTrainPath,
|
|
@@ -9667,29 +10689,29 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9667
10689
|
}
|
|
9668
10690
|
};
|
|
9669
10691
|
};
|
|
9670
|
-
|
|
9671
|
-
const selectedFromCli = parseStrategy(
|
|
10692
|
+
run11 = async () => {
|
|
10693
|
+
const selectedFromCli = parseStrategy(flags13.strategy);
|
|
9672
10694
|
const selected = selectedFromCli ?? await selectStrategy();
|
|
9673
10695
|
const envModelType = (process.env.ML_MODEL_TYPE ?? "random_forest").trim().toLowerCase();
|
|
9674
10696
|
const defaultModelType = MODEL_TYPES.includes(
|
|
9675
10697
|
envModelType
|
|
9676
10698
|
) ? envModelType : "random_forest";
|
|
9677
|
-
const modelFromCli = parseModelType(
|
|
10699
|
+
const modelFromCli = parseModelType(flags13.model);
|
|
9678
10700
|
const modelType = modelFromCli ?? await selectModelType(defaultModelType);
|
|
9679
|
-
const useLatestOnly = Boolean(
|
|
9680
|
-
const exportDir =
|
|
9681
|
-
const modelDirRoot =
|
|
10701
|
+
const useLatestOnly = Boolean(flags13.latestOnly) || asBool(process.env.ML_TRAIN_USE_LATEST_ONLY);
|
|
10702
|
+
const exportDir = import_path6.default.join(process.cwd(), "data", "ml", "export");
|
|
10703
|
+
const modelDirRoot = import_path6.default.join(process.cwd(), "data", "ml");
|
|
9682
10704
|
const datasetFiles = await listDatasetFiles(exportDir, selected);
|
|
9683
10705
|
let inputPath;
|
|
9684
10706
|
let cleanupFiles = [];
|
|
9685
10707
|
if (datasetFiles.length && useLatestOnly) {
|
|
9686
10708
|
const latest = datasetFiles[datasetFiles.length - 1];
|
|
9687
|
-
inputPath =
|
|
9688
|
-
console.log(`Using latest JSONL dataset: ${
|
|
10709
|
+
inputPath = import_path6.default.join(exportDir, latest);
|
|
10710
|
+
console.log(`Using latest JSONL dataset: ${import_path6.default.basename(inputPath)}`);
|
|
9689
10711
|
} else if (datasetFiles.length) {
|
|
9690
|
-
const files = datasetFiles.map((name) =>
|
|
10712
|
+
const files = datasetFiles.map((name) => import_path6.default.join(exportDir, name));
|
|
9691
10713
|
const mergedPrefix = `ml-dataset-merged-${Date.now()}`;
|
|
9692
|
-
inputPath =
|
|
10714
|
+
inputPath = import_path6.default.join(exportDir, `${mergedPrefix}.jsonl`);
|
|
9693
10715
|
await concatFiles(files, inputPath);
|
|
9694
10716
|
cleanupFiles = [inputPath];
|
|
9695
10717
|
console.log(`Merged datasets: ${datasetFiles.length} exports`);
|
|
@@ -9698,7 +10720,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9698
10720
|
process.exit(1);
|
|
9699
10721
|
}
|
|
9700
10722
|
try {
|
|
9701
|
-
const inputStat = await
|
|
10723
|
+
const inputStat = await import_promises5.default.stat(inputPath);
|
|
9702
10724
|
const totalInputBytes = inputStat.size;
|
|
9703
10725
|
const autoIncrementalThresholdGb = asInt3(
|
|
9704
10726
|
process.env.ML_TRAIN_INCREMENTAL_THRESHOLD_GB,
|
|
@@ -9754,10 +10776,10 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9754
10776
|
`${splitMode} split key=${splitFiles.key} export_hash=${splitFiles.exportHash.slice(0, 12)}`
|
|
9755
10777
|
);
|
|
9756
10778
|
console.log(
|
|
9757
|
-
`${splitMode} holdout files: train=${
|
|
10779
|
+
`${splitMode} holdout files: train=${import_path6.default.basename(splitFiles.holdoutTrainPath)} (${splitFiles.counts.holdoutTrainRows} rows), test=${import_path6.default.basename(splitFiles.holdoutTestPath)} (${splitFiles.counts.holdoutTestRows} rows)`
|
|
9758
10780
|
);
|
|
9759
10781
|
console.log(
|
|
9760
|
-
`${splitMode} prod file: ${
|
|
10782
|
+
`${splitMode} prod file: ${import_path6.default.basename(splitFiles.prodPath)} (${splitFiles.counts.prodRows} rows)`
|
|
9761
10783
|
);
|
|
9762
10784
|
if (splitFiles.walkForwardFolds.length) {
|
|
9763
10785
|
console.log(
|
|
@@ -9765,7 +10787,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9765
10787
|
);
|
|
9766
10788
|
for (const foldEntry of splitFiles.walkForwardFolds) {
|
|
9767
10789
|
console.log(
|
|
9768
|
-
`${splitMode} walk-forward fold ${foldEntry.fold}: train=${
|
|
10790
|
+
`${splitMode} walk-forward fold ${foldEntry.fold}: train=${import_path6.default.basename(foldEntry.trainPath)} (${foldEntry.trainRows} rows), test=${import_path6.default.basename(foldEntry.testPath)} (${foldEntry.testRows} rows)`
|
|
9769
10791
|
);
|
|
9770
10792
|
}
|
|
9771
10793
|
} else {
|
|
@@ -9781,11 +10803,11 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9781
10803
|
"python",
|
|
9782
10804
|
"/app/ml/train.py",
|
|
9783
10805
|
"--input",
|
|
9784
|
-
`/app/data/ml/export/${
|
|
10806
|
+
`/app/data/ml/export/${import_path6.default.basename(splitFiles.holdoutTrainPath)}`,
|
|
9785
10807
|
"--test-input",
|
|
9786
|
-
`/app/data/ml/export/${
|
|
10808
|
+
`/app/data/ml/export/${import_path6.default.basename(splitFiles.holdoutTestPath)}`,
|
|
9787
10809
|
"--prod-input",
|
|
9788
|
-
`/app/data/ml/export/${
|
|
10810
|
+
`/app/data/ml/export/${import_path6.default.basename(splitFiles.prodPath)}`,
|
|
9789
10811
|
"--strategy",
|
|
9790
10812
|
selected,
|
|
9791
10813
|
"--model-type",
|
|
@@ -9803,9 +10825,9 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9803
10825
|
...useEnsemble ? ["--ensemble"] : [],
|
|
9804
10826
|
...splitFiles.walkForwardFolds.flatMap((foldEntry) => [
|
|
9805
10827
|
"--walk-forward-fold-train-input",
|
|
9806
|
-
`/app/data/ml/export/${
|
|
10828
|
+
`/app/data/ml/export/${import_path6.default.basename(foldEntry.trainPath)}`,
|
|
9807
10829
|
"--walk-forward-fold-test-input",
|
|
9808
|
-
`/app/data/ml/export/${
|
|
10830
|
+
`/app/data/ml/export/${import_path6.default.basename(foldEntry.testPath)}`
|
|
9809
10831
|
])
|
|
9810
10832
|
];
|
|
9811
10833
|
const startedAt = Date.now();
|
|
@@ -9818,7 +10840,7 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9818
10840
|
90
|
|
9819
10841
|
);
|
|
9820
10842
|
console.log(`Starting train command: docker ${trainArgs.join(" ")}`);
|
|
9821
|
-
const child = (0,
|
|
10843
|
+
const child = (0, import_child_process3.spawn)("docker", trainArgs, {
|
|
9822
10844
|
stdio: ["inherit", "pipe", "pipe"],
|
|
9823
10845
|
env: {
|
|
9824
10846
|
...process.env,
|
|
@@ -9884,10 +10906,10 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9884
10906
|
);
|
|
9885
10907
|
}
|
|
9886
10908
|
if (code === 0) {
|
|
9887
|
-
const modelDir =
|
|
10909
|
+
const modelDir = import_path6.default.join(modelDirRoot, "models");
|
|
9888
10910
|
let savedModels = [];
|
|
9889
10911
|
try {
|
|
9890
|
-
const entries = await
|
|
10912
|
+
const entries = await import_promises5.default.readdir(modelDir, { withFileTypes: true });
|
|
9891
10913
|
savedModels = entries.filter((entry) => entry.isFile()).map((entry) => entry.name).filter(
|
|
9892
10914
|
(name) => name === `${selected}.joblib` || /^.+\.model\d+\.joblib$/.test(name)
|
|
9893
10915
|
).filter((name) => name.startsWith(`${selected}.`)).sort();
|
|
@@ -9909,12 +10931,12 @@ var init_mlTrainLatestSelect = __esm({
|
|
|
9909
10931
|
process.exitCode = finalCode;
|
|
9910
10932
|
} finally {
|
|
9911
10933
|
for (const file of cleanupFiles) {
|
|
9912
|
-
await
|
|
10934
|
+
await import_promises5.default.rm(file, { force: true });
|
|
9913
10935
|
}
|
|
9914
10936
|
}
|
|
9915
10937
|
process.exit(process.exitCode ?? 1);
|
|
9916
10938
|
};
|
|
9917
|
-
|
|
10939
|
+
run11().catch((err) => {
|
|
9918
10940
|
console.error("Failed to train:", err);
|
|
9919
10941
|
process.exit(1);
|
|
9920
10942
|
});
|
|
@@ -9926,32 +10948,32 @@ var results_exports = {};
|
|
|
9926
10948
|
__export(results_exports, {
|
|
9927
10949
|
results: () => results2
|
|
9928
10950
|
});
|
|
9929
|
-
var
|
|
10951
|
+
var import_args14, import_chalk18, import_connectors7, import_cli7, import_redis4, ListIt3, flags14, MIN_PROFIT, MIN_WIN_RATE, MIN_ORDERS_PER_MONTH, createListIt3, formatPercent, getMonthlyReturn, toStrategyResultEntry, toStrategyResults, getTestConfigs, buildBestResults, getSavedProfitsBySymbol, getCoverageRow, results2;
|
|
9930
10952
|
var init_results = __esm({
|
|
9931
10953
|
"src/scripts/results.ts"() {
|
|
9932
10954
|
"use strict";
|
|
9933
|
-
|
|
9934
|
-
|
|
10955
|
+
import_args14 = __toESM(require("args"));
|
|
10956
|
+
import_chalk18 = __toESM(require("chalk"));
|
|
9935
10957
|
import_connectors7 = require("@tradejs/connectors");
|
|
9936
10958
|
import_cli7 = require("@tradejs/node/cli");
|
|
9937
10959
|
import_redis4 = require("@tradejs/infra/redis");
|
|
9938
|
-
|
|
9939
|
-
|
|
10960
|
+
ListIt3 = require_list_it();
|
|
10961
|
+
import_args14.default.example(
|
|
9940
10962
|
"yarn results --strategy TrendLine --coverage",
|
|
9941
10963
|
"Show best results per symbol and coverage"
|
|
9942
10964
|
);
|
|
9943
|
-
|
|
9944
|
-
|
|
9945
|
-
|
|
9946
|
-
|
|
9947
|
-
|
|
9948
|
-
|
|
9949
|
-
|
|
9950
|
-
|
|
10965
|
+
import_args14.default.option(["s", "strategy"], "Strategy name");
|
|
10966
|
+
import_args14.default.option(["C", "coverage"], "Show coverage table", false);
|
|
10967
|
+
import_args14.default.option(["u", "update"], "Update results config in redis", false);
|
|
10968
|
+
import_args14.default.option(["m", "merge"], "Merge results config in redis", false);
|
|
10969
|
+
import_args14.default.option(["c", "clear"], "Clear results config in redis", false);
|
|
10970
|
+
import_args14.default.option(["V", "verbose"], "Verbose output", false);
|
|
10971
|
+
import_args14.default.option(["U", "user"], "Use user config", "root");
|
|
10972
|
+
flags14 = import_args14.default.parse(process.argv);
|
|
9951
10973
|
MIN_PROFIT = 2;
|
|
9952
10974
|
MIN_WIN_RATE = 40;
|
|
9953
10975
|
MIN_ORDERS_PER_MONTH = 1;
|
|
9954
|
-
|
|
10976
|
+
createListIt3 = () => new ListIt3({
|
|
9955
10977
|
autoAlign: true,
|
|
9956
10978
|
headerUnderline: true
|
|
9957
10979
|
});
|
|
@@ -10075,39 +11097,39 @@ var init_results = __esm({
|
|
|
10075
11097
|
};
|
|
10076
11098
|
};
|
|
10077
11099
|
results2 = async () => {
|
|
10078
|
-
if (!
|
|
10079
|
-
console.error(
|
|
11100
|
+
if (!flags14.strategy) {
|
|
11101
|
+
console.error(import_chalk18.default.red("Missing --strategy"));
|
|
10080
11102
|
process.exit(1);
|
|
10081
11103
|
}
|
|
10082
|
-
const strategyName =
|
|
10083
|
-
const userName2 =
|
|
10084
|
-
if (
|
|
11104
|
+
const strategyName = flags14.strategy;
|
|
11105
|
+
const userName2 = flags14.user;
|
|
11106
|
+
if (flags14.clear) {
|
|
10085
11107
|
const cleared = await (0, import_redis4.delKey)(
|
|
10086
11108
|
import_redis4.redisKeys.strategyResults(userName2, strategyName)
|
|
10087
11109
|
);
|
|
10088
11110
|
console.log(
|
|
10089
|
-
cleared ?
|
|
11111
|
+
cleared ? import_chalk18.default.green(`Cleared results:${strategyName}`) : import_chalk18.default.yellow(`No results to clear for ${strategyName}`)
|
|
10090
11112
|
);
|
|
10091
|
-
if (!
|
|
11113
|
+
if (!flags14.coverage && !flags14.update && !flags14.merge) {
|
|
10092
11114
|
process.exit();
|
|
10093
11115
|
}
|
|
10094
11116
|
}
|
|
10095
11117
|
const bestBySymbol = await buildBestResults(userName2, strategyName);
|
|
10096
|
-
if (
|
|
11118
|
+
if (flags14.coverage) {
|
|
10097
11119
|
const rows = [...bestBySymbol.values()].sort((a, b) => b.profit - a.profit).map((row) => [
|
|
10098
|
-
|
|
10099
|
-
|
|
10100
|
-
|
|
10101
|
-
|
|
11120
|
+
import_chalk18.default.yellow(row.symbol),
|
|
11121
|
+
import_chalk18.default.green(formatPercent(row.profit)),
|
|
11122
|
+
import_chalk18.default.magenta(formatPercent(row.winRate)),
|
|
11123
|
+
import_chalk18.default.blue(row.orders.toString())
|
|
10102
11124
|
]);
|
|
10103
11125
|
console.log("");
|
|
10104
|
-
console.log(
|
|
11126
|
+
console.log(import_chalk18.default.gray("RESULTS:"));
|
|
10105
11127
|
console.log(
|
|
10106
|
-
|
|
10107
|
-
|
|
10108
|
-
|
|
10109
|
-
|
|
10110
|
-
|
|
11128
|
+
createListIt3().setHeaderRow([
|
|
11129
|
+
import_chalk18.default.yellow("SYMBOL"),
|
|
11130
|
+
import_chalk18.default.green("PROFIT"),
|
|
11131
|
+
import_chalk18.default.magenta("WINRATE"),
|
|
11132
|
+
import_chalk18.default.blue("ORDERS")
|
|
10111
11133
|
]).d(rows).toString()
|
|
10112
11134
|
);
|
|
10113
11135
|
console.log("");
|
|
@@ -10118,17 +11140,17 @@ var init_results = __esm({
|
|
|
10118
11140
|
goodSymbols
|
|
10119
11141
|
);
|
|
10120
11142
|
console.log(
|
|
10121
|
-
|
|
11143
|
+
import_chalk18.default.gray(
|
|
10122
11144
|
`COVERAGE: ${goodMissing} / ${goodExisting} / ${total} (${coverage.toFixed(2)} %)`
|
|
10123
11145
|
)
|
|
10124
11146
|
);
|
|
10125
11147
|
console.log("");
|
|
10126
11148
|
}
|
|
10127
|
-
if (
|
|
11149
|
+
if (flags14.update || flags14.merge) {
|
|
10128
11150
|
const resultsConfig = toStrategyResults(bestBySymbol.values());
|
|
10129
|
-
if (
|
|
11151
|
+
if (flags14.merge) {
|
|
10130
11152
|
if (Object.keys(resultsConfig).length === 0) {
|
|
10131
|
-
console.log(
|
|
11153
|
+
console.log(import_chalk18.default.yellow("No good results to merge."));
|
|
10132
11154
|
process.exit(0);
|
|
10133
11155
|
}
|
|
10134
11156
|
const current = await (0, import_redis4.getData)(
|
|
@@ -10144,7 +11166,7 @@ var init_results = __esm({
|
|
|
10144
11166
|
);
|
|
10145
11167
|
if (updates.length === 0) {
|
|
10146
11168
|
console.log(
|
|
10147
|
-
|
|
11169
|
+
import_chalk18.default.yellow(
|
|
10148
11170
|
`No symbols with higher profit than saved results:${strategyName}`
|
|
10149
11171
|
)
|
|
10150
11172
|
);
|
|
@@ -10157,7 +11179,7 @@ var init_results = __esm({
|
|
|
10157
11179
|
await (0, import_redis4.setData)(import_redis4.redisKeys.strategyResults(userName2, strategyName), merged, {
|
|
10158
11180
|
expire: 0
|
|
10159
11181
|
});
|
|
10160
|
-
if (
|
|
11182
|
+
if (flags14.verbose) {
|
|
10161
11183
|
const updateRows = updates.sort((a, b) => {
|
|
10162
11184
|
const prevA = savedProfitBySymbol.get(a.symbol) ?? -Infinity;
|
|
10163
11185
|
const prevB = savedProfitBySymbol.get(b.symbol) ?? -Infinity;
|
|
@@ -10166,30 +11188,30 @@ var init_results = __esm({
|
|
|
10166
11188
|
const prevProfit = savedProfitBySymbol.get(row.symbol);
|
|
10167
11189
|
const delta = row.profit - (prevProfit ?? 0);
|
|
10168
11190
|
return [
|
|
10169
|
-
|
|
10170
|
-
|
|
11191
|
+
import_chalk18.default.yellow(row.symbol),
|
|
11192
|
+
import_chalk18.default.gray(
|
|
10171
11193
|
prevProfit === void 0 ? "N/A" : formatPercent(prevProfit)
|
|
10172
11194
|
),
|
|
10173
|
-
|
|
10174
|
-
|
|
11195
|
+
import_chalk18.default.green(formatPercent(row.profit)),
|
|
11196
|
+
import_chalk18.default.blue(
|
|
10175
11197
|
`${delta >= 0 ? "+" : ""}${formatPercent(delta).replace("+", "")}`
|
|
10176
11198
|
)
|
|
10177
11199
|
];
|
|
10178
11200
|
});
|
|
10179
11201
|
console.log("");
|
|
10180
|
-
console.log(
|
|
11202
|
+
console.log(import_chalk18.default.gray("MERGE UPDATES:"));
|
|
10181
11203
|
console.log(
|
|
10182
|
-
|
|
10183
|
-
|
|
10184
|
-
|
|
10185
|
-
|
|
10186
|
-
|
|
11204
|
+
createListIt3().setHeaderRow([
|
|
11205
|
+
import_chalk18.default.yellow("SYMBOL"),
|
|
11206
|
+
import_chalk18.default.gray("PREV_PROFIT"),
|
|
11207
|
+
import_chalk18.default.green("NEW_PROFIT"),
|
|
11208
|
+
import_chalk18.default.blue("DELTA")
|
|
10187
11209
|
]).d(updateRows).toString()
|
|
10188
11210
|
);
|
|
10189
11211
|
console.log("");
|
|
10190
11212
|
}
|
|
10191
11213
|
console.log(
|
|
10192
|
-
|
|
11214
|
+
import_chalk18.default.green(
|
|
10193
11215
|
`Merged ${updates.length} symbols into results:${strategyName} (only higher profit)`
|
|
10194
11216
|
)
|
|
10195
11217
|
);
|
|
@@ -10203,7 +11225,7 @@ var init_results = __esm({
|
|
|
10203
11225
|
}
|
|
10204
11226
|
);
|
|
10205
11227
|
console.log(
|
|
10206
|
-
|
|
11228
|
+
import_chalk18.default.green(
|
|
10207
11229
|
`Updated results:${strategyName} with ${Object.keys(resultsConfig).length} symbols`
|
|
10208
11230
|
)
|
|
10209
11231
|
);
|
|
@@ -10218,46 +11240,52 @@ var init_results = __esm({
|
|
|
10218
11240
|
|
|
10219
11241
|
// src/scripts/signals.ts
|
|
10220
11242
|
var signals_exports = {};
|
|
10221
|
-
|
|
11243
|
+
__export(signals_exports, {
|
|
11244
|
+
signals: () => signals
|
|
11245
|
+
});
|
|
11246
|
+
var import_config5, import_args15, import_progress6, import_connectors8, import_chalk19, import_connectors9, import_cli8, import_async4, import_constants5, import_strategies3, import_time3, import_logger4, import_redis5, PRELOAD_START, projectRoot6, flags15, interval3, formatElapsed, resolveStrategyNameByConfigKey, loadRuntimeStrategies, createStrategySkipStats, recordStrategyReason, logStrategySkipStats, resolveSignalsConnectorName, findSignals, signals;
|
|
10222
11247
|
var init_signals = __esm({
|
|
10223
11248
|
"src/scripts/signals.ts"() {
|
|
10224
11249
|
"use strict";
|
|
10225
11250
|
import_config5 = require("dotenv/config");
|
|
10226
|
-
|
|
10227
|
-
|
|
11251
|
+
import_args15 = __toESM(require("args"));
|
|
11252
|
+
import_progress6 = __toESM(require("progress"));
|
|
10228
11253
|
import_connectors8 = require("@tradejs/connectors");
|
|
10229
|
-
|
|
11254
|
+
import_chalk19 = __toESM(require("chalk"));
|
|
10230
11255
|
import_connectors9 = require("@tradejs/node/connectors");
|
|
10231
11256
|
import_cli8 = require("@tradejs/node/cli");
|
|
10232
|
-
|
|
10233
|
-
|
|
11257
|
+
import_async4 = require("@tradejs/core/async");
|
|
11258
|
+
import_constants5 = require("@tradejs/core/constants");
|
|
10234
11259
|
import_strategies3 = require("@tradejs/node/strategies");
|
|
10235
11260
|
import_time3 = require("@tradejs/core/time");
|
|
10236
11261
|
import_logger4 = require("@tradejs/infra/logger");
|
|
10237
11262
|
import_redis5 = require("@tradejs/infra/redis");
|
|
10238
|
-
|
|
10239
|
-
|
|
10240
|
-
|
|
10241
|
-
|
|
10242
|
-
|
|
10243
|
-
|
|
10244
|
-
|
|
10245
|
-
|
|
10246
|
-
|
|
10247
|
-
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
11263
|
+
import_args15.default.option(["t", "tickers"], "Selected tickers");
|
|
11264
|
+
import_args15.default.option(["e", "exclude"], "Exclude tickers from tests");
|
|
11265
|
+
import_args15.default.option(["l", "tickersLimit"], "Tickers limit");
|
|
11266
|
+
import_args15.default.option(["f", "timeframe"], "Timeframe", 15);
|
|
11267
|
+
import_args15.default.option(["m", "makeOrders"], "Make orders");
|
|
11268
|
+
import_args15.default.option(["N", "notify"], "Send message in Telegram", false);
|
|
11269
|
+
import_args15.default.option(["S", "skipScreenshots"], "Skip screenshot generation", false);
|
|
11270
|
+
import_args15.default.option(["u", "updateOnly"], "Only update tickers history", false);
|
|
11271
|
+
import_args15.default.option(["C", "cacheOnly"], "Do not update tickers history", false);
|
|
11272
|
+
import_args15.default.option(["L", "showTickersList"], "Just show only ticker list", false);
|
|
11273
|
+
import_args15.default.option(
|
|
11274
|
+
["R", "showSkipStats"],
|
|
11275
|
+
"Show aggregated skip stats by strategy",
|
|
11276
|
+
false
|
|
11277
|
+
);
|
|
11278
|
+
import_args15.default.option(["c", "chunk"], "Split by chunks, ex. 1/3");
|
|
11279
|
+
import_args15.default.option(["U", "user"], "Use user confg", "root");
|
|
11280
|
+
import_args15.default.option(
|
|
10251
11281
|
"connector",
|
|
10252
11282
|
"Connector provider or name for signals (e.g. bybit, binance, coinbase, custom)",
|
|
10253
11283
|
"bybit"
|
|
10254
11284
|
);
|
|
10255
|
-
PRELOAD_START = (0, import_time3.getTimestamp)(
|
|
11285
|
+
PRELOAD_START = (0, import_time3.getTimestamp)(import_constants5.SIGNALS_CLI_PRELOAD_DAYS);
|
|
10256
11286
|
projectRoot6 = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
10257
|
-
|
|
10258
|
-
|
|
10259
|
-
offset = parseInt(flags12.offset);
|
|
10260
|
-
interval3 = flags12.timeframe.toString();
|
|
11287
|
+
flags15 = import_args15.default.parse(process.argv);
|
|
11288
|
+
interval3 = flags15.timeframe.toString();
|
|
10261
11289
|
formatElapsed = (startedAt) => `${((Date.now() - startedAt) / 1e3).toFixed(1)}s`;
|
|
10262
11290
|
resolveStrategyNameByConfigKey = (userName2, key) => {
|
|
10263
11291
|
const parts = key.split(":");
|
|
@@ -10297,6 +11325,45 @@ var init_signals = __esm({
|
|
|
10297
11325
|
);
|
|
10298
11326
|
return strategyConfigs.filter(Boolean);
|
|
10299
11327
|
};
|
|
11328
|
+
createStrategySkipStats = (runtimeStrategies) => new Map(
|
|
11329
|
+
runtimeStrategies.map(({ strategyName }) => [
|
|
11330
|
+
strategyName,
|
|
11331
|
+
{
|
|
11332
|
+
evaluated: 0,
|
|
11333
|
+
signals: 0,
|
|
11334
|
+
reasons: /* @__PURE__ */ new Map()
|
|
11335
|
+
}
|
|
11336
|
+
])
|
|
11337
|
+
);
|
|
11338
|
+
recordStrategyReason = (strategyStats, strategyName, reason) => {
|
|
11339
|
+
const stats = strategyStats.get(strategyName);
|
|
11340
|
+
if (!stats) {
|
|
11341
|
+
return;
|
|
11342
|
+
}
|
|
11343
|
+
stats.reasons.set(reason, (stats.reasons.get(reason) ?? 0) + 1);
|
|
11344
|
+
};
|
|
11345
|
+
logStrategySkipStats = (runtimeStrategies, strategyStats) => {
|
|
11346
|
+
import_logger4.logger.info(import_chalk19.default.yellow("skip stats:"));
|
|
11347
|
+
for (const { strategyName } of runtimeStrategies) {
|
|
11348
|
+
const stats = strategyStats.get(strategyName);
|
|
11349
|
+
if (!stats) {
|
|
11350
|
+
continue;
|
|
11351
|
+
}
|
|
11352
|
+
import_logger4.logger.info(
|
|
11353
|
+
`${strategyName}: evaluated=${stats.evaluated}, signals=${stats.signals}`
|
|
11354
|
+
);
|
|
11355
|
+
const sortedReasons = [...stats.reasons.entries()].sort(
|
|
11356
|
+
(left, right) => right[1] - left[1] || left[0].localeCompare(right[0])
|
|
11357
|
+
);
|
|
11358
|
+
if (!sortedReasons.length) {
|
|
11359
|
+
import_logger4.logger.info(" none");
|
|
11360
|
+
continue;
|
|
11361
|
+
}
|
|
11362
|
+
for (const [reason, count] of sortedReasons) {
|
|
11363
|
+
import_logger4.logger.info(` ${reason}: ${count}`);
|
|
11364
|
+
}
|
|
11365
|
+
}
|
|
11366
|
+
};
|
|
10300
11367
|
resolveSignalsConnectorName = async (value) => {
|
|
10301
11368
|
const connectorName = await (0, import_connectors9.resolveConnectorName)(value, projectRoot6);
|
|
10302
11369
|
if (connectorName) {
|
|
@@ -10309,13 +11376,9 @@ var init_signals = __esm({
|
|
|
10309
11376
|
);
|
|
10310
11377
|
return import_connectors9.DEFAULT_CONNECTOR_NAME;
|
|
10311
11378
|
};
|
|
10312
|
-
findSignals = async (symbol, connector, btcBinanceData, btcCoinbaseData, runtimeStrategies) => {
|
|
10313
|
-
const prevSignals = await (0, import_redis5.getKeys)(import_redis5.redisKeys.signalsBySymbol(symbol));
|
|
10314
|
-
if (prevSignals.length) {
|
|
10315
|
-
import_logger4.logger.info("Exit by signal exists %s", symbol);
|
|
10316
|
-
return null;
|
|
10317
|
-
}
|
|
11379
|
+
findSignals = async (symbol, connector, btcBinanceData, btcCoinbaseData, runtimeStrategies, strategyStats) => {
|
|
10318
11380
|
const currentTimestamp = (0, import_time3.getTimestamp)();
|
|
11381
|
+
const strategySignals = [];
|
|
10319
11382
|
const [cachedData, btcCachedData] = await Promise.all([
|
|
10320
11383
|
connector.kline({
|
|
10321
11384
|
symbol,
|
|
@@ -10335,12 +11398,16 @@ var init_signals = __esm({
|
|
|
10335
11398
|
const lastCandle = cachedData.pop();
|
|
10336
11399
|
const btcLastCandle = btcCachedData.pop();
|
|
10337
11400
|
if (!lastCandle || !btcLastCandle) {
|
|
10338
|
-
return;
|
|
11401
|
+
return strategySignals;
|
|
10339
11402
|
}
|
|
10340
11403
|
for (const runtimeStrategy of runtimeStrategies) {
|
|
10341
11404
|
const { strategyName, strategyCreator, strategyConfig } = runtimeStrategy;
|
|
11405
|
+
const stats = strategyStats.get(strategyName);
|
|
11406
|
+
if (stats) {
|
|
11407
|
+
stats.evaluated += 1;
|
|
11408
|
+
}
|
|
10342
11409
|
const strategy = await strategyCreator({
|
|
10343
|
-
userName:
|
|
11410
|
+
userName: flags15.user,
|
|
10344
11411
|
connector,
|
|
10345
11412
|
symbol,
|
|
10346
11413
|
data: [...cachedData],
|
|
@@ -10351,29 +11418,36 @@ var init_signals = __esm({
|
|
|
10351
11418
|
...strategyConfig,
|
|
10352
11419
|
ENV: "CRON",
|
|
10353
11420
|
INTERVAL: interval3,
|
|
10354
|
-
MAKE_ORDERS:
|
|
11421
|
+
MAKE_ORDERS: flags15.makeOrders
|
|
10355
11422
|
}
|
|
10356
11423
|
});
|
|
10357
11424
|
const signal = await strategy(lastCandle, btcLastCandle);
|
|
10358
11425
|
if (!signal || typeof signal === "string") {
|
|
11426
|
+
if (typeof signal === "string") {
|
|
11427
|
+
recordStrategyReason(strategyStats, strategyName, signal);
|
|
11428
|
+
}
|
|
10359
11429
|
continue;
|
|
10360
11430
|
}
|
|
11431
|
+
if (stats) {
|
|
11432
|
+
stats.signals += 1;
|
|
11433
|
+
}
|
|
11434
|
+
strategySignals.push(signal);
|
|
10361
11435
|
await (0, import_redis5.setData)(import_redis5.redisKeys.signal(symbol, signal.signalId), signal, {
|
|
10362
|
-
expire:
|
|
11436
|
+
expire: import_constants5.TTL_1D
|
|
10363
11437
|
});
|
|
10364
11438
|
await (0, import_redis5.setData)(import_redis5.redisKeys.storeSignal(symbol, signal.signalId), signal, {
|
|
10365
|
-
expire:
|
|
11439
|
+
expire: import_constants5.TTL_3M
|
|
10366
11440
|
});
|
|
10367
11441
|
import_logger4.logger.info("Signal found %s by strategy %s", symbol, strategyName);
|
|
10368
|
-
return signal;
|
|
10369
11442
|
}
|
|
11443
|
+
return strategySignals;
|
|
10370
11444
|
};
|
|
10371
11445
|
signals = async () => {
|
|
10372
11446
|
const startedAt = Date.now();
|
|
10373
11447
|
const signals2 = new Array();
|
|
10374
11448
|
let status = "completed";
|
|
10375
11449
|
try {
|
|
10376
|
-
const connectorName = await resolveSignalsConnectorName(
|
|
11450
|
+
const connectorName = await resolveSignalsConnectorName(flags15.connector);
|
|
10377
11451
|
const connectorFactory = await (0, import_connectors9.getConnectorCreatorByName)(
|
|
10378
11452
|
connectorName,
|
|
10379
11453
|
projectRoot6
|
|
@@ -10382,7 +11456,7 @@ var init_signals = __esm({
|
|
|
10382
11456
|
throw new Error(`Connector "${connectorName}" is not registered`);
|
|
10383
11457
|
}
|
|
10384
11458
|
const marketConnector = await connectorFactory({
|
|
10385
|
-
userName:
|
|
11459
|
+
userName: flags15.user
|
|
10386
11460
|
});
|
|
10387
11461
|
let btcBinanceConnector = marketConnector;
|
|
10388
11462
|
let btcCoinbaseConnector = marketConnector;
|
|
@@ -10393,7 +11467,7 @@ var init_signals = __esm({
|
|
|
10393
11467
|
);
|
|
10394
11468
|
if (binanceFactory) {
|
|
10395
11469
|
btcBinanceConnector = await binanceFactory({
|
|
10396
|
-
userName:
|
|
11470
|
+
userName: flags15.user
|
|
10397
11471
|
});
|
|
10398
11472
|
} else {
|
|
10399
11473
|
import_logger4.logger.warn(
|
|
@@ -10407,7 +11481,7 @@ var init_signals = __esm({
|
|
|
10407
11481
|
);
|
|
10408
11482
|
if (coinbaseFactory) {
|
|
10409
11483
|
btcCoinbaseConnector = await coinbaseFactory({
|
|
10410
|
-
userName:
|
|
11484
|
+
userName: flags15.user
|
|
10411
11485
|
});
|
|
10412
11486
|
} else {
|
|
10413
11487
|
import_logger4.logger.warn(
|
|
@@ -10418,22 +11492,37 @@ var init_signals = __esm({
|
|
|
10418
11492
|
}
|
|
10419
11493
|
const tickers = await (0, import_cli8.getTickers)(
|
|
10420
11494
|
marketConnector,
|
|
10421
|
-
|
|
10422
|
-
|
|
10423
|
-
|
|
10424
|
-
|
|
11495
|
+
flags15.tickers,
|
|
11496
|
+
flags15.exclude,
|
|
11497
|
+
flags15.tickersLimit,
|
|
11498
|
+
flags15.chunk
|
|
10425
11499
|
);
|
|
10426
|
-
if (
|
|
10427
|
-
console.log(
|
|
11500
|
+
if (flags15.showTickersList) {
|
|
11501
|
+
console.log(import_chalk19.default.gray(JSON.stringify(tickers.sort(), null, 2)));
|
|
10428
11502
|
return;
|
|
10429
11503
|
}
|
|
10430
|
-
if (!
|
|
10431
|
-
await (0, import_cli8.update)(
|
|
11504
|
+
if (!flags15.cacheOnly) {
|
|
11505
|
+
await (0, import_cli8.update)(
|
|
11506
|
+
marketConnector,
|
|
11507
|
+
interval3,
|
|
11508
|
+
tickers,
|
|
11509
|
+
import_constants5.SIGNALS_CLI_PRELOAD_DAYS
|
|
11510
|
+
);
|
|
10432
11511
|
if (btcBinanceConnector !== marketConnector) {
|
|
10433
|
-
await (0, import_cli8.update)(
|
|
11512
|
+
await (0, import_cli8.update)(
|
|
11513
|
+
btcBinanceConnector,
|
|
11514
|
+
interval3,
|
|
11515
|
+
["BTCUSDT"],
|
|
11516
|
+
import_constants5.SIGNALS_CLI_PRELOAD_DAYS
|
|
11517
|
+
);
|
|
10434
11518
|
}
|
|
10435
11519
|
if (btcCoinbaseConnector !== marketConnector) {
|
|
10436
|
-
await (0, import_cli8.update)(
|
|
11520
|
+
await (0, import_cli8.update)(
|
|
11521
|
+
btcCoinbaseConnector,
|
|
11522
|
+
interval3,
|
|
11523
|
+
["BTCUSDT"],
|
|
11524
|
+
import_constants5.SIGNALS_CLI_PRELOAD_DAYS
|
|
11525
|
+
);
|
|
10437
11526
|
}
|
|
10438
11527
|
}
|
|
10439
11528
|
const currentTimestamp = (0, import_time3.getTimestamp)();
|
|
@@ -10453,52 +11542,53 @@ var init_signals = __esm({
|
|
|
10453
11542
|
interval: interval3
|
|
10454
11543
|
})
|
|
10455
11544
|
]);
|
|
10456
|
-
if (
|
|
11545
|
+
if (flags15.updateOnly) {
|
|
10457
11546
|
return;
|
|
10458
11547
|
}
|
|
10459
|
-
const runtimeStrategies = await loadRuntimeStrategies(
|
|
11548
|
+
const runtimeStrategies = await loadRuntimeStrategies(flags15.user);
|
|
10460
11549
|
if (!runtimeStrategies.length) {
|
|
10461
11550
|
import_logger4.logger.warn(
|
|
10462
11551
|
"No strategy configs found by users:%s:strategies:*:config",
|
|
10463
|
-
|
|
11552
|
+
flags15.user
|
|
10464
11553
|
);
|
|
10465
11554
|
return;
|
|
10466
11555
|
}
|
|
10467
11556
|
import_logger4.logger.info(
|
|
10468
|
-
|
|
11557
|
+
import_chalk19.default.yellow(
|
|
10469
11558
|
`loaded strategies: ${runtimeStrategies.map((s) => s.strategyName).join(", ")}`
|
|
10470
11559
|
)
|
|
10471
11560
|
);
|
|
10472
|
-
const
|
|
11561
|
+
const strategyStats = createStrategySkipStats(runtimeStrategies);
|
|
11562
|
+
const bar = new import_progress6.default(
|
|
10473
11563
|
":current/:total [:bar][:percent] :found :eta(s) :symbol",
|
|
10474
11564
|
{
|
|
10475
11565
|
total: tickers.length,
|
|
10476
11566
|
width: 30
|
|
10477
11567
|
}
|
|
10478
11568
|
);
|
|
10479
|
-
import_logger4.logger.info(
|
|
10480
|
-
await (0,
|
|
10481
|
-
const
|
|
11569
|
+
import_logger4.logger.info(import_chalk19.default.yellow(`tickers: ${tickers.length}`));
|
|
11570
|
+
await (0, import_async4.runWithConcurrency)(tickers, 5, async (symbol) => {
|
|
11571
|
+
const strategySignals = await findSignals(
|
|
10482
11572
|
symbol,
|
|
10483
11573
|
marketConnector,
|
|
10484
11574
|
btcBinanceData,
|
|
10485
11575
|
btcCoinbaseData,
|
|
10486
|
-
runtimeStrategies
|
|
11576
|
+
runtimeStrategies,
|
|
11577
|
+
strategyStats
|
|
10487
11578
|
);
|
|
10488
|
-
if (
|
|
10489
|
-
signals2.push(
|
|
11579
|
+
if (strategySignals.length > 0) {
|
|
11580
|
+
signals2.push(...strategySignals);
|
|
10490
11581
|
}
|
|
10491
11582
|
bar.tick(1, {
|
|
10492
|
-
found:
|
|
10493
|
-
symbol:
|
|
11583
|
+
found: import_chalk19.default.cyan(signals2.length),
|
|
11584
|
+
symbol: import_chalk19.default.gray(symbol)
|
|
10494
11585
|
});
|
|
10495
11586
|
});
|
|
10496
|
-
if (!
|
|
10497
|
-
await (0, import_cli8.makeScreenshots)(signals2, "15");
|
|
10498
|
-
await (0, import_cli8.makeScreenshots)(signals2, "60");
|
|
11587
|
+
if (!flags15.skipScreenshots) {
|
|
11588
|
+
await (0, import_cli8.makeScreenshots)(signals2, "15", flags15.user);
|
|
10499
11589
|
}
|
|
10500
|
-
if (
|
|
10501
|
-
await (0, import_cli8.sendToTG)(signals2, "15");
|
|
11590
|
+
if (flags15.notify) {
|
|
11591
|
+
await (0, import_cli8.sendToTG)(signals2, "15", flags15.user);
|
|
10502
11592
|
}
|
|
10503
11593
|
import_logger4.logger.info(
|
|
10504
11594
|
JSON.stringify(
|
|
@@ -10507,6 +11597,9 @@ var init_signals = __esm({
|
|
|
10507
11597
|
2
|
|
10508
11598
|
)
|
|
10509
11599
|
);
|
|
11600
|
+
if (flags15.showSkipStats) {
|
|
11601
|
+
logStrategySkipStats(runtimeStrategies, strategyStats);
|
|
11602
|
+
}
|
|
10510
11603
|
} catch (error) {
|
|
10511
11604
|
status = "failed";
|
|
10512
11605
|
import_logger4.logger.error(
|
|
@@ -10516,33 +11609,35 @@ var init_signals = __esm({
|
|
|
10516
11609
|
throw error;
|
|
10517
11610
|
} finally {
|
|
10518
11611
|
import_logger4.logger.info(
|
|
10519
|
-
|
|
11612
|
+
import_chalk19.default.yellow(
|
|
10520
11613
|
`signals ${status} in ${formatElapsed(startedAt)} (found=${signals2.length})`
|
|
10521
11614
|
)
|
|
10522
11615
|
);
|
|
10523
11616
|
}
|
|
10524
11617
|
};
|
|
10525
|
-
|
|
10526
|
-
|
|
10527
|
-
|
|
10528
|
-
|
|
10529
|
-
|
|
11618
|
+
if (process.env.NODE_ENV !== "test") {
|
|
11619
|
+
signals().then(() => {
|
|
11620
|
+
process.exit(0);
|
|
11621
|
+
}).catch(() => {
|
|
11622
|
+
process.exit(1);
|
|
11623
|
+
});
|
|
11624
|
+
}
|
|
10530
11625
|
}
|
|
10531
11626
|
});
|
|
10532
11627
|
|
|
10533
11628
|
// src/scripts/test-ml.ts
|
|
10534
11629
|
var test_ml_exports = {};
|
|
10535
|
-
var import_config6,
|
|
11630
|
+
var import_config6, import_constants6, import_ml3, import_strategies4, now, INTERVAL_MIN, CANDLES, makeSeries, makeCandles, buildTrendline, main5;
|
|
10536
11631
|
var init_test_ml = __esm({
|
|
10537
11632
|
"src/scripts/test-ml.ts"() {
|
|
10538
11633
|
"use strict";
|
|
10539
11634
|
import_config6 = require("dotenv/config");
|
|
10540
|
-
|
|
10541
|
-
|
|
11635
|
+
import_constants6 = require("@tradejs/core/constants");
|
|
11636
|
+
import_ml3 = require("@tradejs/infra/ml");
|
|
10542
11637
|
import_strategies4 = require("@tradejs/strategies");
|
|
10543
11638
|
now = Date.now();
|
|
10544
11639
|
INTERVAL_MIN = 15;
|
|
10545
|
-
CANDLES =
|
|
11640
|
+
CANDLES = import_constants6.ML_BASE_CANDLES_WINDOW;
|
|
10546
11641
|
makeSeries = (len, base, step = 0.1) => Array.from({ length: len }, (_4, i) => base + i * step);
|
|
10547
11642
|
makeCandles = (startPrice, startTs) => {
|
|
10548
11643
|
const candles = [];
|
|
@@ -10582,7 +11677,7 @@ var init_test_ml = __esm({
|
|
|
10582
11677
|
{ timestamp: entryTs - 2 * INTERVAL_MIN * 6e4, value: price * 0.995 }
|
|
10583
11678
|
]
|
|
10584
11679
|
});
|
|
10585
|
-
|
|
11680
|
+
main5 = async () => {
|
|
10586
11681
|
const startTs = now - CANDLES * INTERVAL_MIN * 6e4;
|
|
10587
11682
|
const candles = makeCandles(120, startTs);
|
|
10588
11683
|
const btcCandles = makeCandles(42e3, startTs);
|
|
@@ -10608,8 +11703,8 @@ var init_test_ml = __esm({
|
|
|
10608
11703
|
correlation: 0.18,
|
|
10609
11704
|
touches: 6,
|
|
10610
11705
|
distance: 0.7,
|
|
10611
|
-
candles15m: candles.slice(-
|
|
10612
|
-
btcCandles15m: btcCandles.slice(-
|
|
11706
|
+
candles15m: candles.slice(-import_constants6.ML_BASE_CANDLES_WINDOW),
|
|
11707
|
+
btcCandles15m: btcCandles.slice(-import_constants6.ML_BASE_CANDLES_WINDOW),
|
|
10613
11708
|
atr: makeSeries(10, 1.2, 0.02),
|
|
10614
11709
|
atrPct: makeSeries(10, 1.05, 5e-3),
|
|
10615
11710
|
maFast: makeSeries(10, currentPrice * 0.995, 0.01),
|
|
@@ -10634,7 +11729,7 @@ var init_test_ml = __esm({
|
|
|
10634
11729
|
}
|
|
10635
11730
|
};
|
|
10636
11731
|
const { TRENDLINE, HIGHS, LOWS, ML_THRESHOLD } = import_strategies4.trendLineDefaultConfig;
|
|
10637
|
-
const fullRow = (0,
|
|
11732
|
+
const fullRow = (0, import_ml3.buildMlTrainingRow)(
|
|
10638
11733
|
{
|
|
10639
11734
|
signal,
|
|
10640
11735
|
context: {
|
|
@@ -10649,9 +11744,9 @@ var init_test_ml = __esm({
|
|
|
10649
11744
|
},
|
|
10650
11745
|
null
|
|
10651
11746
|
);
|
|
10652
|
-
const row = (0,
|
|
10653
|
-
const features = (0,
|
|
10654
|
-
const mlResult = await (0,
|
|
11747
|
+
const row = (0, import_ml3.trimMlTrainingRowWindows)(fullRow, 5);
|
|
11748
|
+
const features = (0, import_ml3.buildMlFeatures)(row);
|
|
11749
|
+
const mlResult = await (0, import_ml3.fetchMlThreshold)({
|
|
10655
11750
|
strategy: "TrendLine",
|
|
10656
11751
|
features,
|
|
10657
11752
|
threshold: ML_THRESHOLD,
|
|
@@ -10663,7 +11758,7 @@ var init_test_ml = __esm({
|
|
|
10663
11758
|
}
|
|
10664
11759
|
console.log("ML result:", mlResult);
|
|
10665
11760
|
};
|
|
10666
|
-
|
|
11761
|
+
main5().catch((err) => {
|
|
10667
11762
|
console.error("test-ml failed:", err);
|
|
10668
11763
|
process.exit(1);
|
|
10669
11764
|
});
|
|
@@ -10672,7 +11767,7 @@ var init_test_ml = __esm({
|
|
|
10672
11767
|
|
|
10673
11768
|
// src/scripts/test.ts
|
|
10674
11769
|
var test_exports = {};
|
|
10675
|
-
var import_config7, import_connectors10, getPositions,
|
|
11770
|
+
var import_config7, import_connectors10, getPositions, main6;
|
|
10676
11771
|
var init_test = __esm({
|
|
10677
11772
|
"src/scripts/test.ts"() {
|
|
10678
11773
|
"use strict";
|
|
@@ -10682,38 +11777,38 @@ var init_test = __esm({
|
|
|
10682
11777
|
const res = await connector.getPositions();
|
|
10683
11778
|
console.log("res", res);
|
|
10684
11779
|
};
|
|
10685
|
-
|
|
11780
|
+
main6 = async () => {
|
|
10686
11781
|
const byBitConnector = await import_connectors10.connectors.ByBit({
|
|
10687
11782
|
userName: "root"
|
|
10688
11783
|
});
|
|
10689
11784
|
await getPositions(byBitConnector);
|
|
10690
11785
|
};
|
|
10691
|
-
|
|
11786
|
+
main6();
|
|
10692
11787
|
}
|
|
10693
11788
|
});
|
|
10694
11789
|
|
|
10695
11790
|
// src/scripts/user-add.ts
|
|
10696
11791
|
var user_add_exports = {};
|
|
10697
|
-
var
|
|
11792
|
+
var import_args16, import_chalk20, import_bcryptjs, import_crypto2, import_redis6, flags16, run12;
|
|
10698
11793
|
var init_user_add = __esm({
|
|
10699
11794
|
"src/scripts/user-add.ts"() {
|
|
10700
11795
|
"use strict";
|
|
10701
|
-
|
|
10702
|
-
|
|
11796
|
+
import_args16 = __toESM(require("args"));
|
|
11797
|
+
import_chalk20 = __toESM(require("chalk"));
|
|
10703
11798
|
import_bcryptjs = __toESM(require("bcryptjs"));
|
|
10704
11799
|
import_crypto2 = __toESM(require("crypto"));
|
|
10705
11800
|
import_redis6 = require("@tradejs/infra/redis");
|
|
10706
|
-
|
|
10707
|
-
|
|
10708
|
-
|
|
10709
|
-
|
|
10710
|
-
|
|
10711
|
-
|
|
10712
|
-
const userName2 = String(
|
|
10713
|
-
const password = String(
|
|
10714
|
-
const requestedToken = String(
|
|
11801
|
+
import_args16.default.example("yarn user-add -u myname -p 123456", "Create or update user");
|
|
11802
|
+
import_args16.default.option(["u", "user"], "Username", "");
|
|
11803
|
+
import_args16.default.option(["p", "password"], "Password (plain text)", "");
|
|
11804
|
+
import_args16.default.option(["t", "token"], "Persistent token (optional)", "");
|
|
11805
|
+
flags16 = import_args16.default.parse(process.argv);
|
|
11806
|
+
run12 = async () => {
|
|
11807
|
+
const userName2 = String(flags16.user || "").trim();
|
|
11808
|
+
const password = String(flags16.password || "");
|
|
11809
|
+
const requestedToken = String(flags16.token || "").trim();
|
|
10715
11810
|
if (!userName2 || !password) {
|
|
10716
|
-
console.error(
|
|
11811
|
+
console.error(import_chalk20.default.red("Missing -U <user> or -P <password>"));
|
|
10717
11812
|
process.exit(1);
|
|
10718
11813
|
}
|
|
10719
11814
|
const passwordHash = await import_bcryptjs.default.hash(password, 10);
|
|
@@ -10730,18 +11825,20 @@ var init_user_add = __esm({
|
|
|
10730
11825
|
await (0, import_redis6.setData)(import_redis6.redisKeys.user(userName2), next, {
|
|
10731
11826
|
expire: 0
|
|
10732
11827
|
});
|
|
10733
|
-
console.log(
|
|
11828
|
+
console.log(import_chalk20.default.green(`User ${userName2} updated`));
|
|
10734
11829
|
if (token) {
|
|
10735
|
-
console.log(
|
|
11830
|
+
console.log(import_chalk20.default.gray(`token: ${token}`));
|
|
10736
11831
|
}
|
|
10737
11832
|
process.exit(0);
|
|
10738
11833
|
};
|
|
10739
|
-
|
|
11834
|
+
run12();
|
|
10740
11835
|
}
|
|
10741
11836
|
});
|
|
10742
11837
|
|
|
10743
11838
|
// src/cli.ts
|
|
10744
11839
|
var scriptLoaders = {
|
|
11840
|
+
"ai-export": () => Promise.resolve().then(() => (init_aiExport(), aiExport_exports)),
|
|
11841
|
+
"ai-train": () => Promise.resolve().then(() => (init_aiTrain(), aiTrain_exports)),
|
|
10745
11842
|
backtest: () => Promise.resolve().then(() => (init_backtest(), backtest_exports)),
|
|
10746
11843
|
bot: () => Promise.resolve().then(() => (init_bot(), bot_exports)),
|
|
10747
11844
|
"clean-dir": () => Promise.resolve().then(() => (init_cleanDir(), cleanDir_exports)),
|
|
@@ -10755,7 +11852,7 @@ var scriptLoaders = {
|
|
|
10755
11852
|
"infra-init": () => Promise.resolve().then(() => (init_infraInit(), infraInit_exports)),
|
|
10756
11853
|
"infra-up": () => Promise.resolve().then(() => (init_infraUp(), infraUp_exports)),
|
|
10757
11854
|
migration: () => Promise.resolve().then(() => (init_migration(), migration_exports)),
|
|
10758
|
-
"ml-export": () => Promise.resolve().then(() => (
|
|
11855
|
+
"ml-export": () => Promise.resolve().then(() => (init_mlExport(), mlExport_exports)),
|
|
10759
11856
|
"ml-inspect": () => Promise.resolve().then(() => (init_mlInspect(), mlInspect_exports)),
|
|
10760
11857
|
"ml-train:latest": () => Promise.resolve().then(() => (init_mlTrainLatestSelect(), mlTrainLatestSelect_exports)),
|
|
10761
11858
|
results: () => Promise.resolve().then(() => (init_results(), results_exports)),
|
|
@@ -10771,21 +11868,21 @@ var printUsage = () => {
|
|
|
10771
11868
|
`Usage: tradejs <command> [args]\\n\\nCommands:\\n${commands.join("\\n")}`
|
|
10772
11869
|
);
|
|
10773
11870
|
};
|
|
10774
|
-
var
|
|
10775
|
-
const [, scriptPath, command = "", ...
|
|
11871
|
+
var main7 = async () => {
|
|
11872
|
+
const [, scriptPath, command = "", ...args17] = process.argv;
|
|
10776
11873
|
const loader = scriptLoaders[command];
|
|
10777
11874
|
if (!loader) {
|
|
10778
11875
|
printUsage();
|
|
10779
11876
|
process.exit(command ? 1 : 0);
|
|
10780
11877
|
}
|
|
10781
|
-
const nextArgs = [...
|
|
11878
|
+
const nextArgs = [...args17];
|
|
10782
11879
|
if (command === "spread:ingest") {
|
|
10783
11880
|
nextArgs.push("--provider", "binance_coinbase_spread");
|
|
10784
11881
|
}
|
|
10785
11882
|
process.argv = [process.argv[0], scriptPath, ...nextArgs];
|
|
10786
11883
|
await loader();
|
|
10787
11884
|
};
|
|
10788
|
-
|
|
11885
|
+
main7().catch((error) => {
|
|
10789
11886
|
console.error(error);
|
|
10790
11887
|
process.exit(1);
|
|
10791
11888
|
});
|