@tradejs/cli 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/cli.js +1910 -813
- package/dist/lib/aiTrainMetrics.js +150 -0
- package/dist/scripts/aiExport.js +192 -0
- package/dist/scripts/aiExportSelect.js +135 -0
- package/dist/scripts/aiTrain.js +2239 -0
- package/dist/scripts/backtest.js +218 -32
- package/dist/scripts/derivativesIngest.js +5 -0
- package/dist/scripts/derivativesIngestCoinalyzeAll.js +21 -12
- package/dist/scripts/mlExport.js +112 -13
- package/dist/scripts/mlExportSelect.js +43 -9
- package/dist/scripts/mlTrainLatestSelect.js +4 -3
- package/dist/scripts/resolveExportStrategy.js +133 -0
- package/dist/scripts/selectStrategy.js +4 -3
- package/dist/scripts/signals.js +109 -25
- package/dist/scripts/spawnSiblingScript.js +71 -0
- package/dist/workers/testerWorker.js +13 -2
- package/package.json +11 -9
package/dist/scripts/backtest.js
CHANGED
|
@@ -1637,6 +1637,7 @@ import_args.default.option(["t", "tickers"], "Selected tickers");
|
|
|
1637
1637
|
import_args.default.option(["e", "exclude"], "Exclude tickers from tests");
|
|
1638
1638
|
import_args.default.option(["l", "tickersLimit"], "Tickers limit");
|
|
1639
1639
|
import_args.default.option(["n", "tests"], "Tests limit", import_constants.TESTS_LIMIT);
|
|
1640
|
+
import_args.default.option("skip", "Skip first N tests", 0);
|
|
1640
1641
|
import_args.default.option(["p", "parallel"], "Parallel tasks", MAX_PARALLEL);
|
|
1641
1642
|
import_args.default.option(["f", "timeframe"], "Timeframe", 15);
|
|
1642
1643
|
import_args.default.option(["T", "top"], "Return N best tests", import_constants.TESTS_TOP_LIMIT);
|
|
@@ -1656,9 +1657,26 @@ import_args.default.option(
|
|
|
1656
1657
|
"Write ML dataset rows to per-worker JSONL chunks",
|
|
1657
1658
|
false
|
|
1658
1659
|
);
|
|
1659
|
-
|
|
1660
|
+
import_args.default.option(
|
|
1661
|
+
["A", "ai"],
|
|
1662
|
+
"Write AI prompt rows to per-worker JSONL chunks",
|
|
1663
|
+
false
|
|
1664
|
+
);
|
|
1665
|
+
var normalizedArgv = process.argv.map((arg) => {
|
|
1666
|
+
if (arg === "--ML") {
|
|
1667
|
+
return "--ml";
|
|
1668
|
+
}
|
|
1669
|
+
if (arg === "--AI") {
|
|
1670
|
+
return "--ai";
|
|
1671
|
+
}
|
|
1672
|
+
return arg;
|
|
1673
|
+
});
|
|
1674
|
+
var flags = import_args.default.parse(normalizedArgv);
|
|
1660
1675
|
var interval = flags.timeframe.toString();
|
|
1661
|
-
var progressStep = flags.progressStep;
|
|
1676
|
+
var progressStep = Math.max(1, parseInt(String(flags.progressStep), 10));
|
|
1677
|
+
var testsLimit = Math.max(0, parseInt(String(flags.tests), 10));
|
|
1678
|
+
var testsSkip = Math.max(0, parseInt(String(flags.skip ?? 0), 10));
|
|
1679
|
+
var testItemTimeoutMs = 12e4;
|
|
1662
1680
|
var uuid = (len = 12) => (0, import_uuid.v4)().slice(-len);
|
|
1663
1681
|
var projectRoot = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
1664
1682
|
var testerWorkerPathCandidates = [
|
|
@@ -1701,6 +1719,10 @@ var errorTests = 0;
|
|
|
1701
1719
|
var errorMessages = [];
|
|
1702
1720
|
var results = [];
|
|
1703
1721
|
var resultsByTickers = /* @__PURE__ */ new Map();
|
|
1722
|
+
var resultArtifactsByTestName = /* @__PURE__ */ new Map();
|
|
1723
|
+
var artifactRefCountsByTestName = /* @__PURE__ */ new Map();
|
|
1724
|
+
var bestTickerTestNameBySymbol = /* @__PURE__ */ new Map();
|
|
1725
|
+
var topResultNames = /* @__PURE__ */ new Set();
|
|
1704
1726
|
var userName = flags.user;
|
|
1705
1727
|
var runStartedAt = Date.now();
|
|
1706
1728
|
var createListIt = () => new ListIt({
|
|
@@ -1709,10 +1731,108 @@ var createListIt = () => new ListIt({
|
|
|
1709
1731
|
});
|
|
1710
1732
|
var createTable = (headers, rows) => createListIt().setHeaderRow(headers).d(rows).toString();
|
|
1711
1733
|
var createTimestamp = (date) => (0, import_date_fns.format)(date, "yyyyMMddHHmm");
|
|
1712
|
-
var
|
|
1734
|
+
var isGoodTest = (result) => result.stat?.orders > 5 && result.stat?.profit > 10;
|
|
1735
|
+
var getResultAmount = (result) => result.stat.amount ?? 0;
|
|
1713
1736
|
var recordError = (error) => {
|
|
1714
1737
|
errorMessages.push(error);
|
|
1715
1738
|
};
|
|
1739
|
+
var stripInlineLogs = (result) => {
|
|
1740
|
+
const { inlineOrderLog, inlinePositionLog, ...resultWithoutLogs } = result;
|
|
1741
|
+
return resultWithoutLogs;
|
|
1742
|
+
};
|
|
1743
|
+
var getInlineArtifacts = (result) => {
|
|
1744
|
+
if (!result.inlineOrderLog && !result.inlinePositionLog) {
|
|
1745
|
+
return null;
|
|
1746
|
+
}
|
|
1747
|
+
return {
|
|
1748
|
+
orderLog: result.inlineOrderLog,
|
|
1749
|
+
positionLog: result.inlinePositionLog
|
|
1750
|
+
};
|
|
1751
|
+
};
|
|
1752
|
+
var retainArtifacts = (testName, artifacts) => {
|
|
1753
|
+
if (artifacts) {
|
|
1754
|
+
resultArtifactsByTestName.set(testName, artifacts);
|
|
1755
|
+
}
|
|
1756
|
+
artifactRefCountsByTestName.set(
|
|
1757
|
+
testName,
|
|
1758
|
+
(artifactRefCountsByTestName.get(testName) ?? 0) + 1
|
|
1759
|
+
);
|
|
1760
|
+
};
|
|
1761
|
+
var releaseArtifacts = (testName) => {
|
|
1762
|
+
const nextCount = (artifactRefCountsByTestName.get(testName) ?? 0) - 1;
|
|
1763
|
+
if (nextCount <= 0) {
|
|
1764
|
+
artifactRefCountsByTestName.delete(testName);
|
|
1765
|
+
resultArtifactsByTestName.delete(testName);
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
1768
|
+
artifactRefCountsByTestName.set(testName, nextCount);
|
|
1769
|
+
};
|
|
1770
|
+
var insertTopResult = (currentResults, nextResult, limit) => {
|
|
1771
|
+
if (limit <= 0) {
|
|
1772
|
+
return {
|
|
1773
|
+
results: [],
|
|
1774
|
+
added: false
|
|
1775
|
+
};
|
|
1776
|
+
}
|
|
1777
|
+
const nextResults = [...currentResults];
|
|
1778
|
+
const nextAmount = getResultAmount(nextResult);
|
|
1779
|
+
let insertIndex = nextResults.length;
|
|
1780
|
+
for (let index = 0; index < nextResults.length; index += 1) {
|
|
1781
|
+
if (nextAmount > getResultAmount(nextResults[index])) {
|
|
1782
|
+
insertIndex = index;
|
|
1783
|
+
break;
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
if (insertIndex === nextResults.length && nextResults.length >= limit) {
|
|
1787
|
+
return {
|
|
1788
|
+
results: currentResults,
|
|
1789
|
+
added: false
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1792
|
+
nextResults.splice(insertIndex, 0, nextResult);
|
|
1793
|
+
if (nextResults.length > limit) {
|
|
1794
|
+
nextResults.length = limit;
|
|
1795
|
+
}
|
|
1796
|
+
return {
|
|
1797
|
+
results: nextResults,
|
|
1798
|
+
added: nextResults.some(
|
|
1799
|
+
(candidate) => candidate.test.name === nextResult.test.name
|
|
1800
|
+
)
|
|
1801
|
+
};
|
|
1802
|
+
};
|
|
1803
|
+
var updateTopResults = (nextResults, nextResult, artifacts) => {
|
|
1804
|
+
const previousTopNames = topResultNames;
|
|
1805
|
+
const nextTopNames = new Set(nextResults.map((result) => result.test.name));
|
|
1806
|
+
if (artifacts && !previousTopNames.has(nextResult.test.name) && nextTopNames.has(nextResult.test.name)) {
|
|
1807
|
+
retainArtifacts(nextResult.test.name, artifacts);
|
|
1808
|
+
}
|
|
1809
|
+
for (const testName of previousTopNames) {
|
|
1810
|
+
if (!nextTopNames.has(testName)) {
|
|
1811
|
+
releaseArtifacts(testName);
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
results = nextResults;
|
|
1815
|
+
topResultNames = nextTopNames;
|
|
1816
|
+
};
|
|
1817
|
+
var updateBestTickerResult = (result, artifacts) => {
|
|
1818
|
+
if (!isGoodTest(result)) {
|
|
1819
|
+
return false;
|
|
1820
|
+
}
|
|
1821
|
+
const previousResult = resultsByTickers.get(result.test.symbol);
|
|
1822
|
+
if (previousResult && previousResult.stat.profit >= result.stat.profit) {
|
|
1823
|
+
return false;
|
|
1824
|
+
}
|
|
1825
|
+
const previousTestName = bestTickerTestNameBySymbol.get(result.test.symbol);
|
|
1826
|
+
if (previousTestName) {
|
|
1827
|
+
releaseArtifacts(previousTestName);
|
|
1828
|
+
}
|
|
1829
|
+
resultsByTickers.set(result.test.symbol, result);
|
|
1830
|
+
bestTickerTestNameBySymbol.set(result.test.symbol, result.test.name);
|
|
1831
|
+
if (artifacts) {
|
|
1832
|
+
retainArtifacts(result.test.name, artifacts);
|
|
1833
|
+
}
|
|
1834
|
+
return true;
|
|
1835
|
+
};
|
|
1716
1836
|
var isStrategyConfigGrid = (value) => {
|
|
1717
1837
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1718
1838
|
return false;
|
|
@@ -1735,13 +1855,39 @@ var resolveBacktestConnectorName = async (value) => {
|
|
|
1735
1855
|
};
|
|
1736
1856
|
var getLogsById = async (orderLogId) => {
|
|
1737
1857
|
const orderLog = await (0, import_redis.getData)(
|
|
1738
|
-
import_redis.redisKeys.cacheOrders(userName, orderLogId)
|
|
1858
|
+
import_redis.redisKeys.cacheOrders(userName, orderLogId),
|
|
1859
|
+
null
|
|
1739
1860
|
);
|
|
1740
1861
|
const positionLog = await (0, import_redis.getData)(
|
|
1741
|
-
import_redis.redisKeys.cachePositions(userName, orderLogId)
|
|
1862
|
+
import_redis.redisKeys.cachePositions(userName, orderLogId),
|
|
1863
|
+
null
|
|
1742
1864
|
);
|
|
1743
1865
|
return { orderLog, positionLog };
|
|
1744
1866
|
};
|
|
1867
|
+
var cacheArtifactsById = async (orderLogId, artifacts) => {
|
|
1868
|
+
if (!artifacts || !Array.isArray(artifacts.orderLog) || !Array.isArray(artifacts.positionLog)) {
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1871
|
+
await Promise.all([
|
|
1872
|
+
(0, import_redis.setData)(import_redis.redisKeys.cacheOrders(userName, orderLogId), artifacts.orderLog, {
|
|
1873
|
+
expire: import_constants.TTL_1D
|
|
1874
|
+
}),
|
|
1875
|
+
(0, import_redis.setData)(
|
|
1876
|
+
import_redis.redisKeys.cachePositions(userName, orderLogId),
|
|
1877
|
+
artifacts.positionLog,
|
|
1878
|
+
{
|
|
1879
|
+
expire: import_constants.TTL_1D
|
|
1880
|
+
}
|
|
1881
|
+
)
|
|
1882
|
+
]);
|
|
1883
|
+
};
|
|
1884
|
+
var resolveResultArtifacts = async (result) => {
|
|
1885
|
+
const inlineArtifacts = resultArtifactsByTestName.get(result.test.name);
|
|
1886
|
+
if (inlineArtifacts?.orderLog && inlineArtifacts?.positionLog) {
|
|
1887
|
+
return inlineArtifacts;
|
|
1888
|
+
}
|
|
1889
|
+
return getLogsById(result.orderLogId);
|
|
1890
|
+
};
|
|
1745
1891
|
var setTestData = async (test, stat, orderLog) => {
|
|
1746
1892
|
await (0, import_redis.setData)(
|
|
1747
1893
|
import_redis.redisKeys.testOrders(test.userName, test.strategyName, test.name),
|
|
@@ -1838,15 +1984,38 @@ var backtest = async () => {
|
|
|
1838
1984
|
strategyName,
|
|
1839
1985
|
typedBacktestConfig,
|
|
1840
1986
|
connectorName
|
|
1841
|
-
)
|
|
1987
|
+
);
|
|
1842
1988
|
const mlEnabled = Boolean(flags.ml);
|
|
1843
|
-
|
|
1989
|
+
const aiEnabled = Boolean(flags.ai);
|
|
1990
|
+
testSuite = testSuite.map((test) => ({
|
|
1991
|
+
...test,
|
|
1992
|
+
ml: mlEnabled,
|
|
1993
|
+
ai: aiEnabled,
|
|
1994
|
+
timeoutMs: testItemTimeoutMs
|
|
1995
|
+
})).slice(testsSkip, testsSkip + testsLimit);
|
|
1996
|
+
if (!testSuite.length) {
|
|
1997
|
+
console.log(
|
|
1998
|
+
import_chalk.default.yellow(
|
|
1999
|
+
`No tests selected (skip=${testsSkip}, limit=${testsLimit}).`
|
|
2000
|
+
)
|
|
2001
|
+
);
|
|
2002
|
+
return;
|
|
2003
|
+
}
|
|
1844
2004
|
const chunkSize = Math.ceil(testSuite.length / parseInt(flags.parallel));
|
|
1845
2005
|
const chunks = import_lodash.default.chunk(testSuite, chunkSize);
|
|
1846
|
-
let completedWorkers = 0;
|
|
1847
2006
|
let completedTests = 0;
|
|
1848
2007
|
let isFinishing = false;
|
|
1849
2008
|
const workers = /* @__PURE__ */ new Set();
|
|
2009
|
+
const maybeFinish = async () => {
|
|
2010
|
+
if (isFinishing) {
|
|
2011
|
+
return;
|
|
2012
|
+
}
|
|
2013
|
+
if (completedTests !== testSuite.length || workers.size > 0) {
|
|
2014
|
+
return;
|
|
2015
|
+
}
|
|
2016
|
+
isFinishing = true;
|
|
2017
|
+
await finish();
|
|
2018
|
+
};
|
|
1850
2019
|
const stopWorkers = () => {
|
|
1851
2020
|
for (const worker of workers) {
|
|
1852
2021
|
if (!worker.killed) {
|
|
@@ -1886,13 +2055,8 @@ var backtest = async () => {
|
|
|
1886
2055
|
workers.add(tester);
|
|
1887
2056
|
tester.on("message", async (msg) => {
|
|
1888
2057
|
if (msg.done) {
|
|
1889
|
-
completedWorkers++;
|
|
1890
2058
|
workers.delete(tester);
|
|
1891
|
-
|
|
1892
|
-
isFinishing = true;
|
|
1893
|
-
results = (0, import_backtest.sortBestTests)(results, flags.top);
|
|
1894
|
-
await finish();
|
|
1895
|
-
}
|
|
2059
|
+
await maybeFinish();
|
|
1896
2060
|
return;
|
|
1897
2061
|
}
|
|
1898
2062
|
completedTests++;
|
|
@@ -1910,20 +2074,25 @@ var backtest = async () => {
|
|
|
1910
2074
|
} else {
|
|
1911
2075
|
successTests++;
|
|
1912
2076
|
}
|
|
1913
|
-
|
|
1914
|
-
const
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
}
|
|
2077
|
+
const fullResult = msg;
|
|
2078
|
+
const result = stripInlineLogs(fullResult);
|
|
2079
|
+
const inlineArtifacts = getInlineArtifacts(fullResult);
|
|
2080
|
+
const nextTopResults = insertTopResult(results, result, flags.top);
|
|
2081
|
+
const shouldCacheTopArtifacts = nextTopResults.added;
|
|
2082
|
+
if (nextTopResults.added || results !== nextTopResults.results) {
|
|
2083
|
+
updateTopResults(nextTopResults.results, result, inlineArtifacts);
|
|
2084
|
+
}
|
|
2085
|
+
const updatedBestTickerResult = updateBestTickerResult(
|
|
2086
|
+
result,
|
|
2087
|
+
inlineArtifacts
|
|
2088
|
+
);
|
|
2089
|
+
if ((shouldCacheTopArtifacts || updatedBestTickerResult) && inlineArtifacts) {
|
|
2090
|
+
await cacheArtifactsById(result.orderLogId, inlineArtifacts);
|
|
2091
|
+
}
|
|
1921
2092
|
if (completedTests % progressStep === 0 || completedTests === testSuite.length) {
|
|
1922
|
-
|
|
1923
|
-
const
|
|
1924
|
-
|
|
1925
|
-
stat: { profit }
|
|
1926
|
-
} = results[0];
|
|
2093
|
+
const bestResult = results[0];
|
|
2094
|
+
const symbol = bestResult?.test.symbol || "-";
|
|
2095
|
+
const profit = bestResult?.stat.profit || 0;
|
|
1927
2096
|
const profitStr = `${(profit || 0).toFixed(2)}$`;
|
|
1928
2097
|
bar.tick(
|
|
1929
2098
|
completedTests === testSuite.length ? completedTests % progressStep : progressStep,
|
|
@@ -1944,6 +2113,7 @@ var backtest = async () => {
|
|
|
1944
2113
|
recordError({ error: `Worker exited with code ${code}` });
|
|
1945
2114
|
console.error(import_chalk.default.red(`Worker exited with code ${code}`));
|
|
1946
2115
|
}
|
|
2116
|
+
void maybeFinish();
|
|
1947
2117
|
});
|
|
1948
2118
|
await (0, import_redis.setData)(import_redis.redisKeys.cacheChunk(userName, chunkId), chunkWithId, {
|
|
1949
2119
|
expire: import_constants.TTL_1D
|
|
@@ -1954,10 +2124,18 @@ var backtest = async () => {
|
|
|
1954
2124
|
var saveAndPrintResults = async () => {
|
|
1955
2125
|
const colorizedResults = [];
|
|
1956
2126
|
for await (const result of results) {
|
|
1957
|
-
const { test
|
|
2127
|
+
const { test } = result;
|
|
1958
2128
|
const { symbol, name } = test;
|
|
1959
|
-
const { orderLog, positionLog } = await
|
|
2129
|
+
const { orderLog, positionLog } = await resolveResultArtifacts(result);
|
|
2130
|
+
if (!orderLog || !positionLog) {
|
|
2131
|
+
throw new Error(`Logs not found for test ${name}`);
|
|
2132
|
+
}
|
|
1960
2133
|
const stat = (0, import_backtest.calculateStatsFull)(positionLog);
|
|
2134
|
+
if (!stat && result.stat.orders > 0) {
|
|
2135
|
+
throw new Error(
|
|
2136
|
+
`Position log is empty for test ${name} despite ${result.stat.orders} closed orders`
|
|
2137
|
+
);
|
|
2138
|
+
}
|
|
1961
2139
|
await setTestData(test, stat, orderLog);
|
|
1962
2140
|
const statRow = [
|
|
1963
2141
|
import_chalk.default.blue(name),
|
|
@@ -1980,10 +2158,18 @@ var saveAndPrintResults = async () => {
|
|
|
1980
2158
|
var saveAndPrintResultsByTickers = async () => {
|
|
1981
2159
|
const colorizedResultsByTickers = [];
|
|
1982
2160
|
for await (const result of resultsByTickers.values()) {
|
|
1983
|
-
const { test
|
|
2161
|
+
const { test } = result;
|
|
1984
2162
|
const { symbol, name } = test;
|
|
1985
|
-
const { orderLog, positionLog } = await
|
|
2163
|
+
const { orderLog, positionLog } = await resolveResultArtifacts(result);
|
|
2164
|
+
if (!orderLog || !positionLog) {
|
|
2165
|
+
throw new Error(`Logs not found for ticker result ${name}`);
|
|
2166
|
+
}
|
|
1986
2167
|
const stat = (0, import_backtest.calculateStatsFull)(positionLog);
|
|
2168
|
+
if (!stat && result.stat.orders > 0) {
|
|
2169
|
+
throw new Error(
|
|
2170
|
+
`Position log is empty for ticker result ${name} despite ${result.stat.orders} closed orders`
|
|
2171
|
+
);
|
|
2172
|
+
}
|
|
1987
2173
|
await setTestData(test, stat, orderLog);
|
|
1988
2174
|
const statRow = [
|
|
1989
2175
|
import_chalk.default.blue(name),
|
|
@@ -2038,7 +2224,7 @@ var finish = async () => {
|
|
|
2038
2224
|
finishedAt: finishedAt.toISOString(),
|
|
2039
2225
|
durationSeconds,
|
|
2040
2226
|
results,
|
|
2041
|
-
resultsByTickers: resultsByTickers.values(),
|
|
2227
|
+
resultsByTickers: Array.from(resultsByTickers.values()),
|
|
2042
2228
|
bestConfig,
|
|
2043
2229
|
mergedConfig,
|
|
2044
2230
|
successTests,
|
|
@@ -28,6 +28,7 @@ var import_args = __toESM(require("args"));
|
|
|
28
28
|
var import_chalk = __toESM(require("chalk"));
|
|
29
29
|
var import_indicators = require("@tradejs/core/indicators");
|
|
30
30
|
var import_timescale = require("@tradejs/infra/timescale");
|
|
31
|
+
var import_userSettings = require("@tradejs/infra/userSettings");
|
|
31
32
|
var import_connectors = require("@tradejs/connectors");
|
|
32
33
|
import_args.default.example(
|
|
33
34
|
"yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m,1h --days 120",
|
|
@@ -41,6 +42,7 @@ import_args.default.option(
|
|
|
41
42
|
"Provider name: coinalyze | binance_coinbase_spread",
|
|
42
43
|
"coinalyze"
|
|
43
44
|
);
|
|
45
|
+
import_args.default.option(["U", "user"], "User settings profile name from Redis", "root");
|
|
44
46
|
import_args.default.option(["b", "batchDays"], "Request chunk size in days", 7);
|
|
45
47
|
var flags = import_args.default.parse(process.argv);
|
|
46
48
|
var asInt = (value, fallback) => {
|
|
@@ -59,6 +61,8 @@ var run = async () => {
|
|
|
59
61
|
);
|
|
60
62
|
const days = asInt(flags.days, 120);
|
|
61
63
|
const batchDays = asInt(flags.batchDays, 7);
|
|
64
|
+
const userName = String(flags.user || "root").trim() || "root";
|
|
65
|
+
const coinalyzeApiKey = providerName === "coinalyze" ? (await (0, import_userSettings.getUserSettings)(userName)).COINALYZE_API_KEY : void 0;
|
|
62
66
|
if (!symbols.length) throw new Error("No symbols provided");
|
|
63
67
|
if (!intervals.length) throw new Error("No intervals provided");
|
|
64
68
|
await (0, import_timescale.waitForDbReady)();
|
|
@@ -76,6 +80,7 @@ var run = async () => {
|
|
|
76
80
|
);
|
|
77
81
|
const window = await provider.fetchWindow({
|
|
78
82
|
symbol,
|
|
83
|
+
apiKey: coinalyzeApiKey,
|
|
79
84
|
interval,
|
|
80
85
|
fromMs: cursor,
|
|
81
86
|
toMs
|
|
@@ -31,6 +31,7 @@ var import_lodash = __toESM(require("lodash"));
|
|
|
31
31
|
var import_connectors = require("@tradejs/connectors");
|
|
32
32
|
var import_indicators = require("@tradejs/core/indicators");
|
|
33
33
|
var import_async = require("@tradejs/core/async");
|
|
34
|
+
var import_userSettings = require("@tradejs/infra/userSettings");
|
|
34
35
|
var import_cli = require("@tradejs/node/cli");
|
|
35
36
|
var import_timescale = require("@tradejs/infra/timescale");
|
|
36
37
|
var coinalyzeIntervalMap = {
|
|
@@ -41,7 +42,7 @@ import_args.default.example(
|
|
|
41
42
|
"yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m,1h",
|
|
42
43
|
"Fetch derivatives for all getTickers symbols matched to Coinalyze markets"
|
|
43
44
|
);
|
|
44
|
-
import_args.default.option(["U", "user"], "
|
|
45
|
+
import_args.default.option(["U", "user"], "User settings profile name from Redis", "root");
|
|
45
46
|
import_args.default.option(["t", "tickers"], "Comma-separated include symbols");
|
|
46
47
|
import_args.default.option(["e", "exclude"], "Comma-separated exclude symbols");
|
|
47
48
|
import_args.default.option(["l", "tickersLimit"], "Tickers limit");
|
|
@@ -76,14 +77,10 @@ var asInt = (value, fallback) => {
|
|
|
76
77
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
77
78
|
};
|
|
78
79
|
var parseList = (value) => String(value ?? "").split(",").map((item) => item.trim().toUpperCase()).filter(Boolean);
|
|
79
|
-
var apiKey = process.env.COINALYZE_API_KEY?.trim();
|
|
80
|
-
if (!apiKey) {
|
|
81
|
-
throw new Error("Missing COINALYZE_API_KEY");
|
|
82
|
-
}
|
|
83
80
|
var coinalyzeBaseUrl = process.env.COINALYZE_BASE_URL?.trim() || "https://api.coinalyze.net/v1";
|
|
84
81
|
var coinalyzeMaxRetries = asInt(process.env.COINALYZE_MAX_RETRIES, 4);
|
|
85
82
|
var lastRequestTs = 0;
|
|
86
|
-
var fetchJsonWithRateLimit = async (url) => {
|
|
83
|
+
var fetchJsonWithRateLimit = async (url, apiKey) => {
|
|
87
84
|
const requestDelayMs = Math.max(100, asInt(flags.requestDelayMs, 200));
|
|
88
85
|
const requestTimeoutMs = Math.max(
|
|
89
86
|
5e3,
|
|
@@ -135,9 +132,10 @@ var fetchJsonWithRateLimit = async (url) => {
|
|
|
135
132
|
}
|
|
136
133
|
throw new Error("Coinalyze request failed after retries");
|
|
137
134
|
};
|
|
138
|
-
var fetchCoinalyzeMarkets = async () => {
|
|
135
|
+
var fetchCoinalyzeMarkets = async (apiKey) => {
|
|
139
136
|
const raw = await fetchJsonWithRateLimit(
|
|
140
|
-
`${coinalyzeBaseUrl}/future-markets
|
|
137
|
+
`${coinalyzeBaseUrl}/future-markets`,
|
|
138
|
+
apiKey
|
|
141
139
|
);
|
|
142
140
|
return Array.isArray(raw) ? raw : [];
|
|
143
141
|
};
|
|
@@ -220,16 +218,17 @@ var toSeriesMap = (raw, metric) => {
|
|
|
220
218
|
return out;
|
|
221
219
|
};
|
|
222
220
|
var fetchMetricBatch = async (params) => {
|
|
223
|
-
const { endpoint, metric, marketSymbols, interval, fromMs, toMs } = params;
|
|
221
|
+
const { endpoint, metric, marketSymbols, apiKey, interval, fromMs, toMs } = params;
|
|
224
222
|
const url = new URL(`${coinalyzeBaseUrl}${endpoint}`);
|
|
225
223
|
url.searchParams.set("symbols", marketSymbols.join(","));
|
|
226
224
|
url.searchParams.set("interval", coinalyzeIntervalMap[interval] || interval);
|
|
227
225
|
url.searchParams.set("from", String(Math.floor(fromMs / 1e3)));
|
|
228
226
|
url.searchParams.set("to", String(Math.floor(toMs / 1e3)));
|
|
229
|
-
const raw = await fetchJsonWithRateLimit(url.toString());
|
|
227
|
+
const raw = await fetchJsonWithRateLimit(url.toString(), apiKey);
|
|
230
228
|
return toSeriesMap(raw, metric);
|
|
231
229
|
};
|
|
232
230
|
var run = async () => {
|
|
231
|
+
const userName = String(flags.user || "root").trim() || "root";
|
|
233
232
|
const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
|
|
234
233
|
flags.intervals
|
|
235
234
|
);
|
|
@@ -239,9 +238,16 @@ var run = async () => {
|
|
|
239
238
|
const tickersLimit = flags.tickersLimit !== void 0 ? asInt(flags.tickersLimit, 0) || void 0 : void 0;
|
|
240
239
|
const exchangePriority = parseList(flags.exchangePriority || "A,6,0");
|
|
241
240
|
if (!intervals.length) throw new Error("No intervals provided");
|
|
241
|
+
const settings = await (0, import_userSettings.getUserSettings)(userName);
|
|
242
|
+
const coinalyzeApiKey = settings.COINALYZE_API_KEY.trim();
|
|
243
|
+
if (!coinalyzeApiKey) {
|
|
244
|
+
throw new Error(
|
|
245
|
+
`Missing COINALYZE_API_KEY in user settings for user ${userName}`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
242
248
|
const connectorFactory = import_connectors.connectors[import_connectors.ConnectorNames.ByBit];
|
|
243
249
|
const bybit = await connectorFactory({
|
|
244
|
-
userName
|
|
250
|
+
userName
|
|
245
251
|
});
|
|
246
252
|
const tickers = await (0, import_cli.getTickers)(
|
|
247
253
|
bybit,
|
|
@@ -253,7 +259,7 @@ var run = async () => {
|
|
|
253
259
|
if (!tickers.length) {
|
|
254
260
|
throw new Error("No tickers loaded via getTickers");
|
|
255
261
|
}
|
|
256
|
-
const markets = await fetchCoinalyzeMarkets();
|
|
262
|
+
const markets = await fetchCoinalyzeMarkets(coinalyzeApiKey);
|
|
257
263
|
if (!markets.length) {
|
|
258
264
|
throw new Error("No markets returned by Coinalyze /future-markets");
|
|
259
265
|
}
|
|
@@ -321,6 +327,7 @@ var run = async () => {
|
|
|
321
327
|
endpoint: oiPath,
|
|
322
328
|
metric: "oi",
|
|
323
329
|
marketSymbols,
|
|
330
|
+
apiKey: coinalyzeApiKey,
|
|
324
331
|
interval,
|
|
325
332
|
fromMs: cursor,
|
|
326
333
|
toMs
|
|
@@ -329,6 +336,7 @@ var run = async () => {
|
|
|
329
336
|
endpoint: fundingPath,
|
|
330
337
|
metric: "funding",
|
|
331
338
|
marketSymbols,
|
|
339
|
+
apiKey: coinalyzeApiKey,
|
|
332
340
|
interval,
|
|
333
341
|
fromMs: cursor,
|
|
334
342
|
toMs
|
|
@@ -337,6 +345,7 @@ var run = async () => {
|
|
|
337
345
|
endpoint: liqPath,
|
|
338
346
|
metric: "liq",
|
|
339
347
|
marketSymbols,
|
|
348
|
+
apiKey: coinalyzeApiKey,
|
|
340
349
|
interval,
|
|
341
350
|
fromMs: cursor,
|
|
342
351
|
toMs
|
package/dist/scripts/mlExport.js
CHANGED
|
@@ -24,10 +24,106 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
|
|
25
25
|
// src/scripts/mlExport.ts
|
|
26
26
|
var import_args = __toESM(require("args"));
|
|
27
|
-
var
|
|
27
|
+
var import_chalk2 = __toESM(require("chalk"));
|
|
28
28
|
var import_promises = __toESM(require("fs/promises"));
|
|
29
29
|
var import_path = __toESM(require("path"));
|
|
30
30
|
var import_ml = require("@tradejs/infra/ml");
|
|
31
|
+
|
|
32
|
+
// src/scripts/selectStrategy.ts
|
|
33
|
+
var import_readline = __toESM(require("readline"));
|
|
34
|
+
var import_chalk = __toESM(require("chalk"));
|
|
35
|
+
var import_strategies = require("@tradejs/node/strategies");
|
|
36
|
+
var defaultStrategy = "TrendLine";
|
|
37
|
+
var projectRoot = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
|
|
38
|
+
var getStrategyChoices = async () => {
|
|
39
|
+
try {
|
|
40
|
+
const loaded = await (0, import_strategies.getAvailableStrategyNames)(projectRoot);
|
|
41
|
+
if (loaded.length) {
|
|
42
|
+
return loaded;
|
|
43
|
+
}
|
|
44
|
+
} catch (error) {
|
|
45
|
+
console.warn(`Failed to load strategy list: ${String(error)}`);
|
|
46
|
+
}
|
|
47
|
+
return [
|
|
48
|
+
"Breakout",
|
|
49
|
+
"MaStrategy",
|
|
50
|
+
"AdaptiveMomentumRibbon",
|
|
51
|
+
"TrendLine",
|
|
52
|
+
"VolumeDivergence"
|
|
53
|
+
];
|
|
54
|
+
};
|
|
55
|
+
var selectStrategy = async (promptLabel = "Select strategy", options) => {
|
|
56
|
+
const strategies = options?.strategies && options.strategies.length ? options.strategies : await getStrategyChoices();
|
|
57
|
+
const preferredDefault = options?.defaultStrategy || defaultStrategy;
|
|
58
|
+
const fallbackStrategy = strategies.includes(preferredDefault) ? preferredDefault : strategies[0];
|
|
59
|
+
if (!process.stdin.isTTY) {
|
|
60
|
+
return fallbackStrategy;
|
|
61
|
+
}
|
|
62
|
+
console.log(import_chalk.default.cyan("Available strategies:"));
|
|
63
|
+
strategies.forEach((name, index) => {
|
|
64
|
+
const isDefault = name === defaultStrategy;
|
|
65
|
+
const label = isDefault ? import_chalk.default.green(name) : name;
|
|
66
|
+
const suffix = isDefault ? import_chalk.default.gray(" (default)") : "";
|
|
67
|
+
console.log(` ${import_chalk.default.yellow(String(index + 1))}) ${label}${suffix}`);
|
|
68
|
+
});
|
|
69
|
+
const rl = import_readline.default.createInterface({
|
|
70
|
+
input: process.stdin,
|
|
71
|
+
output: process.stdout
|
|
72
|
+
});
|
|
73
|
+
const question = (text) => new Promise((resolve) => rl.question(text, resolve));
|
|
74
|
+
const answer = await question(
|
|
75
|
+
`${promptLabel} [${import_chalk.default.green(fallbackStrategy)}]: `
|
|
76
|
+
);
|
|
77
|
+
rl.close();
|
|
78
|
+
const trimmed = answer.trim();
|
|
79
|
+
if (!trimmed) {
|
|
80
|
+
return fallbackStrategy;
|
|
81
|
+
}
|
|
82
|
+
const asNumber = Number(trimmed);
|
|
83
|
+
if (Number.isFinite(asNumber) && asNumber >= 1 && asNumber <= strategies.length) {
|
|
84
|
+
return strategies[asNumber - 1];
|
|
85
|
+
}
|
|
86
|
+
const byName = strategies.find(
|
|
87
|
+
(name) => name.toLowerCase() === trimmed.toLowerCase()
|
|
88
|
+
);
|
|
89
|
+
if (byName) {
|
|
90
|
+
return byName;
|
|
91
|
+
}
|
|
92
|
+
console.warn(`Unknown strategy "${trimmed}", using ${fallbackStrategy}.`);
|
|
93
|
+
return fallbackStrategy;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
// src/scripts/resolveExportStrategy.ts
|
|
97
|
+
var resolveExportStrategy = async ({
|
|
98
|
+
explicitStrategy,
|
|
99
|
+
outDir,
|
|
100
|
+
datasetLabel,
|
|
101
|
+
promptLabel,
|
|
102
|
+
listStrategies
|
|
103
|
+
}) => {
|
|
104
|
+
const raw = String(explicitStrategy || "").trim();
|
|
105
|
+
if (raw) {
|
|
106
|
+
return raw;
|
|
107
|
+
}
|
|
108
|
+
const availableStrategies = await listStrategies({ outDir });
|
|
109
|
+
if (!availableStrategies.length) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
if (availableStrategies.length === 1) {
|
|
113
|
+
return availableStrategies[0];
|
|
114
|
+
}
|
|
115
|
+
if (!process.stdin.isTTY) {
|
|
116
|
+
throw new Error(
|
|
117
|
+
`Multiple ${datasetLabel} chunk strategies found in ${outDir}: ${availableStrategies.join(", ")}. Pass --strategy.`
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
return selectStrategy(promptLabel, {
|
|
121
|
+
strategies: availableStrategies,
|
|
122
|
+
defaultStrategy: availableStrategies[0]
|
|
123
|
+
});
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
// src/scripts/mlExport.ts
|
|
31
127
|
import_args.default.example(
|
|
32
128
|
"yarn ts-node ./src/scripts/mlExport --strategy trendline",
|
|
33
129
|
"Merge per-worker ML chunks into one dataset JSONL"
|
|
@@ -45,21 +141,24 @@ var listChunkFiles = async (outDir, strategyName) => {
|
|
|
45
141
|
const entries = await import_promises.default.readdir(outDir, { withFileTypes: true });
|
|
46
142
|
return entries.filter((entry) => entry.isFile()).map((entry) => entry.name).filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => import_path.default.join(outDir, name)).sort();
|
|
47
143
|
};
|
|
48
|
-
var resolveStrategyName = async () => {
|
|
49
|
-
const raw = String(flags.strategy || "").trim();
|
|
50
|
-
if (raw) {
|
|
51
|
-
return raw;
|
|
52
|
-
}
|
|
53
|
-
throw new Error("Missing --strategy. Use mlExportSelect or pass --strategy.");
|
|
54
|
-
};
|
|
55
144
|
var main = async () => {
|
|
56
145
|
const outDir = String(flags.outDir || "data/ml/export");
|
|
57
|
-
const strategyName = await resolveStrategyName();
|
|
58
146
|
await import_promises.default.mkdir(outDir, { recursive: true });
|
|
147
|
+
const strategyName = await resolveExportStrategy({
|
|
148
|
+
explicitStrategy: String(flags.strategy || ""),
|
|
149
|
+
outDir,
|
|
150
|
+
datasetLabel: "ML",
|
|
151
|
+
promptLabel: "Select ML export strategy",
|
|
152
|
+
listStrategies: import_ml.listMlChunkStrategies
|
|
153
|
+
});
|
|
154
|
+
if (!strategyName) {
|
|
155
|
+
console.log(import_chalk2.default.yellow(`No ML chunk files found in ${outDir}`));
|
|
156
|
+
process.exit(0);
|
|
157
|
+
}
|
|
59
158
|
const chunkFiles = await listChunkFiles(outDir, strategyName);
|
|
60
159
|
if (!chunkFiles.length) {
|
|
61
160
|
console.log(
|
|
62
|
-
|
|
161
|
+
import_chalk2.default.yellow(
|
|
63
162
|
`No chunk files found for strategy "${strategyName}" in ${outDir}`
|
|
64
163
|
)
|
|
65
164
|
);
|
|
@@ -79,9 +178,9 @@ var main = async () => {
|
|
|
79
178
|
await import_promises.default.rm(filePath, { force: true });
|
|
80
179
|
}
|
|
81
180
|
}
|
|
82
|
-
console.log(
|
|
181
|
+
console.log(import_chalk2.default.green(`Merged dataset saved: ${mergedPath}`));
|
|
83
182
|
console.log(
|
|
84
|
-
|
|
183
|
+
import_chalk2.default.gray(
|
|
85
184
|
`strategy=${strategyName}, source_chunks=${chunkFiles.length}, deleteChunks=${Boolean(
|
|
86
185
|
shouldDeleteChunks
|
|
87
186
|
)}`
|
|
@@ -90,6 +189,6 @@ var main = async () => {
|
|
|
90
189
|
process.exit(0);
|
|
91
190
|
};
|
|
92
191
|
main().catch((error) => {
|
|
93
|
-
console.error(
|
|
192
|
+
console.error(import_chalk2.default.red(error?.message || String(error)));
|
|
94
193
|
process.exit(1);
|
|
95
194
|
});
|