@yuants/app-virtual-exchange 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -11,19 +11,17 @@ const terminal = Terminal.fromNodeEnv();
|
|
|
11
11
|
const listBackwardSeriesIds = async () => {
|
|
12
12
|
const product_ids = await requestSQL(terminal, `select product_id from product`);
|
|
13
13
|
console.time('[SeriesCollector][InterestRate][Forwards] calc');
|
|
14
|
-
const series_ids = new
|
|
14
|
+
const series_ids = new Map();
|
|
15
15
|
for (const terminalInfo of terminal.terminalInfos) {
|
|
16
16
|
for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
|
|
17
17
|
if (serviceInfo.method !== 'IngestInterestRate')
|
|
18
18
|
continue;
|
|
19
19
|
try {
|
|
20
20
|
const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);
|
|
21
|
-
if (meta.direction !== 'forward')
|
|
22
|
-
continue;
|
|
23
21
|
for (const { product_id } of product_ids) {
|
|
24
22
|
if (!product_id.startsWith(meta.product_id_prefix))
|
|
25
23
|
continue;
|
|
26
|
-
series_ids.
|
|
24
|
+
series_ids.set(product_id, meta);
|
|
27
25
|
}
|
|
28
26
|
}
|
|
29
27
|
finally {
|
|
@@ -34,10 +32,9 @@ const listBackwardSeriesIds = async () => {
|
|
|
34
32
|
return series_ids;
|
|
35
33
|
};
|
|
36
34
|
defer(async () => {
|
|
37
|
-
const time = Date.now();
|
|
38
35
|
const series_ids = await listBackwardSeriesIds();
|
|
39
|
-
console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for.
|
|
40
|
-
await Promise.all([...series_ids].map(async (product_id) => {
|
|
36
|
+
console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `);
|
|
37
|
+
await Promise.all([...series_ids].map(async ([product_id, meta]) => {
|
|
41
38
|
var _a, _b, _c, _d;
|
|
42
39
|
try {
|
|
43
40
|
const [datasource_id] = decodePath(product_id);
|
|
@@ -47,20 +44,31 @@ defer(async () => {
|
|
|
47
44
|
capacity: 1,
|
|
48
45
|
}).acquire();
|
|
49
46
|
{
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
47
|
+
let req;
|
|
48
|
+
if (meta.direction === 'forward') {
|
|
49
|
+
const [record] = await requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`);
|
|
50
|
+
const time = record ? new Date(record.end_time).getTime() : 0;
|
|
51
|
+
req = {
|
|
52
|
+
product_id: product_id,
|
|
53
|
+
direction: 'forward',
|
|
54
|
+
time,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
// backward
|
|
59
|
+
req = {
|
|
60
|
+
product_id,
|
|
61
|
+
direction: 'backward',
|
|
62
|
+
time: Date.now(),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
console.info(formatTime(Date.now()), 'DispatchIngestInterestRateRequestForwardTask', `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`);
|
|
58
66
|
const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
|
|
59
67
|
terminal.metrics
|
|
60
68
|
.counter('series_collector_forwards_ingest_count', '')
|
|
61
69
|
.labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
|
|
62
70
|
.inc(res.wrote_count || 0);
|
|
63
|
-
console.info(formatTime(Date.now()), '
|
|
71
|
+
console.info(formatTime(Date.now()), 'DispatchIngestInterestRateRequestForwardTaskResult', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
64
72
|
}
|
|
65
73
|
}
|
|
66
74
|
catch (e) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;AAErC,OAAO,
|
|
1
|
+
{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;AAErC,OAAO,EAIL,0CAA0C,GAC3C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,UAAU,CAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACnE,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,0CAA0C,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAE5E,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;iBAClC;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,KAAK,CAAC,KAAK,IAAI,EAAE;IACf,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,mDAAmD,UAAU,CAAC,IAAI,wCAAwC,CAC3G,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE;;QAC/C,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,EAAE;gBAC3D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,IAAI,GAA+B,CAAC;gBACpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;oBAChC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,4DAA4D,SAAS,CACnE,UAAU,CACX,kEAAkE,CACpE,CAAC;oBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAE9D,GAAG,GAAG;wBACJ,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,SAAS;wBACpB,IAAI;qBACL,CAAC;iBACH;qBAAM;oBACL,WAAW;oBACX,GAAG,GAAG;wBACJ,UAAU;wBACV,SAAS,EAAE,UAAU;wBACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;qBACjB,CAAC;iBACH;gBAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,8CAA8C,EAC9C,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,oDAAoD,EACpD,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,yCAAyC,EACzC,aAAa,UAAU,EAAE,EACzB,CAAC,CACF,CAAC;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n IIngestInterestRateRequest,\n IInterestRateServiceMetadata,\n ISeriesIngestResult,\n parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n console.time('[SeriesCollector][InterestRate][Forwards] calc');\n\n const series_ids = new Map<string, IInterestRateServiceMetadata>();\n for (const terminalInfo of terminal.terminalInfos) {\n for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n if (serviceInfo.method !== 'IngestInterestRate') continue;\n try {\n const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n\n for (const { product_id } of product_ids) {\n if (!product_id.startsWith(meta.product_id_prefix)) continue;\n series_ids.set(product_id, meta);\n }\n } finally {\n }\n }\n }\n\n console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');\n\n return series_ids;\n};\n\ndefer(async () => {\n const series_ids = await listBackwardSeriesIds();\n console.log(\n `[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `,\n );\n\n await Promise.all(\n [...series_ids].map(async ([product_id, meta]) => {\n try {\n const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`interest_rate_forwards:${datasource_id}`, {\n refillInterval: 1000,\n capacity: 1,\n }).acquire();\n {\n let req: IIngestInterestRateRequest;\n if (meta.direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n product_id,\n )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n );\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id: product_id,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n direction: 'backward',\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n 'DispatchIngestInterestRateRequestForwardTask',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n const res = await terminal.client.requestForResponseData<\n IIngestInterestRateRequest,\n ISeriesIngestResult\n >('IngestInterestRate', req);\n\n terminal.metrics\n .counter('series_collector_forwards_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n .inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n 'DispatchIngestInterestRateRequestForwardTaskResult',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n }\n } catch (e) {\n console.info(\n formatTime(Date.now()),\n 'DispatchIngestInterestRateErrorForwards',\n `series_id=${product_id}`,\n e,\n );\n }\n }),\n );\n})\n .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n .subscribe();\n"]}
|
|
@@ -13,19 +13,17 @@ const terminal = protocol_1.Terminal.fromNodeEnv();
|
|
|
13
13
|
const listBackwardSeriesIds = async () => {
|
|
14
14
|
const product_ids = await (0, sql_1.requestSQL)(terminal, `select product_id from product`);
|
|
15
15
|
console.time('[SeriesCollector][InterestRate][Forwards] calc');
|
|
16
|
-
const series_ids = new
|
|
16
|
+
const series_ids = new Map();
|
|
17
17
|
for (const terminalInfo of terminal.terminalInfos) {
|
|
18
18
|
for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
|
|
19
19
|
if (serviceInfo.method !== 'IngestInterestRate')
|
|
20
20
|
continue;
|
|
21
21
|
try {
|
|
22
22
|
const meta = (0, exchange_1.parseInterestRateServiceMetadataFromSchema)(serviceInfo.schema);
|
|
23
|
-
if (meta.direction !== 'forward')
|
|
24
|
-
continue;
|
|
25
23
|
for (const { product_id } of product_ids) {
|
|
26
24
|
if (!product_id.startsWith(meta.product_id_prefix))
|
|
27
25
|
continue;
|
|
28
|
-
series_ids.
|
|
26
|
+
series_ids.set(product_id, meta);
|
|
29
27
|
}
|
|
30
28
|
}
|
|
31
29
|
finally {
|
|
@@ -36,10 +34,9 @@ const listBackwardSeriesIds = async () => {
|
|
|
36
34
|
return series_ids;
|
|
37
35
|
};
|
|
38
36
|
(0, rxjs_1.defer)(async () => {
|
|
39
|
-
const time = Date.now();
|
|
40
37
|
const series_ids = await listBackwardSeriesIds();
|
|
41
|
-
console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for.
|
|
42
|
-
await Promise.all([...series_ids].map(async (product_id) => {
|
|
38
|
+
console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `);
|
|
39
|
+
await Promise.all([...series_ids].map(async ([product_id, meta]) => {
|
|
43
40
|
var _a, _b, _c, _d;
|
|
44
41
|
try {
|
|
45
42
|
const [datasource_id] = (0, utils_1.decodePath)(product_id);
|
|
@@ -49,20 +46,31 @@ const listBackwardSeriesIds = async () => {
|
|
|
49
46
|
capacity: 1,
|
|
50
47
|
}).acquire();
|
|
51
48
|
{
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
49
|
+
let req;
|
|
50
|
+
if (meta.direction === 'forward') {
|
|
51
|
+
const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`);
|
|
52
|
+
const time = record ? new Date(record.end_time).getTime() : 0;
|
|
53
|
+
req = {
|
|
54
|
+
product_id: product_id,
|
|
55
|
+
direction: 'forward',
|
|
56
|
+
time,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
// backward
|
|
61
|
+
req = {
|
|
62
|
+
product_id,
|
|
63
|
+
direction: 'backward',
|
|
64
|
+
time: Date.now(),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateRequestForwardTask', `product_id=${req.product_id}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
|
|
60
68
|
const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
|
|
61
69
|
terminal.metrics
|
|
62
70
|
.counter('series_collector_forwards_ingest_count', '')
|
|
63
71
|
.labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
|
|
64
72
|
.inc(res.wrote_count || 0);
|
|
65
|
-
console.info((0, utils_1.formatTime)(Date.now()), '
|
|
73
|
+
console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateRequestForwardTaskResult', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
66
74
|
}
|
|
67
75
|
}
|
|
68
76
|
catch (e) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;;AAErC,+
|
|
1
|
+
{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;;AAErC,+CAK0B;AAC1B,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AACpE,+BAA4C;AAE5C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,IAAA,gBAAU,EAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACnE,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,IAAA,qDAA0C,EAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAE5E,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;iBAClC;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;IACf,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,mDAAmD,UAAU,CAAC,IAAI,wCAAwC,CAC3G,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE;;QAC/C,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,EAAE;gBAC3D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,IAAI,GAA+B,CAAC;gBACpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;oBAChC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,UAAU,CACX,kEAAkE,CACpE,CAAC;oBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAE9D,GAAG,GAAG;wBACJ,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,SAAS;wBACpB,IAAI;qBACL,CAAC;iBACH;qBAAM;oBACL,WAAW;oBACX,GAAG,GAAG;wBACJ,UAAU;wBACV,SAAS,EAAE,UAAU;wBACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;qBACjB,CAAC;iBACH;gBAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,8CAA8C,EAC9C,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,oDAAoD,EACpD,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,yCAAyC,EACzC,aAAa,UAAU,EAAE,EACzB,CAAC,CACF,CAAC;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n IIngestInterestRateRequest,\n IInterestRateServiceMetadata,\n ISeriesIngestResult,\n parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n console.time('[SeriesCollector][InterestRate][Forwards] calc');\n\n const series_ids = new Map<string, IInterestRateServiceMetadata>();\n for (const terminalInfo of terminal.terminalInfos) {\n for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n if (serviceInfo.method !== 'IngestInterestRate') continue;\n try {\n const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n\n for (const { product_id } of product_ids) {\n if (!product_id.startsWith(meta.product_id_prefix)) continue;\n series_ids.set(product_id, meta);\n }\n } finally {\n }\n }\n }\n\n console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');\n\n return series_ids;\n};\n\ndefer(async () => {\n const series_ids = await listBackwardSeriesIds();\n console.log(\n `[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `,\n );\n\n await Promise.all(\n [...series_ids].map(async ([product_id, meta]) => {\n try {\n const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`interest_rate_forwards:${datasource_id}`, {\n refillInterval: 1000,\n capacity: 1,\n }).acquire();\n {\n let req: IIngestInterestRateRequest;\n if (meta.direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n product_id,\n )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n );\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id: product_id,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n direction: 'backward',\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n 'DispatchIngestInterestRateRequestForwardTask',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n const res = await terminal.client.requestForResponseData<\n IIngestInterestRateRequest,\n ISeriesIngestResult\n >('IngestInterestRate', req);\n\n terminal.metrics\n .counter('series_collector_forwards_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n .inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n 'DispatchIngestInterestRateRequestForwardTaskResult',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n }\n } catch (e) {\n console.info(\n formatTime(Date.now()),\n 'DispatchIngestInterestRateErrorForwards',\n `series_id=${product_id}`,\n e,\n );\n }\n }),\n );\n})\n .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n .subscribe();\n"]}
|