@yuants/app-virtual-exchange 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/series-collector/backwards-interest-rate.js +4 -4
- package/dist/series-collector/backwards-interest-rate.js.map +1 -1
- package/dist/series-collector/backwards-ohlc.js +33 -68
- package/dist/series-collector/backwards-ohlc.js.map +1 -1
- package/dist/series-collector/discovery.js +30 -2
- package/dist/series-collector/discovery.js.map +1 -1
- package/dist/series-collector/forwards-interest-rate.js +5 -5
- package/dist/series-collector/forwards-interest-rate.js.map +1 -1
- package/dist/series-collector/forwards-ohlc.js +33 -68
- package/dist/series-collector/forwards-ohlc.js.map +1 -1
- package/dist/series-collector/index.js +1 -2
- package/dist/series-collector/index.js.map +1 -1
- package/dist/series-collector/interest-rate.js +11 -5
- package/dist/series-collector/interest-rate.js.map +1 -1
- package/dist/series-collector/ohlc.js +49 -0
- package/dist/series-collector/ohlc.js.map +1 -0
- package/dist/series-collector/patch-interest-rate.js +4 -0
- package/dist/series-collector/patch-interest-rate.js.map +1 -1
- package/dist/series-collector/patch-ohlc.js +67 -0
- package/dist/series-collector/patch-ohlc.js.map +1 -0
- package/lib/index.d.ts +0 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +0 -2
- package/lib/index.js.map +1 -1
- package/lib/series-collector/backwards-interest-rate.d.ts.map +1 -1
- package/lib/series-collector/backwards-interest-rate.js +4 -4
- package/lib/series-collector/backwards-interest-rate.js.map +1 -1
- package/lib/series-collector/backwards-ohlc.d.ts +2 -1
- package/lib/series-collector/backwards-ohlc.d.ts.map +1 -1
- package/lib/series-collector/backwards-ohlc.js +34 -67
- package/lib/series-collector/backwards-ohlc.js.map +1 -1
- package/lib/series-collector/discovery.d.ts +5 -0
- package/lib/series-collector/discovery.d.ts.map +1 -1
- package/lib/series-collector/discovery.js +31 -2
- package/lib/series-collector/discovery.js.map +1 -1
- package/lib/series-collector/forwards-interest-rate.d.ts.map +1 -1
- package/lib/series-collector/forwards-interest-rate.js +5 -5
- package/lib/series-collector/forwards-interest-rate.js.map +1 -1
- package/lib/series-collector/forwards-ohlc.d.ts +2 -1
- package/lib/series-collector/forwards-ohlc.d.ts.map +1 -1
- package/lib/series-collector/forwards-ohlc.js +34 -67
- package/lib/series-collector/forwards-ohlc.js.map +1 -1
- package/lib/series-collector/index.d.ts +1 -2
- package/lib/series-collector/index.d.ts.map +1 -1
- package/lib/series-collector/index.js +1 -2
- package/lib/series-collector/index.js.map +1 -1
- package/lib/series-collector/interest-rate.js +9 -3
- package/lib/series-collector/interest-rate.js.map +1 -1
- package/lib/series-collector/ohlc.d.ts +2 -0
- package/lib/series-collector/ohlc.d.ts.map +1 -0
- package/lib/series-collector/ohlc.js +51 -0
- package/lib/series-collector/ohlc.js.map +1 -0
- package/lib/series-collector/patch-interest-rate.d.ts.map +1 -1
- package/lib/series-collector/patch-interest-rate.js +4 -0
- package/lib/series-collector/patch-interest-rate.js.map +1 -1
- package/lib/series-collector/patch-ohlc.d.ts +3 -0
- package/lib/series-collector/patch-ohlc.d.ts.map +1 -0
- package/lib/series-collector/patch-ohlc.js +71 -0
- package/lib/series-collector/patch-ohlc.js.map +1 -0
- package/package.json +1 -1
|
@@ -4,78 +4,45 @@
|
|
|
4
4
|
// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。
|
|
5
5
|
// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.handleIngestOHLCForward = void 0;
|
|
7
8
|
const data_ohlc_1 = require("@yuants/data-ohlc");
|
|
8
|
-
const exchange_1 = require("@yuants/exchange");
|
|
9
9
|
const protocol_1 = require("@yuants/protocol");
|
|
10
10
|
const sql_1 = require("@yuants/sql");
|
|
11
11
|
const utils_1 = require("@yuants/utils");
|
|
12
|
-
const rxjs_1 = require("rxjs");
|
|
13
12
|
const terminal = protocol_1.Terminal.fromNodeEnv();
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
finally {
|
|
36
|
-
}
|
|
37
|
-
}
|
|
13
|
+
const ingestCounter = terminal.metrics
|
|
14
|
+
.counter('series_collector_ingest_count', '')
|
|
15
|
+
.labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });
|
|
16
|
+
const handleIngestOHLCForward = async (series_id, direction, signal) => {
|
|
17
|
+
var _a, _b, _c, _d;
|
|
18
|
+
const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
|
|
19
|
+
const [datasource_id] = (0, utils_1.decodePath)(product_id);
|
|
20
|
+
// 控制速率：每个数据源每秒钟只能请求一次
|
|
21
|
+
await (0, utils_1.tokenBucket)(`ohlc:forward:${datasource_id}`).acquire(1, signal);
|
|
22
|
+
let req;
|
|
23
|
+
if (direction === 'forward') {
|
|
24
|
+
const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`);
|
|
25
|
+
const time = record ? new Date(record.end_time).getTime() : 0;
|
|
26
|
+
req = {
|
|
27
|
+
product_id,
|
|
28
|
+
duration,
|
|
29
|
+
direction: 'forward',
|
|
30
|
+
time,
|
|
31
|
+
};
|
|
38
32
|
}
|
|
39
|
-
|
|
40
|
-
|
|
33
|
+
else {
|
|
34
|
+
// backward
|
|
35
|
+
req = {
|
|
36
|
+
product_id,
|
|
37
|
+
duration,
|
|
38
|
+
direction,
|
|
39
|
+
time: Date.now(),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Forward]', 'Request', `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
|
|
43
|
+
const res = await terminal.client.requestForResponseData('IngestOHLC', req);
|
|
44
|
+
ingestCounter.inc(res.wrote_count || 0);
|
|
45
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Forward]', 'Response', `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
41
46
|
};
|
|
42
|
-
|
|
43
|
-
const time = Date.now();
|
|
44
|
-
const series_ids = await listForwardSeriesIds();
|
|
45
|
-
console.log(`[SeriesCollector][OHLC][Forwards] Found ${series_ids.size} series to collect forwards data for. (${(0, utils_1.formatTime)(Date.now() - time)})`);
|
|
46
|
-
await Promise.all([...series_ids].map(async (series_id) => {
|
|
47
|
-
var _a, _b, _c, _d;
|
|
48
|
-
try {
|
|
49
|
-
const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
|
|
50
|
-
const [datasource_id] = (0, utils_1.decodePath)(product_id);
|
|
51
|
-
// 控制速率：每个数据源每秒钟只能请求一次
|
|
52
|
-
await (0, utils_1.tokenBucket)(`ohlc_forwards_target_${datasource_id}`, {
|
|
53
|
-
refillInterval: 1000,
|
|
54
|
-
capacity: 1,
|
|
55
|
-
}).acquire();
|
|
56
|
-
{
|
|
57
|
-
const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`);
|
|
58
|
-
const time = record ? new Date(record.end_time).getTime() : 0;
|
|
59
|
-
const req = {
|
|
60
|
-
product_id,
|
|
61
|
-
duration,
|
|
62
|
-
direction: 'forward',
|
|
63
|
-
time,
|
|
64
|
-
};
|
|
65
|
-
console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestOHLC_Forwards', `product_id=${product_id}, duration=${duration}, time=${(0, utils_1.formatTime)(time)}`);
|
|
66
|
-
const res = await terminal.client.requestForResponseData('IngestOHLC', req);
|
|
67
|
-
terminal.metrics
|
|
68
|
-
.counter('series_collector_forwards_ingest_count', '')
|
|
69
|
-
.labels({ terminal_id: terminal.terminal_id, type: 'ohlc' })
|
|
70
|
-
.inc(res.wrote_count || 0);
|
|
71
|
-
console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestOHLCResult_Forwards', `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
catch (e) {
|
|
75
|
-
console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestOHLCError_Forwards', `series_id=${series_id}`, e);
|
|
76
|
-
}
|
|
77
|
-
}));
|
|
78
|
-
})
|
|
79
|
-
.pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 1000 }))
|
|
80
|
-
.subscribe();
|
|
47
|
+
exports.handleIngestOHLCForward = handleIngestOHLCForward;
|
|
81
48
|
//# sourceMappingURL=forwards-ohlc.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC
|
|
1
|
+
{"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;;;AAErC,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzE,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,gBAAgB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACtE,IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAhEW,QAAA,uBAAuB,2BAgElC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\nexport const handleIngestOHLCForward = async (\n series_id: string,\n direction: 'forward' | 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`ohlc:forward:${datasource_id}`).acquire(1, signal);\n let req: IIngestOHLCRequest;\n if (direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n );\n\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id,\n duration,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n duration,\n direction,\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,iBAAiB,CAAC;AACzB,OAAO,QAAQ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,2BAAyB;AACzB,kBAAgB","sourcesContent":["import './interest-rate';\nimport './ohlc';\n"]}
|
|
@@ -17,7 +17,9 @@ const patch_interest_rate_1 = require("./patch-interest-rate");
|
|
|
17
17
|
const forwardTask = (0, rxjs_1.defer)(async () => {
|
|
18
18
|
await (0, forwards_interest_rate_1.handleIngestInterestRateForward)(product_id, meta, abortController.signal);
|
|
19
19
|
})
|
|
20
|
-
.pipe((0, rxjs_1.
|
|
20
|
+
.pipe((0, rxjs_1.tap)({
|
|
21
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][InterestRate][Forward]`, 'Error', err),
|
|
22
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
21
23
|
.subscribe();
|
|
22
24
|
sub.add(() => {
|
|
23
25
|
forwardTask.unsubscribe();
|
|
@@ -26,7 +28,9 @@ const patch_interest_rate_1 = require("./patch-interest-rate");
|
|
|
26
28
|
const backwardTask = (0, rxjs_1.defer)(async () => {
|
|
27
29
|
await (0, backwards_interest_rate_1.handleIngestInterestRateBackward)(product_id, meta, abortController.signal);
|
|
28
30
|
})
|
|
29
|
-
.pipe((0, rxjs_1.
|
|
31
|
+
.pipe((0, rxjs_1.tap)({
|
|
32
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][InterestRate][Backward]`, 'Error', err),
|
|
33
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
30
34
|
.subscribe();
|
|
31
35
|
sub.add(() => {
|
|
32
36
|
backwardTask.unsubscribe();
|
|
@@ -35,7 +39,9 @@ const patch_interest_rate_1 = require("./patch-interest-rate");
|
|
|
35
39
|
const patchTask = (0, rxjs_1.defer)(async () => {
|
|
36
40
|
await (0, patch_interest_rate_1.handleInterestRatePatch)(product_id, meta, abortController.signal);
|
|
37
41
|
})
|
|
38
|
-
.pipe((0, rxjs_1.
|
|
42
|
+
.pipe((0, rxjs_1.tap)({
|
|
43
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][InterestRate][Patch]`, 'Error', err),
|
|
44
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
39
45
|
.subscribe();
|
|
40
46
|
sub.add(() => {
|
|
41
47
|
patchTask.unsubscribe();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":";;AAAA,
|
|
1
|
+
{"version":3,"file":"interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":";;AAAA,yCAAsD;AACtD,+BAAkE;AAClE,uEAA6E;AAC7E,2CAAwD;AACxD,qEAA2E;AAC3E,+DAAgE;AAEhE,IAAA,YAAK,EAAC,GAAG,EAAE,CAAC,IAAA,qCAAyB,GAAE,CAAC;KACrC,IAAI,CACH,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtB,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EACxB,IAAA,UAAG,EAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EACnC,IAAA,iBAAS,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EACX,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,CACrB,IAAI,iBAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,mDAAmD;IACnD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAE9C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,MAAM,WAAW,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACnC,MAAM,IAAA,wDAA+B,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAClF,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,OAAO,EACP,GAAG,CACJ;KACJ,CAAC,EAEF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,YAAY,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACpC,MAAM,IAAA,0DAAgC,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IACnF,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,OAAO,EACP,GAAG,CACJ;KACJ,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,YAAY,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,SAAS,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACjC,MAAM,IAAA,6CAAuB,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,OAAO,EACP,GAAG,CACJ;KACJ,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,SAAS,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CACL,CACF;KACA,SAAS,EAAE,CAAC","sourcesContent":["import { formatTime, listWatch } from '@yuants/utils';\nimport { defer, map, Observable, repeat, retry, tap } from 'rxjs';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { listInterestRateSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleInterestRatePatch } from './patch-interest-rate';\n\ndefer(() => listInterestRateSeriesIds())\n .pipe(\n retry({ delay: 1000 }),\n repeat({ delay: 60000 }),\n map((x) => Array.from(x.entries())),\n listWatch(\n (x) => x[0],\n ([product_id, meta]) =>\n new Observable((sub) => {\n // 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度\n const abortController = new AbortController();\n\n sub.add(() => {\n abortController.abort();\n });\n\n // 先处理前向任务\n const forwardTask = defer(async () => {\n await handleIngestInterestRateForward(product_id, meta, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(\n formatTime(Date.now()),\n `[SeriesCollector][InterestRate][Forward]`,\n 'Error',\n err,\n ),\n }),\n\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n forwardTask.unsubscribe();\n });\n\n // 设置后向任务\n const backwardTask = defer(async () => {\n await handleIngestInterestRateBackward(product_id, meta, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(\n formatTime(Date.now()),\n `[SeriesCollector][InterestRate][Backward]`,\n 'Error',\n err,\n ),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n backwardTask.unsubscribe();\n });\n\n // 设置补齐任务\n const patchTask = defer(async () => {\n await handleInterestRatePatch(product_id, meta, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(\n formatTime(Date.now()),\n `[SeriesCollector][InterestRate][Patch]`,\n 'Error',\n err,\n ),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n patchTask.unsubscribe();\n });\n }),\n ),\n )\n .subscribe();\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/ohlc.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const utils_1 = require("@yuants/utils");
|
|
4
|
+
const rxjs_1 = require("rxjs");
|
|
5
|
+
const discovery_1 = require("./discovery");
|
|
6
|
+
const forwards_ohlc_1 = require("./forwards-ohlc");
|
|
7
|
+
const backwards_ohlc_1 = require("./backwards-ohlc");
|
|
8
|
+
const patch_ohlc_1 = require("./patch-ohlc");
|
|
9
|
+
(0, rxjs_1.defer)(() => (0, discovery_1.listOHLCSeriesIds)())
|
|
10
|
+
.pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 60000 }), (0, rxjs_1.map)((x) => Array.from(x.entries())), (0, utils_1.listWatch)((x) => x[0], ([product_id, direction]) => new rxjs_1.Observable((sub) => {
|
|
11
|
+
// 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度
|
|
12
|
+
const abortController = new AbortController();
|
|
13
|
+
sub.add(() => {
|
|
14
|
+
abortController.abort();
|
|
15
|
+
});
|
|
16
|
+
// 先处理前向任务
|
|
17
|
+
const forwardTask = (0, rxjs_1.defer)(async () => {
|
|
18
|
+
await (0, forwards_ohlc_1.handleIngestOHLCForward)(product_id, direction, abortController.signal);
|
|
19
|
+
})
|
|
20
|
+
.pipe((0, rxjs_1.tap)({
|
|
21
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][OHLC][Forward]`, 'Error', err),
|
|
22
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
23
|
+
.subscribe();
|
|
24
|
+
sub.add(() => {
|
|
25
|
+
forwardTask.unsubscribe();
|
|
26
|
+
});
|
|
27
|
+
// 设置后向任务
|
|
28
|
+
const backwardTask = (0, rxjs_1.defer)(async () => {
|
|
29
|
+
await (0, backwards_ohlc_1.handleIngestOHLCBackward)(product_id, direction, abortController.signal);
|
|
30
|
+
})
|
|
31
|
+
.pipe((0, rxjs_1.tap)({
|
|
32
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][OHLC][Backward]`, 'Error', err),
|
|
33
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
34
|
+
.subscribe();
|
|
35
|
+
sub.add(() => {
|
|
36
|
+
backwardTask.unsubscribe();
|
|
37
|
+
});
|
|
38
|
+
// 设置补齐任务
|
|
39
|
+
const patchTask = (0, rxjs_1.defer)(async () => {
|
|
40
|
+
await (0, patch_ohlc_1.handleOHLCPatch)(product_id, direction, abortController.signal);
|
|
41
|
+
})
|
|
42
|
+
.pipe((0, rxjs_1.tap)({
|
|
43
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][OHLC][Patch]`, 'Error', err),
|
|
44
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
45
|
+
.subscribe();
|
|
46
|
+
sub.add(() => {
|
|
47
|
+
patchTask.unsubscribe();
|
|
48
|
+
});
|
|
49
|
+
})))
|
|
50
|
+
.subscribe();
|
|
51
|
+
//# sourceMappingURL=ohlc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ohlc.js","sourceRoot":"","sources":["../../src/series-collector/ohlc.ts"],"names":[],"mappings":";;AAAA,yCAAsD;AACtD,+BAAkE;AAElE,2CAAgD;AAGhD,mDAA0D;AAC1D,qDAA4D;AAC5D,6CAA+C;AAE/C,IAAA,YAAK,EAAC,GAAG,EAAE,CAAC,IAAA,6BAAiB,GAAE,CAAC;KAC7B,IAAI,CACH,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtB,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EACxB,IAAA,UAAG,EAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EACnC,IAAA,iBAAS,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EACX,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,CAC1B,IAAI,iBAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,mDAAmD;IACnD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAE9C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,MAAM,WAAW,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACnC,MAAM,IAAA,uCAAuB,EAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC/E,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,kCAAkC,EAAE,OAAO,EAAE,GAAG,CAAC;KACzF,CAAC,EAEF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,YAAY,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACpC,MAAM,IAAA,yCAAwB,EAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAChF,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,mCAAmC,EAAE,OAAO,EAAE,GAAG,CAAC;KAC1F,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,YAAY,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,SAAS,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACjC,MAAM,IAAA,4BAAe,EAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IACvE,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,gCAAgC,EAAE,OAAO,EAAE,GAAG,CAAC;KACvF,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,SAAS,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CACL,CACF;KACA,SAAS,EAAE,CAAC","sourcesContent":["import { formatTime, listWatch } from '@yuants/utils';\nimport { defer, map, Observable, repeat, retry, tap } from 'rxjs';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { listOHLCSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleInterestRatePatch } from './patch-interest-rate';\nimport { handleIngestOHLCForward } from './forwards-ohlc';\nimport { handleIngestOHLCBackward } from './backwards-ohlc';\nimport { handleOHLCPatch } from './patch-ohlc';\n\ndefer(() => listOHLCSeriesIds())\n .pipe(\n retry({ delay: 1000 }),\n repeat({ delay: 60000 }),\n map((x) => Array.from(x.entries())),\n listWatch(\n (x) => x[0],\n ([product_id, direction]) =>\n new Observable((sub) => {\n // 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度\n const abortController = new AbortController();\n\n sub.add(() => {\n abortController.abort();\n });\n\n // 先处理前向任务\n const forwardTask = defer(async () => {\n await handleIngestOHLCForward(product_id, direction, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(formatTime(Date.now()), `[SeriesCollector][OHLC][Forward]`, 'Error', err),\n }),\n\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n forwardTask.unsubscribe();\n });\n\n // 设置后向任务\n const backwardTask = defer(async () => {\n await handleIngestOHLCBackward(product_id, direction, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(formatTime(Date.now()), `[SeriesCollector][OHLC][Backward]`, 'Error', err),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n backwardTask.unsubscribe();\n });\n\n // 设置补齐任务\n const patchTask = defer(async () => {\n await handleOHLCPatch(product_id, direction, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(formatTime(Date.now()), `[SeriesCollector][OHLC][Patch]`, 'Error', err),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n patchTask.unsubscribe();\n });\n }),\n ),\n )\n .subscribe();\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"patch-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";AAAA,OAAO,EAEL,4BAA4B,EAE7B,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"patch-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";AAAA,OAAO,EAEL,4BAA4B,EAE7B,MAAM,kBAAkB,CAAC;AAY1B,eAAO,MAAM,uBAAuB,eACtB,MAAM,QACZ,4BAA4B,UAC1B,WAAW,kBAqFpB,CAAC"}
|
|
@@ -5,6 +5,9 @@ const protocol_1 = require("@yuants/protocol");
|
|
|
5
5
|
const sql_1 = require("@yuants/sql");
|
|
6
6
|
const utils_1 = require("@yuants/utils");
|
|
7
7
|
const terminal = protocol_1.Terminal.fromNodeEnv();
|
|
8
|
+
const ingestCounter = terminal.metrics
|
|
9
|
+
.counter('series_collector_ingest_count', '')
|
|
10
|
+
.labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });
|
|
8
11
|
// Patch 任务：查找数据缺口并进行补齐
|
|
9
12
|
const handleInterestRatePatch = async (product_id, meta, signal) => {
|
|
10
13
|
var _a, _b, _c, _d;
|
|
@@ -57,6 +60,7 @@ LIMIT 1;
|
|
|
57
60
|
}
|
|
58
61
|
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchRequest', `direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
|
|
59
62
|
const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
|
|
63
|
+
ingestCounter.inc(res.wrote_count || 0);
|
|
60
64
|
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchBackwardResult', `ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
61
65
|
};
|
|
62
66
|
exports.handleInterestRatePatch = handleInterestRatePatch;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";;;AAKA,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,uBAAuB;AAChB,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,IAAA,mBAAW,EAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;QAChC,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";;;AAKA,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,uBAAuB;AAChB,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,IAAA,mBAAW,EAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;QAChC,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAxFW,QAAA,uBAAuB,2BAwFlC","sourcesContent":["import {\n IIngestInterestRateRequest,\n IInterestRateServiceMetadata,\n ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n product_id: string,\n meta: IInterestRateServiceMetadata,\n signal: AbortSignal,\n) => {\n const [datasource_id] = decodePath(product_id);\n await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'interest_rate' \n AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n 'FindGap',\n `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n let req: IIngestInterestRateRequest;\n\n if (meta.direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"patch-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";AAaA,eAAO,MAAM,eAAe,cACf,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,kBAyFpB,CAAC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleOHLCPatch = void 0;
|
|
4
|
+
const data_ohlc_1 = require("@yuants/data-ohlc");
|
|
5
|
+
const protocol_1 = require("@yuants/protocol");
|
|
6
|
+
const sql_1 = require("@yuants/sql");
|
|
7
|
+
const utils_1 = require("@yuants/utils");
|
|
8
|
+
const terminal = protocol_1.Terminal.fromNodeEnv();
|
|
9
|
+
const ingestCounter = terminal.metrics
|
|
10
|
+
.counter('series_collector_ingest_count', '')
|
|
11
|
+
.labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });
|
|
12
|
+
// Patch 任务：查找数据缺口并进行补齐
|
|
13
|
+
const handleOHLCPatch = async (series_id, direction, signal) => {
|
|
14
|
+
var _a, _b, _c, _d;
|
|
15
|
+
const [datasource_id] = (0, utils_1.decodePath)(series_id);
|
|
16
|
+
await (0, utils_1.tokenBucket)(`ohlc:patch:${datasource_id}`).acquire(1, signal);
|
|
17
|
+
const [record] = await (0, sql_1.requestSQL)(terminal, `
|
|
18
|
+
WITH reversed_ranges AS (
|
|
19
|
+
SELECT
|
|
20
|
+
start_time,
|
|
21
|
+
end_time,
|
|
22
|
+
LEAD(end_time) OVER (
|
|
23
|
+
PARTITION BY table_name, series_id
|
|
24
|
+
ORDER BY start_time DESC
|
|
25
|
+
) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间
|
|
26
|
+
FROM series_data_range
|
|
27
|
+
WHERE table_name = 'ohlc_v2'
|
|
28
|
+
AND series_id = ${(0, sql_1.escapeSQL)(series_id)}
|
|
29
|
+
)
|
|
30
|
+
SELECT
|
|
31
|
+
next_end_time AS gap_start_time, -- 前一个区间的结束时间
|
|
32
|
+
start_time AS gap_end_time -- 当前区间的开始时间
|
|
33
|
+
FROM reversed_ranges
|
|
34
|
+
WHERE next_end_time IS NOT NULL
|
|
35
|
+
AND start_time > next_end_time -- 有空缺
|
|
36
|
+
ORDER BY start_time DESC -- 从最新开始
|
|
37
|
+
LIMIT 1;
|
|
38
|
+
`);
|
|
39
|
+
// no gap
|
|
40
|
+
if (!record)
|
|
41
|
+
return;
|
|
42
|
+
const gapStartTime = new Date(record.gap_start_time).getTime();
|
|
43
|
+
const gapEndTime = new Date(record.gap_end_time).getTime();
|
|
44
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Patch]', 'FindGap', `series=${series_id}, from=${(0, utils_1.formatTime)(gapStartTime)}, to=${(0, utils_1.formatTime)(gapEndTime)}`);
|
|
45
|
+
const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
|
|
46
|
+
let req;
|
|
47
|
+
if (direction === 'forward') {
|
|
48
|
+
// forward patch
|
|
49
|
+
req = {
|
|
50
|
+
product_id: product_id,
|
|
51
|
+
duration,
|
|
52
|
+
direction: 'forward',
|
|
53
|
+
time: gapStartTime,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
// backward patch
|
|
58
|
+
req = {
|
|
59
|
+
product_id: product_id,
|
|
60
|
+
duration,
|
|
61
|
+
direction: 'backward',
|
|
62
|
+
time: gapEndTime,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Patch]', series_id, 'PatchRequest', `direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
|
|
66
|
+
const res = await terminal.client.requestForResponseData('IngestOHLC', req);
|
|
67
|
+
ingestCounter.inc(res.wrote_count || 0);
|
|
68
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Patch]', series_id, 'PatchBackwardResult', `ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
69
|
+
};
|
|
70
|
+
exports.handleOHLCPatch = handleOHLCPatch;
|
|
71
|
+
//# sourceMappingURL=patch-ohlc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"patch-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";;;AAAA,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,uBAAuB;AAChB,MAAM,eAAe,GAAG,KAAK,EAClC,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,IAAA,mBAAW,EAAC,cAAc,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,SAAS,CAAC;;;;;;;;;;KAUvC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,UAAU,SAAS,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACtF,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAE5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA5FW,QAAA,eAAe,mBA4F1B","sourcesContent":["import { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleOHLCPatch = async (\n series_id: string,\n direction: 'forward' | 'backward',\n signal: AbortSignal,\n) => {\n const [datasource_id] = decodePath(series_id);\n await tokenBucket(`ohlc:patch:${datasource_id}`).acquire(1, signal);\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'ohlc_v2' \n AND series_id = ${escapeSQL(series_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n 'FindGap',\n `series=${series_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
|