@yuants/app-virtual-exchange 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/series-collector/backwards-interest-rate.js +29 -63
- package/dist/series-collector/backwards-interest-rate.js.map +1 -1
- package/dist/series-collector/backwards-ohlc.js +33 -68
- package/dist/series-collector/backwards-ohlc.js.map +1 -1
- package/dist/series-collector/discovery.js +62 -0
- package/dist/series-collector/discovery.js.map +1 -0
- package/dist/series-collector/forwards-interest-rate.js +31 -70
- package/dist/series-collector/forwards-interest-rate.js.map +1 -1
- package/dist/series-collector/forwards-ohlc.js +33 -68
- package/dist/series-collector/forwards-ohlc.js.map +1 -1
- package/dist/series-collector/index.js +2 -4
- package/dist/series-collector/index.js.map +1 -1
- package/dist/series-collector/interest-rate.js +49 -0
- package/dist/series-collector/interest-rate.js.map +1 -0
- package/dist/series-collector/ohlc.js +49 -0
- package/dist/series-collector/ohlc.js.map +1 -0
- package/dist/series-collector/patch-interest-rate.js +63 -0
- package/dist/series-collector/patch-interest-rate.js.map +1 -0
- package/dist/series-collector/patch-ohlc.js +67 -0
- package/dist/series-collector/patch-ohlc.js.map +1 -0
- package/dist/series-collector/sql-helpers.js +11 -0
- package/dist/series-collector/sql-helpers.js.map +1 -0
- package/lib/series-collector/backwards-interest-rate.d.ts +3 -1
- package/lib/series-collector/backwards-interest-rate.d.ts.map +1 -1
- package/lib/series-collector/backwards-interest-rate.js +31 -63
- package/lib/series-collector/backwards-interest-rate.js.map +1 -1
- package/lib/series-collector/backwards-ohlc.d.ts +2 -1
- package/lib/series-collector/backwards-ohlc.d.ts.map +1 -1
- package/lib/series-collector/backwards-ohlc.js +34 -67
- package/lib/series-collector/backwards-ohlc.js.map +1 -1
- package/lib/series-collector/discovery.d.ts +12 -0
- package/lib/series-collector/discovery.d.ts.map +1 -0
- package/lib/series-collector/discovery.js +67 -0
- package/lib/series-collector/discovery.js.map +1 -0
- package/lib/series-collector/forwards-interest-rate.d.ts +3 -1
- package/lib/series-collector/forwards-interest-rate.d.ts.map +1 -1
- package/lib/series-collector/forwards-interest-rate.js +33 -70
- package/lib/series-collector/forwards-interest-rate.js.map +1 -1
- package/lib/series-collector/forwards-ohlc.d.ts +2 -1
- package/lib/series-collector/forwards-ohlc.d.ts.map +1 -1
- package/lib/series-collector/forwards-ohlc.js +34 -67
- package/lib/series-collector/forwards-ohlc.js.map +1 -1
- package/lib/series-collector/index.d.ts +2 -4
- package/lib/series-collector/index.d.ts.map +1 -1
- package/lib/series-collector/index.js +2 -4
- package/lib/series-collector/index.js.map +1 -1
- package/lib/series-collector/interest-rate.d.ts +2 -0
- package/lib/series-collector/interest-rate.d.ts.map +1 -0
- package/lib/series-collector/interest-rate.js +51 -0
- package/lib/series-collector/interest-rate.js.map +1 -0
- package/lib/series-collector/ohlc.d.ts +2 -0
- package/lib/series-collector/ohlc.d.ts.map +1 -0
- package/lib/series-collector/ohlc.js +51 -0
- package/lib/series-collector/ohlc.js.map +1 -0
- package/lib/series-collector/patch-interest-rate.d.ts +4 -0
- package/lib/series-collector/patch-interest-rate.d.ts.map +1 -0
- package/lib/series-collector/patch-interest-rate.js +67 -0
- package/lib/series-collector/patch-interest-rate.js.map +1 -0
- package/lib/series-collector/patch-ohlc.d.ts +3 -0
- package/lib/series-collector/patch-ohlc.d.ts.map +1 -0
- package/lib/series-collector/patch-ohlc.js +71 -0
- package/lib/series-collector/patch-ohlc.js.map +1 -0
- package/lib/series-collector/sql-helpers.d.ts +11 -0
- package/lib/series-collector/sql-helpers.d.ts.map +1 -0
- package/lib/series-collector/sql-helpers.js +17 -0
- package/lib/series-collector/sql-helpers.js.map +1 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC
|
|
1
|
+
{"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;;;AAErC,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzE,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,gBAAgB,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7D,IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAhEW,QAAA,uBAAuB,2BAgElC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo,提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列,向对应的 IngestOHLC Service 发送拉取请求,补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率,避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\nexport const handleIngestOHLCForward = async (\n series_id: string,\n direction: 'forward' | 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n const [datasource_id] = decodePath(product_id);\n // 控制速率:每个数据源每秒钟只能请求一次\n await tokenBucket(`ohlc:forward:${datasource_id}`).acquire();\n let req: IIngestOHLCRequest;\n if (direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n );\n\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id,\n duration,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n duration,\n direction,\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,iBAAiB,CAAC;AACzB,OAAO,QAAQ,CAAC"}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
require("./
|
|
4
|
-
require("./
|
|
5
|
-
require("./forwards-interest-rate");
|
|
6
|
-
require("./forwards-ohlc");
|
|
3
|
+
require("./interest-rate");
|
|
4
|
+
require("./ohlc");
|
|
7
5
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,2BAAyB;AACzB,kBAAgB","sourcesContent":["import './interest-rate';\nimport './ohlc';\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const utils_1 = require("@yuants/utils");
|
|
4
|
+
const rxjs_1 = require("rxjs");
|
|
5
|
+
const backwards_interest_rate_1 = require("./backwards-interest-rate");
|
|
6
|
+
const discovery_1 = require("./discovery");
|
|
7
|
+
const forwards_interest_rate_1 = require("./forwards-interest-rate");
|
|
8
|
+
const patch_interest_rate_1 = require("./patch-interest-rate");
|
|
9
|
+
(0, rxjs_1.defer)(() => (0, discovery_1.listInterestRateSeriesIds)())
|
|
10
|
+
.pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 60000 }), (0, rxjs_1.map)((x) => Array.from(x.entries())), (0, utils_1.listWatch)((x) => x[0], ([product_id, meta]) => new rxjs_1.Observable((sub) => {
|
|
11
|
+
// 处理每个利率品种任务: (forward / backward / patch),都需要独立调度
|
|
12
|
+
const abortController = new AbortController();
|
|
13
|
+
sub.add(() => {
|
|
14
|
+
abortController.abort();
|
|
15
|
+
});
|
|
16
|
+
// 先处理前向任务
|
|
17
|
+
const forwardTask = (0, rxjs_1.defer)(async () => {
|
|
18
|
+
await (0, forwards_interest_rate_1.handleIngestInterestRateForward)(product_id, meta, abortController.signal);
|
|
19
|
+
})
|
|
20
|
+
.pipe((0, rxjs_1.tap)({
|
|
21
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][InterestRate][Forward]`, 'Error', err),
|
|
22
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
23
|
+
.subscribe();
|
|
24
|
+
sub.add(() => {
|
|
25
|
+
forwardTask.unsubscribe();
|
|
26
|
+
});
|
|
27
|
+
// 设置后向任务
|
|
28
|
+
const backwardTask = (0, rxjs_1.defer)(async () => {
|
|
29
|
+
await (0, backwards_interest_rate_1.handleIngestInterestRateBackward)(product_id, meta, abortController.signal);
|
|
30
|
+
})
|
|
31
|
+
.pipe((0, rxjs_1.tap)({
|
|
32
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][InterestRate][Backward]`, 'Error', err),
|
|
33
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
34
|
+
.subscribe();
|
|
35
|
+
sub.add(() => {
|
|
36
|
+
backwardTask.unsubscribe();
|
|
37
|
+
});
|
|
38
|
+
// 设置补齐任务
|
|
39
|
+
const patchTask = (0, rxjs_1.defer)(async () => {
|
|
40
|
+
await (0, patch_interest_rate_1.handleInterestRatePatch)(product_id, meta, abortController.signal);
|
|
41
|
+
})
|
|
42
|
+
.pipe((0, rxjs_1.tap)({
|
|
43
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][InterestRate][Patch]`, 'Error', err),
|
|
44
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
45
|
+
.subscribe();
|
|
46
|
+
sub.add(() => {
|
|
47
|
+
patchTask.unsubscribe();
|
|
48
|
+
});
|
|
49
|
+
})))
|
|
50
|
+
.subscribe();
|
|
51
|
+
//# sourceMappingURL=interest-rate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":";;AAAA,yCAAsD;AACtD,+BAAkE;AAClE,uEAA6E;AAC7E,2CAAwD;AACxD,qEAA2E;AAC3E,+DAAgE;AAEhE,IAAA,YAAK,EAAC,GAAG,EAAE,CAAC,IAAA,qCAAyB,GAAE,CAAC;KACrC,IAAI,CACH,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtB,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EACxB,IAAA,UAAG,EAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EACnC,IAAA,iBAAS,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EACX,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,CACrB,IAAI,iBAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,mDAAmD;IACnD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAE9C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,MAAM,WAAW,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACnC,MAAM,IAAA,wDAA+B,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAClF,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,OAAO,EACP,GAAG,CACJ;KACJ,CAAC,EAEF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,YAAY,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACpC,MAAM,IAAA,0DAAgC,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IACnF,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,OAAO,EACP,GAAG,CACJ;KACJ,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,YAAY,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,SAAS,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACjC,MAAM,IAAA,6CAAuB,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,OAAO,EACP,GAAG,CACJ;KACJ,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,SAAS,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CACL,CACF;KACA,SAAS,EAAE,CAAC","sourcesContent":["import { formatTime, listWatch } from '@yuants/utils';\nimport { defer, map, Observable, repeat, retry, tap } from 'rxjs';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { listInterestRateSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleInterestRatePatch } from './patch-interest-rate';\n\ndefer(() => listInterestRateSeriesIds())\n .pipe(\n retry({ delay: 1000 }),\n repeat({ delay: 60000 }),\n map((x) => Array.from(x.entries())),\n listWatch(\n (x) => x[0],\n ([product_id, meta]) =>\n new Observable((sub) => {\n // 处理每个利率品种任务: (forward / backward / patch),都需要独立调度\n const abortController = new AbortController();\n\n sub.add(() => {\n abortController.abort();\n });\n\n // 先处理前向任务\n const forwardTask = defer(async () => {\n await handleIngestInterestRateForward(product_id, meta, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(\n formatTime(Date.now()),\n `[SeriesCollector][InterestRate][Forward]`,\n 'Error',\n err,\n ),\n }),\n\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n forwardTask.unsubscribe();\n });\n\n // 设置后向任务\n const backwardTask = defer(async () => {\n await handleIngestInterestRateBackward(product_id, meta, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(\n formatTime(Date.now()),\n `[SeriesCollector][InterestRate][Backward]`,\n 'Error',\n err,\n ),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n backwardTask.unsubscribe();\n });\n\n // 设置补齐任务\n const patchTask = defer(async () => {\n await handleInterestRatePatch(product_id, meta, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(\n formatTime(Date.now()),\n `[SeriesCollector][InterestRate][Patch]`,\n 'Error',\n err,\n ),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n patchTask.unsubscribe();\n });\n }),\n ),\n )\n .subscribe();\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/ohlc.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const utils_1 = require("@yuants/utils");
|
|
4
|
+
const rxjs_1 = require("rxjs");
|
|
5
|
+
const discovery_1 = require("./discovery");
|
|
6
|
+
const forwards_ohlc_1 = require("./forwards-ohlc");
|
|
7
|
+
const backwards_ohlc_1 = require("./backwards-ohlc");
|
|
8
|
+
const patch_ohlc_1 = require("./patch-ohlc");
|
|
9
|
+
(0, rxjs_1.defer)(() => (0, discovery_1.listOHLCSeriesIds)())
|
|
10
|
+
.pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 60000 }), (0, rxjs_1.map)((x) => Array.from(x.entries())), (0, utils_1.listWatch)((x) => x[0], ([product_id, direction]) => new rxjs_1.Observable((sub) => {
|
|
11
|
+
// 处理每个利率品种任务: (forward / backward / patch),都需要独立调度
|
|
12
|
+
const abortController = new AbortController();
|
|
13
|
+
sub.add(() => {
|
|
14
|
+
abortController.abort();
|
|
15
|
+
});
|
|
16
|
+
// 先处理前向任务
|
|
17
|
+
const forwardTask = (0, rxjs_1.defer)(async () => {
|
|
18
|
+
await (0, forwards_ohlc_1.handleIngestOHLCForward)(product_id, direction, abortController.signal);
|
|
19
|
+
})
|
|
20
|
+
.pipe((0, rxjs_1.tap)({
|
|
21
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][OHLC][Forward]`, 'Error', err),
|
|
22
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
23
|
+
.subscribe();
|
|
24
|
+
sub.add(() => {
|
|
25
|
+
forwardTask.unsubscribe();
|
|
26
|
+
});
|
|
27
|
+
// 设置后向任务
|
|
28
|
+
const backwardTask = (0, rxjs_1.defer)(async () => {
|
|
29
|
+
await (0, backwards_ohlc_1.handleIngestOHLCBackward)(product_id, direction, abortController.signal);
|
|
30
|
+
})
|
|
31
|
+
.pipe((0, rxjs_1.tap)({
|
|
32
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][OHLC][Backward]`, 'Error', err),
|
|
33
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
34
|
+
.subscribe();
|
|
35
|
+
sub.add(() => {
|
|
36
|
+
backwardTask.unsubscribe();
|
|
37
|
+
});
|
|
38
|
+
// 设置补齐任务
|
|
39
|
+
const patchTask = (0, rxjs_1.defer)(async () => {
|
|
40
|
+
await (0, patch_ohlc_1.handleOHLCPatch)(product_id, direction, abortController.signal);
|
|
41
|
+
})
|
|
42
|
+
.pipe((0, rxjs_1.tap)({
|
|
43
|
+
error: (err) => console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][OHLC][Patch]`, 'Error', err),
|
|
44
|
+
}), (0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
|
|
45
|
+
.subscribe();
|
|
46
|
+
sub.add(() => {
|
|
47
|
+
patchTask.unsubscribe();
|
|
48
|
+
});
|
|
49
|
+
})))
|
|
50
|
+
.subscribe();
|
|
51
|
+
//# sourceMappingURL=ohlc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ohlc.js","sourceRoot":"","sources":["../../src/series-collector/ohlc.ts"],"names":[],"mappings":";;AAAA,yCAAsD;AACtD,+BAAkE;AAElE,2CAAgD;AAGhD,mDAA0D;AAC1D,qDAA4D;AAC5D,6CAA+C;AAE/C,IAAA,YAAK,EAAC,GAAG,EAAE,CAAC,IAAA,6BAAiB,GAAE,CAAC;KAC7B,IAAI,CACH,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtB,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EACxB,IAAA,UAAG,EAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EACnC,IAAA,iBAAS,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EACX,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,CAC1B,IAAI,iBAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,mDAAmD;IACnD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAE9C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,MAAM,WAAW,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACnC,MAAM,IAAA,uCAAuB,EAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC/E,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,kCAAkC,EAAE,OAAO,EAAE,GAAG,CAAC;KACzF,CAAC,EAEF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,YAAY,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACpC,MAAM,IAAA,yCAAwB,EAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAChF,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,mCAAmC,EAAE,OAAO,EAAE,GAAG,CAAC;KAC1F,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,YAAY,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,SAAS,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACjC,MAAM,IAAA,4BAAe,EAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IACvE,CAAC,CAAC;SACC,IAAI,CACH,IAAA,UAAG,EAAC;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CACb,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,gCAAgC,EAAE,OAAO,EAAE,GAAG,CAAC;KACvF,CAAC,EACF,IAAA,YAAK,GAAE,EACP,IAAA,aAAM,GAAE,CACT;SACA,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,SAAS,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CACL,CACF;KACA,SAAS,EAAE,CAAC","sourcesContent":["import { formatTime, listWatch } from '@yuants/utils';\nimport { defer, map, Observable, repeat, retry, tap } from 'rxjs';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { listOHLCSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleInterestRatePatch } from './patch-interest-rate';\nimport { handleIngestOHLCForward } from './forwards-ohlc';\nimport { handleIngestOHLCBackward } from './backwards-ohlc';\nimport { handleOHLCPatch } from './patch-ohlc';\n\ndefer(() => listOHLCSeriesIds())\n .pipe(\n retry({ delay: 1000 }),\n repeat({ delay: 60000 }),\n map((x) => Array.from(x.entries())),\n listWatch(\n (x) => x[0],\n ([product_id, direction]) =>\n new Observable((sub) => {\n // 处理每个利率品种任务: (forward / backward / patch),都需要独立调度\n const abortController = new AbortController();\n\n sub.add(() => {\n abortController.abort();\n });\n\n // 先处理前向任务\n const forwardTask = defer(async () => {\n await handleIngestOHLCForward(product_id, direction, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(formatTime(Date.now()), `[SeriesCollector][OHLC][Forward]`, 'Error', err),\n }),\n\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n forwardTask.unsubscribe();\n });\n\n // 设置后向任务\n const backwardTask = defer(async () => {\n await handleIngestOHLCBackward(product_id, direction, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(formatTime(Date.now()), `[SeriesCollector][OHLC][Backward]`, 'Error', err),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n backwardTask.unsubscribe();\n });\n\n // 设置补齐任务\n const patchTask = defer(async () => {\n await handleOHLCPatch(product_id, direction, abortController.signal);\n })\n .pipe(\n tap({\n error: (err) =>\n console.info(formatTime(Date.now()), `[SeriesCollector][OHLC][Patch]`, 'Error', err),\n }),\n retry(),\n repeat(),\n )\n .subscribe();\n\n sub.add(() => {\n patchTask.unsubscribe();\n });\n }),\n ),\n )\n .subscribe();\n"]}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { IInterestRateServiceMetadata } from '@yuants/exchange';
|
|
3
|
+
export declare const handleInterestRatePatch: (product_id: string, meta: IInterestRateServiceMetadata, signal: AbortSignal) => Promise<void>;
|
|
4
|
+
//# sourceMappingURL=patch-interest-rate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"patch-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";AAAA,OAAO,EAEL,4BAA4B,EAE7B,MAAM,kBAAkB,CAAC;AAY1B,eAAO,MAAM,uBAAuB,eACtB,MAAM,QACZ,4BAA4B,UAC1B,WAAW,kBAqFpB,CAAC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleInterestRatePatch = void 0;
|
|
4
|
+
const protocol_1 = require("@yuants/protocol");
|
|
5
|
+
const sql_1 = require("@yuants/sql");
|
|
6
|
+
const utils_1 = require("@yuants/utils");
|
|
7
|
+
const terminal = protocol_1.Terminal.fromNodeEnv();
|
|
8
|
+
const ingestCounter = terminal.metrics
|
|
9
|
+
.counter('series_collector_ingest_count', '')
|
|
10
|
+
.labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });
|
|
11
|
+
// Patch 任务:查找数据缺口并进行补齐
|
|
12
|
+
const handleInterestRatePatch = async (product_id, meta, signal) => {
|
|
13
|
+
var _a, _b, _c, _d;
|
|
14
|
+
const [datasource_id] = (0, utils_1.decodePath)(product_id);
|
|
15
|
+
await (0, utils_1.tokenBucket)(`interest_rate:patch:${datasource_id}`).acquire(1, signal);
|
|
16
|
+
const [record] = await (0, sql_1.requestSQL)(terminal, `
|
|
17
|
+
WITH reversed_ranges AS (
|
|
18
|
+
SELECT
|
|
19
|
+
start_time,
|
|
20
|
+
end_time,
|
|
21
|
+
LEAD(end_time) OVER (
|
|
22
|
+
PARTITION BY table_name, series_id
|
|
23
|
+
ORDER BY start_time DESC
|
|
24
|
+
) AS next_end_time -- 注意:倒序时 LEAD 是前一个区间
|
|
25
|
+
FROM series_data_range
|
|
26
|
+
WHERE table_name = 'interest_rate'
|
|
27
|
+
AND series_id = ${(0, sql_1.escapeSQL)(product_id)}
|
|
28
|
+
)
|
|
29
|
+
SELECT
|
|
30
|
+
next_end_time AS gap_start_time, -- 前一个区间的结束时间
|
|
31
|
+
start_time AS gap_end_time -- 当前区间的开始时间
|
|
32
|
+
FROM reversed_ranges
|
|
33
|
+
WHERE next_end_time IS NOT NULL
|
|
34
|
+
AND start_time > next_end_time -- 有空缺
|
|
35
|
+
ORDER BY start_time DESC -- 从最新开始
|
|
36
|
+
LIMIT 1;
|
|
37
|
+
`);
|
|
38
|
+
// no gap
|
|
39
|
+
if (!record)
|
|
40
|
+
return;
|
|
41
|
+
const gapStartTime = new Date(record.gap_start_time).getTime();
|
|
42
|
+
const gapEndTime = new Date(record.gap_end_time).getTime();
|
|
43
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', 'FindGap', `series=${product_id}, from=${(0, utils_1.formatTime)(gapStartTime)}, to=${(0, utils_1.formatTime)(gapEndTime)}`);
|
|
44
|
+
let req;
|
|
45
|
+
if (meta.direction === 'forward') {
|
|
46
|
+
// forward patch
|
|
47
|
+
req = {
|
|
48
|
+
product_id: product_id,
|
|
49
|
+
direction: 'forward',
|
|
50
|
+
time: gapStartTime,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
// backward patch
|
|
55
|
+
req = {
|
|
56
|
+
product_id: product_id,
|
|
57
|
+
direction: 'backward',
|
|
58
|
+
time: gapEndTime,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchRequest', `direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
|
|
62
|
+
const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
|
|
63
|
+
ingestCounter.inc(res.wrote_count || 0);
|
|
64
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchBackwardResult', `ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
65
|
+
};
|
|
66
|
+
exports.handleInterestRatePatch = handleInterestRatePatch;
|
|
67
|
+
//# sourceMappingURL=patch-interest-rate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";;;AAKA,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,uBAAuB;AAChB,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,IAAA,mBAAW,EAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;QAChC,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAxFW,QAAA,uBAAuB,2BAwFlC","sourcesContent":["import {\n IIngestInterestRateRequest,\n IInterestRateServiceMetadata,\n ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\n// Patch 任务:查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n product_id: string,\n meta: IInterestRateServiceMetadata,\n signal: AbortSignal,\n) => {\n const [datasource_id] = decodePath(product_id);\n await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意:倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'interest_rate' \n AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n 'FindGap',\n `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n let req: IIngestInterestRateRequest;\n\n if (meta.direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"patch-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";AAaA,eAAO,MAAM,eAAe,cACf,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,kBAyFpB,CAAC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleOHLCPatch = void 0;
|
|
4
|
+
const data_ohlc_1 = require("@yuants/data-ohlc");
|
|
5
|
+
const protocol_1 = require("@yuants/protocol");
|
|
6
|
+
const sql_1 = require("@yuants/sql");
|
|
7
|
+
const utils_1 = require("@yuants/utils");
|
|
8
|
+
const terminal = protocol_1.Terminal.fromNodeEnv();
|
|
9
|
+
const ingestCounter = terminal.metrics
|
|
10
|
+
.counter('series_collector_ingest_count', '')
|
|
11
|
+
.labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });
|
|
12
|
+
// Patch 任务:查找数据缺口并进行补齐
|
|
13
|
+
const handleOHLCPatch = async (series_id, direction, signal) => {
|
|
14
|
+
var _a, _b, _c, _d;
|
|
15
|
+
const [datasource_id] = (0, utils_1.decodePath)(series_id);
|
|
16
|
+
await (0, utils_1.tokenBucket)(`ohlc:patch:${datasource_id}`).acquire(1, signal);
|
|
17
|
+
const [record] = await (0, sql_1.requestSQL)(terminal, `
|
|
18
|
+
WITH reversed_ranges AS (
|
|
19
|
+
SELECT
|
|
20
|
+
start_time,
|
|
21
|
+
end_time,
|
|
22
|
+
LEAD(end_time) OVER (
|
|
23
|
+
PARTITION BY table_name, series_id
|
|
24
|
+
ORDER BY start_time DESC
|
|
25
|
+
) AS next_end_time -- 注意:倒序时 LEAD 是前一个区间
|
|
26
|
+
FROM series_data_range
|
|
27
|
+
WHERE table_name = 'ohlc_v2'
|
|
28
|
+
AND series_id = ${(0, sql_1.escapeSQL)(series_id)}
|
|
29
|
+
)
|
|
30
|
+
SELECT
|
|
31
|
+
next_end_time AS gap_start_time, -- 前一个区间的结束时间
|
|
32
|
+
start_time AS gap_end_time -- 当前区间的开始时间
|
|
33
|
+
FROM reversed_ranges
|
|
34
|
+
WHERE next_end_time IS NOT NULL
|
|
35
|
+
AND start_time > next_end_time -- 有空缺
|
|
36
|
+
ORDER BY start_time DESC -- 从最新开始
|
|
37
|
+
LIMIT 1;
|
|
38
|
+
`);
|
|
39
|
+
// no gap
|
|
40
|
+
if (!record)
|
|
41
|
+
return;
|
|
42
|
+
const gapStartTime = new Date(record.gap_start_time).getTime();
|
|
43
|
+
const gapEndTime = new Date(record.gap_end_time).getTime();
|
|
44
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Patch]', 'FindGap', `series=${series_id}, from=${(0, utils_1.formatTime)(gapStartTime)}, to=${(0, utils_1.formatTime)(gapEndTime)}`);
|
|
45
|
+
const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
|
|
46
|
+
let req;
|
|
47
|
+
if (direction === 'forward') {
|
|
48
|
+
// forward patch
|
|
49
|
+
req = {
|
|
50
|
+
product_id: product_id,
|
|
51
|
+
duration,
|
|
52
|
+
direction: 'forward',
|
|
53
|
+
time: gapStartTime,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
// backward patch
|
|
58
|
+
req = {
|
|
59
|
+
product_id: product_id,
|
|
60
|
+
duration,
|
|
61
|
+
direction: 'backward',
|
|
62
|
+
time: gapEndTime,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Patch]', series_id, 'PatchRequest', `direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
|
|
66
|
+
const res = await terminal.client.requestForResponseData('IngestOHLC', req);
|
|
67
|
+
ingestCounter.inc(res.wrote_count || 0);
|
|
68
|
+
console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][OHLC][Patch]', series_id, 'PatchBackwardResult', `ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
|
|
69
|
+
};
|
|
70
|
+
exports.handleOHLCPatch = handleOHLCPatch;
|
|
71
|
+
//# sourceMappingURL=patch-ohlc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"patch-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";;;AAAA,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,uBAAuB;AAChB,MAAM,eAAe,GAAG,KAAK,EAClC,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,IAAA,mBAAW,EAAC,cAAc,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,SAAS,CAAC;;;;;;;;;;KAUvC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,UAAU,SAAS,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACtF,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAE5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA5FW,QAAA,eAAe,mBA4F1B","sourcesContent":["import { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\n// Patch 任务:查找数据缺口并进行补齐\nexport const handleOHLCPatch = async (\n series_id: string,\n direction: 'forward' | 'backward',\n signal: AbortSignal,\n) => {\n const [datasource_id] = decodePath(series_id);\n await tokenBucket(`ohlc:patch:${datasource_id}`).acquire(1, signal);\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意:倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'ohlc_v2' \n AND series_id = ${escapeSQL(series_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n 'FindGap',\n `series=${series_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count || 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Terminal } from '@yuants/protocol';
|
|
2
|
+
/**
|
|
3
|
+
* 查找某个利率品种系列ID的最新结束时间(for forward task)
|
|
4
|
+
* @param terminal
|
|
5
|
+
* @param product_id
|
|
6
|
+
* @returns
|
|
7
|
+
*/
|
|
8
|
+
export declare const findInterestRateEndTimeForward: (terminal: Terminal, product_id: string) => Promise<string>;
|
|
9
|
+
export declare const findInterestRateStartTimeBackward: (terminal: Terminal, product_id: string) => Promise<string>;
|
|
10
|
+
export declare const findOHLCEndTimeForward: (terminal: Terminal, series_id: string) => Promise<string>;
|
|
11
|
+
//# sourceMappingURL=sql-helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sql-helpers.d.ts","sourceRoot":"","sources":["../../src/series-collector/sql-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAG5C;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,aAAc,QAAQ,cAAc,MAAM,oBAUxC,CAAC;AAE9C,eAAO,MAAM,iCAAiC,aAAc,QAAQ,cAAc,MAAM,oBAUzC,CAAC;AAEhD,eAAO,MAAM,sBAAsB,aAAc,QAAQ,aAAa,MAAM,oBAU/B,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.findOHLCEndTimeForward = exports.findInterestRateStartTimeBackward = exports.findInterestRateEndTimeForward = void 0;
|
|
4
|
+
const sql_1 = require("@yuants/sql");
|
|
5
|
+
/**
|
|
6
|
+
* 查找某个利率品种系列ID的最新结束时间(for forward task)
|
|
7
|
+
* @param terminal
|
|
8
|
+
* @param product_id
|
|
9
|
+
* @returns
|
|
10
|
+
*/
|
|
11
|
+
const findInterestRateEndTimeForward = (terminal, product_id) => (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.end_time; });
|
|
12
|
+
exports.findInterestRateEndTimeForward = findInterestRateEndTimeForward;
|
|
13
|
+
const findInterestRateStartTimeBackward = (terminal, product_id) => (0, sql_1.requestSQL)(terminal, `select start_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by start_time asc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.start_time; });
|
|
14
|
+
exports.findInterestRateStartTimeBackward = findInterestRateStartTimeBackward;
|
|
15
|
+
const findOHLCEndTimeForward = (terminal, series_id) => (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.end_time; });
|
|
16
|
+
exports.findOHLCEndTimeForward = findOHLCEndTimeForward;
|
|
17
|
+
//# sourceMappingURL=sql-helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sql-helpers.js","sourceRoot":"","sources":["../../src/series-collector/sql-helpers.ts"],"names":[],"mappings":";;;AACA,qCAAoD;AAEpD;;;;;GAKG;AACI,MAAM,8BAA8B,GAAG,CAAC,QAAkB,EAAE,UAAkB,EAAE,EAAE,CACvF,IAAA,gBAAU,EAKR,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,UAAU,CACX,kEAAkE,CACpE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,QAAQ,CAAA,EAAA,CAAC,CAAC;AAVjC,QAAA,8BAA8B,kCAUG;AAEvC,MAAM,iCAAiC,GAAG,CAAC,QAAkB,EAAE,UAAkB,EAAE,EAAE,CAC1F,IAAA,gBAAU,EAKR,QAAQ,EACR,8DAA8D,IAAA,eAAS,EACrE,UAAU,CACX,mEAAmE,CACrE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,UAAU,CAAA,EAAA,CAAC,CAAC;AAVnC,QAAA,iCAAiC,qCAUE;AAEzC,MAAM,sBAAsB,GAAG,CAAC,QAAkB,EAAE,SAAiB,EAAE,EAAE,CAC9E,IAAA,gBAAU,EAKR,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,QAAQ,CAAA,EAAA,CAAC,CAAC;AAVjC,QAAA,sBAAsB,0BAUW","sourcesContent":["import { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\n\n/**\n * 查找某个利率品种系列ID的最新结束时间(for forward task)\n * @param terminal\n * @param product_id\n * @returns\n */\nexport const findInterestRateEndTimeForward = (terminal: Terminal, product_id: string) =>\n requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n product_id,\n )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n ).then((records) => records?.[0]?.end_time);\n\nexport const findInterestRateStartTimeBackward = (terminal: Terminal, product_id: string) =>\n requestSQL<\n {\n start_time: string;\n }[]\n >(\n terminal,\n `select start_time from series_data_range where series_id = ${escapeSQL(\n product_id,\n )} and table_name = 'interest_rate' order by start_time asc limit 1`,\n ).then((records) => records?.[0]?.start_time);\n\nexport const findOHLCEndTimeForward = (terminal: Terminal, series_id: string) =>\n requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n ).then((records) => records?.[0]?.end_time);\n"]}
|