@yuants/app-virtual-exchange 0.13.0 → 0.15.0

package/dist/series-collector/forwards-interest-rate.js

@@ -0,0 +1,73 @@
+// 解决 Forwards 拉取历史数据的调度器
+// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。
+// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。
+// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
+import { parseInterestRateServiceMetadataFromSchema, } from '@yuants/exchange';
+import { Terminal } from '@yuants/protocol';
+import { escapeSQL, requestSQL } from '@yuants/sql';
+import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { defer, repeat, retry } from 'rxjs';
+const terminal = Terminal.fromNodeEnv();
+const listBackwardSeriesIds = async () => {
+    const product_ids = await requestSQL(terminal, `select product_id from product`);
+    console.time('[SeriesCollector][InterestRate][Forwards] calc');
+    const series_ids = new Set();
+    for (const terminalInfo of terminal.terminalInfos) {
+        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
+            if (serviceInfo.method !== 'IngestInterestRate')
+                continue;
+            try {
+                const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);
+                if (meta.direction !== 'forward')
+                    continue;
+                for (const { product_id } of product_ids) {
+                    if (!product_id.startsWith(meta.product_id_prefix))
+                        continue;
+                    series_ids.add(product_id);
+                }
+            }
+            finally {
+            }
+        }
+    }
+    console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');
+    return series_ids;
+};
+defer(async () => {
+    const time = Date.now();
+    const series_ids = await listBackwardSeriesIds();
+    console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. (${formatTime(Date.now() - time)})`);
+    await Promise.all([...series_ids].map(async (product_id) => {
+        var _a, _b, _c, _d;
+        try {
+            const [datasource_id] = decodePath(product_id);
+            // 控制速率：每个数据源每秒钟只能请求一次
+            await tokenBucket(`interest_rate_forwards:${datasource_id}`, {
+                refillInterval: 1000,
+                capacity: 1,
+            }).acquire();
+            {
+                const [record] = await requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`);
+                const time = record ? new Date(record.end_time).getTime() : 0;
+                const req = {
+                    product_id: product_id,
+                    direction: 'forward',
+                    time,
+                };
+                console.info(formatTime(Date.now()), 'DispatchIngestInterestRateRequestForwards', `product_id=${product_id}, time=${formatTime(time)}`);
+                const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+                terminal.metrics
+                    .counter('series_collector_forwards_ingest_count', '')
+                    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
+                    .inc(res.wrote_count || 0);
+                console.info(formatTime(Date.now()), 'DispatchIngestInterestRateResultForwards', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+            }
+        }
+        catch (e) {
+            console.info(formatTime(Date.now()), 'DispatchIngestInterestRateErrorForwards', `series_id=${product_id}`, e);
+        }
+    }));
+})
+    .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))
+    .subscribe();
+//# sourceMappingURL=forwards-interest-rate.js.map

package/dist/series-collector/forwards-interest-rate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;AAErC,OAAO,EAGL,0CAA0C,GAC3C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,UAAU,CAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,0CAA0C,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC5E,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;oBAAE,SAAS;gBAE3C,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;iBAC5B;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,KAAK,CAAC,KAAK,IAAI,EAAE;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,mDACE,UAAU,CAAC,IACb,0CAA0C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAC3E,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;;QACvC,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,EAAE;gBAC3D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,4DAA4D,SAAS,CACnE,UAAU,CACX,kEAAkE,CACpE,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE9D,MAAM,GAAG,GAA+B;oBACtC,UAAU,EAAE,UAAU;oBACtB,SAAS,EAAE,SAAS;oBACpB,IAAI;iBACL,CAAC;gBAEF,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,cAAc,UAAU,UAAU,UAAU,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC;gBAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,yCAAyC,EACzC,aAAa,UAAU,EAAE,EACzB,CAAC,CACF,CAAC;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  ISeriesIngestResult,\n  parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][InterestRate][Forwards] calc');\n\n  const series_ids = new Set<string>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n        if (meta.direction !== 'forward') continue;\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.add(product_id);\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const time = Date.now();\n  const series_ids = await listBackwardSeriesIds();\n  console.log(\n    `[SeriesCollector][InterestRate][Forwards] Found ${\n      series_ids.size\n    } series to collect forwards data for. (${formatTime(Date.now() - time)})`,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async (product_id) => {\n      try {\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`interest_rate_forwards:${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          const [record] = await requestSQL<\n            {\n              end_time: string;\n            }[]\n          >(\n            terminal,\n            `select end_time from series_data_range where series_id = ${escapeSQL(\n              product_id,\n            )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n          );\n          const time = record ? new Date(record.end_time).getTime() : 0;\n\n          const req: IIngestInterestRateRequest = {\n            product_id: product_id,\n            direction: 'forward',\n            time,\n          };\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequestForwards',\n            `product_id=${product_id}, time=${formatTime(time)}`,\n          );\n\n          const res = await terminal.client.requestForResponseData<\n            IIngestInterestRateRequest,\n            ISeriesIngestResult\n          >('IngestInterestRate', req);\n\n          terminal.metrics\n            .counter('series_collector_forwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateResultForwards',\n            `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(\n          formatTime(Date.now()),\n          'DispatchIngestInterestRateErrorForwards',\n          `series_id=${product_id}`,\n          e,\n        );\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}

package/dist/series-collector/forwards-ohlc.js

@@ -0,0 +1,79 @@
+// 解决 Forwards 拉取历史数据的调度器
+// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。
+// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。
+// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
+import { decodeOHLCSeriesId, encodeOHLCSeriesId } from '@yuants/data-ohlc';
+import { parseOHLCServiceMetadataFromSchema, } from '@yuants/exchange';
+import { Terminal } from '@yuants/protocol';
+import { escapeSQL, requestSQL } from '@yuants/sql';
+import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { defer, repeat, retry } from 'rxjs';
+const terminal = Terminal.fromNodeEnv();
+const listForwardSeriesIds = async () => {
+    const product_ids = await requestSQL(terminal, `select product_id from product`);
+    console.time('[SeriesCollector][OHLC][Forwards] calc');
+    const series_ids = new Set();
+    for (const terminalInfo of terminal.terminalInfos) {
+        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
+            if (serviceInfo.method !== 'IngestOHLC')
+                continue;
+            try {
+                const meta = parseOHLCServiceMetadataFromSchema(serviceInfo.schema);
+                if (meta.direction !== 'forward')
+                    continue;
+                for (const { product_id } of product_ids) {
+                    if (!product_id.startsWith(meta.product_id_prefix))
+                        continue;
+                    for (const duration of meta.duration_list) {
+                        const series_id = encodeOHLCSeriesId(product_id, duration);
+                        series_ids.add(series_id);
+                    }
+                }
+            }
+            finally {
+            }
+        }
+    }
+    console.timeEnd('[SeriesCollector][OHLC][Forwards] calc');
+    return series_ids;
+};
+defer(async () => {
+    const time = Date.now();
+    const series_ids = await listForwardSeriesIds();
+    console.log(`[SeriesCollector][OHLC][Forwards] Found ${series_ids.size} series to collect forwards data for. (${formatTime(Date.now() - time)})`);
+    await Promise.all([...series_ids].map(async (series_id) => {
+        var _a, _b, _c, _d;
+        try {
+            const { product_id, duration } = decodeOHLCSeriesId(series_id);
+            const [datasource_id] = decodePath(product_id);
+            // 控制速率：每个数据源每秒钟只能请求一次
+            await tokenBucket(`ohlc_forwards_target_${datasource_id}`, {
+                refillInterval: 1000,
+                capacity: 1,
+            }).acquire();
+            {
+                const [record] = await requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`);
+                const time = record ? new Date(record.end_time).getTime() : 0;
+                const req = {
+                    product_id,
+                    duration,
+                    direction: 'forward',
+                    time,
+                };
+                console.info(formatTime(Date.now()), 'DispatchIngestOHLC_Forwards', `product_id=${product_id}, duration=${duration}, time=${formatTime(time)}`);
+                const res = await terminal.client.requestForResponseData('IngestOHLC', req);
+                terminal.metrics
+                    .counter('series_collector_forwards_ingest_count', '')
+                    .labels({ terminal_id: terminal.terminal_id, type: 'ohlc' })
+                    .inc(res.wrote_count || 0);
+                console.info(formatTime(Date.now()), 'DispatchIngestOHLCResult_Forwards', `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+            }
+        }
+        catch (e) {
+            console.info(formatTime(Date.now()), 'DispatchIngestOHLCError_Forwards', `series_id=${series_id}`, e);
+        }
+    }));
+})
+    .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))
+    .subscribe();
+//# sourceMappingURL=forwards-ohlc.js.map

package/dist/series-collector/forwards-ohlc.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;AAErC,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EAGL,kCAAkC,GACnC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAG,KAAK,IAAI,EAAE;IACtC,MAAM,WAAW,GAAG,MAAM,UAAU,CAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IAEvD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,YAAY;gBAAE,SAAS;YAClD,IAAI;gBACF,MAAM,IAAI,GAAG,kCAAkC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBACpE,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;oBAAE,SAAS;gBAE3C,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,EAAE;wBACzC,MAAM,SAAS,GAAG,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC3D,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;qBAC3B;iBACF;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;IAE1D,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,KAAK,CAAC,KAAK,IAAI,EAAE;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,MAAM,oBAAoB,EAAE,CAAC;IAChD,OAAO,CAAC,GAAG,CACT,2CACE,UAAU,CAAC,IACb,0CAA0C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAC3E,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;;QACtC,IAAI;YACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC/D,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,WAAW,CAAC,wBAAwB,aAAa,EAAE,EAAE;gBACzD,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,4DAA4D,SAAS,CACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;gBAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE9D,MAAM,GAAG,GAAuB;oBAC9B,UAAU;oBACV,QAAQ;oBACR,SAAS,EAAE,SAAS;oBACpB,IAAI;iBACL,CAAC;gBAEF,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,6BAA6B,EAC7B,cAAc,UAAU,cAAc,QAAQ,UAAU,UAAU,CAAC,IAAI,CAAC,EAAE,CAC3E,CAAC;gBAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;gBAEF,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;qBAC3D,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,kCAAkC,EAAE,aAAa,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;SACvG;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId, encodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport {\n  IIngestOHLCRequest,\n  ISeriesIngestResult,\n  parseOHLCServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listForwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][OHLC][Forwards] calc');\n\n  const series_ids = new Set<string>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestOHLC') continue;\n      try {\n        const meta = parseOHLCServiceMetadataFromSchema(serviceInfo.schema);\n        if (meta.direction !== 'forward') continue;\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          for (const duration of meta.duration_list) {\n            const series_id = encodeOHLCSeriesId(product_id, duration);\n            series_ids.add(series_id);\n          }\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][OHLC][Forwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const time = Date.now();\n  const series_ids = await listForwardSeriesIds();\n  console.log(\n    `[SeriesCollector][OHLC][Forwards] Found ${\n      series_ids.size\n    } series to collect forwards data for. (${formatTime(Date.now() - time)})`,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async (series_id) => {\n      try {\n        const { product_id, duration } = decodeOHLCSeriesId(series_id);\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`ohlc_forwards_target_${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          const [record] = await requestSQL<\n            {\n              end_time: string;\n            }[]\n          >(\n            terminal,\n            `select end_time from series_data_range where series_id = ${escapeSQL(\n              series_id,\n            )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n          );\n\n          const time = record ? new Date(record.end_time).getTime() : 0;\n\n          const req: IIngestOHLCRequest = {\n            product_id,\n            duration,\n            direction: 'forward',\n            time,\n          };\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestOHLC_Forwards',\n            `product_id=${product_id}, duration=${duration}, time=${formatTime(time)}`,\n          );\n\n          const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n            'IngestOHLC',\n            req,\n          );\n\n          terminal.metrics\n            .counter('series_collector_forwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'ohlc' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestOHLCResult_Forwards',\n            `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(formatTime(Date.now()), 'DispatchIngestOHLCError_Forwards', `series_id=${series_id}`, e);\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}

package/dist/series-collector/index.js

@@ -1,3 +1,5 @@
 import './backwards-interest-rate';
 import './backwards-ohlc';
+import './forwards-interest-rate';
+import './forwards-ohlc';
 //# sourceMappingURL=index.js.map

package/dist/series-collector/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,2BAA2B,CAAC;AACnC,OAAO,kBAAkB,CAAC","sourcesContent":["import './backwards-interest-rate';\nimport './backwards-ohlc';\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,2BAA2B,CAAC;AACnC,OAAO,kBAAkB,CAAC;AAC1B,OAAO,0BAA0B,CAAC;AAClC,OAAO,iBAAiB,CAAC","sourcesContent":["import './backwards-interest-rate';\nimport './backwards-ohlc';\nimport './forwards-interest-rate';\nimport './forwards-ohlc';\n"]}

package/lib/series-collector/forwards-interest-rate.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=forwards-interest-rate.d.ts.map

package/lib/series-collector/forwards-interest-rate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":""}

package/lib/series-collector/forwards-interest-rate.js

@@ -0,0 +1,75 @@
+"use strict";
+// 解决 Forwards 拉取历史数据的调度器
+// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。
+// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。
+// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
+Object.defineProperty(exports, "__esModule", { value: true });
+const exchange_1 = require("@yuants/exchange");
+const protocol_1 = require("@yuants/protocol");
+const sql_1 = require("@yuants/sql");
+const utils_1 = require("@yuants/utils");
+const rxjs_1 = require("rxjs");
+const terminal = protocol_1.Terminal.fromNodeEnv();
+const listBackwardSeriesIds = async () => {
+    const product_ids = await (0, sql_1.requestSQL)(terminal, `select product_id from product`);
+    console.time('[SeriesCollector][InterestRate][Forwards] calc');
+    const series_ids = new Set();
+    for (const terminalInfo of terminal.terminalInfos) {
+        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
+            if (serviceInfo.method !== 'IngestInterestRate')
+                continue;
+            try {
+                const meta = (0, exchange_1.parseInterestRateServiceMetadataFromSchema)(serviceInfo.schema);
+                if (meta.direction !== 'forward')
+                    continue;
+                for (const { product_id } of product_ids) {
+                    if (!product_id.startsWith(meta.product_id_prefix))
+                        continue;
+                    series_ids.add(product_id);
+                }
+            }
+            finally {
+            }
+        }
+    }
+    console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');
+    return series_ids;
+};
+(0, rxjs_1.defer)(async () => {
+    const time = Date.now();
+    const series_ids = await listBackwardSeriesIds();
+    console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. (${(0, utils_1.formatTime)(Date.now() - time)})`);
+    await Promise.all([...series_ids].map(async (product_id) => {
+        var _a, _b, _c, _d;
+        try {
+            const [datasource_id] = (0, utils_1.decodePath)(product_id);
+            // 控制速率：每个数据源每秒钟只能请求一次
+            await (0, utils_1.tokenBucket)(`interest_rate_forwards:${datasource_id}`, {
+                refillInterval: 1000,
+                capacity: 1,
+            }).acquire();
+            {
+                const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`);
+                const time = record ? new Date(record.end_time).getTime() : 0;
+                const req = {
+                    product_id: product_id,
+                    direction: 'forward',
+                    time,
+                };
+                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateRequestForwards', `product_id=${product_id}, time=${(0, utils_1.formatTime)(time)}`);
+                const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+                terminal.metrics
+                    .counter('series_collector_forwards_ingest_count', '')
+                    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
+                    .inc(res.wrote_count || 0);
+                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateResultForwards', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+            }
+        }
+        catch (e) {
+            console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateErrorForwards', `series_id=${product_id}`, e);
+        }
+    }));
+})
+    .pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 1000 }))
+    .subscribe();
+//# sourceMappingURL=forwards-interest-rate.js.map

package/lib/series-collector/forwards-interest-rate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;;AAErC,+CAI0B;AAC1B,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AACpE,+BAA4C;AAE5C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,IAAA,gBAAU,EAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,IAAA,qDAA0C,EAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC5E,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;oBAAE,SAAS;gBAE3C,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;iBAC5B;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,mDACE,UAAU,CAAC,IACb,0CAA0C,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAC3E,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;;QACvC,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,EAAE;gBAC3D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,UAAU,CACX,kEAAkE,CACpE,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE9D,MAAM,GAAG,GAA+B;oBACtC,UAAU,EAAE,UAAU;oBACtB,SAAS,EAAE,SAAS;oBACpB,IAAI;iBACL,CAAC;gBAEF,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,cAAc,UAAU,UAAU,IAAA,kBAAU,EAAC,IAAI,CAAC,EAAE,CACrD,CAAC;gBAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,yCAAyC,EACzC,aAAa,UAAU,EAAE,EACzB,CAAC,CACF,CAAC;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  ISeriesIngestResult,\n  parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][InterestRate][Forwards] calc');\n\n  const series_ids = new Set<string>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n        if (meta.direction !== 'forward') continue;\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.add(product_id);\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const time = Date.now();\n  const series_ids = await listBackwardSeriesIds();\n  console.log(\n    `[SeriesCollector][InterestRate][Forwards] Found ${\n      series_ids.size\n    } series to collect forwards data for. (${formatTime(Date.now() - time)})`,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async (product_id) => {\n      try {\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`interest_rate_forwards:${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          const [record] = await requestSQL<\n            {\n              end_time: string;\n            }[]\n          >(\n            terminal,\n            `select end_time from series_data_range where series_id = ${escapeSQL(\n              product_id,\n            )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n          );\n          const time = record ? new Date(record.end_time).getTime() : 0;\n\n          const req: IIngestInterestRateRequest = {\n            product_id: product_id,\n            direction: 'forward',\n            time,\n          };\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequestForwards',\n            `product_id=${product_id}, time=${formatTime(time)}`,\n          );\n\n          const res = await terminal.client.requestForResponseData<\n            IIngestInterestRateRequest,\n            ISeriesIngestResult\n          >('IngestInterestRate', req);\n\n          terminal.metrics\n            .counter('series_collector_forwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateResultForwards',\n            `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(\n          formatTime(Date.now()),\n          'DispatchIngestInterestRateErrorForwards',\n          `series_id=${product_id}`,\n          e,\n        );\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}

package/lib/series-collector/forwards-ohlc.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=forwards-ohlc.d.ts.map

package/lib/series-collector/forwards-ohlc.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwards-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":""}

package/lib/series-collector/forwards-ohlc.js

@@ -0,0 +1,81 @@
+"use strict";
+// 解决 Forwards 拉取历史数据的调度器
+// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。
+// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。
+// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
+Object.defineProperty(exports, "__esModule", { value: true });
+const data_ohlc_1 = require("@yuants/data-ohlc");
+const exchange_1 = require("@yuants/exchange");
+const protocol_1 = require("@yuants/protocol");
+const sql_1 = require("@yuants/sql");
+const utils_1 = require("@yuants/utils");
+const rxjs_1 = require("rxjs");
+const terminal = protocol_1.Terminal.fromNodeEnv();
+const listForwardSeriesIds = async () => {
+    const product_ids = await (0, sql_1.requestSQL)(terminal, `select product_id from product`);
+    console.time('[SeriesCollector][OHLC][Forwards] calc');
+    const series_ids = new Set();
+    for (const terminalInfo of terminal.terminalInfos) {
+        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
+            if (serviceInfo.method !== 'IngestOHLC')
+                continue;
+            try {
+                const meta = (0, exchange_1.parseOHLCServiceMetadataFromSchema)(serviceInfo.schema);
+                if (meta.direction !== 'forward')
+                    continue;
+                for (const { product_id } of product_ids) {
+                    if (!product_id.startsWith(meta.product_id_prefix))
+                        continue;
+                    for (const duration of meta.duration_list) {
+                        const series_id = (0, data_ohlc_1.encodeOHLCSeriesId)(product_id, duration);
+                        series_ids.add(series_id);
+                    }
+                }
+            }
+            finally {
+            }
+        }
+    }
+    console.timeEnd('[SeriesCollector][OHLC][Forwards] calc');
+    return series_ids;
+};
+(0, rxjs_1.defer)(async () => {
+    const time = Date.now();
+    const series_ids = await listForwardSeriesIds();
+    console.log(`[SeriesCollector][OHLC][Forwards] Found ${series_ids.size} series to collect forwards data for. (${(0, utils_1.formatTime)(Date.now() - time)})`);
+    await Promise.all([...series_ids].map(async (series_id) => {
+        var _a, _b, _c, _d;
+        try {
+            const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
+            const [datasource_id] = (0, utils_1.decodePath)(product_id);
+            // 控制速率：每个数据源每秒钟只能请求一次
+            await (0, utils_1.tokenBucket)(`ohlc_forwards_target_${datasource_id}`, {
+                refillInterval: 1000,
+                capacity: 1,
+            }).acquire();
+            {
+                const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`);
+                const time = record ? new Date(record.end_time).getTime() : 0;
+                const req = {
+                    product_id,
+                    duration,
+                    direction: 'forward',
+                    time,
+                };
+                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestOHLC_Forwards', `product_id=${product_id}, duration=${duration}, time=${(0, utils_1.formatTime)(time)}`);
+                const res = await terminal.client.requestForResponseData('IngestOHLC', req);
+                terminal.metrics
+                    .counter('series_collector_forwards_ingest_count', '')
+                    .labels({ terminal_id: terminal.terminal_id, type: 'ohlc' })
+                    .inc(res.wrote_count || 0);
+                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestOHLCResult_Forwards', `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+            }
+        }
+        catch (e) {
+            console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestOHLCError_Forwards', `series_id=${series_id}`, e);
+        }
+    }));
+})
+    .pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 1000 }))
+    .subscribe();
+//# sourceMappingURL=forwards-ohlc.js.map

package/lib/series-collector/forwards-ohlc.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;;AAErC,iDAA2E;AAC3E,+CAI0B;AAC1B,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AACpE,+BAA4C;AAE5C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAG,KAAK,IAAI,EAAE;IACtC,MAAM,WAAW,GAAG,MAAM,IAAA,gBAAU,EAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IAEvD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,YAAY;gBAAE,SAAS;YAClD,IAAI;gBACF,MAAM,IAAI,GAAG,IAAA,6CAAkC,EAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBACpE,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;oBAAE,SAAS;gBAE3C,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,EAAE;wBACzC,MAAM,SAAS,GAAG,IAAA,8BAAkB,EAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC3D,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;qBAC3B;iBACF;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;IAE1D,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,MAAM,oBAAoB,EAAE,CAAC;IAChD,OAAO,CAAC,GAAG,CACT,2CACE,UAAU,CAAC,IACb,0CAA0C,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAC3E,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;;QACtC,IAAI;YACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;YAC/D,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,IAAA,mBAAW,EAAC,wBAAwB,aAAa,EAAE,EAAE;gBACzD,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;gBAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE9D,MAAM,GAAG,GAAuB;oBAC9B,UAAU;oBACV,QAAQ;oBACR,SAAS,EAAE,SAAS;oBACpB,IAAI;iBACL,CAAC;gBAEF,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,6BAA6B,EAC7B,cAAc,UAAU,cAAc,QAAQ,UAAU,IAAA,kBAAU,EAAC,IAAI,CAAC,EAAE,CAC3E,CAAC;gBAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;gBAEF,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;qBAC3D,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,kCAAkC,EAAE,aAAa,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;SACvG;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId, encodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport {\n  IIngestOHLCRequest,\n  ISeriesIngestResult,\n  parseOHLCServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listForwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][OHLC][Forwards] calc');\n\n  const series_ids = new Set<string>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestOHLC') continue;\n      try {\n        const meta = parseOHLCServiceMetadataFromSchema(serviceInfo.schema);\n        if (meta.direction !== 'forward') continue;\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          for (const duration of meta.duration_list) {\n            const series_id = encodeOHLCSeriesId(product_id, duration);\n            series_ids.add(series_id);\n          }\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][OHLC][Forwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const time = Date.now();\n  const series_ids = await listForwardSeriesIds();\n  console.log(\n    `[SeriesCollector][OHLC][Forwards] Found ${\n      series_ids.size\n    } series to collect forwards data for. (${formatTime(Date.now() - time)})`,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async (series_id) => {\n      try {\n        const { product_id, duration } = decodeOHLCSeriesId(series_id);\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`ohlc_forwards_target_${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          const [record] = await requestSQL<\n            {\n              end_time: string;\n            }[]\n          >(\n            terminal,\n            `select end_time from series_data_range where series_id = ${escapeSQL(\n              series_id,\n            )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n          );\n\n          const time = record ? new Date(record.end_time).getTime() : 0;\n\n          const req: IIngestOHLCRequest = {\n            product_id,\n            duration,\n            direction: 'forward',\n            time,\n          };\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestOHLC_Forwards',\n            `product_id=${product_id}, duration=${duration}, time=${formatTime(time)}`,\n          );\n\n          const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n            'IngestOHLC',\n            req,\n          );\n\n          terminal.metrics\n            .counter('series_collector_forwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'ohlc' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestOHLCResult_Forwards',\n            `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(formatTime(Date.now()), 'DispatchIngestOHLCError_Forwards', `series_id=${series_id}`, e);\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}

package/lib/series-collector/index.d.ts

@@ -1,3 +1,5 @@
 import './backwards-interest-rate';
 import './backwards-ohlc';
+import './forwards-interest-rate';
+import './forwards-ohlc';
 //# sourceMappingURL=index.d.ts.map

package/lib/series-collector/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,2BAA2B,CAAC;AACnC,OAAO,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,2BAA2B,CAAC;AACnC,OAAO,kBAAkB,CAAC;AAC1B,OAAO,0BAA0B,CAAC;AAClC,OAAO,iBAAiB,CAAC"}

package/lib/series-collector/index.js

@@ -2,4 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 require("./backwards-interest-rate");
 require("./backwards-ohlc");
+require("./forwards-interest-rate");
+require("./forwards-ohlc");
 //# sourceMappingURL=index.js.map

package/lib/series-collector/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,qCAAmC;AACnC,4BAA0B","sourcesContent":["import './backwards-interest-rate';\nimport './backwards-ohlc';\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,qCAAmC;AACnC,4BAA0B;AAC1B,oCAAkC;AAClC,2BAAyB","sourcesContent":["import './backwards-interest-rate';\nimport './backwards-ohlc';\nimport './forwards-interest-rate';\nimport './forwards-ohlc';\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@yuants/app-virtual-exchange",
-	"version": "0.13.0",
+	"version": "0.15.0",
 	"main": "lib/index.js",
 	"files": [
 		"dist",
@@ -14,11 +14,11 @@
 		"@yuants/data-account": "0.11.4",
 		"@yuants/data-order": "0.7.5",
 		"@yuants/data-quote": "0.4.4",
-		"@yuants/data-ohlc": "0.5.3",
+		"@yuants/data-ohlc": "0.6.0",
 		"@yuants/data-interest-rate": "0.2.3",
 		"@yuants/secret": "0.4.5",
 		"@yuants/sql": "0.9.35",
-		"@yuants/exchange": "0.8.5",
+		"@yuants/exchange": "0.8.6",
 		"@yuants/cache": "0.3.8",
 		"rxjs": "~7.5.6",
 		"ajv": "~8.12.0"