@yuants/app-virtual-exchange 0.16.0 → 0.17.0

package/dist/series-collector/backwards-interest-rate.js

@@ -2,72 +2,38 @@
 // 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。
 // 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。
 // 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
-import { parseInterestRateServiceMetadataFromSchema, } from '@yuants/exchange';
 import { Terminal } from '@yuants/protocol';
-import { escapeSQL, requestSQL } from '@yuants/sql';
 import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
-import { defer, repeat, retry } from 'rxjs';
+import { findInterestRateStartTimeBackward } from './sql-helpers';
 const terminal = Terminal.fromNodeEnv();
-const listBackwardSeriesIds = async () => {
-    const product_ids = await requestSQL(terminal, `select product_id from product`);
-    console.time('[SeriesCollector][InterestRate][Backwards] calc');
-    const series_ids = new Set();
-    for (const terminalInfo of terminal.terminalInfos) {
-        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
-            if (serviceInfo.method !== 'IngestInterestRate')
-                continue;
-            try {
-                const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);
-                if (meta.direction !== 'backward')
-                    continue;
-                for (const { product_id } of product_ids) {
-                    if (!product_id.startsWith(meta.product_id_prefix))
-                        continue;
-                    series_ids.add(product_id);
-                }
-            }
-            finally {
-            }
-        }
+export const handleIngestInterestRateBackward = async (product_id, meta, signal) => {
+    var _a, _b, _c, _d;
+    const [datasource_id] = decodePath(product_id);
+    // 控制速率：每个数据源每秒钟只能请求一次
+    await tokenBucket(`interest_rate:backward:${datasource_id}`).acquire(1, signal);
+    let req;
+    if (meta.direction === 'backward') {
+        const startTime = await findInterestRateStartTimeBackward(terminal, product_id);
+        const start_time = startTime ? new Date(startTime).getTime() : Date.now();
+        req = {
+            product_id: product_id,
+            direction: 'backward',
+            time: start_time,
+        };
     }
-    console.timeEnd('[SeriesCollector][InterestRate][Backwards] calc');
-    return series_ids;
+    else {
+        req = {
+            product_id: product_id,
+            direction: 'forward',
+            time: 0,
+        };
+    }
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Backward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`);
+    const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+    terminal.metrics
+        .counter('series_collector_ingest_count', '')
+        .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' })
+        .inc(res.wrote_count || 0);
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Backward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
 };
-defer(async () => {
-    const time = Date.now();
-    const series_ids = await listBackwardSeriesIds();
-    console.log(`[SeriesCollector][InterestRate][Backwards] Found ${series_ids.size} series to collect backwards data for. (${formatTime(Date.now() - time)})`);
-    await Promise.all([...series_ids].map(async (product_id) => {
-        var _a, _b, _c, _d;
-        try {
-            const [datasource_id] = decodePath(product_id);
-            // 控制速率：每个数据源每秒钟只能请求一次
-            await tokenBucket(`interest_rate_backwards:${datasource_id}`, {
-                refillInterval: 1000,
-                capacity: 1,
-            }).acquire();
-            {
-                const [record] = await requestSQL(terminal, `select start_time from series_data_range where series_id = ${escapeSQL(product_id)} and table_name = 'interest_rate' order by start_time limit 1`);
-                const start_time = record ? new Date(record.start_time).getTime() : Date.now();
-                const req = {
-                    product_id: product_id,
-                    direction: 'backward',
-                    time: start_time,
-                };
-                console.info(formatTime(Date.now()), 'DispatchIngestInterestRateRequest', `product_id=${product_id}, time=${formatTime(start_time)}`);
-                const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
-                terminal.metrics
-                    .counter('series_collector_backwards_ingest_count', '')
-                    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
-                    .inc(res.wrote_count || 0);
-                console.info(formatTime(Date.now()), 'DispatchIngestInterestRateResult', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
-            }
-        }
-        catch (e) {
-            console.info(formatTime(Date.now()), 'DispatchIngestInterestRateError', `series_id=${product_id}`, e);
-        }
-    }));
-})
-    .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))
-    .subscribe();
 //# sourceMappingURL=backwards-interest-rate.js.map

package/dist/series-collector/backwards-interest-rate.js.map

@@ -1 +1 @@
-{"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,yDAAyD;AACzD,qCAAqC;AAErC,OAAO,EAGL,0CAA0C,GAC3C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,UAAU,CAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,0CAA0C,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC5E,IAAI,IAAI,CAAC,SAAS,KAAK,UAAU;oBAAE,SAAS;gBAE5C,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;iBAC5B;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAEnE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,KAAK,CAAC,KAAK,IAAI,EAAE;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,oDACE,UAAU,CAAC,IACb,2CAA2C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAC5E,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;;QACvC,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,WAAW,CAAC,2BAA2B,aAAa,EAAE,EAAE;gBAC5D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,8DAA8D,SAAS,CACrE,UAAU,CACX,+DAA+D,CACjE,CAAC;gBACF,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE/E,MAAM,GAAG,GAA+B;oBACtC,UAAU,EAAE,UAAU;oBACtB,SAAS,EAAE,UAAU;oBACrB,IAAI,EAAE,UAAU;iBACjB,CAAC;gBAEF,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,cAAc,UAAU,UAAU,UAAU,CAAC,UAAU,CAAC,EAAE,CAC3D,CAAC;gBAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC;qBACtD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,iCAAiC,EAAE,aAAa,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;SACvG;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  ISeriesIngestResult,\n  parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][InterestRate][Backwards] calc');\n\n  const series_ids = new Set<string>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n        if (meta.direction !== 'backward') continue;\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.add(product_id);\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][InterestRate][Backwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const time = Date.now();\n  const series_ids = await listBackwardSeriesIds();\n  console.log(\n    `[SeriesCollector][InterestRate][Backwards] Found ${\n      series_ids.size\n    } series to collect backwards data for. (${formatTime(Date.now() - time)})`,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async (product_id) => {\n      try {\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`interest_rate_backwards:${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          const [record] = await requestSQL<\n            {\n              start_time: string;\n            }[]\n          >(\n            terminal,\n            `select start_time from series_data_range where series_id = ${escapeSQL(\n              product_id,\n            )} and table_name = 'interest_rate' order by start_time limit 1`,\n          );\n          const start_time = record ? new Date(record.start_time).getTime() : Date.now();\n\n          const req: IIngestInterestRateRequest = {\n            product_id: product_id,\n            direction: 'backward',\n            time: start_time,\n          };\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequest',\n            `product_id=${product_id}, time=${formatTime(start_time)}`,\n          );\n\n          const res = await terminal.client.requestForResponseData<\n            IIngestInterestRateRequest,\n            ISeriesIngestResult\n          >('IngestInterestRate', req);\n\n          terminal.metrics\n            .counter('series_collector_backwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateResult',\n            `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(formatTime(Date.now()), 'DispatchIngestInterestRateError', `series_id=${product_id}`, e);\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}
+{"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,yDAAyD;AACzD,qCAAqC;AAOrC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,iCAAiC,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,CAAC,MAAM,gCAAgC,GAAG,KAAK,EACnD,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAChF,IAAI,GAA+B,CAAC;IACpC,IAAI,IAAI,CAAC,SAAS,KAAK,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,MAAM,iCAAiC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1E,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,QAAQ,CAAC,OAAO;SACb,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;SAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;SACtF,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { findInterestRateStartTimeBackward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nexport const handleIngestInterestRateBackward = async (\n  product_id: string,\n  meta: IInterestRateServiceMetadata,\n  signal: AbortSignal,\n) => {\n  const [datasource_id] = decodePath(product_id);\n  // 控制速率：每个数据源每秒钟只能请求一次\n  await tokenBucket(`interest_rate:backward:${datasource_id}`).acquire(1, signal);\n  let req: IIngestInterestRateRequest;\n  if (meta.direction === 'backward') {\n    const startTime = await findInterestRateStartTimeBackward(terminal, product_id);\n    const start_time = startTime ? new Date(startTime).getTime() : Date.now();\n\n    req = {\n      product_id: product_id,\n      direction: 'backward',\n      time: start_time,\n    };\n  } else {\n    req = {\n      product_id: product_id,\n      direction: 'forward',\n      time: 0,\n    };\n  }\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Backward]',\n    'Request',\n    `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n  );\n\n  const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n    'IngestInterestRate',\n    req,\n  );\n\n  terminal.metrics\n    .counter('series_collector_ingest_count', '')\n    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' })\n    .inc(res.wrote_count || 0);\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Backward]',\n    'Result',\n    `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n      res.range?.start_time ?? NaN,\n    )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n  );\n};\n"]}

package/dist/series-collector/discovery.js

@@ -0,0 +1,33 @@
+// 发现所有支持利率的品种系列ID
+import { parseInterestRateServiceMetadataFromSchema } from '@yuants/exchange';
+import { Terminal } from '@yuants/protocol';
+import { requestSQL } from '@yuants/sql';
+const terminal = Terminal.fromNodeEnv();
+/**
+ * 列出所有支持利率的品种系列ID以及对应的 Service Metadata
+ * @returns
+ */
+export const listInterestRateSeriesIds = async () => {
+    const product_ids = await requestSQL(terminal, 
+    // 必须是支持利率的品种
+    `select product_id from product where no_interest_rate = false`);
+    const series_ids = new Map();
+    for (const terminalInfo of terminal.terminalInfos) {
+        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
+            if (serviceInfo.method !== 'IngestInterestRate')
+                continue;
+            try {
+                const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);
+                for (const { product_id } of product_ids) {
+                    if (!product_id.startsWith(meta.product_id_prefix))
+                        continue;
+                    series_ids.set(product_id, meta);
+                }
+            }
+            finally {
+            }
+        }
+    }
+    return series_ids;
+};
+//# sourceMappingURL=discovery.js.map

package/dist/series-collector/discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.js","sourceRoot":"","sources":["../../src/series-collector/discovery.ts"],"names":[],"mappings":"AAAA,kBAAkB;AAElB,OAAO,EAAgC,0CAA0C,EAAE,MAAM,kBAAkB,CAAC;AAC5G,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,KAAK,IAAI,EAAE;IAClD,MAAM,WAAW,GAAG,MAAM,UAAU,CAClC,QAAQ;IACR,aAAa;IACb,+DAA+D,CAChE,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACnE,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,0CAA0C,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAE5E,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;iBAClC;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC","sourcesContent":["// 发现所有支持利率的品种系列ID\n\nimport { IInterestRateServiceMetadata, parseInterestRateServiceMetadataFromSchema } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { requestSQL } from '@yuants/sql';\n\nconst terminal = Terminal.fromNodeEnv();\n\n/**\n * 列出所有支持利率的品种系列ID以及对应的 Service Metadata\n * @returns\n */\nexport const listInterestRateSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(\n    terminal,\n    // 必须是支持利率的品种\n    `select product_id from product where no_interest_rate = false`,\n  );\n\n  const series_ids = new Map<string, IInterestRateServiceMetadata>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.set(product_id, meta);\n        }\n      } finally {\n      }\n    }\n  }\n\n  return series_ids;\n};\n"]}

package/dist/series-collector/forwards-interest-rate.js

@@ -2,80 +2,41 @@
 // 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。
 // 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。
 // 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
-import { parseInterestRateServiceMetadataFromSchema, } from '@yuants/exchange';
 import { Terminal } from '@yuants/protocol';
-import { escapeSQL, requestSQL } from '@yuants/sql';
 import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
-import { defer, repeat, retry } from 'rxjs';
+import { findInterestRateEndTimeForward } from './sql-helpers';
 const terminal = Terminal.fromNodeEnv();
-const listBackwardSeriesIds = async () => {
-    const product_ids = await requestSQL(terminal, `select product_id from product`);
-    console.time('[SeriesCollector][InterestRate][Forwards] calc');
-    const series_ids = new Map();
-    for (const terminalInfo of terminal.terminalInfos) {
-        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
-            if (serviceInfo.method !== 'IngestInterestRate')
-                continue;
-            try {
-                const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);
-                for (const { product_id } of product_ids) {
-                    if (!product_id.startsWith(meta.product_id_prefix))
-                        continue;
-                    series_ids.set(product_id, meta);
-                }
-            }
-            finally {
-            }
+export const handleIngestInterestRateForward = async (product_id, meta, signal) => {
+    var _a, _b, _c, _d;
+    const [datasource_id] = decodePath(product_id);
+    // 控制速率：每个数据源每秒钟只能请求一次
+    await tokenBucket(`interest_rate_forwards:${datasource_id}`).acquire(1, signal);
+    {
+        let req;
+        if (meta.direction === 'forward') {
+            const endTime = await findInterestRateEndTimeForward(terminal, product_id);
+            const time = endTime ? new Date(endTime).getTime() : 0;
+            req = {
+                product_id: product_id,
+                direction: 'forward',
+                time,
+            };
         }
+        else {
+            // backward
+            req = {
+                product_id,
+                direction: 'backward',
+                time: Date.now(),
+            };
+        }
+        console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`);
+        const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+        terminal.metrics
+            .counter('series_collector_forwards_ingest_count', '')
+            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
+            .inc(res.wrote_count || 0);
+        console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
     }
-    console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');
-    return series_ids;
 };
-defer(async () => {
-    const series_ids = await listBackwardSeriesIds();
-    console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `);
-    await Promise.all([...series_ids].map(async ([product_id, meta]) => {
-        var _a, _b, _c, _d;
-        try {
-            const [datasource_id] = decodePath(product_id);
-            // 控制速率：每个数据源每秒钟只能请求一次
-            await tokenBucket(`interest_rate_forwards:${datasource_id}`, {
-                refillInterval: 1000,
-                capacity: 1,
-            }).acquire();
-            {
-                let req;
-                if (meta.direction === 'forward') {
-                    const [record] = await requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`);
-                    const time = record ? new Date(record.end_time).getTime() : 0;
-                    req = {
-                        product_id: product_id,
-                        direction: 'forward',
-                        time,
-                    };
-                }
-                else {
-                    // backward
-                    req = {
-                        product_id,
-                        direction: 'backward',
-                        time: Date.now(),
-                    };
-                }
-                console.info(formatTime(Date.now()), 'DispatchIngestInterestRateRequestForwardTask', `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`);
-                const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
-                terminal.metrics
-                    .counter('series_collector_forwards_ingest_count', '')
-                    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
-                    .inc(res.wrote_count || 0);
-                console.info(formatTime(Date.now()), 'DispatchIngestInterestRateRequestForwardTaskResult', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
-            }
-        }
-        catch (e) {
-            console.info(formatTime(Date.now()), 'DispatchIngestInterestRateErrorForwards', `series_id=${product_id}`, e);
-        }
-    }));
-})
-    .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))
-    .subscribe();
 //# sourceMappingURL=forwards-interest-rate.js.map

package/dist/series-collector/forwards-interest-rate.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;AAErC,OAAO,EAIL,0CAA0C,GAC3C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,UAAU,CAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACnE,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,0CAA0C,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAE5E,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;iBAClC;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,KAAK,CAAC,KAAK,IAAI,EAAE;IACf,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,mDAAmD,UAAU,CAAC,IAAI,wCAAwC,CAC3G,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE;;QAC/C,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,EAAE;gBAC3D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,IAAI,GAA+B,CAAC;gBACpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;oBAChC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,4DAA4D,SAAS,CACnE,UAAU,CACX,kEAAkE,CACpE,CAAC;oBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAE9D,GAAG,GAAG;wBACJ,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,SAAS;wBACpB,IAAI;qBACL,CAAC;iBACH;qBAAM;oBACL,WAAW;oBACX,GAAG,GAAG;wBACJ,UAAU;wBACV,SAAS,EAAE,UAAU;wBACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;qBACjB,CAAC;iBACH;gBAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,8CAA8C,EAC9C,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,oDAAoD,EACpD,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,yCAAyC,EACzC,aAAa,UAAU,EAAE,EACzB,CAAC,CACF,CAAC;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n  parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][InterestRate][Forwards] calc');\n\n  const series_ids = new Map<string, IInterestRateServiceMetadata>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.set(product_id, meta);\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const series_ids = await listBackwardSeriesIds();\n  console.log(\n    `[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async ([product_id, meta]) => {\n      try {\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`interest_rate_forwards:${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          let req: IIngestInterestRateRequest;\n          if (meta.direction === 'forward') {\n            const [record] = await requestSQL<\n              {\n                end_time: string;\n              }[]\n            >(\n              terminal,\n              `select end_time from series_data_range where series_id = ${escapeSQL(\n                product_id,\n              )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n            );\n            const time = record ? new Date(record.end_time).getTime() : 0;\n\n            req = {\n              product_id: product_id,\n              direction: 'forward',\n              time,\n            };\n          } else {\n            // backward\n            req = {\n              product_id,\n              direction: 'backward',\n              time: Date.now(),\n            };\n          }\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequestForwardTask',\n            `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n          );\n          const res = await terminal.client.requestForResponseData<\n            IIngestInterestRateRequest,\n            ISeriesIngestResult\n          >('IngestInterestRate', req);\n\n          terminal.metrics\n            .counter('series_collector_forwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequestForwardTaskResult',\n            `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(\n          formatTime(Date.now()),\n          'DispatchIngestInterestRateErrorForwards',\n          `series_id=${product_id}`,\n          e,\n        );\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}
+{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;AAOrC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,8BAA8B,EAAE,MAAM,eAAe,CAAC;AAE/D,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,EAClD,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEhF;QACE,IAAI,GAA+B,CAAC;QACpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;YAChC,MAAM,OAAO,GAAG,MAAM,8BAA8B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAEvD,GAAG,GAAG;gBACJ,UAAU,EAAE,UAAU;gBACtB,SAAS,EAAE,SAAS;gBACpB,IAAI;aACL,CAAC;SACH;aAAM;YACL,WAAW;YACX,GAAG,GAAG;gBACJ,UAAU;gBACV,SAAS,EAAE,UAAU;gBACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;aACjB,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;QAEF,QAAQ,CAAC,OAAO;aACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;aACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;aACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAE7B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;KACH;AACH,CAAC,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { findInterestRateEndTimeForward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nexport const handleIngestInterestRateForward = async (\n  product_id: string,\n  meta: IInterestRateServiceMetadata,\n  signal: AbortSignal,\n) => {\n  const [datasource_id] = decodePath(product_id);\n  // 控制速率：每个数据源每秒钟只能请求一次\n  await tokenBucket(`interest_rate_forwards:${datasource_id}`).acquire(1, signal);\n\n  {\n    let req: IIngestInterestRateRequest;\n    if (meta.direction === 'forward') {\n      const endTime = await findInterestRateEndTimeForward(terminal, product_id);\n      const time = endTime ? new Date(endTime).getTime() : 0;\n\n      req = {\n        product_id: product_id,\n        direction: 'forward',\n        time,\n      };\n    } else {\n      // backward\n      req = {\n        product_id,\n        direction: 'backward',\n        time: Date.now(),\n      };\n    }\n\n    console.info(\n      formatTime(Date.now()),\n      '[SeriesCollector][InterestRate][Forward]',\n      'Request',\n      `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n    );\n    const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n      'IngestInterestRate',\n      req,\n    );\n\n    terminal.metrics\n      .counter('series_collector_forwards_ingest_count', '')\n      .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n      .inc(res.wrote_count || 0);\n\n    console.info(\n      formatTime(Date.now()),\n      '[SeriesCollector][InterestRate][Forward]',\n      'Result',\n      `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n        res.range?.start_time ?? NaN,\n      )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n    );\n  }\n};\n"]}

package/dist/series-collector/index.js

@@ -1,5 +1,4 @@
-import './backwards-interest-rate';
 import './backwards-ohlc';
-import './forwards-interest-rate';
 import './forwards-ohlc';
+import './interest-rate';
 //# sourceMappingURL=index.js.map

package/dist/series-collector/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,2BAA2B,CAAC;AACnC,OAAO,kBAAkB,CAAC;AAC1B,OAAO,0BAA0B,CAAC;AAClC,OAAO,iBAAiB,CAAC","sourcesContent":["import './backwards-interest-rate';\nimport './backwards-ohlc';\nimport './forwards-interest-rate';\nimport './forwards-ohlc';\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAC1B,OAAO,iBAAiB,CAAC;AACzB,OAAO,iBAAiB,CAAC","sourcesContent":["import './backwards-ohlc';\nimport './forwards-ohlc';\nimport './interest-rate';\n"]}

package/dist/series-collector/interest-rate.js

@@ -0,0 +1,43 @@
+import { listWatch } from '@yuants/utils';
+import { defer, map, Observable, repeat, retry } from 'rxjs';
+import { handleIngestInterestRateBackward } from './backwards-interest-rate';
+import { listInterestRateSeriesIds } from './discovery';
+import { handleIngestInterestRateForward } from './forwards-interest-rate';
+import { handleInterestRatePatch } from './patch-interest-rate';
+defer(() => listInterestRateSeriesIds())
+    .pipe(retry({ delay: 1000 }), repeat({ delay: 60000 }), map((x) => Array.from(x.entries())), listWatch((x) => x[0], ([product_id, meta]) => new Observable((sub) => {
+    // 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度
+    const abortController = new AbortController();
+    sub.add(() => {
+        abortController.abort();
+    });
+    // 先处理前向任务
+    const forwardTask = defer(async () => {
+        await handleIngestInterestRateForward(product_id, meta, abortController.signal);
+    })
+        .pipe(retry(), repeat())
+        .subscribe();
+    sub.add(() => {
+        forwardTask.unsubscribe();
+    });
+    // 设置后向任务
+    const backwardTask = defer(async () => {
+        await handleIngestInterestRateBackward(product_id, meta, abortController.signal);
+    })
+        .pipe(retry(), repeat())
+        .subscribe();
+    sub.add(() => {
+        backwardTask.unsubscribe();
+    });
+    // 设置补齐任务
+    const patchTask = defer(async () => {
+        await handleInterestRatePatch(product_id, meta, abortController.signal);
+    })
+        .pipe(retry(), repeat())
+        .subscribe();
+    sub.add(() => {
+        patchTask.unsubscribe();
+    });
+})))
+    .subscribe();
+//# sourceMappingURL=interest-rate.js.map

package/dist/series-collector/interest-rate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7D,OAAO,EAAE,gCAAgC,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,+BAA+B,EAAE,MAAM,0BAA0B,CAAC;AAC3E,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAEhE,KAAK,CAAC,GAAG,EAAE,CAAC,yBAAyB,EAAE,CAAC;KACrC,IAAI,CACH,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtB,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EACnC,SAAS,CACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EACX,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,CACrB,IAAI,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,mDAAmD;IACnD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAE9C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE;QACnC,MAAM,+BAA+B,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAClF,CAAC,CAAC;SACC,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;SACvB,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE;QACpC,MAAM,gCAAgC,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IACnF,CAAC,CAAC;SACC,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;SACvB,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,YAAY,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE;QACjC,MAAM,uBAAuB,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC,CAAC;SACC,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;SACvB,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,SAAS,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CACL,CACF;KACA,SAAS,EAAE,CAAC","sourcesContent":["import { listWatch } from '@yuants/utils';\nimport { defer, map, Observable, repeat, retry } from 'rxjs';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { listInterestRateSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleInterestRatePatch } from './patch-interest-rate';\n\ndefer(() => listInterestRateSeriesIds())\n  .pipe(\n    retry({ delay: 1000 }),\n    repeat({ delay: 60000 }),\n    map((x) => Array.from(x.entries())),\n    listWatch(\n      (x) => x[0],\n      ([product_id, meta]) =>\n        new Observable((sub) => {\n          // 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度\n          const abortController = new AbortController();\n\n          sub.add(() => {\n            abortController.abort();\n          });\n\n          // 先处理前向任务\n          const forwardTask = defer(async () => {\n            await handleIngestInterestRateForward(product_id, meta, abortController.signal);\n          })\n            .pipe(retry(), repeat())\n            .subscribe();\n\n          sub.add(() => {\n            forwardTask.unsubscribe();\n          });\n\n          // 设置后向任务\n          const backwardTask = defer(async () => {\n            await handleIngestInterestRateBackward(product_id, meta, abortController.signal);\n          })\n            .pipe(retry(), repeat())\n            .subscribe();\n\n          sub.add(() => {\n            backwardTask.unsubscribe();\n          });\n\n          // 设置补齐任务\n          const patchTask = defer(async () => {\n            await handleInterestRatePatch(product_id, meta, abortController.signal);\n          })\n            .pipe(retry(), repeat())\n            .subscribe();\n\n          sub.add(() => {\n            patchTask.unsubscribe();\n          });\n        }),\n    ),\n  )\n  .subscribe();\n"]}

package/dist/series-collector/patch-interest-rate.js

@@ -0,0 +1,59 @@
+import { Terminal } from '@yuants/protocol';
+import { escapeSQL, requestSQL } from '@yuants/sql';
+import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+const terminal = Terminal.fromNodeEnv();
+// Patch 任务：查找数据缺口并进行补齐
+export const handleInterestRatePatch = async (product_id, meta, signal) => {
+    var _a, _b, _c, _d;
+    const [datasource_id] = decodePath(product_id);
+    await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);
+    const [record] = await requestSQL(terminal, `
+WITH reversed_ranges AS (
+    SELECT 
+        start_time,
+        end_time,
+        LEAD(end_time) OVER (
+            PARTITION BY table_name, series_id 
+            ORDER BY start_time DESC
+        ) AS next_end_time  -- 注意：倒序时 LEAD 是前一个区间
+    FROM series_data_range
+    WHERE table_name = 'interest_rate' 
+      AND series_id = ${escapeSQL(product_id)}
+)
+SELECT 
+    next_end_time AS gap_start_time,  -- 前一个区间的结束时间
+    start_time AS gap_end_time        -- 当前区间的开始时间
+FROM reversed_ranges
+WHERE next_end_time IS NOT NULL 
+  AND start_time > next_end_time      -- 有空缺
+ORDER BY start_time DESC              -- 从最新开始
+LIMIT 1;
+    `);
+    // no gap
+    if (!record)
+        return;
+    const gapStartTime = new Date(record.gap_start_time).getTime();
+    const gapEndTime = new Date(record.gap_end_time).getTime();
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Patch]', 'FindGap', `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`);
+    let req;
+    if (meta.direction === 'forward') {
+        // forward patch
+        req = {
+            product_id: product_id,
+            direction: 'forward',
+            time: gapStartTime,
+        };
+    }
+    else {
+        // backward patch
+        req = {
+            product_id: product_id,
+            direction: 'backward',
+            time: gapEndTime,
+        };
+    }
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchRequest', `direction=${req.direction}, time=${formatTime(req.time)}`);
+    const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchBackwardResult', `ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+};
+//# sourceMappingURL=patch-interest-rate.js.map

package/dist/series-collector/patch-interest-rate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEpE,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,uBAAuB;AACvB,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,WAAW,CAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,SAAS,CAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,UAAU,CAAC,YAAY,CAAC,QAAQ,UAAU,CAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;QAChC,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n  product_id: string,\n  meta: IInterestRateServiceMetadata,\n  signal: AbortSignal,\n) => {\n  const [datasource_id] = decodePath(product_id);\n  await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);\n  const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n    terminal,\n    `\nWITH reversed_ranges AS (\n    SELECT \n        start_time,\n        end_time,\n        LEAD(end_time) OVER (\n            PARTITION BY table_name, series_id \n            ORDER BY start_time DESC\n        ) AS next_end_time  -- 注意：倒序时 LEAD 是前一个区间\n    FROM series_data_range\n    WHERE table_name = 'interest_rate' \n      AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n    next_end_time AS gap_start_time,  -- 前一个区间的结束时间\n    start_time AS gap_end_time        -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n  AND start_time > next_end_time      -- 有空缺\nORDER BY start_time DESC              -- 从最新开始\nLIMIT 1;\n    `,\n  );\n\n  // no gap\n  if (!record) return;\n\n  const gapStartTime = new Date(record.gap_start_time).getTime();\n  const gapEndTime = new Date(record.gap_end_time).getTime();\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Patch]',\n    'FindGap',\n    `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n  );\n\n  let req: IIngestInterestRateRequest;\n\n  if (meta.direction === 'forward') {\n    // forward patch\n    req = {\n      product_id: product_id,\n      direction: 'forward' as const,\n      time: gapStartTime,\n    };\n  } else {\n    // backward patch\n    req = {\n      product_id: product_id,\n      direction: 'backward' as const,\n      time: gapEndTime,\n    };\n  }\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Patch]',\n    product_id,\n    'PatchRequest',\n    `direction=${req.direction}, time=${formatTime(req.time)}`,\n  );\n\n  const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n    'IngestInterestRate',\n    req,\n  );\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Patch]',\n    product_id,\n    'PatchBackwardResult',\n    `ingested_count=${res.wrote_count}, start_time=${formatTime(\n      res.range?.start_time ?? NaN,\n    )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n  );\n};\n"]}

package/dist/series-collector/sql-helpers.js

@@ -0,0 +1,11 @@
+import { escapeSQL, requestSQL } from '@yuants/sql';
+/**
+ * 查找某个利率品种系列ID的最新结束时间（for forward task）
+ * @param terminal
+ * @param product_id
+ * @returns
+ */
+export const findInterestRateEndTimeForward = (terminal, product_id) => requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.end_time; });
+export const findInterestRateStartTimeBackward = (terminal, product_id) => requestSQL(terminal, `select start_time from series_data_range where series_id = ${escapeSQL(product_id)} and table_name = 'interest_rate' order by start_time asc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.start_time; });
+export const findOHLCEndTimeForward = (terminal, series_id) => requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.end_time; });
+//# sourceMappingURL=sql-helpers.js.map

package/dist/series-collector/sql-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-helpers.js","sourceRoot":"","sources":["../../src/series-collector/sql-helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,QAAkB,EAAE,UAAkB,EAAE,EAAE,CACvF,UAAU,CAKR,QAAQ,EACR,4DAA4D,SAAS,CACnE,UAAU,CACX,kEAAkE,CACpE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,QAAQ,CAAA,EAAA,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,QAAkB,EAAE,UAAkB,EAAE,EAAE,CAC1F,UAAU,CAKR,QAAQ,EACR,8DAA8D,SAAS,CACrE,UAAU,CACX,mEAAmE,CACrE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,UAAU,CAAA,EAAA,CAAC,CAAC;AAEhD,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,QAAkB,EAAE,SAAiB,EAAE,EAAE,CAC9E,UAAU,CAKR,QAAQ,EACR,4DAA4D,SAAS,CACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,QAAQ,CAAA,EAAA,CAAC,CAAC","sourcesContent":["import { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\n\n/**\n * 查找某个利率品种系列ID的最新结束时间（for forward task）\n * @param terminal\n * @param product_id\n * @returns\n */\nexport const findInterestRateEndTimeForward = (terminal: Terminal, product_id: string) =>\n  requestSQL<\n    {\n      end_time: string;\n    }[]\n  >(\n    terminal,\n    `select end_time from series_data_range where series_id = ${escapeSQL(\n      product_id,\n    )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n  ).then((records) => records?.[0]?.end_time);\n\nexport const findInterestRateStartTimeBackward = (terminal: Terminal, product_id: string) =>\n  requestSQL<\n    {\n      start_time: string;\n    }[]\n  >(\n    terminal,\n    `select start_time from series_data_range where series_id = ${escapeSQL(\n      product_id,\n    )} and table_name = 'interest_rate' order by start_time asc limit 1`,\n  ).then((records) => records?.[0]?.start_time);\n\nexport const findOHLCEndTimeForward = (terminal: Terminal, series_id: string) =>\n  requestSQL<\n    {\n      end_time: string;\n    }[]\n  >(\n    terminal,\n    `select end_time from series_data_range where series_id = ${escapeSQL(\n      series_id,\n    )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n  ).then((records) => records?.[0]?.end_time);\n"]}

package/lib/series-collector/backwards-interest-rate.d.ts

@@ -1,2 +1,4 @@
-export {};
+/// <reference types="node" />
+import { IInterestRateServiceMetadata } from '@yuants/exchange';
+export declare const handleIngestInterestRateBackward: (product_id: string, meta: IInterestRateServiceMetadata, signal: AbortSignal) => Promise<void>;
 //# sourceMappingURL=backwards-interest-rate.d.ts.map

package/lib/series-collector/backwards-interest-rate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":""}
+{"version":3,"file":"backwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";AAKA,OAAO,EAEL,4BAA4B,EAE7B,MAAM,kBAAkB,CAAC;AAO1B,eAAO,MAAM,gCAAgC,eAC/B,MAAM,QACZ,4BAA4B,UAC1B,WAAW,kBAgDpB,CAAC"}

package/lib/series-collector/backwards-interest-rate.js

@@ -4,72 +4,40 @@
 // 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。
 // 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
 Object.defineProperty(exports, "__esModule", { value: true });
-const exchange_1 = require("@yuants/exchange");
+exports.handleIngestInterestRateBackward = void 0;
 const protocol_1 = require("@yuants/protocol");
-const sql_1 = require("@yuants/sql");
 const utils_1 = require("@yuants/utils");
-const rxjs_1 = require("rxjs");
+const sql_helpers_1 = require("./sql-helpers");
 const terminal = protocol_1.Terminal.fromNodeEnv();
-const listBackwardSeriesIds = async () => {
-    const product_ids = await (0, sql_1.requestSQL)(terminal, `select product_id from product`);
-    console.time('[SeriesCollector][InterestRate][Backwards] calc');
-    const series_ids = new Set();
-    for (const terminalInfo of terminal.terminalInfos) {
-        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
-            if (serviceInfo.method !== 'IngestInterestRate')
-                continue;
-            try {
-                const meta = (0, exchange_1.parseInterestRateServiceMetadataFromSchema)(serviceInfo.schema);
-                if (meta.direction !== 'backward')
-                    continue;
-                for (const { product_id } of product_ids) {
-                    if (!product_id.startsWith(meta.product_id_prefix))
-                        continue;
-                    series_ids.add(product_id);
-                }
-            }
-            finally {
-            }
-        }
+const handleIngestInterestRateBackward = async (product_id, meta, signal) => {
+    var _a, _b, _c, _d;
+    const [datasource_id] = (0, utils_1.decodePath)(product_id);
+    // 控制速率：每个数据源每秒钟只能请求一次
+    await (0, utils_1.tokenBucket)(`interest_rate:backward:${datasource_id}`).acquire(1, signal);
+    let req;
+    if (meta.direction === 'backward') {
+        const startTime = await (0, sql_helpers_1.findInterestRateStartTimeBackward)(terminal, product_id);
+        const start_time = startTime ? new Date(startTime).getTime() : Date.now();
+        req = {
+            product_id: product_id,
+            direction: 'backward',
+            time: start_time,
+        };
     }
-    console.timeEnd('[SeriesCollector][InterestRate][Backwards] calc');
-    return series_ids;
+    else {
+        req = {
+            product_id: product_id,
+            direction: 'forward',
+            time: 0,
+        };
+    }
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Backward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
+    const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+    terminal.metrics
+        .counter('series_collector_ingest_count', '')
+        .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' })
+        .inc(res.wrote_count || 0);
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Backward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
 };
-(0, rxjs_1.defer)(async () => {
-    const time = Date.now();
-    const series_ids = await listBackwardSeriesIds();
-    console.log(`[SeriesCollector][InterestRate][Backwards] Found ${series_ids.size} series to collect backwards data for. (${(0, utils_1.formatTime)(Date.now() - time)})`);
-    await Promise.all([...series_ids].map(async (product_id) => {
-        var _a, _b, _c, _d;
-        try {
-            const [datasource_id] = (0, utils_1.decodePath)(product_id);
-            // 控制速率：每个数据源每秒钟只能请求一次
-            await (0, utils_1.tokenBucket)(`interest_rate_backwards:${datasource_id}`, {
-                refillInterval: 1000,
-                capacity: 1,
-            }).acquire();
-            {
-                const [record] = await (0, sql_1.requestSQL)(terminal, `select start_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by start_time limit 1`);
-                const start_time = record ? new Date(record.start_time).getTime() : Date.now();
-                const req = {
-                    product_id: product_id,
-                    direction: 'backward',
-                    time: start_time,
-                };
-                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateRequest', `product_id=${product_id}, time=${(0, utils_1.formatTime)(start_time)}`);
-                const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
-                terminal.metrics
-                    .counter('series_collector_backwards_ingest_count', '')
-                    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
-                    .inc(res.wrote_count || 0);
-                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateResult', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
-            }
-        }
-        catch (e) {
-            console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateError', `series_id=${product_id}`, e);
-        }
-    }));
-})
-    .pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 1000 }))
-    .subscribe();
+exports.handleIngestInterestRateBackward = handleIngestInterestRateBackward;
 //# sourceMappingURL=backwards-interest-rate.js.map

package/lib/series-collector/backwards-interest-rate.js.map

@@ -1 +1 @@
-{"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,yDAAyD;AACzD,qCAAqC;;AAErC,+CAI0B;AAC1B,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AACpE,+BAA4C;AAE5C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,IAAA,gBAAU,EAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,IAAA,qDAA0C,EAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC5E,IAAI,IAAI,CAAC,SAAS,KAAK,UAAU;oBAAE,SAAS;gBAE5C,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;iBAC5B;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAEnE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,oDACE,UAAU,CAAC,IACb,2CAA2C,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAC5E,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;;QACvC,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,IAAA,mBAAW,EAAC,2BAA2B,aAAa,EAAE,EAAE;gBAC5D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,8DAA8D,IAAA,eAAS,EACrE,UAAU,CACX,+DAA+D,CACjE,CAAC;gBACF,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE/E,MAAM,GAAG,GAA+B;oBACtC,UAAU,EAAE,UAAU;oBACtB,SAAS,EAAE,UAAU;oBACrB,IAAI,EAAE,UAAU;iBACjB,CAAC;gBAEF,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,cAAc,UAAU,UAAU,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CAC3D,CAAC;gBAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC;qBACtD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,iCAAiC,EAAE,aAAa,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;SACvG;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  ISeriesIngestResult,\n  parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][InterestRate][Backwards] calc');\n\n  const series_ids = new Set<string>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n        if (meta.direction !== 'backward') continue;\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.add(product_id);\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][InterestRate][Backwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const time = Date.now();\n  const series_ids = await listBackwardSeriesIds();\n  console.log(\n    `[SeriesCollector][InterestRate][Backwards] Found ${\n      series_ids.size\n    } series to collect backwards data for. (${formatTime(Date.now() - time)})`,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async (product_id) => {\n      try {\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`interest_rate_backwards:${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          const [record] = await requestSQL<\n            {\n              start_time: string;\n            }[]\n          >(\n            terminal,\n            `select start_time from series_data_range where series_id = ${escapeSQL(\n              product_id,\n            )} and table_name = 'interest_rate' order by start_time limit 1`,\n          );\n          const start_time = record ? new Date(record.start_time).getTime() : Date.now();\n\n          const req: IIngestInterestRateRequest = {\n            product_id: product_id,\n            direction: 'backward',\n            time: start_time,\n          };\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequest',\n            `product_id=${product_id}, time=${formatTime(start_time)}`,\n          );\n\n          const res = await terminal.client.requestForResponseData<\n            IIngestInterestRateRequest,\n            ISeriesIngestResult\n          >('IngestInterestRate', req);\n\n          terminal.metrics\n            .counter('series_collector_backwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateResult',\n            `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(formatTime(Date.now()), 'DispatchIngestInterestRateError', `series_id=${product_id}`, e);\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}
+{"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,yDAAyD;AACzD,qCAAqC;;;AAOrC,+CAA4C;AAC5C,yCAAoE;AACpE,+CAAkE;AAElE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAEjC,MAAM,gCAAgC,GAAG,KAAK,EACnD,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAChF,IAAI,GAA+B,CAAC;IACpC,IAAI,IAAI,CAAC,SAAS,KAAK,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAiC,EAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1E,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,QAAQ,CAAC,OAAO;SACb,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;SAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;SACtF,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAnDW,QAAA,gCAAgC,oCAmD3C","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { findInterestRateStartTimeBackward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nexport const handleIngestInterestRateBackward = async (\n  product_id: string,\n  meta: IInterestRateServiceMetadata,\n  signal: AbortSignal,\n) => {\n  const [datasource_id] = decodePath(product_id);\n  // 控制速率：每个数据源每秒钟只能请求一次\n  await tokenBucket(`interest_rate:backward:${datasource_id}`).acquire(1, signal);\n  let req: IIngestInterestRateRequest;\n  if (meta.direction === 'backward') {\n    const startTime = await findInterestRateStartTimeBackward(terminal, product_id);\n    const start_time = startTime ? new Date(startTime).getTime() : Date.now();\n\n    req = {\n      product_id: product_id,\n      direction: 'backward',\n      time: start_time,\n    };\n  } else {\n    req = {\n      product_id: product_id,\n      direction: 'forward',\n      time: 0,\n    };\n  }\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Backward]',\n    'Request',\n    `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n  );\n\n  const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n    'IngestInterestRate',\n    req,\n  );\n\n  terminal.metrics\n    .counter('series_collector_ingest_count', '')\n    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' })\n    .inc(res.wrote_count || 0);\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Backward]',\n    'Result',\n    `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n      res.range?.start_time ?? NaN,\n    )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n  );\n};\n"]}

package/lib/series-collector/discovery.d.ts

@@ -0,0 +1,7 @@
+import { IInterestRateServiceMetadata } from '@yuants/exchange';
+/**
+ * 列出所有支持利率的品种系列ID以及对应的 Service Metadata
+ * @returns
+ */
+export declare const listInterestRateSeriesIds: () => Promise<Map<string, IInterestRateServiceMetadata>>;
+//# sourceMappingURL=discovery.d.ts.map

package/lib/series-collector/discovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/series-collector/discovery.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAA8C,MAAM,kBAAkB,CAAC;AAM5G;;;GAGG;AACH,eAAO,MAAM,yBAAyB,0DAwBrC,CAAC"}

package/lib/series-collector/discovery.js

@@ -0,0 +1,37 @@
+"use strict";
+// 发现所有支持利率的品种系列ID
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listInterestRateSeriesIds = void 0;
+const exchange_1 = require("@yuants/exchange");
+const protocol_1 = require("@yuants/protocol");
+const sql_1 = require("@yuants/sql");
+const terminal = protocol_1.Terminal.fromNodeEnv();
+/**
+ * 列出所有支持利率的品种系列ID以及对应的 Service Metadata
+ * @returns
+ */
+const listInterestRateSeriesIds = async () => {
+    const product_ids = await (0, sql_1.requestSQL)(terminal, 
+    // 必须是支持利率的品种
+    `select product_id from product where no_interest_rate = false`);
+    const series_ids = new Map();
+    for (const terminalInfo of terminal.terminalInfos) {
+        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
+            if (serviceInfo.method !== 'IngestInterestRate')
+                continue;
+            try {
+                const meta = (0, exchange_1.parseInterestRateServiceMetadataFromSchema)(serviceInfo.schema);
+                for (const { product_id } of product_ids) {
+                    if (!product_id.startsWith(meta.product_id_prefix))
+                        continue;
+                    series_ids.set(product_id, meta);
+                }
+            }
+            finally {
+            }
+        }
+    }
+    return series_ids;
+};
+exports.listInterestRateSeriesIds = listInterestRateSeriesIds;
+//# sourceMappingURL=discovery.js.map

package/lib/series-collector/discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.js","sourceRoot":"","sources":["../../src/series-collector/discovery.ts"],"names":[],"mappings":";AAAA,kBAAkB;;;AAElB,+CAA4G;AAC5G,+CAA4C;AAC5C,qCAAyC;AAEzC,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC;;;GAGG;AACI,MAAM,yBAAyB,GAAG,KAAK,IAAI,EAAE;IAClD,MAAM,WAAW,GAAG,MAAM,IAAA,gBAAU,EAClC,QAAQ;IACR,aAAa;IACb,+DAA+D,CAChE,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACnE,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,IAAA,qDAA0C,EAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAE5E,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;iBAClC;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAxBW,QAAA,yBAAyB,6BAwBpC","sourcesContent":["// 发现所有支持利率的品种系列ID\n\nimport { IInterestRateServiceMetadata, parseInterestRateServiceMetadataFromSchema } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { requestSQL } from '@yuants/sql';\n\nconst terminal = Terminal.fromNodeEnv();\n\n/**\n * 列出所有支持利率的品种系列ID以及对应的 Service Metadata\n * @returns\n */\nexport const listInterestRateSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(\n    terminal,\n    // 必须是支持利率的品种\n    `select product_id from product where no_interest_rate = false`,\n  );\n\n  const series_ids = new Map<string, IInterestRateServiceMetadata>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.set(product_id, meta);\n        }\n      } finally {\n      }\n    }\n  }\n\n  return series_ids;\n};\n"]}

package/lib/series-collector/forwards-interest-rate.d.ts

@@ -1,2 +1,4 @@
-export {};
+/// <reference types="node" />
+import { IInterestRateServiceMetadata } from '@yuants/exchange';
+export declare const handleIngestInterestRateForward: (product_id: string, meta: IInterestRateServiceMetadata, signal: AbortSignal) => Promise<void>;
 //# sourceMappingURL=forwards-interest-rate.d.ts.map

package/lib/series-collector/forwards-interest-rate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":""}
+{"version":3,"file":"forwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAKA,OAAO,EAEL,4BAA4B,EAE7B,MAAM,kBAAkB,CAAC;AAO1B,eAAO,MAAM,+BAA+B,eAC9B,MAAM,QACZ,4BAA4B,UAC1B,WAAW,kBAmDpB,CAAC"}

package/lib/series-collector/forwards-interest-rate.js

@@ -4,80 +4,43 @@
 // 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。
 // 使用 Token Bucket 控制每个数据源的请求速率，避免过载。
 Object.defineProperty(exports, "__esModule", { value: true });
-const exchange_1 = require("@yuants/exchange");
+exports.handleIngestInterestRateForward = void 0;
 const protocol_1 = require("@yuants/protocol");
-const sql_1 = require("@yuants/sql");
 const utils_1 = require("@yuants/utils");
-const rxjs_1 = require("rxjs");
+const sql_helpers_1 = require("./sql-helpers");
 const terminal = protocol_1.Terminal.fromNodeEnv();
-const listBackwardSeriesIds = async () => {
-    const product_ids = await (0, sql_1.requestSQL)(terminal, `select product_id from product`);
-    console.time('[SeriesCollector][InterestRate][Forwards] calc');
-    const series_ids = new Map();
-    for (const terminalInfo of terminal.terminalInfos) {
-        for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {
-            if (serviceInfo.method !== 'IngestInterestRate')
-                continue;
-            try {
-                const meta = (0, exchange_1.parseInterestRateServiceMetadataFromSchema)(serviceInfo.schema);
-                for (const { product_id } of product_ids) {
-                    if (!product_id.startsWith(meta.product_id_prefix))
-                        continue;
-                    series_ids.set(product_id, meta);
-                }
-            }
-            finally {
-            }
+const handleIngestInterestRateForward = async (product_id, meta, signal) => {
+    var _a, _b, _c, _d;
+    const [datasource_id] = (0, utils_1.decodePath)(product_id);
+    // 控制速率：每个数据源每秒钟只能请求一次
+    await (0, utils_1.tokenBucket)(`interest_rate_forwards:${datasource_id}`).acquire(1, signal);
+    {
+        let req;
+        if (meta.direction === 'forward') {
+            const endTime = await (0, sql_helpers_1.findInterestRateEndTimeForward)(terminal, product_id);
+            const time = endTime ? new Date(endTime).getTime() : 0;
+            req = {
+                product_id: product_id,
+                direction: 'forward',
+                time,
+            };
         }
+        else {
+            // backward
+            req = {
+                product_id,
+                direction: 'backward',
+                time: Date.now(),
+            };
+        }
+        console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
+        const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+        terminal.metrics
+            .counter('series_collector_forwards_ingest_count', '')
+            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
+            .inc(res.wrote_count || 0);
+        console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
     }
-    console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');
-    return series_ids;
 };
-(0, rxjs_1.defer)(async () => {
-    const series_ids = await listBackwardSeriesIds();
-    console.log(`[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `);
-    await Promise.all([...series_ids].map(async ([product_id, meta]) => {
-        var _a, _b, _c, _d;
-        try {
-            const [datasource_id] = (0, utils_1.decodePath)(product_id);
-            // 控制速率：每个数据源每秒钟只能请求一次
-            await (0, utils_1.tokenBucket)(`interest_rate_forwards:${datasource_id}`, {
-                refillInterval: 1000,
-                capacity: 1,
-            }).acquire();
-            {
-                let req;
-                if (meta.direction === 'forward') {
-                    const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`);
-                    const time = record ? new Date(record.end_time).getTime() : 0;
-                    req = {
-                        product_id: product_id,
-                        direction: 'forward',
-                        time,
-                    };
-                }
-                else {
-                    // backward
-                    req = {
-                        product_id,
-                        direction: 'backward',
-                        time: Date.now(),
-                    };
-                }
-                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateRequestForwardTask', `product_id=${req.product_id}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
-                const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
-                terminal.metrics
-                    .counter('series_collector_forwards_ingest_count', '')
-                    .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })
-                    .inc(res.wrote_count || 0);
-                console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateRequestForwardTaskResult', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
-            }
-        }
-        catch (e) {
-            console.info((0, utils_1.formatTime)(Date.now()), 'DispatchIngestInterestRateErrorForwards', `series_id=${product_id}`, e);
-        }
-    }));
-})
-    .pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 1000 }))
-    .subscribe();
+exports.handleIngestInterestRateForward = handleIngestInterestRateForward;
 //# sourceMappingURL=forwards-interest-rate.js.map

package/lib/series-collector/forwards-interest-rate.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;;AAErC,+CAK0B;AAC1B,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AACpE,+BAA4C;AAE5C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,qBAAqB,GAAG,KAAK,IAAI,EAAE;IACvC,MAAM,WAAW,GAAG,MAAM,IAAA,gBAAU,EAA2B,QAAQ,EAAE,gCAAgC,CAAC,CAAC;IAE3G,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACnE,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,oBAAoB;gBAAE,SAAS;YAC1D,IAAI;gBACF,MAAM,IAAI,GAAG,IAAA,qDAA0C,EAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAE5E,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE;oBACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;wBAAE,SAAS;oBAC7D,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;iBAClC;aACF;oBAAS;aACT;SACF;KACF;IAED,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;IACf,MAAM,UAAU,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CACT,mDAAmD,UAAU,CAAC,IAAI,wCAAwC,CAC3G,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE;;QAC/C,IAAI;YACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;YAC/C,sBAAsB;YACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,EAAE;gBAC3D,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC,OAAO,EAAE,CAAC;YACb;gBACE,IAAI,GAA+B,CAAC;gBACpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;oBAChC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,UAAU,CACX,kEAAkE,CACpE,CAAC;oBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAE9D,GAAG,GAAG;wBACJ,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,SAAS;wBACpB,IAAI;qBACL,CAAC;iBACH;qBAAM;oBACL,WAAW;oBACX,GAAG,GAAG;wBACJ,UAAU;wBACV,SAAS,EAAE,UAAU;wBACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;qBACjB,CAAC;iBACH;gBAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,8CAA8C,EAC9C,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAGtD,oBAAoB,EAAE,GAAG,CAAC,CAAC;gBAE7B,QAAQ,CAAC,OAAO;qBACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;qBACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;gBAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,oDAAoD,EACpD,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;aACH;SACF;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,yCAAyC,EACzC,aAAa,UAAU,EAAE,EACzB,CAAC,CACF,CAAC;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;KACC,IAAI,CAAC,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KACrD,SAAS,EAAE,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n  parseInterestRateServiceMetadataFromSchema,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { defer, repeat, retry } from 'rxjs';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst listBackwardSeriesIds = async () => {\n  const product_ids = await requestSQL<{ product_id: string }[]>(terminal, `select product_id from product`);\n\n  console.time('[SeriesCollector][InterestRate][Forwards] calc');\n\n  const series_ids = new Map<string, IInterestRateServiceMetadata>();\n  for (const terminalInfo of terminal.terminalInfos) {\n    for (const serviceInfo of Object.values(terminalInfo.serviceInfo || {})) {\n      if (serviceInfo.method !== 'IngestInterestRate') continue;\n      try {\n        const meta = parseInterestRateServiceMetadataFromSchema(serviceInfo.schema);\n\n        for (const { product_id } of product_ids) {\n          if (!product_id.startsWith(meta.product_id_prefix)) continue;\n          series_ids.set(product_id, meta);\n        }\n      } finally {\n      }\n    }\n  }\n\n  console.timeEnd('[SeriesCollector][InterestRate][Forwards] calc');\n\n  return series_ids;\n};\n\ndefer(async () => {\n  const series_ids = await listBackwardSeriesIds();\n  console.log(\n    `[SeriesCollector][InterestRate][Forwards] Found ${series_ids.size} series to collect forwards data for. `,\n  );\n\n  await Promise.all(\n    [...series_ids].map(async ([product_id, meta]) => {\n      try {\n        const [datasource_id] = decodePath(product_id);\n        // 控制速率：每个数据源每秒钟只能请求一次\n        await tokenBucket(`interest_rate_forwards:${datasource_id}`, {\n          refillInterval: 1000,\n          capacity: 1,\n        }).acquire();\n        {\n          let req: IIngestInterestRateRequest;\n          if (meta.direction === 'forward') {\n            const [record] = await requestSQL<\n              {\n                end_time: string;\n              }[]\n            >(\n              terminal,\n              `select end_time from series_data_range where series_id = ${escapeSQL(\n                product_id,\n              )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n            );\n            const time = record ? new Date(record.end_time).getTime() : 0;\n\n            req = {\n              product_id: product_id,\n              direction: 'forward',\n              time,\n            };\n          } else {\n            // backward\n            req = {\n              product_id,\n              direction: 'backward',\n              time: Date.now(),\n            };\n          }\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequestForwardTask',\n            `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n          );\n          const res = await terminal.client.requestForResponseData<\n            IIngestInterestRateRequest,\n            ISeriesIngestResult\n          >('IngestInterestRate', req);\n\n          terminal.metrics\n            .counter('series_collector_forwards_ingest_count', '')\n            .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n            .inc(res.wrote_count || 0);\n\n          console.info(\n            formatTime(Date.now()),\n            'DispatchIngestInterestRateRequestForwardTaskResult',\n            `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n              res.range?.start_time ?? NaN,\n            )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n          );\n        }\n      } catch (e) {\n        console.info(\n          formatTime(Date.now()),\n          'DispatchIngestInterestRateErrorForwards',\n          `series_id=${product_id}`,\n          e,\n        );\n      }\n    }),\n  );\n})\n  .pipe(retry({ delay: 1000 }), repeat({ delay: 1000 }))\n  .subscribe();\n"]}
+{"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,yDAAyD;AACzD,qCAAqC;;;AAOrC,+CAA4C;AAC5C,yCAAoE;AACpE,+CAA+D;AAE/D,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAEjC,MAAM,+BAA+B,GAAG,KAAK,EAClD,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEhF;QACE,IAAI,GAA+B,CAAC;QACpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;YAChC,MAAM,OAAO,GAAG,MAAM,IAAA,4CAA8B,EAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAEvD,GAAG,GAAG;gBACJ,UAAU,EAAE,UAAU;gBACtB,SAAS,EAAE,SAAS;gBACpB,IAAI;aACL,CAAC;SACH;aAAM;YACL,WAAW;YACX,GAAG,GAAG;gBACJ,UAAU;gBACV,SAAS,EAAE,UAAU;gBACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;aACjB,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;QAEF,QAAQ,CAAC,OAAO;aACb,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC;aACrD,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;aACpE,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAE7B,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;KACH;AACH,CAAC,CAAC;AAtDW,QAAA,+BAA+B,mCAsD1C","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestInterestRate Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { findInterestRateEndTimeForward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nexport const handleIngestInterestRateForward = async (\n  product_id: string,\n  meta: IInterestRateServiceMetadata,\n  signal: AbortSignal,\n) => {\n  const [datasource_id] = decodePath(product_id);\n  // 控制速率：每个数据源每秒钟只能请求一次\n  await tokenBucket(`interest_rate_forwards:${datasource_id}`).acquire(1, signal);\n\n  {\n    let req: IIngestInterestRateRequest;\n    if (meta.direction === 'forward') {\n      const endTime = await findInterestRateEndTimeForward(terminal, product_id);\n      const time = endTime ? new Date(endTime).getTime() : 0;\n\n      req = {\n        product_id: product_id,\n        direction: 'forward',\n        time,\n      };\n    } else {\n      // backward\n      req = {\n        product_id,\n        direction: 'backward',\n        time: Date.now(),\n      };\n    }\n\n    console.info(\n      formatTime(Date.now()),\n      '[SeriesCollector][InterestRate][Forward]',\n      'Request',\n      `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n    );\n    const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n      'IngestInterestRate',\n      req,\n    );\n\n    terminal.metrics\n      .counter('series_collector_forwards_ingest_count', '')\n      .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate' })\n      .inc(res.wrote_count || 0);\n\n    console.info(\n      formatTime(Date.now()),\n      '[SeriesCollector][InterestRate][Forward]',\n      'Result',\n      `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n        res.range?.start_time ?? NaN,\n      )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n    );\n  }\n};\n"]}

package/lib/series-collector/index.d.ts

@@ -1,5 +1,4 @@
-import './backwards-interest-rate';
 import './backwards-ohlc';
-import './forwards-interest-rate';
 import './forwards-ohlc';
+import './interest-rate';
 //# sourceMappingURL=index.d.ts.map

package/lib/series-collector/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,2BAA2B,CAAC;AACnC,OAAO,kBAAkB,CAAC;AAC1B,OAAO,0BAA0B,CAAC;AAClC,OAAO,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAC1B,OAAO,iBAAiB,CAAC;AACzB,OAAO,iBAAiB,CAAC"}

package/lib/series-collector/index.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-require("./backwards-interest-rate");
 require("./backwards-ohlc");
-require("./forwards-interest-rate");
 require("./forwards-ohlc");
+require("./interest-rate");
 //# sourceMappingURL=index.js.map

package/lib/series-collector/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,qCAAmC;AACnC,4BAA0B;AAC1B,oCAAkC;AAClC,2BAAyB","sourcesContent":["import './backwards-interest-rate';\nimport './backwards-ohlc';\nimport './forwards-interest-rate';\nimport './forwards-ohlc';\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/series-collector/index.ts"],"names":[],"mappings":";;AAAA,4BAA0B;AAC1B,2BAAyB;AACzB,2BAAyB","sourcesContent":["import './backwards-ohlc';\nimport './forwards-ohlc';\nimport './interest-rate';\n"]}

package/lib/series-collector/interest-rate.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=interest-rate.d.ts.map

package/lib/series-collector/interest-rate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":""}

package/lib/series-collector/interest-rate.js

@@ -0,0 +1,45 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("@yuants/utils");
+const rxjs_1 = require("rxjs");
+const backwards_interest_rate_1 = require("./backwards-interest-rate");
+const discovery_1 = require("./discovery");
+const forwards_interest_rate_1 = require("./forwards-interest-rate");
+const patch_interest_rate_1 = require("./patch-interest-rate");
+(0, rxjs_1.defer)(() => (0, discovery_1.listInterestRateSeriesIds)())
+    .pipe((0, rxjs_1.retry)({ delay: 1000 }), (0, rxjs_1.repeat)({ delay: 60000 }), (0, rxjs_1.map)((x) => Array.from(x.entries())), (0, utils_1.listWatch)((x) => x[0], ([product_id, meta]) => new rxjs_1.Observable((sub) => {
+    // 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度
+    const abortController = new AbortController();
+    sub.add(() => {
+        abortController.abort();
+    });
+    // 先处理前向任务
+    const forwardTask = (0, rxjs_1.defer)(async () => {
+        await (0, forwards_interest_rate_1.handleIngestInterestRateForward)(product_id, meta, abortController.signal);
+    })
+        .pipe((0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
+        .subscribe();
+    sub.add(() => {
+        forwardTask.unsubscribe();
+    });
+    // 设置后向任务
+    const backwardTask = (0, rxjs_1.defer)(async () => {
+        await (0, backwards_interest_rate_1.handleIngestInterestRateBackward)(product_id, meta, abortController.signal);
+    })
+        .pipe((0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
+        .subscribe();
+    sub.add(() => {
+        backwardTask.unsubscribe();
+    });
+    // 设置补齐任务
+    const patchTask = (0, rxjs_1.defer)(async () => {
+        await (0, patch_interest_rate_1.handleInterestRatePatch)(product_id, meta, abortController.signal);
+    })
+        .pipe((0, rxjs_1.retry)(), (0, rxjs_1.repeat)())
+        .subscribe();
+    sub.add(() => {
+        patchTask.unsubscribe();
+    });
+})))
+    .subscribe();
+//# sourceMappingURL=interest-rate.js.map

package/lib/series-collector/interest-rate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/interest-rate.ts"],"names":[],"mappings":";;AAAA,yCAA0C;AAC1C,+BAA6D;AAC7D,uEAA6E;AAC7E,2CAAwD;AACxD,qEAA2E;AAC3E,+DAAgE;AAEhE,IAAA,YAAK,EAAC,GAAG,EAAE,CAAC,IAAA,qCAAyB,GAAE,CAAC;KACrC,IAAI,CACH,IAAA,YAAK,EAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtB,IAAA,aAAM,EAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EACxB,IAAA,UAAG,EAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EACnC,IAAA,iBAAS,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EACX,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,CACrB,IAAI,iBAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,mDAAmD;IACnD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAE9C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,MAAM,WAAW,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACnC,MAAM,IAAA,wDAA+B,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAClF,CAAC,CAAC;SACC,IAAI,CAAC,IAAA,YAAK,GAAE,EAAE,IAAA,aAAM,GAAE,CAAC;SACvB,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,YAAY,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACpC,MAAM,IAAA,0DAAgC,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IACnF,CAAC,CAAC;SACC,IAAI,CAAC,IAAA,YAAK,GAAE,EAAE,IAAA,aAAM,GAAE,CAAC;SACvB,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,YAAY,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,SAAS,GAAG,IAAA,YAAK,EAAC,KAAK,IAAI,EAAE;QACjC,MAAM,IAAA,6CAAuB,EAAC,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC,CAAC;SACC,IAAI,CAAC,IAAA,YAAK,GAAE,EAAE,IAAA,aAAM,GAAE,CAAC;SACvB,SAAS,EAAE,CAAC;IAEf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE;QACX,SAAS,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CACL,CACF;KACA,SAAS,EAAE,CAAC","sourcesContent":["import { listWatch } from '@yuants/utils';\nimport { defer, map, Observable, repeat, retry } from 'rxjs';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { listInterestRateSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleInterestRatePatch } from './patch-interest-rate';\n\ndefer(() => listInterestRateSeriesIds())\n  .pipe(\n    retry({ delay: 1000 }),\n    repeat({ delay: 60000 }),\n    map((x) => Array.from(x.entries())),\n    listWatch(\n      (x) => x[0],\n      ([product_id, meta]) =>\n        new Observable((sub) => {\n          // 处理每个利率品种任务: (forward / backward / patch)，都需要独立调度\n          const abortController = new AbortController();\n\n          sub.add(() => {\n            abortController.abort();\n          });\n\n          // 先处理前向任务\n          const forwardTask = defer(async () => {\n            await handleIngestInterestRateForward(product_id, meta, abortController.signal);\n          })\n            .pipe(retry(), repeat())\n            .subscribe();\n\n          sub.add(() => {\n            forwardTask.unsubscribe();\n          });\n\n          // 设置后向任务\n          const backwardTask = defer(async () => {\n            await handleIngestInterestRateBackward(product_id, meta, abortController.signal);\n          })\n            .pipe(retry(), repeat())\n            .subscribe();\n\n          sub.add(() => {\n            backwardTask.unsubscribe();\n          });\n\n          // 设置补齐任务\n          const patchTask = defer(async () => {\n            await handleInterestRatePatch(product_id, meta, abortController.signal);\n          })\n            .pipe(retry(), repeat())\n            .subscribe();\n\n          sub.add(() => {\n            patchTask.unsubscribe();\n          });\n        }),\n    ),\n  )\n  .subscribe();\n"]}

package/lib/series-collector/patch-interest-rate.d.ts

@@ -0,0 +1,4 @@
+/// <reference types="node" />
+import { IInterestRateServiceMetadata } from '@yuants/exchange';
+export declare const handleInterestRatePatch: (product_id: string, meta: IInterestRateServiceMetadata, signal: AbortSignal) => Promise<void>;
+//# sourceMappingURL=patch-interest-rate.d.ts.map

package/lib/series-collector/patch-interest-rate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"patch-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";AAAA,OAAO,EAEL,4BAA4B,EAE7B,MAAM,kBAAkB,CAAC;AAQ1B,eAAO,MAAM,uBAAuB,eACtB,MAAM,QACZ,4BAA4B,UAC1B,WAAW,kBAmFpB,CAAC"}

package/lib/series-collector/patch-interest-rate.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleInterestRatePatch = void 0;
+const protocol_1 = require("@yuants/protocol");
+const sql_1 = require("@yuants/sql");
+const utils_1 = require("@yuants/utils");
+const terminal = protocol_1.Terminal.fromNodeEnv();
+// Patch 任务：查找数据缺口并进行补齐
+const handleInterestRatePatch = async (product_id, meta, signal) => {
+    var _a, _b, _c, _d;
+    const [datasource_id] = (0, utils_1.decodePath)(product_id);
+    await (0, utils_1.tokenBucket)(`interest_rate:patch:${datasource_id}`).acquire(1, signal);
+    const [record] = await (0, sql_1.requestSQL)(terminal, `
+WITH reversed_ranges AS (
+    SELECT 
+        start_time,
+        end_time,
+        LEAD(end_time) OVER (
+            PARTITION BY table_name, series_id 
+            ORDER BY start_time DESC
+        ) AS next_end_time  -- 注意：倒序时 LEAD 是前一个区间
+    FROM series_data_range
+    WHERE table_name = 'interest_rate' 
+      AND series_id = ${(0, sql_1.escapeSQL)(product_id)}
+)
+SELECT 
+    next_end_time AS gap_start_time,  -- 前一个区间的结束时间
+    start_time AS gap_end_time        -- 当前区间的开始时间
+FROM reversed_ranges
+WHERE next_end_time IS NOT NULL 
+  AND start_time > next_end_time      -- 有空缺
+ORDER BY start_time DESC              -- 从最新开始
+LIMIT 1;
+    `);
+    // no gap
+    if (!record)
+        return;
+    const gapStartTime = new Date(record.gap_start_time).getTime();
+    const gapEndTime = new Date(record.gap_end_time).getTime();
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', 'FindGap', `series=${product_id}, from=${(0, utils_1.formatTime)(gapStartTime)}, to=${(0, utils_1.formatTime)(gapEndTime)}`);
+    let req;
+    if (meta.direction === 'forward') {
+        // forward patch
+        req = {
+            product_id: product_id,
+            direction: 'forward',
+            time: gapStartTime,
+        };
+    }
+    else {
+        // backward patch
+        req = {
+            product_id: product_id,
+            direction: 'backward',
+            time: gapEndTime,
+        };
+    }
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchRequest', `direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
+    const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Patch]', product_id, 'PatchBackwardResult', `ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+};
+exports.handleInterestRatePatch = handleInterestRatePatch;
+//# sourceMappingURL=patch-interest-rate.js.map

package/lib/series-collector/patch-interest-rate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";;;AAKA,+CAA4C;AAC5C,qCAAoD;AACpD,yCAAoE;AAEpE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,uBAAuB;AAChB,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,IAAkC,EAClC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,IAAA,mBAAW,EAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;QAChC,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAtFW,QAAA,uBAAuB,2BAsFlC","sourcesContent":["import {\n  IIngestInterestRateRequest,\n  IInterestRateServiceMetadata,\n  ISeriesIngestResult,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { decodePath, formatTime, tokenBucket } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n  product_id: string,\n  meta: IInterestRateServiceMetadata,\n  signal: AbortSignal,\n) => {\n  const [datasource_id] = decodePath(product_id);\n  await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);\n  const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n    terminal,\n    `\nWITH reversed_ranges AS (\n    SELECT \n        start_time,\n        end_time,\n        LEAD(end_time) OVER (\n            PARTITION BY table_name, series_id \n            ORDER BY start_time DESC\n        ) AS next_end_time  -- 注意：倒序时 LEAD 是前一个区间\n    FROM series_data_range\n    WHERE table_name = 'interest_rate' \n      AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n    next_end_time AS gap_start_time,  -- 前一个区间的结束时间\n    start_time AS gap_end_time        -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n  AND start_time > next_end_time      -- 有空缺\nORDER BY start_time DESC              -- 从最新开始\nLIMIT 1;\n    `,\n  );\n\n  // no gap\n  if (!record) return;\n\n  const gapStartTime = new Date(record.gap_start_time).getTime();\n  const gapEndTime = new Date(record.gap_end_time).getTime();\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Patch]',\n    'FindGap',\n    `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n  );\n\n  let req: IIngestInterestRateRequest;\n\n  if (meta.direction === 'forward') {\n    // forward patch\n    req = {\n      product_id: product_id,\n      direction: 'forward' as const,\n      time: gapStartTime,\n    };\n  } else {\n    // backward patch\n    req = {\n      product_id: product_id,\n      direction: 'backward' as const,\n      time: gapEndTime,\n    };\n  }\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Patch]',\n    product_id,\n    'PatchRequest',\n    `direction=${req.direction}, time=${formatTime(req.time)}`,\n  );\n\n  const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n    'IngestInterestRate',\n    req,\n  );\n\n  console.info(\n    formatTime(Date.now()),\n    '[SeriesCollector][InterestRate][Patch]',\n    product_id,\n    'PatchBackwardResult',\n    `ingested_count=${res.wrote_count}, start_time=${formatTime(\n      res.range?.start_time ?? NaN,\n    )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n  );\n};\n"]}

package/lib/series-collector/sql-helpers.d.ts

@@ -0,0 +1,11 @@
+import { Terminal } from '@yuants/protocol';
+/**
+ * 查找某个利率品种系列ID的最新结束时间（for forward task）
+ * @param terminal
+ * @param product_id
+ * @returns
+ */
+export declare const findInterestRateEndTimeForward: (terminal: Terminal, product_id: string) => Promise<string>;
+export declare const findInterestRateStartTimeBackward: (terminal: Terminal, product_id: string) => Promise<string>;
+export declare const findOHLCEndTimeForward: (terminal: Terminal, series_id: string) => Promise<string>;
+//# sourceMappingURL=sql-helpers.d.ts.map

package/lib/series-collector/sql-helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-helpers.d.ts","sourceRoot":"","sources":["../../src/series-collector/sql-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAG5C;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,aAAc,QAAQ,cAAc,MAAM,oBAUxC,CAAC;AAE9C,eAAO,MAAM,iCAAiC,aAAc,QAAQ,cAAc,MAAM,oBAUzC,CAAC;AAEhD,eAAO,MAAM,sBAAsB,aAAc,QAAQ,aAAa,MAAM,oBAU/B,CAAC"}

package/lib/series-collector/sql-helpers.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findOHLCEndTimeForward = exports.findInterestRateStartTimeBackward = exports.findInterestRateEndTimeForward = void 0;
+const sql_1 = require("@yuants/sql");
+/**
+ * 查找某个利率品种系列ID的最新结束时间（for forward task）
+ * @param terminal
+ * @param product_id
+ * @returns
+ */
+const findInterestRateEndTimeForward = (terminal, product_id) => (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by end_time desc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.end_time; });
+exports.findInterestRateEndTimeForward = findInterestRateEndTimeForward;
+const findInterestRateStartTimeBackward = (terminal, product_id) => (0, sql_1.requestSQL)(terminal, `select start_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(product_id)} and table_name = 'interest_rate' order by start_time asc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.start_time; });
+exports.findInterestRateStartTimeBackward = findInterestRateStartTimeBackward;
+const findOHLCEndTimeForward = (terminal, series_id) => (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`).then((records) => { var _a; return (_a = records === null || records === void 0 ? void 0 : records[0]) === null || _a === void 0 ? void 0 : _a.end_time; });
+exports.findOHLCEndTimeForward = findOHLCEndTimeForward;
+//# sourceMappingURL=sql-helpers.js.map

package/lib/series-collector/sql-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-helpers.js","sourceRoot":"","sources":["../../src/series-collector/sql-helpers.ts"],"names":[],"mappings":";;;AACA,qCAAoD;AAEpD;;;;;GAKG;AACI,MAAM,8BAA8B,GAAG,CAAC,QAAkB,EAAE,UAAkB,EAAE,EAAE,CACvF,IAAA,gBAAU,EAKR,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,UAAU,CACX,kEAAkE,CACpE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,QAAQ,CAAA,EAAA,CAAC,CAAC;AAVjC,QAAA,8BAA8B,kCAUG;AAEvC,MAAM,iCAAiC,GAAG,CAAC,QAAkB,EAAE,UAAkB,EAAE,EAAE,CAC1F,IAAA,gBAAU,EAKR,QAAQ,EACR,8DAA8D,IAAA,eAAS,EACrE,UAAU,CACX,mEAAmE,CACrE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,UAAU,CAAA,EAAA,CAAC,CAAC;AAVnC,QAAA,iCAAiC,qCAUE;AAEzC,MAAM,sBAAsB,GAAG,CAAC,QAAkB,EAAE,SAAiB,EAAE,EAAE,CAC9E,IAAA,gBAAU,EAKR,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,CAAC,0CAAE,QAAQ,CAAA,EAAA,CAAC,CAAC;AAVjC,QAAA,sBAAsB,0BAUW","sourcesContent":["import { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\n\n/**\n * 查找某个利率品种系列ID的最新结束时间（for forward task）\n * @param terminal\n * @param product_id\n * @returns\n */\nexport const findInterestRateEndTimeForward = (terminal: Terminal, product_id: string) =>\n  requestSQL<\n    {\n      end_time: string;\n    }[]\n  >(\n    terminal,\n    `select end_time from series_data_range where series_id = ${escapeSQL(\n      product_id,\n    )} and table_name = 'interest_rate' order by end_time desc limit 1`,\n  ).then((records) => records?.[0]?.end_time);\n\nexport const findInterestRateStartTimeBackward = (terminal: Terminal, product_id: string) =>\n  requestSQL<\n    {\n      start_time: string;\n    }[]\n  >(\n    terminal,\n    `select start_time from series_data_range where series_id = ${escapeSQL(\n      product_id,\n    )} and table_name = 'interest_rate' order by start_time asc limit 1`,\n  ).then((records) => records?.[0]?.start_time);\n\nexport const findOHLCEndTimeForward = (terminal: Terminal, series_id: string) =>\n  requestSQL<\n    {\n      end_time: string;\n    }[]\n  >(\n    terminal,\n    `select end_time from series_data_range where series_id = ${escapeSQL(\n      series_id,\n    )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n  ).then((records) => records?.[0]?.end_time);\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@yuants/app-virtual-exchange",
-	"version": "0.16.0",
+	"version": "0.17.0",
 	"main": "lib/index.js",
 	"files": [
 		"dist",