npm - @yuants/app-virtual-exchange - Versions diffs - 0.18.3 → 0.18.4 - Mend

@yuants/app-virtual-exchange 0.18.3 → 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

package/dist/series-collector/backwards-interest-rate.js CHANGED Viewed

@@ -1,5 +1,5 @@
 import { Terminal } from '@yuants/protocol';
-import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { formatTime } from '@yuants/utils';
 import { findInterestRateStartTimeBackward } from './sql-helpers';
 const terminal = Terminal.fromNodeEnv();
 const ingestCounter = terminal.metrics
@@ -7,9 +7,6 @@ const ingestCounter = terminal.metrics
     .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' });
 export const handleIngestInterestRateBackward = async (product_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = decodePath(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await tokenBucket(`interest_rate:backward:${datasource_id}`).acquire(1, signal);
     let req;
     if (direction === 'backward') {
         const startTime = await findInterestRateStartTimeBackward(terminal, product_id);

package/dist/series-collector/backwards-interest-rate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,~~UAAU,EAAE,WAAW,EAAE,~~MAAM,eAAe,CAAC;~~AACpE~~,OAAO,EAAE,iCAAiC,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AACxC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAE1F,MAAM,CAAC,MAAM,gCAAgC,GAAG,KAAK,EACnD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,~~MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAChF,~~IAAI,GAA+B,CAAC;IACpC,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,SAAS,GAAG,MAAM,iCAAiC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1E,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\nimport { findInterestRateStartTimeBackward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' });\n\nexport const handleIngestInterestRateBackward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n ~~const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`interest_rate:backward:${datasource_id}`).acquire(1, signal);\n~~ let req: IIngestInterestRateRequest;\n if (direction === 'backward') {\n const startTime = await findInterestRateStartTimeBackward(terminal, product_id);\n const start_time = startTime ? new Date(startTime).getTime() : Date.now();\n\n req = {\n product_id: product_id,\n direction: 'backward',\n time: start_time,\n };\n } else {\n req = {\n product_id: product_id,\n direction: 'forward',\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,iCAAiC,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AACxC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAE1F,MAAM,CAAC,MAAM,gCAAgC,GAAG,KAAK,EACnD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,IAAI,GAA+B,CAAC;IACpC,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,SAAS,GAAG,MAAM,iCAAiC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1E,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { formatTime } from '@yuants/utils';\nimport { findInterestRateStartTimeBackward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' });\n\nexport const handleIngestInterestRateBackward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n let req: IIngestInterestRateRequest;\n if (direction === 'backward') {\n const startTime = await findInterestRateStartTimeBackward(terminal, product_id);\n const start_time = startTime ? new Date(startTime).getTime() : Date.now();\n\n req = {\n product_id: product_id,\n direction: 'backward',\n time: start_time,\n };\n } else {\n req = {\n product_id: product_id,\n direction: 'forward',\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/dist/series-collector/backwards-ohlc.js CHANGED Viewed

@@ -5,7 +5,7 @@
 import { decodeOHLCSeriesId } from '@yuants/data-ohlc';
 import { Terminal } from '@yuants/protocol';
 import { escapeSQL, requestSQL } from '@yuants/sql';
-import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { formatTime } from '@yuants/utils';
 const terminal = Terminal.fromNodeEnv();
 const ingestCounter = terminal.metrics
     .counter('series_collector_ingest_count', '')
@@ -13,9 +13,6 @@ const ingestCounter = terminal.metrics
 export const handleIngestOHLCBackward = async (series_id, direction, signal) => {
     var _a, _b, _c, _d;
     const { product_id, duration } = decodeOHLCSeriesId(series_id);
-    const [datasource_id] = decodePath(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await tokenBucket(`ohlc:backward:${datasource_id}`).acquire(1, signal);
     let req;
     if (direction === 'backward') {
         const [record] = await requestSQL(terminal, `select start_time from series_data_range where series_id = ${escapeSQL(series_id)} and table_name = 'ohlc_v2' order by start_time limit 1`);

package/dist/series-collector/backwards-ohlc.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/backwards-ohlc.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,iDAAiD;AACjD,qCAAqC;AAErC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,~~UAAU,EAAE,WAAW,EAAE,~~MAAM,eAAe,CAAC;~~AAEpE~~,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,wBAAwB,GAAG,KAAK,EAC3C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;~~IAC~~/D,~~MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,WAAW,CAAC,iBAAiB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEvE,~~IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,8DAA8D,SAAS,CACrE,SAAS,CACV,yDAAyD,CAC3D,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/E,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,UAAU;QACV,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'backward' });\n\nexport const handleIngestOHLCBackward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n ~~const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`ohlc:backward:${datasource_id}`).acquire(1, signal);\n~~\n let req: IIngestOHLCRequest;\n if (direction === 'backward') {\n const [record] = await requestSQL<\n {\n start_time: string;\n }[]\n >(\n terminal,\n `select start_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by start_time limit 1`,\n );\n const start_time = record ? new Date(record.start_time).getTime() : Date.now();\n\n req = {\n product_id,\n duration,\n direction: 'backward',\n time: start_time,\n };\n } else {\n // forward\n req = {\n product_id,\n duration,\n direction,\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"backwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/backwards-ohlc.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,iDAAiD;AACjD,qCAAqC;AAErC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,wBAAwB,GAAG,KAAK,EAC3C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,8DAA8D,SAAS,CACrE,SAAS,CACV,yDAAyD,CAC3D,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/E,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,UAAU;QACV,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'backward' });\n\nexport const handleIngestOHLCBackward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n if (direction === 'backward') {\n const [record] = await requestSQL<\n {\n start_time: string;\n }[]\n >(\n terminal,\n `select start_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by start_time limit 1`,\n );\n const start_time = record ? new Date(record.start_time).getTime() : Date.now();\n\n req = {\n product_id,\n duration,\n direction: 'backward',\n time: start_time,\n };\n } else {\n // forward\n req = {\n product_id,\n duration,\n direction,\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/dist/series-collector/forwards-interest-rate.js CHANGED Viewed

@@ -1,5 +1,5 @@
 import { Terminal } from '@yuants/protocol';
-import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { formatTime } from '@yuants/utils';
 import { findInterestRateEndTimeForward } from './sql-helpers';
 const terminal = Terminal.fromNodeEnv();
 const ingestCounter = terminal.metrics
@@ -7,32 +7,27 @@ const ingestCounter = terminal.metrics
     .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });
 export const handleIngestInterestRateForward = async (product_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = decodePath(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await tokenBucket(`interest_rate:forwards:${datasource_id}`).acquire(1, signal);
-    {
-        let req;
-        if (direction === 'forward') {
-            const endTime = await findInterestRateEndTimeForward(terminal, product_id);
-            const time = endTime ? new Date(endTime).getTime() : 0;
-            req = {
-                product_id: product_id,
-                direction: 'forward',
-                time,
-            };
-        }
-        else {
-            // backward
-            req = {
-                product_id,
-                direction: 'backward',
-                time: Date.now(),
-            };
-        }
-        console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`);
-        const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
-        ingestCounter.inc(res.wrote_count || 0);
-        console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+    let req;
+    if (direction === 'forward') {
+        const endTime = await findInterestRateEndTimeForward(terminal, product_id);
+        const time = endTime ? new Date(endTime).getTime() : 0;
+        req = {
+            product_id: product_id,
+            direction: 'forward',
+            time,
+        };
     }
+    else {
+        // backward
+        req = {
+            product_id,
+            direction: 'backward',
+            time: Date.now(),
+        };
+    }
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`);
+    const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+    ingestCounter.inc(res.wrote_count || 0);
+    console.info(formatTime(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${formatTime((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
 };
 //# sourceMappingURL=forwards-interest-rate.js.map

package/dist/series-collector/forwards-interest-rate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,~~UAAU,EAAE,WAAW,EAAE,~~MAAM,eAAe,CAAC;~~AACpE~~,OAAO,EAAE,8BAA8B,EAAE,MAAM,eAAe,CAAC;AAE/D,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,EAClD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,~~MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,WAAW,CAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEhF;QACE,~~IAAI,GAA+B,CAAC;~~QACpC~~,IAAI,SAAS,KAAK,SAAS,EAAE;~~YAC3B~~,MAAM,OAAO,GAAG,MAAM,8BAA8B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;~~YAC3E~~,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;~~YAEvD~~,GAAG,GAAG;~~gBACJ~~,UAAU,EAAE,UAAU;~~gBACtB~~,SAAS,EAAE,SAAS;~~gBACpB~~,IAAI;~~aACL~~,CAAC;~~SACH~~;~~aAAM~~;~~YACL~~,WAAW;~~YACX~~,GAAG,GAAG;~~gBACJ~~,UAAU;~~gBACV~~,SAAS,EAAE,UAAU;~~gBACrB~~,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;~~aACjB~~,CAAC;~~SACH~~;~~QAED~~,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;~~QACF~~,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;~~QAEF~~,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;~~QAExC~~,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;~~KACH;AACH~~,CAAC,CAAC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\nimport { findInterestRateEndTimeForward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\nexport const handleIngestInterestRateForward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`interest_rate:forwards:${datasource_id}`).acquire(1, signal);\n\n {\n let req: IIngestInterestRateRequest;\n if (direction === 'forward') {\n const endTime = await findInterestRateEndTimeForward(terminal, product_id);\n const time = endTime ? new Date(endTime).getTime() : 0;\n\n req = {\n product_id: product_id,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n direction: 'backward',\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n }~~\n}~~;\n"]}
1	+ {"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,8BAA8B,EAAE,MAAM,eAAe,CAAC;AAE/D,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,EAClD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,IAAI,GAA+B,CAAC;IACpC,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,OAAO,GAAG,MAAM,8BAA8B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { formatTime } from '@yuants/utils';\nimport { findInterestRateEndTimeForward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\nexport const handleIngestInterestRateForward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n let req: IIngestInterestRateRequest;\n if (direction === 'forward') {\n const endTime = await findInterestRateEndTimeForward(terminal, product_id);\n const time = endTime ? new Date(endTime).getTime() : 0;\n\n req = {\n product_id: product_id,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n direction: 'backward',\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/dist/series-collector/forwards-ohlc.js CHANGED Viewed

@@ -5,7 +5,7 @@
 import { decodeOHLCSeriesId } from '@yuants/data-ohlc';
 import { Terminal } from '@yuants/protocol';
 import { escapeSQL, requestSQL } from '@yuants/sql';
-import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { formatTime } from '@yuants/utils';
 const terminal = Terminal.fromNodeEnv();
 const ingestCounter = terminal.metrics
     .counter('series_collector_ingest_count', '')
@@ -13,9 +13,6 @@ const ingestCounter = terminal.metrics
 export const handleIngestOHLCForward = async (series_id, direction, signal) => {
     var _a, _b, _c, _d;
     const { product_id, duration } = decodeOHLCSeriesId(series_id);
-    const [datasource_id] = decodePath(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await tokenBucket(`ohlc:forward:${datasource_id}`).acquire(1, signal);
     let req;
     if (direction === 'forward') {
         const [record] = await requestSQL(terminal, `select end_time from series_data_range where series_id = ${escapeSQL(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`);

package/dist/series-collector/forwards-ohlc.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;AAErC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,~~UAAU,EAAE,WAAW,EAAE,~~MAAM,eAAe,CAAC;~~AAEpE~~,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC/D,~~MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,WAAW,CAAC,gBAAgB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACtE,~~IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,4DAA4D,SAAS,CACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\nexport const handleIngestOHLCForward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n ~~const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`ohlc:forward:${datasource_id}`).acquire(1, signal);\n~~ let req: IIngestOHLCRequest;\n if (direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n );\n\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id,\n duration,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n duration,\n direction,\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;AAErC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAK/B,QAAQ,EACR,4DAA4D,SAAS,CACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,UAAU,CACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\nexport const handleIngestOHLCForward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n let req: IIngestOHLCRequest;\n if (direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n );\n\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id,\n duration,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n duration,\n direction,\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/dist/series-collector/patch-interest-rate.js CHANGED Viewed

@@ -1,6 +1,6 @@
 import { Terminal } from '@yuants/protocol';
 import { escapeSQL, requestSQL } from '@yuants/sql';
-import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { formatTime } from '@yuants/utils';
 const terminal = Terminal.fromNodeEnv();
 const ingestCounter = terminal.metrics
     .counter('series_collector_ingest_count', '')
@@ -8,8 +8,6 @@ const ingestCounter = terminal.metrics
 // Patch 任务：查找数据缺口并进行补齐
 export const handleInterestRatePatch = async (product_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = decodePath(product_id);
-    await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);
     const [record] = await requestSQL(terminal, `
 WITH reversed_ranges AS (
     SELECT

package/dist/series-collector/patch-interest-rate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,~~UAAU,EAAE,WAAW,EAAE,~~MAAM,eAAe,CAAC;~~AAEpE~~,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,uBAAuB;AACvB,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,~~aAAa,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC/C,~~MAAM,~~WAAW,~~CAAC,~~uBAAuB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,~~GAAG,MAAM,UAAU,CAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,SAAS,CAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,UAAU,CAAC,YAAY,CAAC,QAAQ,UAAU,CAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [~~datasource_id] = decodePath(product_id);\n await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);\n const [~~record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'interest_rate' \n AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n 'FindGap',\n `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n let req: IIngestInterestRateRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,uBAAuB;AACvB,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,SAAS,CAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,UAAU,CAAC,YAAY,CAAC,QAAQ,UAAU,CAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'interest_rate' \n AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n 'FindGap',\n `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n let req: IIngestInterestRateRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/dist/series-collector/patch-ohlc.js CHANGED Viewed

@@ -1,7 +1,7 @@
 import { decodeOHLCSeriesId } from '@yuants/data-ohlc';
 import { Terminal } from '@yuants/protocol';
 import { escapeSQL, requestSQL } from '@yuants/sql';
-import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
+import { formatTime } from '@yuants/utils';
 const terminal = Terminal.fromNodeEnv();
 const ingestCounter = terminal.metrics
     .counter('series_collector_ingest_count', '')
@@ -9,8 +9,6 @@ const ingestCounter = terminal.metrics
 // Patch 任务：查找数据缺口并进行补齐
 export const handleIngestOHLCPatch = async (series_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = decodePath(series_id);
-    await tokenBucket(`ohlc:patch:${datasource_id}`).acquire(1, signal);
     const [record] = await requestSQL(terminal, `
 WITH reversed_ranges AS (
     SELECT

package/dist/series-collector/patch-ohlc.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"patch-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,~~UAAU,EAAE,WAAW,EAAE,~~MAAM,eAAe,CAAC;~~AAEpE~~,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,uBAAuB;AACvB,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,~~aAAa,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9C,~~MAAM,~~WAAW,~~CAAC,~~cAAc,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,CAAC,MAAM,CAAC,~~GAAG,MAAM,UAAU,CAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,SAAS,CAAC,SAAS,CAAC;;;;;;;;;;KAUvC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,UAAU,SAAS,UAAU,UAAU,CAAC,YAAY,CAAC,QAAQ,UAAU,CAAC,UAAU,CAAC,EAAE,CACtF,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAE5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleIngestOHLCPatch = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [~~datasource_id] = decodePath(series_id);\n await tokenBucket(`ohlc:patch:${datasource_id}`).acquire(1, signal);\n const [~~record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'ohlc_v2' \n AND series_id = ${escapeSQL(series_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n 'FindGap',\n `series=${series_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"patch-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,uBAAuB;AACvB,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,SAAS,CAAC,SAAS,CAAC;;;;;;;;;;KAUvC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,UAAU,SAAS,UAAU,UAAU,CAAC,YAAY,CAAC,QAAQ,UAAU,CAAC,UAAU,CAAC,EAAE,CACtF,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAE5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,UAAU,CACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,UAAU,CAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleIngestOHLCPatch = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'ohlc_v2' \n AND series_id = ${escapeSQL(series_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n 'FindGap',\n `series=${series_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/dist/series-collector/setup.js CHANGED Viewed

@@ -1,4 +1,4 @@
-import { formatTime, tokenBucket } from '@yuants/utils';
+import { decodePath, formatTime, tokenBucket } from '@yuants/utils';
 import { handleIngestInterestRateBackward } from './backwards-interest-rate';
 import { handleIngestOHLCBackward } from './backwards-ohlc';
 import { listInterestRateSeriesIds, listOHLCSeriesIds } from './discovery';
@@ -32,20 +32,25 @@ const api = {
                     await tokenBucket(`${type}:${task}`).acquire(1, signal);
                     try {
                         const tasks = await list();
-                        // 并行
-                        // await Promise.all(
-                        //   Array.from(tasks.entries()).map(([series_id, direction]) =>
-                        //     handler(series_id, direction, signal).catch((err) => {
-                        //       console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);
-                        //     }),
-                        //   ),
-                        // );
-                        // 串行调度
-                        for (const [series_id, direction] of tasks.entries()) {
-                            await handler(series_id, direction, signal).catch((err) => {
-                                console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);
-                            });
+                        // const groups = Map.groupBy(tasks, item => decodePath(item[0])[0]);
+                        const groups = new Map();
+                        for (const item of tasks) {
+                            const [datasource_id] = decodePath(item[0]);
+                            let items = groups.get(datasource_id);
+                            if (!items) {
+                                items = [];
+                                groups.set(datasource_id, items);
+                            }
+                            items.push(item);
                         }
+                        await Promise.all(Array.from(groups.entries()).map(async ([datasource_id, tasks]) => {
+                            for (const [series_id, direction] of tasks) {
+                                await tokenBucket(`${type}:${task}:${datasource_id}`).acquire(1, signal);
+                                await handler(series_id, direction, signal).catch((err) => {
+                                    console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);
+                                });
+                            }
+                        }));
                     }
                     catch (e) { }
                 }

package/dist/series-collector/setup.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/series-collector/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;~~AACxD~~,OAAO,EAAE,gCAAgC,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,+BAA+B,EAAE,MAAM,0BAA0B,CAAC;AAC3E,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD,MAAM,GAAG,GAAG;IACV,IAAI,EAAE;QACJ,IAAI,EAAE,iBAAiB;QACvB,OAAO,EAAE,uBAAuB;QAChC,QAAQ,EAAE,wBAAwB;QAClC,KAAK,EAAE,qBAAqB;KAC7B;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,yBAAyB;QAC/B,OAAO,EAAE,+BAA+B;QACxC,QAAQ,EAAE,gCAAgC;QAC1C,KAAK,EAAE,uBAAuB;KAC/B;CACF,CAAC;AAEF,CAAC,KAAK,IAAI,EAAE;IACV,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,cAAc,CAAU,EAAE;QACpD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE;YAC5D,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,KAAK,IAAI,EAAE;gBACV,OAAO,IAAI,EAAE;oBACX,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBACxD,IAAI;wBACF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;~~wBAC3B~~,~~KAAK~~;~~wBACL~~,~~qBAAqB~~;~~wBACrB~~,~~gEAAgE~~;~~wBAChE~~,~~6DAA6D~~;~~wBAC7D~~,~~mGAAmG~~;~~wBACnG~~,~~UAAU~~;~~wBACV~~,~~OAAO~~;~~wBACP~~,KAAK;~~wBACL~~,OAAO;~~wBACP~~,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,~~EAAE~~;~~4BACpD~~,MAAM,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;~~gCACxD~~,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,IAAI,KAAK,IAAI,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;~~4BAC5F~~,CAAC,CAAC,CAAC;~~yBACJ~~;~~qBACF~~;oBAAC,OAAO,CAAC,EAAE,GAAE;iBACf;YACH,CAAC,CAAC,EAAE,CAAC;SACN;KACF;AACH,CAAC,CAAC,EAAE,CAAC","sourcesContent":["import { formatTime, tokenBucket } from '@yuants/utils';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { handleIngestOHLCBackward } from './backwards-ohlc';\nimport { listInterestRateSeriesIds, listOHLCSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleIngestOHLCForward } from './forwards-ohlc';\nimport { handleInterestRatePatch } from './patch-interest-rate';\nimport { handleIngestOHLCPatch } from './patch-ohlc';\n\nconst api = {\n OHLC: {\n list: listOHLCSeriesIds,\n forward: handleIngestOHLCForward,\n backward: handleIngestOHLCBackward,\n patch: handleIngestOHLCPatch,\n },\n InterestRate: {\n list: listInterestRateSeriesIds,\n forward: handleIngestInterestRateForward,\n backward: handleIngestInterestRateBackward,\n patch: handleInterestRatePatch,\n },\n};\n\n(async () => {\n const abortController = new AbortController();\n const signal = abortController.signal;\n for (const type of ['OHLC', 'InterestRate'] as const) {\n const list = api[type].list;\n for (const task of ['forward', 'backward', 'patch'] as const) {\n const handler = api[type][task];\n (async () => {\n while (true) {\n await tokenBucket(`${type}:${task}`).acquire(1, signal);\n try {\n const tasks = await list();\n ~~// 并行\~~n // ~~await~~ ~~Promise~~.~~all~~(~~\n // Array.from(~~tasks~~.entries~~())~~.map((~~[~~series_id~~, ~~direction~~]) ~~=>\~~n ~~// handler~~(~~series_id,~~ ~~direction,~~ ~~signal~~)~~.catch((err)~~ => {\n ~~// console.info~~(~~formatTime(Date~~.~~now~~()), ~~`[SeriesCollector][$~~{~~type}]~~[~~${task}~~]~~`, 'Error'~~, ~~err~~);\n // }),\n ~~// ),\~~n // )~~;\n //~~ ~~串行调度\~~n for (const [series_id, direction] of tasks~~.entries(~~)) {\n await handler(series_id, direction, signal).catch((err) => {\n console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);\n });\n }\n } catch (e) {}\n }\n })();\n }\n }\n})();\n"]}
1	+ {"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/series-collector/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,gCAAgC,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,+BAA+B,EAAE,MAAM,0BAA0B,CAAC;AAC3E,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD,MAAM,GAAG,GAAG;IACV,IAAI,EAAE;QACJ,IAAI,EAAE,iBAAiB;QACvB,OAAO,EAAE,uBAAuB;QAChC,QAAQ,EAAE,wBAAwB;QAClC,KAAK,EAAE,qBAAqB;KAC7B;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,yBAAyB;QAC/B,OAAO,EAAE,+BAA+B;QACxC,QAAQ,EAAE,gCAAgC;QAC1C,KAAK,EAAE,uBAAuB;KAC/B;CACF,CAAC;AAEF,CAAC,KAAK,IAAI,EAAE;IACV,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,cAAc,CAAU,EAAE;QACpD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE;YAC5D,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,KAAK,IAAI,EAAE;gBACV,OAAO,IAAI,EAAE;oBACX,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBACxD,IAAI;wBACF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;wBAE3B,qEAAqE;wBACrE,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8C,CAAC;wBACrE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;4BACxB,MAAM,CAAC,aAAa,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;4BAC5C,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;4BACtC,IAAI,CAAC,KAAK,EAAE;gCACV,KAAK,GAAG,EAAE,CAAC;gCACX,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;6BAClC;4BACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;yBAClB;wBAED,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,EAAE;4BAChE,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE;gCAC1C,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gCACzE,MAAM,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oCACxD,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,IAAI,KAAK,IAAI,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gCAC5F,CAAC,CAAC,CAAC;6BACJ;wBACH,CAAC,CAAC,CACH,CAAC;qBACH;oBAAC,OAAO,CAAC,EAAE,GAAE;iBACf;YACH,CAAC,CAAC,EAAE,CAAC;SACN;KACF;AACH,CAAC,CAAC,EAAE,CAAC","sourcesContent":["import { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { handleIngestOHLCBackward } from './backwards-ohlc';\nimport { listInterestRateSeriesIds, listOHLCSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleIngestOHLCForward } from './forwards-ohlc';\nimport { handleInterestRatePatch } from './patch-interest-rate';\nimport { handleIngestOHLCPatch } from './patch-ohlc';\n\nconst api = {\n OHLC: {\n list: listOHLCSeriesIds,\n forward: handleIngestOHLCForward,\n backward: handleIngestOHLCBackward,\n patch: handleIngestOHLCPatch,\n },\n InterestRate: {\n list: listInterestRateSeriesIds,\n forward: handleIngestInterestRateForward,\n backward: handleIngestInterestRateBackward,\n patch: handleInterestRatePatch,\n },\n};\n\n(async () => {\n const abortController = new AbortController();\n const signal = abortController.signal;\n for (const type of ['OHLC', 'InterestRate'] as const) {\n const list = api[type].list;\n for (const task of ['forward', 'backward', 'patch'] as const) {\n const handler = api[type][task];\n (async () => {\n while (true) {\n await tokenBucket(`${type}:${task}`).acquire(1, signal);\n try {\n const tasks = await list();\n\n // const groups = Map.groupBy(tasks, item => decodePath(item[0])[0]);\n const groups = new Map<string, [string, 'forward' \| 'backward'][]>();\n for (const item of tasks) {\n const [datasource_id] = decodePath(item[0]);\n let items = groups.get(datasource_id);\n if (!items) {\n items = [];\n groups.set(datasource_id, items);\n }\n items.push(item);\n }\n\n await Promise.all(\n Array.from(groups.entries()).map(async ([datasource_id, tasks]) => {\n for (const [series_id, direction] of tasks) {\n await tokenBucket(`${type}:${task}:${datasource_id}`).acquire(1, signal);\n await handler(series_id, direction, signal).catch((err) => {\n console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);\n });\n }\n }),\n );\n } catch (e) {}\n }\n })();\n }\n }\n})();\n"]}

package/lib/series-collector/backwards-interest-rate.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";AAUA,eAAO,MAAM,gCAAgC,eAC/B,MAAM,aACP,SAAS,GAAG,UAAU,UACzB,WAAW,~~kBA6CpB~~,CAAC"}
1	+ {"version":3,"file":"backwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";AAUA,eAAO,MAAM,gCAAgC,eAC/B,MAAM,aACP,SAAS,GAAG,UAAU,UACzB,WAAW,kBA0CpB,CAAC"}

package/lib/series-collector/backwards-interest-rate.js CHANGED Viewed

@@ -10,9 +10,6 @@ const ingestCounter = terminal.metrics
     .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' });
 const handleIngestInterestRateBackward = async (product_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = (0, utils_1.decodePath)(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await (0, utils_1.tokenBucket)(`interest_rate:backward:${datasource_id}`).acquire(1, signal);
     let req;
     if (direction === 'backward') {
         const startTime = await (0, sql_helpers_1.findInterestRateStartTimeBackward)(terminal, product_id);

package/lib/series-collector/backwards-interest-rate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";;;AACA,+CAA4C;AAC5C,~~yCAAoE~~;~~AACpE~~,+CAAkE;AAElE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AACxC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAEnF,MAAM,gCAAgC,GAAG,KAAK,EACnD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,~~MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAChF,~~IAAI,GAA+B,CAAC;IACpC,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAiC,EAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1E,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;~~AAhDW~~,QAAA,gCAAgC,~~oCAgD3C~~","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\nimport { findInterestRateStartTimeBackward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' });\n\nexport const handleIngestInterestRateBackward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n ~~const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`interest_rate:backward:${datasource_id}`).acquire(1, signal);\n~~ let req: IIngestInterestRateRequest;\n if (direction === 'backward') {\n const startTime = await findInterestRateStartTimeBackward(terminal, product_id);\n const start_time = startTime ? new Date(startTime).getTime() : Date.now();\n\n req = {\n product_id: product_id,\n direction: 'backward',\n time: start_time,\n };\n } else {\n req = {\n product_id: product_id,\n direction: 'forward',\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"backwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/backwards-interest-rate.ts"],"names":[],"mappings":";;;AACA,+CAA4C;AAC5C,yCAA2C;AAC3C,+CAAkE;AAElE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AACxC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAEnF,MAAM,gCAAgC,GAAG,KAAK,EACnD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,IAAI,GAA+B,CAAC;IACpC,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAiC,EAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE1E,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,2CAA2C,EAC3C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA7CW,QAAA,gCAAgC,oCA6C3C","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { formatTime } from '@yuants/utils';\nimport { findInterestRateStartTimeBackward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'backward' });\n\nexport const handleIngestInterestRateBackward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n let req: IIngestInterestRateRequest;\n if (direction === 'backward') {\n const startTime = await findInterestRateStartTimeBackward(terminal, product_id);\n const start_time = startTime ? new Date(startTime).getTime() : Date.now();\n\n req = {\n product_id: product_id,\n direction: 'backward',\n time: start_time,\n };\n } else {\n req = {\n product_id: product_id,\n direction: 'forward',\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Backward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/lib/series-collector/backwards-ohlc.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backwards-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/backwards-ohlc.ts"],"names":[],"mappings":";AAiBA,eAAO,MAAM,wBAAwB,cACxB,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,~~kBA6DpB~~,CAAC"}
1	+ {"version":3,"file":"backwards-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/backwards-ohlc.ts"],"names":[],"mappings":";AAiBA,eAAO,MAAM,wBAAwB,cACxB,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,kBA0DpB,CAAC"}

package/lib/series-collector/backwards-ohlc.js CHANGED Viewed

@@ -16,9 +16,6 @@ const ingestCounter = terminal.metrics
 const handleIngestOHLCBackward = async (series_id, direction, signal) => {
     var _a, _b, _c, _d;
     const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
-    const [datasource_id] = (0, utils_1.decodePath)(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await (0, utils_1.tokenBucket)(`ohlc:backward:${datasource_id}`).acquire(1, signal);
     let req;
     if (direction === 'backward') {
         const [record] = await (0, sql_1.requestSQL)(terminal, `select start_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by start_time limit 1`);

package/lib/series-collector/backwards-ohlc.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/backwards-ohlc.ts"],"names":[],"mappings":";AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,iDAAiD;AACjD,qCAAqC;;;AAErC,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,~~yCAAoE~~;~~AAEpE~~,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAE1E,MAAM,wBAAwB,GAAG,KAAK,EAC3C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;~~IAC~~/D,~~MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,iBAAiB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEvE,~~IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,8DAA8D,IAAA,eAAS,EACrE,SAAS,CACV,yDAAyD,CAC3D,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/E,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,UAAU;QACV,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;~~AAhEW~~,QAAA,wBAAwB,~~4BAgEnC~~","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'backward' });\n\nexport const handleIngestOHLCBackward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n ~~const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`ohlc:backward:${datasource_id}`).acquire(1, signal);\n~~\n let req: IIngestOHLCRequest;\n if (direction === 'backward') {\n const [record] = await requestSQL<\n {\n start_time: string;\n }[]\n >(\n terminal,\n `select start_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by start_time limit 1`,\n );\n const start_time = record ? new Date(record.start_time).getTime() : Date.now();\n\n req = {\n product_id,\n duration,\n direction: 'backward',\n time: start_time,\n };\n } else {\n // forward\n req = {\n product_id,\n duration,\n direction,\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"backwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/backwards-ohlc.ts"],"names":[],"mappings":";AAAA,0BAA0B;AAC1B,6DAA6D;AAC7D,iDAAiD;AACjD,qCAAqC;;;AAErC,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAA2C;AAE3C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;AAE1E,MAAM,wBAAwB,GAAG,KAAK,EAC3C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,UAAU,EAAE;QAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,8DAA8D,IAAA,eAAS,EACrE,SAAS,CACV,yDAAyD,CAC3D,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/E,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;SAAM;QACL,UAAU;QACV,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,CAAC;SACR,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA7DW,QAAA,wBAAwB,4BA6DnC","sourcesContent":["// 解决 Backwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Backwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'backward' });\n\nexport const handleIngestOHLCBackward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n if (direction === 'backward') {\n const [record] = await requestSQL<\n {\n start_time: string;\n }[]\n >(\n terminal,\n `select start_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by start_time limit 1`,\n );\n const start_time = record ? new Date(record.start_time).getTime() : Date.now();\n\n req = {\n product_id,\n duration,\n direction: 'backward',\n time: start_time,\n };\n } else {\n // forward\n req = {\n product_id,\n duration,\n direction,\n time: 0,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Backward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/lib/series-collector/forwards-interest-rate.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAWA,eAAO,MAAM,+BAA+B,eAC9B,MAAM,aACP,SAAS,GAAG,UAAU,UACzB,WAAW,~~kBAgDpB~~,CAAC"}
1	+ {"version":3,"file":"forwards-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";AAWA,eAAO,MAAM,+BAA+B,eAC9B,MAAM,aACP,SAAS,GAAG,UAAU,UACzB,WAAW,kBA2CpB,CAAC"}

package/lib/series-collector/forwards-interest-rate.js CHANGED Viewed

@@ -10,33 +10,28 @@ const ingestCounter = terminal.metrics
     .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });
 const handleIngestInterestRateForward = async (product_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = (0, utils_1.decodePath)(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await (0, utils_1.tokenBucket)(`interest_rate:forwards:${datasource_id}`).acquire(1, signal);
-    {
-        let req;
-        if (direction === 'forward') {
-            const endTime = await (0, sql_helpers_1.findInterestRateEndTimeForward)(terminal, product_id);
-            const time = endTime ? new Date(endTime).getTime() : 0;
-            req = {
-                product_id: product_id,
-                direction: 'forward',
-                time,
-            };
-        }
-        else {
-            // backward
-            req = {
-                product_id,
-                direction: 'backward',
-                time: Date.now(),
-            };
-        }
-        console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
-        const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
-        ingestCounter.inc(res.wrote_count || 0);
-        console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
+    let req;
+    if (direction === 'forward') {
+        const endTime = await (0, sql_helpers_1.findInterestRateEndTimeForward)(terminal, product_id);
+        const time = endTime ? new Date(endTime).getTime() : 0;
+        req = {
+            product_id: product_id,
+            direction: 'forward',
+            time,
+        };
     }
+    else {
+        // backward
+        req = {
+            product_id,
+            direction: 'backward',
+            time: Date.now(),
+        };
+    }
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Request', `product_id=${req.product_id}, direction=${req.direction}, time=${(0, utils_1.formatTime)(req.time)}`);
+    const res = await terminal.client.requestForResponseData('IngestInterestRate', req);
+    ingestCounter.inc(res.wrote_count || 0);
+    console.info((0, utils_1.formatTime)(Date.now()), '[SeriesCollector][InterestRate][Forward]', 'Result', `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${(0, utils_1.formatTime)((_b = (_a = res.range) === null || _a === void 0 ? void 0 : _a.start_time) !== null && _b !== void 0 ? _b : NaN)}, end_time=${(0, utils_1.formatTime)((_d = (_c = res.range) === null || _c === void 0 ? void 0 : _c.end_time) !== null && _d !== void 0 ? _d : NaN)}`);
 };
 exports.handleIngestInterestRateForward = handleIngestInterestRateForward;
 //# sourceMappingURL=forwards-interest-rate.js.map

package/lib/series-collector/forwards-interest-rate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";;;AACA,+CAA4C;AAC5C,~~yCAAoE~~;~~AACpE~~,+CAA+D;AAE/D,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAElF,MAAM,+BAA+B,GAAG,KAAK,EAClD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,~~MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAEhF;QACE,~~IAAI,GAA+B,CAAC;~~QACpC~~,IAAI,SAAS,KAAK,SAAS,EAAE;~~YAC3B~~,MAAM,OAAO,GAAG,MAAM,IAAA,4CAA8B,EAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;~~YAC3E~~,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;~~YAEvD~~,GAAG,GAAG;~~gBACJ~~,UAAU,EAAE,UAAU;~~gBACtB~~,SAAS,EAAE,SAAS;~~gBACpB~~,IAAI;~~aACL~~,CAAC;~~SACH~~;~~aAAM~~;~~YACL~~,WAAW;~~YACX~~,GAAG,GAAG;~~gBACJ~~,UAAU;~~gBACV~~,SAAS,EAAE,UAAU;~~gBACrB~~,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;~~aACjB~~,CAAC;~~SACH~~;~~QAED~~,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;~~QACF~~,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;~~QAEF~~,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;~~QAExC~~,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;~~KACH;AACH~~,CAAC,CAAC;~~AAnDW~~,QAAA,+BAA+B,~~mCAmD1C~~","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\nimport { findInterestRateEndTimeForward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\nexport const handleIngestInterestRateForward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`interest_rate:forwards:${datasource_id}`).acquire(1, signal);\n\n {\n let req: IIngestInterestRateRequest;\n if (direction === 'forward') {\n const endTime = await findInterestRateEndTimeForward(terminal, product_id);\n const time = endTime ? new Date(endTime).getTime() : 0;\n\n req = {\n product_id: product_id,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n direction: 'backward',\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n }~~\n}~~;\n"]}
1	+ {"version":3,"file":"forwards-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/forwards-interest-rate.ts"],"names":[],"mappings":";;;AACA,+CAA4C;AAC5C,yCAA2C;AAC3C,+CAA+D;AAE/D,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAElF,MAAM,+BAA+B,GAAG,KAAK,EAClD,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,IAAI,GAA+B,CAAC;IACpC,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,OAAO,GAAG,MAAM,IAAA,4CAA8B,EAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,SAAS,EAAE,UAAU;YACrB,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,0CAA0C,EAC1C,QAAQ,EACR,aAAa,UAAU,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EAClF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA9CW,QAAA,+BAA+B,mCA8C1C","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { formatTime } from '@yuants/utils';\nimport { findInterestRateEndTimeForward } from './sql-helpers';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\nexport const handleIngestInterestRateForward = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n let req: IIngestInterestRateRequest;\n if (direction === 'forward') {\n const endTime = await findInterestRateEndTimeForward(terminal, product_id);\n const time = endTime ? new Date(endTime).getTime() : 0;\n\n req = {\n product_id: product_id,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n direction: 'backward',\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Request',\n `product_id=${req.product_id}, direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Forward]',\n 'Result',\n `series_id=${product_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/lib/series-collector/forwards-ohlc.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwards-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAiBA,eAAO,MAAM,uBAAuB,cACvB,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,~~kBA6DpB~~,CAAC"}
1	+ {"version":3,"file":"forwards-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAiBA,eAAO,MAAM,uBAAuB,cACvB,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,kBA0DpB,CAAC"}

package/lib/series-collector/forwards-ohlc.js CHANGED Viewed

@@ -16,9 +16,6 @@ const ingestCounter = terminal.metrics
 const handleIngestOHLCForward = async (series_id, direction, signal) => {
     var _a, _b, _c, _d;
     const { product_id, duration } = (0, data_ohlc_1.decodeOHLCSeriesId)(series_id);
-    const [datasource_id] = (0, utils_1.decodePath)(product_id);
-    // 控制速率：每个数据源每秒钟只能请求一次
-    await (0, utils_1.tokenBucket)(`ohlc:forward:${datasource_id}`).acquire(1, signal);
     let req;
     if (direction === 'forward') {
         const [record] = await (0, sql_1.requestSQL)(terminal, `select end_time from series_data_range where series_id = ${(0, sql_1.escapeSQL)(series_id)} and table_name = 'ohlc_v2' order by end_time desc limit 1`);

package/lib/series-collector/forwards-ohlc.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;;;AAErC,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,~~yCAAoE~~;~~AAEpE~~,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzE,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAC/D,~~MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,sBAAsB;IACtB,MAAM,IAAA,mBAAW,EAAC,gBAAgB,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACtE,~~IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;~~AAhEW~~,QAAA,uBAAuB,~~2BAgElC~~","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\nexport const handleIngestOHLCForward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n ~~const [datasource_id] = decodePath(product_id);\n // 控制速率：每个数据源每秒钟只能请求一次\n await tokenBucket(`ohlc:forward:${datasource_id}`).acquire(1, signal);\n~~ let req: IIngestOHLCRequest;\n if (direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n );\n\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id,\n duration,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n duration,\n direction,\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"forwards-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/forwards-ohlc.ts"],"names":[],"mappings":";AAAA,yBAAyB;AACzB,4DAA4D;AAC5D,iDAAiD;AACjD,qCAAqC;;;AAErC,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAA2C;AAE3C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzE,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,GAAuB,CAAC;IAC5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAK/B,QAAQ,EACR,4DAA4D,IAAA,eAAS,EACnE,SAAS,CACV,4DAA4D,CAC9D,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,IAAI;SACL,CAAC;KACH;SAAM;QACL,WAAW;QACX,GAAG,GAAG;YACJ,UAAU;YACV,QAAQ;YACR,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,SAAS,EACT,cAAc,GAAG,CAAC,UAAU,cAAc,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EACpG,GAAG,CAAC,IAAI,CACT,EAAE,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,kCAAkC,EAClC,UAAU,EACV,aAAa,SAAS,oBAAoB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACjF,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA7DW,QAAA,uBAAuB,2BA6DlC","sourcesContent":["// 解决 Forwards 拉取历史数据的调度器\n// 该文件会定期扫描所有 Terminal 的 ServiceInfo，提取出所有支持 Forwards 拉取的序列。\n// 然后对每个序列，向对应的 IngestOHLC Service 发送拉取请求，补齐历史数据。\n// 使用 Token Bucket 控制每个数据源的请求速率，避免过载。\n\nimport { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\nexport const handleIngestOHLCForward = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n let req: IIngestOHLCRequest;\n if (direction === 'forward') {\n const [record] = await requestSQL<\n {\n end_time: string;\n }[]\n >(\n terminal,\n `select end_time from series_data_range where series_id = ${escapeSQL(\n series_id,\n )} and table_name = 'ohlc_v2' order by end_time desc limit 1`,\n );\n\n const time = record ? new Date(record.end_time).getTime() : 0;\n\n req = {\n product_id,\n duration,\n direction: 'forward',\n time,\n };\n } else {\n // backward\n req = {\n product_id,\n duration,\n direction,\n time: Date.now(),\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Request',\n `product_id=${req.product_id}, duration=${req.duration}, direction=${req.direction}, time=${formatTime(\n req.time,\n )}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Forward]',\n 'Response',\n `series_id=${series_id}, ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/lib/series-collector/patch-interest-rate.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"patch-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";AAYA,eAAO,MAAM,uBAAuB,eACtB,MAAM,aACP,SAAS,GAAG,UAAU,UACzB,WAAW,~~kBAqFpB~~,CAAC"}
1	+ {"version":3,"file":"patch-interest-rate.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";AAYA,eAAO,MAAM,uBAAuB,eACtB,MAAM,aACP,SAAS,GAAG,UAAU,UACzB,WAAW,kBAmFpB,CAAC"}

package/lib/series-collector/patch-interest-rate.js CHANGED Viewed

@@ -11,8 +11,6 @@ const ingestCounter = terminal.metrics
 // Patch 任务：查找数据缺口并进行补齐
 const handleInterestRatePatch = async (product_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = (0, utils_1.decodePath)(product_id);
-    await (0, utils_1.tokenBucket)(`interest_rate:patch:${datasource_id}`).acquire(1, signal);
     const [record] = await (0, sql_1.requestSQL)(terminal, `
 WITH reversed_ranges AS (
     SELECT

package/lib/series-collector/patch-interest-rate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";;;AACA,+CAA4C;AAC5C,qCAAoD;AACpD,~~yCAAoE~~;~~AAEpE~~,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,uBAAuB;AAChB,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,~~aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,~~MAAM,~~IAAA,mBAAW,EAAC,uBAAuB,aAAa,EAAE,~~CAAC,~~CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7E,MAAM,CAAC,MAAM,CAAC,~~GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;~~AAxFW~~,QAAA,uBAAuB,~~2BAwFlC~~","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [~~datasource_id] = decodePath(product_id);\n await tokenBucket(`interest_rate:patch:${datasource_id}`).acquire(1, signal);\n const [~~record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'interest_rate' \n AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n 'FindGap',\n `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n let req: IIngestInterestRateRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"patch-interest-rate.js","sourceRoot":"","sources":["../../src/series-collector/patch-interest-rate.ts"],"names":[],"mappings":";;;AACA,+CAA4C;AAC5C,qCAAoD;AACpD,yCAA2C;AAE3C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEzF,uBAAuB;AAChB,MAAM,uBAAuB,GAAG,KAAK,EAC1C,UAAkB,EAClB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,UAAU,CAAC;;;;;;;;;;KAUxC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,SAAS,EACT,UAAU,UAAU,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACvF,CAAC;IAEF,IAAI,GAA+B,CAAC;IAEpC,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,oBAAoB,EACpB,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,wCAAwC,EACxC,UAAU,EACV,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AAtFW,QAAA,uBAAuB,2BAsFlC","sourcesContent":["import { IIngestInterestRateRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'interest_rate', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleInterestRatePatch = async (\n product_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'interest_rate' \n AND series_id = ${escapeSQL(product_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n 'FindGap',\n `series=${product_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n let req: IIngestInterestRateRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestInterestRateRequest, ISeriesIngestResult>(\n 'IngestInterestRate',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][InterestRate][Patch]',\n product_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/lib/series-collector/patch-ohlc.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"patch-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";AAaA,eAAO,MAAM,qBAAqB,cACrB,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,~~kBAyFpB~~,CAAC"}
1	+ {"version":3,"file":"patch-ohlc.d.ts","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";AAaA,eAAO,MAAM,qBAAqB,cACrB,MAAM,aACN,SAAS,GAAG,UAAU,UACzB,WAAW,kBAuFpB,CAAC"}

package/lib/series-collector/patch-ohlc.js CHANGED Viewed

@@ -12,8 +12,6 @@ const ingestCounter = terminal.metrics
 // Patch 任务：查找数据缺口并进行补齐
 const handleIngestOHLCPatch = async (series_id, direction, signal) => {
     var _a, _b, _c, _d;
-    const [datasource_id] = (0, utils_1.decodePath)(series_id);
-    await (0, utils_1.tokenBucket)(`ohlc:patch:${datasource_id}`).acquire(1, signal);
     const [record] = await (0, sql_1.requestSQL)(terminal, `
 WITH reversed_ranges AS (
     SELECT

package/lib/series-collector/patch-ohlc.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"patch-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";;;AAAA,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,~~yCAAoE~~;~~AAEpE~~,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,uBAAuB;AAChB,MAAM,qBAAqB,GAAG,KAAK,EACxC,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,~~aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,SAAS,CAAC,CAAC;IAC9C,~~MAAM,~~IAAA,mBAAW,EAAC,cAAc,aAAa,EAAE,~~CAAC,~~CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,CAAC,MAAM,CAAC,~~GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,SAAS,CAAC;;;;;;;;;;KAUvC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,UAAU,SAAS,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACtF,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAE5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;~~AA5FW~~,QAAA,qBAAqB,~~yBA4FhC~~","sourcesContent":["import { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { ~~decodePath,~~ formatTime, ~~tokenBucket~~ } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleIngestOHLCPatch = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [~~datasource_id] = decodePath(series_id);\n await tokenBucket(`ohlc:patch:${datasource_id}`).acquire(1, signal);\n const [~~record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'ohlc_v2' \n AND series_id = ${escapeSQL(series_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n 'FindGap',\n `series=${series_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}
1	+ {"version":3,"file":"patch-ohlc.js","sourceRoot":"","sources":["../../src/series-collector/patch-ohlc.ts"],"names":[],"mappings":";;;AAAA,iDAAuD;AAEvD,+CAA4C;AAC5C,qCAAoD;AACpD,yCAA2C;AAE3C,MAAM,QAAQ,GAAG,mBAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;KACnC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;KAC5C,MAAM,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAEhF,uBAAuB;AAChB,MAAM,qBAAqB,GAAG,KAAK,EACxC,SAAiB,EACjB,SAAiC,EACjC,MAAmB,EACnB,EAAE;;IACF,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAA,gBAAU,EAC/B,QAAQ,EACR;;;;;;;;;;;wBAWoB,IAAA,eAAS,EAAC,SAAS,CAAC;;;;;;;;;;KAUvC,CACF,CAAC;IAEF,SAAS;IACT,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,UAAU,SAAS,UAAU,IAAA,kBAAU,EAAC,YAAY,CAAC,QAAQ,IAAA,kBAAU,EAAC,UAAU,CAAC,EAAE,CACtF,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAA,8BAAkB,EAAC,SAAS,CAAC,CAAC;IAE/D,IAAI,GAAuB,CAAC;IAE5B,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,gBAAgB;QAChB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,YAAY;SACnB,CAAC;KACH;SAAM;QACL,iBAAiB;QACjB,GAAG,GAAG;YACJ,UAAU,EAAE,UAAU;YACtB,QAAQ;YACR,SAAS,EAAE,UAAmB;YAC9B,IAAI,EAAE,UAAU;SACjB,CAAC;KACH;IAED,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,cAAc,EACd,aAAa,GAAG,CAAC,SAAS,UAAU,IAAA,kBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3D,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACtD,YAAY,EACZ,GAAG,CACJ,CAAC;IAEF,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CACV,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,gCAAgC,EAChC,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GAAG,CAAC,WAAW,gBAAgB,IAAA,kBAAU,EACzD,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,UAAU,mCAAI,GAAG,CAC7B,cAAc,IAAA,kBAAU,EAAC,MAAA,MAAA,GAAG,CAAC,KAAK,0CAAE,QAAQ,mCAAI,GAAG,CAAC,EAAE,CACxD,CAAC;AACJ,CAAC,CAAC;AA1FW,QAAA,qBAAqB,yBA0FhC","sourcesContent":["import { decodeOHLCSeriesId } from '@yuants/data-ohlc';\nimport { IIngestOHLCRequest, ISeriesIngestResult } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { escapeSQL, requestSQL } from '@yuants/sql';\nimport { formatTime } from '@yuants/utils';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst ingestCounter = terminal.metrics\n .counter('series_collector_ingest_count', '')\n .labels({ terminal_id: terminal.terminal_id, type: 'ohlc', task: 'forward' });\n\n// Patch 任务：查找数据缺口并进行补齐\nexport const handleIngestOHLCPatch = async (\n series_id: string,\n direction: 'forward' \| 'backward',\n signal: AbortSignal,\n) => {\n const [record] = await requestSQL<{ gap_start_time: string; gap_end_time: string }[]>(\n terminal,\n `\nWITH reversed_ranges AS (\n SELECT \n start_time,\n end_time,\n LEAD(end_time) OVER (\n PARTITION BY table_name, series_id \n ORDER BY start_time DESC\n ) AS next_end_time -- 注意：倒序时 LEAD 是前一个区间\n FROM series_data_range\n WHERE table_name = 'ohlc_v2' \n AND series_id = ${escapeSQL(series_id)}\n)\nSELECT \n next_end_time AS gap_start_time, -- 前一个区间的结束时间\n start_time AS gap_end_time -- 当前区间的开始时间\nFROM reversed_ranges\nWHERE next_end_time IS NOT NULL \n AND start_time > next_end_time -- 有空缺\nORDER BY start_time DESC -- 从最新开始\nLIMIT 1;\n `,\n );\n\n // no gap\n if (!record) return;\n\n const gapStartTime = new Date(record.gap_start_time).getTime();\n const gapEndTime = new Date(record.gap_end_time).getTime();\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n 'FindGap',\n `series=${series_id}, from=${formatTime(gapStartTime)}, to=${formatTime(gapEndTime)}`,\n );\n\n const { product_id, duration } = decodeOHLCSeriesId(series_id);\n\n let req: IIngestOHLCRequest;\n\n if (direction === 'forward') {\n // forward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'forward' as const,\n time: gapStartTime,\n };\n } else {\n // backward patch\n req = {\n product_id: product_id,\n duration,\n direction: 'backward' as const,\n time: gapEndTime,\n };\n }\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchRequest',\n `direction=${req.direction}, time=${formatTime(req.time)}`,\n );\n\n const res = await terminal.client.requestForResponseData<IIngestOHLCRequest, ISeriesIngestResult>(\n 'IngestOHLC',\n req,\n );\n\n ingestCounter.inc(res.wrote_count \|\| 0);\n\n console.info(\n formatTime(Date.now()),\n '[SeriesCollector][OHLC][Patch]',\n series_id,\n 'PatchBackwardResult',\n `ingested_count=${res.wrote_count}, start_time=${formatTime(\n res.range?.start_time ?? NaN,\n )}, end_time=${formatTime(res.range?.end_time ?? NaN)}`,\n );\n};\n"]}

package/lib/series-collector/setup.js CHANGED Viewed

@@ -34,20 +34,25 @@ const api = {
                     await (0, utils_1.tokenBucket)(`${type}:${task}`).acquire(1, signal);
                     try {
                         const tasks = await list();
-                        // 并行
-                        // await Promise.all(
-                        //   Array.from(tasks.entries()).map(([series_id, direction]) =>
-                        //     handler(series_id, direction, signal).catch((err) => {
-                        //       console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);
-                        //     }),
-                        //   ),
-                        // );
-                        // 串行调度
-                        for (const [series_id, direction] of tasks.entries()) {
-                            await handler(series_id, direction, signal).catch((err) => {
-                                console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);
-                            });
+                        // const groups = Map.groupBy(tasks, item => decodePath(item[0])[0]);
+                        const groups = new Map();
+                        for (const item of tasks) {
+                            const [datasource_id] = (0, utils_1.decodePath)(item[0]);
+                            let items = groups.get(datasource_id);
+                            if (!items) {
+                                items = [];
+                                groups.set(datasource_id, items);
+                            }
+                            items.push(item);
                         }
+                        await Promise.all(Array.from(groups.entries()).map(async ([datasource_id, tasks]) => {
+                            for (const [series_id, direction] of tasks) {
+                                await (0, utils_1.tokenBucket)(`${type}:${task}:${datasource_id}`).acquire(1, signal);
+                                await handler(series_id, direction, signal).catch((err) => {
+                                    console.info((0, utils_1.formatTime)(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);
+                                });
+                            }
+                        }));
                     }
                     catch (e) { }
                 }

package/lib/series-collector/setup.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/series-collector/setup.ts"],"names":[],"mappings":";;AAAA,~~yCAAwD~~;~~AACxD~~,uEAA6E;AAC7E,qDAA4D;AAC5D,2CAA2E;AAC3E,qEAA2E;AAC3E,mDAA0D;AAC1D,+DAAgE;AAChE,6CAAqD;AAErD,MAAM,GAAG,GAAG;IACV,IAAI,EAAE;QACJ,IAAI,EAAE,6BAAiB;QACvB,OAAO,EAAE,uCAAuB;QAChC,QAAQ,EAAE,yCAAwB;QAClC,KAAK,EAAE,kCAAqB;KAC7B;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,qCAAyB;QAC/B,OAAO,EAAE,wDAA+B;QACxC,QAAQ,EAAE,0DAAgC;QAC1C,KAAK,EAAE,6CAAuB;KAC/B;CACF,CAAC;AAEF,CAAC,KAAK,IAAI,EAAE;IACV,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,cAAc,CAAU,EAAE;QACpD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE;YAC5D,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,KAAK,IAAI,EAAE;gBACV,OAAO,IAAI,EAAE;oBACX,MAAM,IAAA,mBAAW,EAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBACxD,IAAI;wBACF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;~~wBAC3B~~,~~KAAK~~;~~wBACL~~,~~qBAAqB~~;~~wBACrB~~,~~gEAAgE~~;~~wBAChE~~,~~6DAA6D~~;~~wBAC7D~~,~~mGAAmG~~;~~wBACnG~~,~~UAAU~~;~~wBACV~~,~~OAAO~~;~~wBACP~~,KAAK;~~wBACL~~,OAAO;~~wBACP~~,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,~~EAAE~~;~~4BACpD~~,MAAM,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;~~gCACxD~~,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,IAAI,KAAK,IAAI,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;~~4BAC5F~~,CAAC,CAAC,CAAC;~~yBACJ~~;~~qBACF~~;oBAAC,OAAO,CAAC,EAAE,GAAE;iBACf;YACH,CAAC,CAAC,EAAE,CAAC;SACN;KACF;AACH,CAAC,CAAC,EAAE,CAAC","sourcesContent":["import { formatTime, tokenBucket } from '@yuants/utils';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { handleIngestOHLCBackward } from './backwards-ohlc';\nimport { listInterestRateSeriesIds, listOHLCSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleIngestOHLCForward } from './forwards-ohlc';\nimport { handleInterestRatePatch } from './patch-interest-rate';\nimport { handleIngestOHLCPatch } from './patch-ohlc';\n\nconst api = {\n OHLC: {\n list: listOHLCSeriesIds,\n forward: handleIngestOHLCForward,\n backward: handleIngestOHLCBackward,\n patch: handleIngestOHLCPatch,\n },\n InterestRate: {\n list: listInterestRateSeriesIds,\n forward: handleIngestInterestRateForward,\n backward: handleIngestInterestRateBackward,\n patch: handleInterestRatePatch,\n },\n};\n\n(async () => {\n const abortController = new AbortController();\n const signal = abortController.signal;\n for (const type of ['OHLC', 'InterestRate'] as const) {\n const list = api[type].list;\n for (const task of ['forward', 'backward', 'patch'] as const) {\n const handler = api[type][task];\n (async () => {\n while (true) {\n await tokenBucket(`${type}:${task}`).acquire(1, signal);\n try {\n const tasks = await list();\n ~~// 并行\~~n // ~~await~~ ~~Promise~~.~~all~~(~~\n // Array.from(~~tasks~~.entries~~())~~.map((~~[~~series_id~~, ~~direction~~]) ~~=>\~~n ~~// handler~~(~~series_id,~~ ~~direction,~~ ~~signal~~)~~.catch((err)~~ => {\n ~~// console.info~~(~~formatTime(Date~~.~~now~~()), ~~`[SeriesCollector][$~~{~~type}]~~[~~${task}~~]~~`, 'Error'~~, ~~err~~);\n // }),\n ~~// ),\~~n // )~~;\n //~~ ~~串行调度\~~n for (const [series_id, direction] of tasks~~.entries(~~)) {\n await handler(series_id, direction, signal).catch((err) => {\n console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);\n });\n }\n } catch (e) {}\n }\n })();\n }\n }\n})();\n"]}
1	+ {"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/series-collector/setup.ts"],"names":[],"mappings":";;AAAA,yCAAoE;AACpE,uEAA6E;AAC7E,qDAA4D;AAC5D,2CAA2E;AAC3E,qEAA2E;AAC3E,mDAA0D;AAC1D,+DAAgE;AAChE,6CAAqD;AAErD,MAAM,GAAG,GAAG;IACV,IAAI,EAAE;QACJ,IAAI,EAAE,6BAAiB;QACvB,OAAO,EAAE,uCAAuB;QAChC,QAAQ,EAAE,yCAAwB;QAClC,KAAK,EAAE,kCAAqB;KAC7B;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,qCAAyB;QAC/B,OAAO,EAAE,wDAA+B;QACxC,QAAQ,EAAE,0DAAgC;QAC1C,KAAK,EAAE,6CAAuB;KAC/B;CACF,CAAC;AAEF,CAAC,KAAK,IAAI,EAAE;IACV,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,cAAc,CAAU,EAAE;QACpD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE;YAC5D,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,KAAK,IAAI,EAAE;gBACV,OAAO,IAAI,EAAE;oBACX,MAAM,IAAA,mBAAW,EAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBACxD,IAAI;wBACF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;wBAE3B,qEAAqE;wBACrE,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8C,CAAC;wBACrE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;4BACxB,MAAM,CAAC,aAAa,CAAC,GAAG,IAAA,kBAAU,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;4BAC5C,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;4BACtC,IAAI,CAAC,KAAK,EAAE;gCACV,KAAK,GAAG,EAAE,CAAC;gCACX,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;6BAClC;4BACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;yBAClB;wBAED,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,EAAE;4BAChE,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE;gCAC1C,MAAM,IAAA,mBAAW,EAAC,GAAG,IAAI,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gCACzE,MAAM,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oCACxD,OAAO,CAAC,IAAI,CAAC,IAAA,kBAAU,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,IAAI,KAAK,IAAI,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gCAC5F,CAAC,CAAC,CAAC;6BACJ;wBACH,CAAC,CAAC,CACH,CAAC;qBACH;oBAAC,OAAO,CAAC,EAAE,GAAE;iBACf;YACH,CAAC,CAAC,EAAE,CAAC;SACN;KACF;AACH,CAAC,CAAC,EAAE,CAAC","sourcesContent":["import { decodePath, formatTime, tokenBucket } from '@yuants/utils';\nimport { handleIngestInterestRateBackward } from './backwards-interest-rate';\nimport { handleIngestOHLCBackward } from './backwards-ohlc';\nimport { listInterestRateSeriesIds, listOHLCSeriesIds } from './discovery';\nimport { handleIngestInterestRateForward } from './forwards-interest-rate';\nimport { handleIngestOHLCForward } from './forwards-ohlc';\nimport { handleInterestRatePatch } from './patch-interest-rate';\nimport { handleIngestOHLCPatch } from './patch-ohlc';\n\nconst api = {\n OHLC: {\n list: listOHLCSeriesIds,\n forward: handleIngestOHLCForward,\n backward: handleIngestOHLCBackward,\n patch: handleIngestOHLCPatch,\n },\n InterestRate: {\n list: listInterestRateSeriesIds,\n forward: handleIngestInterestRateForward,\n backward: handleIngestInterestRateBackward,\n patch: handleInterestRatePatch,\n },\n};\n\n(async () => {\n const abortController = new AbortController();\n const signal = abortController.signal;\n for (const type of ['OHLC', 'InterestRate'] as const) {\n const list = api[type].list;\n for (const task of ['forward', 'backward', 'patch'] as const) {\n const handler = api[type][task];\n (async () => {\n while (true) {\n await tokenBucket(`${type}:${task}`).acquire(1, signal);\n try {\n const tasks = await list();\n\n // const groups = Map.groupBy(tasks, item => decodePath(item[0])[0]);\n const groups = new Map<string, [string, 'forward' \| 'backward'][]>();\n for (const item of tasks) {\n const [datasource_id] = decodePath(item[0]);\n let items = groups.get(datasource_id);\n if (!items) {\n items = [];\n groups.set(datasource_id, items);\n }\n items.push(item);\n }\n\n await Promise.all(\n Array.from(groups.entries()).map(async ([datasource_id, tasks]) => {\n for (const [series_id, direction] of tasks) {\n await tokenBucket(`${type}:${task}:${datasource_id}`).acquire(1, signal);\n await handler(series_id, direction, signal).catch((err) => {\n console.info(formatTime(Date.now()), `[SeriesCollector][${type}][${task}]`, 'Error', err);\n });\n }\n }),\n );\n } catch (e) {}\n }\n })();\n }\n }\n})();\n"]}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
 	"name": "@yuants/app-virtual-exchange",
-	"version": "0.18.3",
+	"version": "0.18.4",
 	"main": "lib/index.js",
 	"files": [
 		"dist",
@@ -8,18 +8,18 @@
 		"temp"
 	],
 	"dependencies": {
-		"@yuants/protocol": "0.53.9",
-		"@yuants/utils": "0.19.2",
-		"@yuants/data-product": "0.5.7",
-		"@yuants/data-account": "0.11.6",
-		"@yuants/data-order": "0.7.7",
-		"@yuants/data-quote": "0.4.6",
-		"@yuants/data-ohlc": "0.6.2",
-		"@yuants/data-interest-rate": "0.2.5",
-		"@yuants/secret": "0.4.7",
-		"@yuants/sql": "0.9.37",
-		"@yuants/exchange": "0.8.8",
-		"@yuants/cache": "0.3.10",
+		"@yuants/protocol": "0.54.0",
+		"@yuants/utils": "0.19.3",
+		"@yuants/data-product": "0.5.8",
+		"@yuants/data-account": "0.11.7",
+		"@yuants/data-order": "0.7.8",
+		"@yuants/data-quote": "0.4.7",
+		"@yuants/data-ohlc": "0.6.3",
+		"@yuants/data-interest-rate": "0.2.6",
+		"@yuants/secret": "0.4.8",
+		"@yuants/sql": "0.9.38",
+		"@yuants/exchange": "0.8.9",
+		"@yuants/cache": "0.3.11",
 		"rxjs": "~7.5.6",
 		"ajv": "~8.12.0"
 	},