@yuants/app-virtual-exchange 0.10.0 → 0.10.1

package/dist/quote/service.js

@@ -1,29 +1,107 @@
 import { Terminal } from '@yuants/protocol';
-import { newError } from '@yuants/utils';
+import { formatTime, newError } from '@yuants/utils';
+import { Subject, concatMap, defer } from 'rxjs';
 import { createQuoteState } from './state';
-import { fillQuoteStateFromUpstream } from './upstream-routing';
+import { createQuoteProviderRegistry } from './upstream';
 const terminal = Terminal.fromNodeEnv();
 const quoteState = createQuoteState();
-const assertFreshnessSatisfied = (data, params) => {
-    var _a;
-    console.info('[VEX][Quote] Asserting freshness satisfied for requested quotes.', JSON.stringify(params), JSON.stringify(data));
-    const { product_ids, fields, updated_at } = params;
-    const stillMissed = [];
+const quoteProviderRegistry = createQuoteProviderRegistry(terminal);
+const normalizeStrings = (values) => [...new Set(values)].sort();
+const normalizeFields = (values) => [...new Set(values)].sort();
+const analyzeRequestedQuotes = (quoteState, product_ids, fields, updated_at) => {
+    const missing = [];
+    const needUpdate = [];
     for (const product_id of product_ids) {
         for (const field of fields) {
-            if (!((_a = data[product_id]) === null || _a === void 0 ? void 0 : _a[field])) {
-                stillMissed.push({ product_id, field });
+            const tuple = quoteState.getValueTuple(product_id, field);
+            if (tuple === undefined) {
+                missing.push({ product_id, field });
+                needUpdate.push({ product_id, field });
+                continue;
+            }
+            if (tuple[1] < updated_at) {
+                needUpdate.push({ product_id, field });
+            }
+        }
+    }
+    return { missing, needUpdate };
+};
+const filterLatest = (quoteState, product_ids, fields) => {
+    const result = {};
+    for (const product_id of product_ids) {
+        result[product_id] = {};
+        for (const field of fields) {
+            const tuple = quoteState.getValueTuple(product_id, field);
+            if (tuple) {
+                result[product_id][field] = tuple;
             }
         }
     }
-    if (stillMissed.length > 0) {
+    return result;
+};
+const assertNotMissing = (missing, updated_at) => {
+    if (missing.length > 0) {
         throw newError('VEX_QUOTE_FRESHNESS_NOT_SATISFIED', {
             updated_at,
-            missed: stillMissed.slice(0, 200),
-            missed_total: stillMissed.length,
+            missed: missing.slice(0, 200),
+            missed_total: missing.length,
         });
     }
 };
+const updateQueue$ = new Subject();
+const updateQueueStats = {
+    queued_total: 0,
+    started_total: 0,
+    processed_total: 0,
+};
+const getQueueStatus = () => {
+    const pending = updateQueueStats.queued_total - updateQueueStats.started_total;
+    const in_flight = updateQueueStats.started_total - updateQueueStats.processed_total;
+    return Object.assign({ pending,
+        in_flight }, updateQueueStats);
+};
+const enqueueUpdateTask = (task) => {
+    updateQueueStats.queued_total++;
+    updateQueueStats.last_enqueued_at = Date.now();
+    updateQueue$.next(task);
+};
+const summarizeError = (error) => {
+    if (typeof error === 'object' && error !== null) {
+        const code = 'code' in error ? error.code : undefined;
+        const message = 'message' in error ? error.message : undefined;
+        return {
+            code: typeof code === 'string' ? code : undefined,
+            message: typeof message === 'string' ? message : undefined,
+        };
+    }
+    return {};
+};
+const processUpdateTask = async (task) => {
+    const { needUpdate } = analyzeRequestedQuotes(quoteState, task.product_ids, task.fields, task.updated_at);
+    await quoteProviderRegistry.fillQuoteStateFromUpstream({
+        quoteState,
+        cacheMissed: needUpdate,
+        updated_at: task.updated_at,
+    });
+};
+updateQueue$
+    .pipe(concatMap((task) => defer(async () => {
+    updateQueueStats.started_total++;
+    updateQueueStats.last_started_at = Date.now();
+    try {
+        await processUpdateTask(task);
+    }
+    catch (error) {
+        const summary = summarizeError(error);
+        updateQueueStats.last_error = Object.assign({ at: Date.now() }, summary);
+        console.info(formatTime(Date.now()), `[VEX][Quote]UpdateQueueTaskFailed`, `product_ids=${task.product_ids.length} fields=${task.fields.length} updated_at=${task.updated_at}`, JSON.stringify(summary));
+    }
+    finally {
+        updateQueueStats.processed_total++;
+        updateQueueStats.last_processed_at = Date.now();
+    }
+})))
+    .subscribe();
 terminal.server.provideService('VEX/UpdateQuotes', {}, async (msg) => {
     quoteState.update(msg.req);
     return { res: { code: 0, message: 'OK' } };
@@ -31,19 +109,7 @@ terminal.server.provideService('VEX/UpdateQuotes', {}, async (msg) => {
 terminal.server.provideService('VEX/DumpQuoteState', {}, async () => {
     return { res: { code: 0, message: 'OK', data: quoteState.dumpAsObject() } };
 });
-const computeCacheMissed = (quoteState, product_ids, fields, updated_at) => {
-    const cacheMissed = [];
-    for (const product_id of product_ids) {
-        for (const field of fields) {
-            const tuple = quoteState.getValueTuple(product_id, field);
-            if (tuple === undefined || tuple[1] < updated_at) {
-                cacheMissed.push({ product_id, field });
-            }
-        }
-    }
-    return cacheMissed;
-};
-terminal.server.provideService('VEX/QueryQuotes', {
+terminal.server.provideService('VEX/QueryQuotes/C1', {
     type: 'object',
     required: ['product_ids', 'fields', 'updated_at'],
     properties: {
@@ -58,11 +124,18 @@ terminal.server.provideService('VEX/QueryQuotes', {
         updated_at: { type: 'number' },
     },
 }, async (msg) => {
-    const { product_ids, fields, updated_at } = msg.req;
-    const cacheMissed = computeCacheMissed(quoteState, product_ids, fields, updated_at);
-    await fillQuoteStateFromUpstream({ terminal, quoteState, cacheMissed, updated_at });
-    const data = quoteState.filter(product_ids, fields, updated_at);
-    assertFreshnessSatisfied(data, { product_ids, fields, updated_at });
+    const product_ids = normalizeStrings(msg.req.product_ids);
+    const fields = normalizeFields(msg.req.fields);
+    const { updated_at } = msg.req;
+    const { missing, needUpdate } = analyzeRequestedQuotes(quoteState, product_ids, fields, updated_at);
+    if (needUpdate.length > 0) {
+        enqueueUpdateTask({ product_ids, fields, updated_at });
+    }
+    const data = filterLatest(quoteState, product_ids, fields);
+    assertNotMissing(missing, updated_at);
     return { res: { code: 0, message: 'OK', data } };
 });
+terminal.server.provideService('VEX/QuoteUpdateQueueStatus', {}, async () => {
+    return { res: { code: 0, message: 'OK', data: getQueueStatus() } };
+});
 //# sourceMappingURL=service.js.map

package/dist/quote/service.js.map

@@ -1 +1 @@
-{"version":3,"file":"service.js","sourceRoot":"","sources":["../../src/quote/service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EAAE,0BAA0B,EAAc,MAAM,oBAAoB,CAAC;AAE5E,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;AAEtC,MAAM,wBAAwB,GAAG,CAC/B,IAAwB,EACxB,MAA0E,EAC1E,EAAE;;IACF,OAAO,CAAC,IAAI,CACV,kEAAkE,EAClE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EACtB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACrB,CAAC;IACF,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IACnD,MAAM,WAAW,GAAoD,EAAE,CAAC;IACxE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,IAAI,CAAC,CAAA,MAAA,IAAI,CAAC,UAAU,CAAC,0CAAG,KAAK,CAAC,CAAA,EAAE;gBAC9B,WAAW,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;aACzC;SACF;KACF;IACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;QAC1B,MAAM,QAAQ,CAAC,mCAAmC,EAAE;YAClD,UAAU;YACV,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACjC,YAAY,EAAE,WAAW,CAAC,MAAM;SACjC,CAAC,CAAC;KACJ;AACH,CAAC,CAAC;AAEF,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAqB,kBAAkB,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IACvF,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAyB,oBAAoB,EAAE,EAAE,EAAE,KAAK,IAAI,EAAE;IAC1F,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,YAAY,EAAE,EAAE,EAAE,CAAC;AAC9E,CAAC,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CACzB,UAAuB,EACvB,WAAqB,EACrB,MAAmB,EACnB,UAAkB,EACJ,EAAE;IAChB,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC1D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,EAAE;gBAChD,WAAW,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;aACzC;SACF;KACF;IACD,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAEF,QAAQ,CAAC,MAAM,CAAC,cAAc,CAI5B,iBAAiB,EACjB;IACE,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,CAAC;IACjD,UAAU,EAAE;QACV,WAAW,EAAE;YACX,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC1B;QACD,MAAM,EAAE;YACN,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC1B;QACD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;KAC/B;CACF,EACD,KAAK,EAAE,GAAG,EAAE,EAAE;IACZ,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC;IAEpD,MAAM,WAAW,GAAG,kBAAkB,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IACpF,MAAM,0BAA0B,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAEpF,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAChE,wBAAwB,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACpE,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;AACnD,CAAC,CACF,CAAC","sourcesContent":["import { Terminal } from '@yuants/protocol';\nimport { newError } from '@yuants/utils';\nimport { createQuoteState } from './state';\nimport { IQuoteKey, IQuoteState, IQuoteUpdateAction } from './types';\nimport { fillQuoteStateFromUpstream, IQuoteMiss } from './upstream-routing';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst quoteState = createQuoteState();\n\nconst assertFreshnessSatisfied = (\n  data: IQuoteUpdateAction,\n  params: { product_ids: string[]; fields: IQuoteKey[]; updated_at: number },\n) => {\n  console.info(\n    '[VEX][Quote] Asserting freshness satisfied for requested quotes.',\n    JSON.stringify(params),\n    JSON.stringify(data),\n  );\n  const { product_ids, fields, updated_at } = params;\n  const stillMissed: Array<{ product_id: string; field: IQuoteKey }> = [];\n  for (const product_id of product_ids) {\n    for (const field of fields) {\n      if (!data[product_id]?.[field]) {\n        stillMissed.push({ product_id, field });\n      }\n    }\n  }\n  if (stillMissed.length > 0) {\n    throw newError('VEX_QUOTE_FRESHNESS_NOT_SATISFIED', {\n      updated_at,\n      missed: stillMissed.slice(0, 200),\n      missed_total: stillMissed.length,\n    });\n  }\n};\n\nterminal.server.provideService<IQuoteUpdateAction>('VEX/UpdateQuotes', {}, async (msg) => {\n  quoteState.update(msg.req);\n  return { res: { code: 0, message: 'OK' } };\n});\n\nterminal.server.provideService<{}, IQuoteUpdateAction>('VEX/DumpQuoteState', {}, async () => {\n  return { res: { code: 0, message: 'OK', data: quoteState.dumpAsObject() } };\n});\n\nconst computeCacheMissed = (\n  quoteState: IQuoteState,\n  product_ids: string[],\n  fields: IQuoteKey[],\n  updated_at: number,\n): IQuoteMiss[] => {\n  const cacheMissed: IQuoteMiss[] = [];\n  for (const product_id of product_ids) {\n    for (const field of fields) {\n      const tuple = quoteState.getValueTuple(product_id, field);\n      if (tuple === undefined || tuple[1] < updated_at) {\n        cacheMissed.push({ product_id, field });\n      }\n    }\n  }\n  return cacheMissed;\n};\n\nterminal.server.provideService<\n  { product_ids: string[]; fields: IQuoteKey[]; updated_at: number },\n  IQuoteUpdateAction\n>(\n  'VEX/QueryQuotes',\n  {\n    type: 'object',\n    required: ['product_ids', 'fields', 'updated_at'],\n    properties: {\n      product_ids: {\n        type: 'array',\n        items: { type: 'string' },\n      },\n      fields: {\n        type: 'array',\n        items: { type: 'string' },\n      },\n      updated_at: { type: 'number' },\n    },\n  },\n  async (msg) => {\n    const { product_ids, fields, updated_at } = msg.req;\n\n    const cacheMissed = computeCacheMissed(quoteState, product_ids, fields, updated_at);\n    await fillQuoteStateFromUpstream({ terminal, quoteState, cacheMissed, updated_at });\n\n    const data = quoteState.filter(product_ids, fields, updated_at);\n    assertFreshnessSatisfied(data, { product_ids, fields, updated_at });\n    return { res: { code: 0, message: 'OK', data } };\n  },\n);\n"]}
+{"version":3,"file":"service.js","sourceRoot":"","sources":["../../src/quote/service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EAAE,2BAA2B,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;AAExC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;AACtC,MAAM,qBAAqB,GAAG,2BAA2B,CAAC,QAAQ,CAAC,CAAC;AAgBpE,MAAM,gBAAgB,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3E,MAAM,eAAe,GAAG,CAAC,MAAmB,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAE7E,MAAM,sBAAsB,GAAG,CAC7B,UAAuB,EACvB,WAAqB,EACrB,MAAmB,EACnB,UAAkB,EACyC,EAAE;IAC7D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC1D,IAAI,KAAK,KAAK,SAAS,EAAE;gBACvB,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpC,UAAU,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;gBACvC,SAAS;aACV;YACD,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,EAAE;gBACzB,UAAU,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;aACxC;SACF;KACF;IACD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACjC,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CACnB,UAAuB,EACvB,WAAqB,EACrB,MAAmB,EACC,EAAE;IACtB,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC1D,IAAI,KAAK,EAAE;gBACT,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;aACnC;SACF;KACF;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,OAAwB,EAAE,UAAkB,EAAE,EAAE;IACxE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;QACtB,MAAM,QAAQ,CAAC,mCAAmC,EAAE;YAClD,UAAU;YACV,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC7B,YAAY,EAAE,OAAO,CAAC,MAAM;SAC7B,CAAC,CAAC;KACJ;AACH,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,OAAO,EAAc,CAAC;AAC/C,MAAM,gBAAgB,GAA2D;IAC/E,YAAY,EAAE,CAAC;IACf,aAAa,EAAE,CAAC;IAChB,eAAe,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,cAAc,GAAG,GAA4B,EAAE;IACnD,MAAM,OAAO,GAAG,gBAAgB,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC;IAC/E,MAAM,SAAS,GAAG,gBAAgB,CAAC,aAAa,GAAG,gBAAgB,CAAC,eAAe,CAAC;IACpF,uBACE,OAAO;QACP,SAAS,IACN,gBAAgB,EACnB;AACJ,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAgB,EAAE,EAAE;IAC7C,gBAAgB,CAAC,YAAY,EAAE,CAAC;IAChC,gBAAgB,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/C,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAuC,EAAE;IAC7E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAE,KAAa,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/D,MAAM,OAAO,GAAG,SAAS,IAAI,KAAK,CAAC,CAAC,CAAE,KAAa,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QACxE,OAAO;YACL,IAAI,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YACjD,OAAO,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;SAC3D,CAAC;KACH;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,KAAK,EAAE,IAAgB,EAAE,EAAE;IACnD,MAAM,EAAE,UAAU,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1G,MAAM,qBAAqB,CAAC,0BAA0B,CAAC;QACrD,UAAU;QACV,WAAW,EAAE,UAAU;QACvB,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,YAAY;KACT,IAAI,CACH,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,KAAK,CAAC,KAAK,IAAI,EAAE;IACf,gBAAgB,CAAC,aAAa,EAAE,CAAC;IACjC,gBAAgB,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9C,IAAI;QACF,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;KAC/B;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,gBAAgB,CAAC,UAAU,mBAAK,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,IAAK,OAAO,CAAE,CAAC;QAC7D,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,mCAAmC,EACnC,eAAe,IAAI,CAAC,WAAW,CAAC,MAAM,WAAW,IAAI,CAAC,MAAM,CAAC,MAAM,eAAe,IAAI,CAAC,UAAU,EAAE,EACnG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CACxB,CAAC;KACH;YAAS;QACR,gBAAgB,CAAC,eAAe,EAAE,CAAC;QACnC,gBAAgB,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;KACjD;AACH,CAAC,CAAC,CACH,CACF;KACA,SAAS,EAAE,CAAC;AAEf,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAqB,kBAAkB,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IACvF,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAyB,oBAAoB,EAAE,EAAE,EAAE,KAAK,IAAI,EAAE;IAC1F,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,YAAY,EAAE,EAAE,EAAE,CAAC;AAC9E,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,MAAM,CAAC,cAAc,CAI5B,oBAAoB,EACpB;IACE,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,CAAC;IACjD,UAAU,EAAE;QACV,WAAW,EAAE;YACX,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC1B;QACD,MAAM,EAAE;YACN,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC1B;QACD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;KAC/B;CACF,EACD,KAAK,EAAE,GAAG,EAAE,EAAE;IACZ,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC;IAE/B,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IACpG,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;QACzB,iBAAiB,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;KACxD;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAC3D,gBAAgB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACtC,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;AACnD,CAAC,CACF,CAAC;AAEF,QAAQ,CAAC,MAAM,CAAC,cAAc,CAA8B,4BAA4B,EAAE,EAAE,EAAE,KAAK,IAAI,EAAE;IACvG,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;AACrE,CAAC,CAAC,CAAC","sourcesContent":["import { Terminal } from '@yuants/protocol';\nimport { formatTime, newError } from '@yuants/utils';\nimport { Subject, concatMap, defer } from 'rxjs';\nimport { createQuoteState } from './state';\nimport { IQuoteKey, IQuoteRequire, IQuoteState, IQuoteUpdateAction } from './types';\nimport { createQuoteProviderRegistry } from './upstream';\n\nconst terminal = Terminal.fromNodeEnv();\n\nconst quoteState = createQuoteState();\nconst quoteProviderRegistry = createQuoteProviderRegistry(terminal);\n\ntype UpdateTask = { product_ids: string[]; fields: IQuoteKey[]; updated_at: number };\n\ntype IQuoteUpdateQueueStatus = {\n  pending: number;\n  in_flight: number;\n  queued_total: number;\n  started_total: number;\n  processed_total: number;\n  last_enqueued_at?: number;\n  last_started_at?: number;\n  last_processed_at?: number;\n  last_error?: { at: number; code?: string; message?: string };\n};\n\nconst normalizeStrings = (values: string[]) => [...new Set(values)].sort();\nconst normalizeFields = (values: IQuoteKey[]) => [...new Set(values)].sort();\n\nconst analyzeRequestedQuotes = (\n  quoteState: IQuoteState,\n  product_ids: string[],\n  fields: IQuoteKey[],\n  updated_at: number,\n): { missing: IQuoteRequire[]; needUpdate: IQuoteRequire[] } => {\n  const missing: IQuoteRequire[] = [];\n  const needUpdate: IQuoteRequire[] = [];\n  for (const product_id of product_ids) {\n    for (const field of fields) {\n      const tuple = quoteState.getValueTuple(product_id, field);\n      if (tuple === undefined) {\n        missing.push({ product_id, field });\n        needUpdate.push({ product_id, field });\n        continue;\n      }\n      if (tuple[1] < updated_at) {\n        needUpdate.push({ product_id, field });\n      }\n    }\n  }\n  return { missing, needUpdate };\n};\n\nconst filterLatest = (\n  quoteState: IQuoteState,\n  product_ids: string[],\n  fields: IQuoteKey[],\n): IQuoteUpdateAction => {\n  const result: IQuoteUpdateAction = {};\n  for (const product_id of product_ids) {\n    result[product_id] = {};\n    for (const field of fields) {\n      const tuple = quoteState.getValueTuple(product_id, field);\n      if (tuple) {\n        result[product_id][field] = tuple;\n      }\n    }\n  }\n  return result;\n};\n\nconst assertNotMissing = (missing: IQuoteRequire[], updated_at: number) => {\n  if (missing.length > 0) {\n    throw newError('VEX_QUOTE_FRESHNESS_NOT_SATISFIED', {\n      updated_at,\n      missed: missing.slice(0, 200),\n      missed_total: missing.length,\n    });\n  }\n};\n\nconst updateQueue$ = new Subject<UpdateTask>();\nconst updateQueueStats: Omit<IQuoteUpdateQueueStatus, 'pending' | 'in_flight'> = {\n  queued_total: 0,\n  started_total: 0,\n  processed_total: 0,\n};\n\nconst getQueueStatus = (): IQuoteUpdateQueueStatus => {\n  const pending = updateQueueStats.queued_total - updateQueueStats.started_total;\n  const in_flight = updateQueueStats.started_total - updateQueueStats.processed_total;\n  return {\n    pending,\n    in_flight,\n    ...updateQueueStats,\n  };\n};\n\nconst enqueueUpdateTask = (task: UpdateTask) => {\n  updateQueueStats.queued_total++;\n  updateQueueStats.last_enqueued_at = Date.now();\n  updateQueue$.next(task);\n};\n\nconst summarizeError = (error: unknown): { code?: string; message?: string } => {\n  if (typeof error === 'object' && error !== null) {\n    const code = 'code' in error ? (error as any).code : undefined;\n    const message = 'message' in error ? (error as any).message : undefined;\n    return {\n      code: typeof code === 'string' ? code : undefined,\n      message: typeof message === 'string' ? message : undefined,\n    };\n  }\n  return {};\n};\n\nconst processUpdateTask = async (task: UpdateTask) => {\n  const { needUpdate } = analyzeRequestedQuotes(quoteState, task.product_ids, task.fields, task.updated_at);\n  await quoteProviderRegistry.fillQuoteStateFromUpstream({\n    quoteState,\n    cacheMissed: needUpdate,\n    updated_at: task.updated_at,\n  });\n};\n\nupdateQueue$\n  .pipe(\n    concatMap((task) =>\n      defer(async () => {\n        updateQueueStats.started_total++;\n        updateQueueStats.last_started_at = Date.now();\n\n        try {\n          await processUpdateTask(task);\n        } catch (error) {\n          const summary = summarizeError(error);\n          updateQueueStats.last_error = { at: Date.now(), ...summary };\n          console.info(\n            formatTime(Date.now()),\n            `[VEX][Quote]UpdateQueueTaskFailed`,\n            `product_ids=${task.product_ids.length} fields=${task.fields.length} updated_at=${task.updated_at}`,\n            JSON.stringify(summary),\n          );\n        } finally {\n          updateQueueStats.processed_total++;\n          updateQueueStats.last_processed_at = Date.now();\n        }\n      }),\n    ),\n  )\n  .subscribe();\n\nterminal.server.provideService<IQuoteUpdateAction>('VEX/UpdateQuotes', {}, async (msg) => {\n  quoteState.update(msg.req);\n  return { res: { code: 0, message: 'OK' } };\n});\n\nterminal.server.provideService<{}, IQuoteUpdateAction>('VEX/DumpQuoteState', {}, async () => {\n  return { res: { code: 0, message: 'OK', data: quoteState.dumpAsObject() } };\n});\n\nterminal.server.provideService<\n  { product_ids: string[]; fields: IQuoteKey[]; updated_at: number },\n  IQuoteUpdateAction\n>(\n  'VEX/QueryQuotes/C1',\n  {\n    type: 'object',\n    required: ['product_ids', 'fields', 'updated_at'],\n    properties: {\n      product_ids: {\n        type: 'array',\n        items: { type: 'string' },\n      },\n      fields: {\n        type: 'array',\n        items: { type: 'string' },\n      },\n      updated_at: { type: 'number' },\n    },\n  },\n  async (msg) => {\n    const product_ids = normalizeStrings(msg.req.product_ids);\n    const fields = normalizeFields(msg.req.fields);\n    const { updated_at } = msg.req;\n\n    const { missing, needUpdate } = analyzeRequestedQuotes(quoteState, product_ids, fields, updated_at);\n    if (needUpdate.length > 0) {\n      enqueueUpdateTask({ product_ids, fields, updated_at });\n    }\n\n    const data = filterLatest(quoteState, product_ids, fields);\n    assertNotMissing(missing, updated_at);\n    return { res: { code: 0, message: 'OK', data } };\n  },\n);\n\nterminal.server.provideService<{}, IQuoteUpdateQueueStatus>('VEX/QuoteUpdateQueueStatus', {}, async () => {\n  return { res: { code: 0, message: 'OK', data: getQueueStatus() } };\n});\n"]}

package/dist/quote/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/quote/types.ts"],"names":[],"mappings":"","sourcesContent":["import { IQuote } from '@yuants/data-quote';\n\nexport type IQuoteKey = Exclude<keyof IQuote, 'datasource_id' | 'product_id' | 'updated_at'>;\n\n/**\n * 用于批量更新的数据结构\n * 结构为：\n * product_id -> field_name (keyof IQuote) -> [value: string, updated_at: number]\n * 这样设计的目的是为了减少更新的数据量，同时保留每个字段的更新时间\n *\n * 例如：\n * ```json\n * {\n *   \"product_123\": {\n *     \"last_price\": [\"100.5\", 1627890123456],\n *     \"volume\": [\"1500\", 1627890123456]\n *   },\n *   \"product_456\": {\n *     \"last_price\": [\"200.0\", 1627890123456]\n *   }\n * }\n * ```\n * @public\n */\nexport type IQuoteUpdateAction = Record<\n  string,\n  Partial<Record<IQuoteKey, [value: string, updated_at: number]>>\n>;\nexport interface IQuoteState {\n  update: (action: IQuoteUpdateAction) => void;\n  dumpAsObject: () => IQuoteUpdateAction;\n  getValueTuple: (product_id: string, field: IQuoteKey) => [string, number] | undefined;\n  filter: (product_ids: string[], fields: IQuoteKey[], updated_at: number) => IQuoteUpdateAction;\n}\n"]}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/quote/types.ts"],"names":[],"mappings":"","sourcesContent":["import { IQuote } from '@yuants/data-quote';\nimport { IQuoteServiceMetadata } from '@yuants/exchange';\n\nexport type IQuoteKey = Exclude<keyof IQuote, 'datasource_id' | 'product_id' | 'updated_at'>;\n\n/**\n * 用于批量更新的数据结构\n * 结构为：\n * product_id -> field_name (keyof IQuote) -> [value: string, updated_at: number]\n * 这样设计的目的是为了减少更新的数据量，同时保留每个字段的更新时间\n *\n * 例如：\n * ```json\n * {\n *   \"product_123\": {\n *     \"last_price\": [\"100.5\", 1627890123456],\n *     \"volume\": [\"1500\", 1627890123456]\n *   },\n *   \"product_456\": {\n *     \"last_price\": [\"200.0\", 1627890123456]\n *   }\n * }\n * ```\n * @public\n */\nexport type IQuoteUpdateAction = Record<\n  string,\n  Partial<Record<IQuoteKey, [value: string, updated_at: number]>>\n>;\nexport interface IQuoteState {\n  update: (action: IQuoteUpdateAction) => void;\n  dumpAsObject: () => IQuoteUpdateAction;\n  getValueTuple: (product_id: string, field: IQuoteKey) => [string, number] | undefined;\n  filter: (product_ids: string[], fields: IQuoteKey[], updated_at: number) => IQuoteUpdateAction;\n}\n\nexport interface IQuoteProviderInstance {\n  terminal_id: string;\n  service_id: string;\n}\n\n/**\n * A \"provider group\" is a capability signature of `GetQuotes`.\n * Multiple vendor terminals may provide the same capability; VEX load-balances across instances.\n */\nexport interface IQuoteProviderGroup {\n  group_id: string;\n  meta: IQuoteServiceMetadata;\n  mapTerminalIdToInstance: Map<string, IQuoteProviderInstance>;\n}\n\nexport interface IQuoteRequire {\n  product_id: string;\n  field: IQuoteKey;\n}\n"]}

package/dist/quote/upstream/executor.js

@@ -0,0 +1,98 @@
+import { newError } from '@yuants/utils';
+import { filter, firstValueFrom, map } from 'rxjs';
+const createConcurrencyLimiter = (concurrency) => {
+    const queue = [];
+    let active = 0;
+    const next = () => {
+        if (active >= concurrency)
+            return;
+        const task = queue.shift();
+        if (!task)
+            return;
+        active++;
+        task();
+    };
+    return async (fn) => await new Promise((resolve, reject) => {
+        queue.push(async () => {
+            try {
+                resolve(await fn());
+            }
+            catch (e) {
+                reject(e);
+            }
+            finally {
+                active--;
+                next();
+            }
+        });
+        next();
+    });
+};
+const pickInstance = (params) => {
+    var _a;
+    const { group_id, instances, mapGroupIdToRoundRobinIndex } = params;
+    if (instances.length === 0)
+        throw newError('VEX_QUOTE_PROVIDER_INSTANCE_EMPTY', { group_id });
+    const nextIndex = ((_a = mapGroupIdToRoundRobinIndex.get(group_id)) !== null && _a !== void 0 ? _a : 0) % instances.length;
+    mapGroupIdToRoundRobinIndex.set(group_id, nextIndex + 1);
+    return instances[nextIndex];
+};
+const requestGetQuotes = async (params) => {
+    const { terminal, instance, req } = params;
+    const res = await firstValueFrom(terminal.client
+        .request('GetQuotes', instance.terminal_id, req, instance.service_id)
+        .pipe(map((msg) => msg.res), filter((v) => v !== undefined)));
+    if (res.code !== 0)
+        throw newError('VEX_QUOTE_PROVIDER_ERROR', { instance, res });
+    if (res.data === undefined)
+        throw newError('VEX_QUOTE_PROVIDER_DATA_MISSING', { instance, res });
+    return res.data;
+};
+const runWithProviderGroupConcurrencyLimit1 = async (params) => {
+    var _a;
+    const { group_id, mapGroupIdToTailPromise, fn } = params;
+    const prev = (_a = mapGroupIdToTailPromise.get(group_id)) !== null && _a !== void 0 ? _a : Promise.resolve();
+    let resolveCurrent = () => { };
+    const current = new Promise((resolve) => {
+        resolveCurrent = resolve;
+    });
+    mapGroupIdToTailPromise.set(group_id, prev.then(() => current));
+    await prev;
+    try {
+        return await fn();
+    }
+    finally {
+        resolveCurrent();
+    }
+};
+export const createGetQuotesExecutor = (terminal) => {
+    const mapGroupIdToRoundRobinIndex = new Map();
+    const mapGroupIdToTailPromise = new Map();
+    const limitGetQuotes = createConcurrencyLimiter(32);
+    const mapKeyToInFlightGetQuotesPromise = new Map();
+    const requestGetQuotesInFlight = (key, planned) => {
+        const existing = mapKeyToInFlightGetQuotesPromise.get(key);
+        if (existing)
+            return existing;
+        const promise = limitGetQuotes(() => runWithProviderGroupConcurrencyLimit1({
+            group_id: planned.group_id,
+            mapGroupIdToTailPromise,
+            fn: async () => {
+                const instance = pickInstance({
+                    group_id: planned.group_id,
+                    instances: planned.instances,
+                    mapGroupIdToRoundRobinIndex,
+                });
+                return await requestGetQuotes({ terminal, instance, req: planned.req });
+            },
+        })).finally(() => {
+            mapKeyToInFlightGetQuotesPromise.delete(key);
+        });
+        mapKeyToInFlightGetQuotesPromise.set(key, promise);
+        return promise;
+    };
+    return {
+        execute: async (requests) => await Promise.all(requests.map(async ({ key, planned }) => await requestGetQuotesInFlight(key, planned))),
+    };
+};
+//# sourceMappingURL=executor.js.map

package/dist/quote/upstream/executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.js","sourceRoot":"","sources":["../../../src/quote/upstream/executor.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAMnD,MAAM,wBAAwB,GAAG,CAAC,WAAmB,EAAE,EAAE;IACvD,MAAM,KAAK,GAAsB,EAAE,CAAC;IACpC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,IAAI,MAAM,IAAI,WAAW;YAAE,OAAO;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,EAAE,CAAC;QACT,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;IACF,OAAO,KAAK,EAAK,EAAoB,EAAc,EAAE,CACnD,MAAM,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACvC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACpB,IAAI;gBACF,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;aACrB;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,CAAC,CAAC,CAAC;aACX;oBAAS;gBACR,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,CAAC;aACR;QACH,CAAC,CAAC,CAAC;QACH,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAIrB,EAA0B,EAAE;;IAC3B,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,2BAA2B,EAAE,GAAG,MAAM,CAAC;IACpE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,QAAQ,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC9F,MAAM,SAAS,GAAG,CAAC,MAAA,2BAA2B,CAAC,GAAG,CAAC,QAAQ,CAAC,mCAAI,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IACtF,2BAA2B,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO,SAAS,CAAC,SAAS,CAAC,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAI/B,EAAuC,EAAE;IACxC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;IAC3C,MAAM,GAAG,GAAG,MAAM,cAAc,CAC9B,QAAQ,CAAC,MAAM;SACZ,OAAO,CACN,WAAW,EACX,QAAQ,CAAC,WAAW,EACpB,GAAG,EACH,QAAQ,CAAC,UAAU,CACpB;SACA,IAAI,CACH,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EACrB,MAAM,CAAC,CAAC,CAAC,EAAqC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAClE,CACJ,CAAC;IACF,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;QAAE,MAAM,QAAQ,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;IAClF,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS;QAAE,MAAM,QAAQ,CAAC,iCAAiC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;IACjG,OAAO,GAAG,CAAC,IAAW,CAAC;AACzB,CAAC,CAAC;AAEF,MAAM,qCAAqC,GAAG,KAAK,EAAK,MAIvD,EAAc,EAAE;;IACf,MAAM,EAAE,QAAQ,EAAE,uBAAuB,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC;IACzD,MAAM,IAAI,GAAG,MAAA,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,mCAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IACxE,IAAI,cAAc,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC5C,cAAc,GAAG,OAAO,CAAC;IAC3B,CAAC,CAAC,CAAC;IACH,uBAAuB,CAAC,GAAG,CACzB,QAAQ,EACR,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CACzB,CAAC;IACF,MAAM,IAAI,CAAC;IACX,IAAI;QACF,OAAO,MAAM,EAAE,EAAE,CAAC;KACnB;YAAS;QACR,cAAc,EAAE,CAAC;KAClB;AACH,CAAC,CAAC;AAMF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,QAAkB,EAAsB,EAAE;IAChF,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9D,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjE,MAAM,cAAc,GAAG,wBAAwB,CAAC,EAAE,CAAC,CAAC;IAEpD,MAAM,gCAAgC,GAAG,IAAI,GAAG,EAA+C,CAAC;IAChG,MAAM,wBAAwB,GAAG,CAAC,GAAW,EAAE,OAAwB,EAAE,EAAE;QACzE,MAAM,QAAQ,GAAG,gCAAgC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3D,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE,CAClC,qCAAqC,CAAC;YACpC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,uBAAuB;YACvB,EAAE,EAAE,KAAK,IAAI,EAAE;gBACb,MAAM,QAAQ,GAAG,YAAY,CAAC;oBAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,2BAA2B;iBAC5B,CAAC,CAAC;gBACH,OAAO,MAAM,gBAAgB,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1E,CAAC;SACF,CAAC,CACH,CAAC,OAAO,CAAC,GAAG,EAAE;YACb,gCAAgC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QACH,gCAAgC,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,CAC1B,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,MAAM,wBAAwB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CACvF;KACJ,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import {\n  IQuoteUpdateAction as IExchangeQuoteUpdateAction,\n  IQuoteServiceRequestByVEX,\n} from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { newError } from '@yuants/utils';\nimport { filter, firstValueFrom, map } from 'rxjs';\nimport { IQuoteProviderInstance } from '../types';\nimport { IPlannedRequestWithKey } from './router';\n\ntype IPlannedRequest = IPlannedRequestWithKey['planned'];\n\nconst createConcurrencyLimiter = (concurrency: number) => {\n  const queue: Array<() => void> = [];\n  let active = 0;\n  const next = () => {\n    if (active >= concurrency) return;\n    const task = queue.shift();\n    if (!task) return;\n    active++;\n    task();\n  };\n  return async <T>(fn: () => Promise<T>): Promise<T> =>\n    await new Promise<T>((resolve, reject) => {\n      queue.push(async () => {\n        try {\n          resolve(await fn());\n        } catch (e) {\n          reject(e);\n        } finally {\n          active--;\n          next();\n        }\n      });\n      next();\n    });\n};\n\nconst pickInstance = (params: {\n  group_id: string;\n  instances: IQuoteProviderInstance[];\n  mapGroupIdToRoundRobinIndex: Map<string, number>;\n}): IQuoteProviderInstance => {\n  const { group_id, instances, mapGroupIdToRoundRobinIndex } = params;\n  if (instances.length === 0) throw newError('VEX_QUOTE_PROVIDER_INSTANCE_EMPTY', { group_id });\n  const nextIndex = (mapGroupIdToRoundRobinIndex.get(group_id) ?? 0) % instances.length;\n  mapGroupIdToRoundRobinIndex.set(group_id, nextIndex + 1);\n  return instances[nextIndex];\n};\n\nconst requestGetQuotes = async (params: {\n  terminal: Terminal;\n  instance: IQuoteProviderInstance;\n  req: IQuoteServiceRequestByVEX;\n}): Promise<IExchangeQuoteUpdateAction> => {\n  const { terminal, instance, req } = params;\n  const res = await firstValueFrom(\n    terminal.client\n      .request<IQuoteServiceRequestByVEX, IExchangeQuoteUpdateAction>(\n        'GetQuotes',\n        instance.terminal_id,\n        req,\n        instance.service_id,\n      )\n      .pipe(\n        map((msg) => msg.res),\n        filter((v): v is Exclude<typeof v, undefined> => v !== undefined),\n      ),\n  );\n  if (res.code !== 0) throw newError('VEX_QUOTE_PROVIDER_ERROR', { instance, res });\n  if (res.data === undefined) throw newError('VEX_QUOTE_PROVIDER_DATA_MISSING', { instance, res });\n  return res.data as any;\n};\n\nconst runWithProviderGroupConcurrencyLimit1 = async <T>(params: {\n  group_id: string;\n  mapGroupIdToTailPromise: Map<string, Promise<void>>;\n  fn: () => Promise<T>;\n}): Promise<T> => {\n  const { group_id, mapGroupIdToTailPromise, fn } = params;\n  const prev = mapGroupIdToTailPromise.get(group_id) ?? Promise.resolve();\n  let resolveCurrent: () => void = () => {};\n  const current = new Promise<void>((resolve) => {\n    resolveCurrent = resolve;\n  });\n  mapGroupIdToTailPromise.set(\n    group_id,\n    prev.then(() => current),\n  );\n  await prev;\n  try {\n    return await fn();\n  } finally {\n    resolveCurrent();\n  }\n};\n\nexport interface IGetQuotesExecutor {\n  execute: (requests: IPlannedRequestWithKey[]) => Promise<IExchangeQuoteUpdateAction[]>;\n}\n\nexport const createGetQuotesExecutor = (terminal: Terminal): IGetQuotesExecutor => {\n  const mapGroupIdToRoundRobinIndex = new Map<string, number>();\n  const mapGroupIdToTailPromise = new Map<string, Promise<void>>();\n  const limitGetQuotes = createConcurrencyLimiter(32);\n\n  const mapKeyToInFlightGetQuotesPromise = new Map<string, Promise<IExchangeQuoteUpdateAction>>();\n  const requestGetQuotesInFlight = (key: string, planned: IPlannedRequest) => {\n    const existing = mapKeyToInFlightGetQuotesPromise.get(key);\n    if (existing) return existing;\n    const promise = limitGetQuotes(() =>\n      runWithProviderGroupConcurrencyLimit1({\n        group_id: planned.group_id,\n        mapGroupIdToTailPromise,\n        fn: async () => {\n          const instance = pickInstance({\n            group_id: planned.group_id,\n            instances: planned.instances,\n            mapGroupIdToRoundRobinIndex,\n          });\n          return await requestGetQuotes({ terminal, instance, req: planned.req });\n        },\n      }),\n    ).finally(() => {\n      mapKeyToInFlightGetQuotesPromise.delete(key);\n    });\n    mapKeyToInFlightGetQuotesPromise.set(key, promise);\n    return promise;\n  };\n\n  return {\n    execute: async (requests) =>\n      await Promise.all(\n        requests.map(async ({ key, planned }) => await requestGetQuotesInFlight(key, planned)),\n      ),\n  };\n};\n"]}

package/dist/quote/upstream/index.js

@@ -0,0 +1,2 @@
+export { createQuoteProviderRegistry } from './registry';
+//# sourceMappingURL=index.js.map

package/dist/quote/upstream/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/quote/upstream/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,YAAY,CAAC","sourcesContent":["export { createQuoteProviderRegistry } from './registry';\nexport type { IQuoteProviderRegistry } from './registry';\n"]}

package/dist/quote/upstream/prefix-matcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prefix-matcher.js","sourceRoot":"","sources":["../../../src/quote/upstream/prefix-matcher.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,OAA4C,EACzB,EAAE;IACrB,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9E,OAAO;QACL,KAAK,EAAE,CAAC,KAAa,EAAO,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;KACpG,CAAC;AACJ,CAAC,CAAC","sourcesContent":["export interface IPrefixMatcher<T> {\n  match: (value: string) => T[];\n}\n\nexport const createSortedPrefixMatcher = <T>(\n  entries: Array<{ prefix: string; value: T }>,\n): IPrefixMatcher<T> => {\n  const sorted = [...entries].sort((a, b) => b.prefix.length - a.prefix.length);\n  return {\n    match: (value: string): T[] => sorted.filter((x) => value.startsWith(x.prefix)).map((x) => x.value),\n  };\n};\n"]}

package/dist/quote/upstream/registry.js

@@ -0,0 +1,116 @@
+import { parseQuoteServiceMetadataFromSchema } from '@yuants/exchange';
+import { encodePath, formatTime, listWatch, newError } from '@yuants/utils';
+import { EMPTY, map, of, tap } from 'rxjs';
+import { createGetQuotesExecutor } from './executor';
+import { buildProviderIndices, createQuoteRouter, } from './router';
+const normalizeMetadataForGroup = (metadata) => {
+    const fields = [...metadata.fields].sort();
+    return Object.assign(Object.assign({}, metadata), { fields });
+};
+export const createQuoteProviderRegistry = (terminal) => {
+    const router = createQuoteRouter();
+    const executor = createGetQuotesExecutor(terminal);
+    const mapGroupIdToGroup = new Map();
+    let version = 0;
+    let cachedIndices;
+    const quoteServiceInfos$ = terminal.terminalInfos$.pipe(map((infos) => infos.flatMap((info) => {
+        var _a;
+        return Object.values((_a = info.serviceInfo) !== null && _a !== void 0 ? _a : {})
+            .filter((serviceInfo) => serviceInfo.method === 'GetQuotes')
+            .map((serviceInfo) => ({ terminal_id: info.terminal_id, serviceInfo }));
+    })));
+    quoteServiceInfos$
+        .pipe(listWatch((v) => v.serviceInfo.service_id, (v) => {
+        var _a, _b;
+        console.info(formatTime(Date.now()), `[VEX][Quote]DiscoveringGetQuotesProvider...`, `from terminal ${v.terminal_id}`, `service ${v.serviceInfo.service_id}`, `schema:  ${JSON.stringify(v.serviceInfo.schema)}`);
+        try {
+            const metadata = normalizeMetadataForGroup(parseQuoteServiceMetadataFromSchema(v.serviceInfo.schema));
+            const group_id = encodePath(metadata.product_id_prefix, metadata.fields.join(','), (_a = metadata.max_products_per_request) !== null && _a !== void 0 ? _a : '');
+            const provider = {
+                terminal_id: v.terminal_id,
+                service_id: v.serviceInfo.service_id || v.serviceInfo.method,
+            };
+            const group = (_b = mapGroupIdToGroup.get(group_id)) !== null && _b !== void 0 ? _b : (() => {
+                const next = {
+                    group_id,
+                    meta: metadata,
+                    mapTerminalIdToInstance: new Map(),
+                };
+                mapGroupIdToGroup.set(group_id, next);
+                return next;
+            })();
+            group.mapTerminalIdToInstance.set(provider.terminal_id, provider);
+            version++;
+            return of(void 0).pipe(tap({
+                unsubscribe: () => {
+                    var _a, _b;
+                    (_a = mapGroupIdToGroup.get(group_id)) === null || _a === void 0 ? void 0 : _a.mapTerminalIdToInstance.delete(v.terminal_id);
+                    if (((_b = mapGroupIdToGroup.get(group_id)) === null || _b === void 0 ? void 0 : _b.mapTerminalIdToInstance.size) === 0) {
+                        mapGroupIdToGroup.delete(group_id);
+                    }
+                    version++;
+                },
+            }));
+        }
+        catch (_c) {
+            console.info(formatTime(Date.now()), `[VEX][Quote]IgnoredGetQuotesProvider`, `Ignored GetQuotes provider from terminal ${v.terminal_id}`, `service ${v.serviceInfo.service_id} due to invalid schema.`);
+            return EMPTY;
+        }
+    }))
+        .subscribe();
+    const snapshot = () => {
+        const groups = Array.from(mapGroupIdToGroup.values());
+        if (!cachedIndices || cachedIndices.version !== version) {
+            cachedIndices = { version, indices: buildProviderIndices(groups) };
+        }
+        return { groups, indices: cachedIndices.indices };
+    };
+    const planOrThrow = (misses, updated_at) => {
+        const { groups, indices } = snapshot();
+        if (groups.length === 0)
+            throw newError('VEX_QUOTE_PROVIDER_NOT_FOUND', { method: 'GetQuotes' });
+        return router.planOrThrow(misses, indices, updated_at);
+    };
+    const execute = async (requests) => (await executor.execute(requests));
+    const fillQuoteStateFromUpstream = async (params) => {
+        const { quoteState, cacheMissed, updated_at } = params;
+        if (cacheMissed.length === 0)
+            return;
+        const { groups } = snapshot();
+        console.info(formatTime(Date.now()), `[VEX][Quote]UpstreamProviderDiscovery`, ` Discovered ${groups.length} GetQuotes provider groups from terminal infos.`, JSON.stringify(groups));
+        const plan = planOrThrow(cacheMissed, updated_at);
+        const mapGroupIdToProductIds = new Map();
+        for (const { planned } of plan.requests) {
+            let productIds = mapGroupIdToProductIds.get(planned.group_id);
+            if (!productIds) {
+                productIds = new Set();
+                mapGroupIdToProductIds.set(planned.group_id, productIds);
+            }
+            for (const product_id of planned.req.product_ids)
+                productIds.add(product_id);
+        }
+        console.info(formatTime(Date.now()), `[VEX][Quote]RouteDispatched`, ` Routed ${cacheMissed.length} missed quotes to ${mapGroupIdToProductIds.size} provider groups.`, JSON.stringify({
+            productsByGroupId: [...mapGroupIdToProductIds.entries()].map(([group_id, productIds]) => ({
+                group_id,
+                product_ids: [...productIds],
+            })),
+            default_action: plan.defaultAction,
+        }));
+        if (Object.keys(plan.defaultAction).length > 0) {
+            quoteState.update(plan.defaultAction);
+        }
+        console.info(formatTime(Date.now()), `[VEX][Quote]RequestPlanned`, `Planned ${plan.requests.length} upstream GetQuotes requests.`, JSON.stringify(plan.requests.map(({ key, planned }) => ({
+            key,
+            group_id: planned.group_id,
+            product_ids: planned.req.product_ids,
+            fields: planned.req.fields,
+        }))));
+        const actions = await execute(plan.requests);
+        console.debug(formatTime(Date.now()), `[VEX][Quote]RequestReceived`, `Received ${actions.length} upstream GetQuotes responses.`, JSON.stringify(actions));
+        for (const action of actions) {
+            quoteState.update(action);
+        }
+    };
+    return { snapshot, planOrThrow, execute, fillQuoteStateFromUpstream };
+};
+//# sourceMappingURL=registry.js.map

package/dist/quote/upstream/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../../src/quote/upstream/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,mCAAmC,EAAE,MAAM,kBAAkB,CAAC;AAE9F,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAS3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EACL,oBAAoB,EACpB,iBAAiB,GAIlB,MAAM,UAAU,CAAC;AAalB,MAAM,yBAAyB,GAAG,CAAC,QAA+B,EAAyB,EAAE;IAC3F,MAAM,MAAM,GAAG,CAAC,GAAI,QAAQ,CAAC,MAAiC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvE,uCAAY,QAAQ,KAAE,MAAM,IAAG;AACjC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,QAAkB,EAA0B,EAAE;IACxF,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA+B,CAAC;IACjE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,aAAyE,CAAC;IAE9E,MAAM,kBAAkB,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,CACrD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACZ,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;;QACrB,OAAA,MAAM,CAAC,MAAM,CAAC,MAAA,IAAI,CAAC,WAAW,mCAAI,EAAE,CAAC;aAClC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,KAAK,WAAW,CAAC;aAC3D,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC,CAAA;KAAA,CAC1E,CACF,CACF,CAAC;IAEF,kBAAkB;SACf,IAAI,CACH,SAAS,CACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,EAC/B,CAAC,CAAC,EAAE,EAAE;;QACJ,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,6CAA6C,EAC7C,iBAAiB,CAAC,CAAC,WAAW,EAAE,EAChC,WAAW,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,EACrC,YAAY,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CACnD,CAAC;QACF,IAAI;YACF,MAAM,QAAQ,GAAG,yBAAyB,CACxC,mCAAmC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAC1D,CAAC;YACF,MAAM,QAAQ,GAAG,UAAU,CACzB,QAAQ,CAAC,iBAAiB,EACzB,QAAQ,CAAC,MAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EACpC,MAAA,QAAQ,CAAC,wBAAwB,mCAAI,EAAE,CACxC,CAAC;YACF,MAAM,QAAQ,GAA2B;gBACvC,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,UAAU,EAAE,CAAC,CAAC,WAAW,CAAC,UAAU,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM;aAC7D,CAAC;YACF,MAAM,KAAK,GACT,MAAA,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,mCAC/B,CAAC,GAAG,EAAE;gBACJ,MAAM,IAAI,GAAwB;oBAChC,QAAQ;oBACR,IAAI,EAAE,QAAQ;oBACd,uBAAuB,EAAE,IAAI,GAAG,EAAkC;iBACnE,CAAC;gBACF,iBAAiB,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACtC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,EAAE,CAAC;YACP,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAClE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CACpB,GAAG,CAAC;gBACF,WAAW,EAAE,GAAG,EAAE;;oBAChB,MAAA,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,0CAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;oBAC/E,IAAI,CAAA,MAAA,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,0CAAE,uBAAuB,CAAC,IAAI,MAAK,CAAC,EAAE;wBACvE,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBACpC;oBACD,OAAO,EAAE,CAAC;gBACZ,CAAC;aACF,CAAC,CACH,CAAC;SACH;QAAC,WAAM;YACN,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,sCAAsC,EACtC,4CAA4C,CAAC,CAAC,WAAW,EAAE,EAC3D,WAAW,CAAC,CAAC,WAAW,CAAC,UAAU,yBAAyB,CAC7D,CAAC;YACF,OAAO,KAAK,CAAC;SACd;IACH,CAAC,CACF,CACF;SACA,SAAS,EAAE,CAAC;IAEf,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,OAAO,KAAK,OAAO,EAAE;YACvD,aAAa,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;SACpE;QACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC;IACpD,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,MAAuB,EAAE,UAAkB,EAAqB,EAAE;QACrF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,QAAQ,CAAC,8BAA8B,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QACjG,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACzD,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,QAAkC,EAAiC,EAAE,CAC1F,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAQ,CAAC;IAE5C,MAAM,0BAA0B,GAAyD,KAAK,EAAE,MAAM,EAAE,EAAE;QACxG,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;QACvD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAErC,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,uCAAuC,EACvC,eAAe,MAAM,CAAC,MAAM,iDAAiD,EAC7E,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CACvB,CAAC;QAEF,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAElD,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC9D,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE;YACvC,IAAI,UAAU,GAAG,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC9D,IAAI,CAAC,UAAU,EAAE;gBACf,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;gBAC/B,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;aAC1D;YACD,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW;gBAAE,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;SAC9E;QACD,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,6BAA6B,EAC7B,WAAW,WAAW,CAAC,MAAM,qBAAqB,sBAAsB,CAAC,IAAI,mBAAmB,EAChG,IAAI,CAAC,SAAS,CAAC;YACb,iBAAiB,EAAE,CAAC,GAAG,sBAAsB,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxF,QAAQ;gBACR,WAAW,EAAE,CAAC,GAAG,UAAU,CAAC;aAC7B,CAAC,CAAC;YACH,cAAc,EAAE,IAAI,CAAC,aAAa;SACnC,CAAC,CACH,CAAC;QAEF,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACvC;QAED,OAAO,CAAC,IAAI,CACV,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,4BAA4B,EAC5B,WAAW,IAAI,CAAC,QAAQ,CAAC,MAAM,+BAA+B,EAC9D,IAAI,CAAC,SAAS,CACZ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACvC,GAAG;YACH,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW;YACpC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM;SAC3B,CAAC,CAAC,CACJ,CACF,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,OAAO,CAAC,KAAK,CACX,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EACtB,6BAA6B,EAC7B,YAAY,OAAO,CAAC,MAAM,gCAAgC,EAC1D,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CACxB,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC5B,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SAC3B;IACH,CAAC,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,0BAA0B,EAAE,CAAC;AACxE,CAAC,CAAC","sourcesContent":["import { IQuoteServiceMetadata, parseQuoteServiceMetadataFromSchema } from '@yuants/exchange';\nimport { Terminal } from '@yuants/protocol';\nimport { encodePath, formatTime, listWatch, newError } from '@yuants/utils';\nimport { EMPTY, map, of, tap } from 'rxjs';\nimport {\n  IQuoteKey,\n  IQuoteProviderGroup,\n  IQuoteProviderInstance,\n  IQuoteRequire,\n  IQuoteState,\n  IQuoteUpdateAction,\n} from '../types';\nimport { createGetQuotesExecutor } from './executor';\nimport {\n  buildProviderIndices,\n  createQuoteRouter,\n  IProviderIndices,\n  IPlannedRequestWithKey,\n  QuoteUpstreamPlan,\n} from './router';\n\nexport interface IQuoteProviderRegistry {\n  snapshot: () => { groups: IQuoteProviderGroup[]; indices: IProviderIndices };\n  planOrThrow: (misses: IQuoteRequire[], updated_at: number) => QuoteUpstreamPlan;\n  execute: (requests: IPlannedRequestWithKey[]) => Promise<IQuoteUpdateAction[]>;\n  fillQuoteStateFromUpstream: (params: {\n    quoteState: IQuoteState;\n    cacheMissed: IQuoteRequire[];\n    updated_at: number;\n  }) => Promise<void>;\n}\n\nconst normalizeMetadataForGroup = (metadata: IQuoteServiceMetadata): IQuoteServiceMetadata => {\n  const fields = [...(metadata.fields as unknown as IQuoteKey[])].sort();\n  return { ...metadata, fields };\n};\n\nexport const createQuoteProviderRegistry = (terminal: Terminal): IQuoteProviderRegistry => {\n  const router = createQuoteRouter();\n  const executor = createGetQuotesExecutor(terminal);\n\n  const mapGroupIdToGroup = new Map<string, IQuoteProviderGroup>();\n  let version = 0;\n  let cachedIndices: { version: number; indices: IProviderIndices } | undefined;\n\n  const quoteServiceInfos$ = terminal.terminalInfos$.pipe(\n    map((infos) =>\n      infos.flatMap((info) =>\n        Object.values(info.serviceInfo ?? {})\n          .filter((serviceInfo) => serviceInfo.method === 'GetQuotes')\n          .map((serviceInfo) => ({ terminal_id: info.terminal_id, serviceInfo })),\n      ),\n    ),\n  );\n\n  quoteServiceInfos$\n    .pipe(\n      listWatch(\n        (v) => v.serviceInfo.service_id,\n        (v) => {\n          console.info(\n            formatTime(Date.now()),\n            `[VEX][Quote]DiscoveringGetQuotesProvider...`,\n            `from terminal ${v.terminal_id}`,\n            `service ${v.serviceInfo.service_id}`,\n            `schema:  ${JSON.stringify(v.serviceInfo.schema)}`,\n          );\n          try {\n            const metadata = normalizeMetadataForGroup(\n              parseQuoteServiceMetadataFromSchema(v.serviceInfo.schema),\n            );\n            const group_id = encodePath(\n              metadata.product_id_prefix,\n              (metadata.fields as any[]).join(','),\n              metadata.max_products_per_request ?? '',\n            );\n            const provider: IQuoteProviderInstance = {\n              terminal_id: v.terminal_id,\n              service_id: v.serviceInfo.service_id || v.serviceInfo.method,\n            };\n            const group =\n              mapGroupIdToGroup.get(group_id) ??\n              (() => {\n                const next: IQuoteProviderGroup = {\n                  group_id,\n                  meta: metadata,\n                  mapTerminalIdToInstance: new Map<string, IQuoteProviderInstance>(),\n                };\n                mapGroupIdToGroup.set(group_id, next);\n                return next;\n              })();\n            group.mapTerminalIdToInstance.set(provider.terminal_id, provider);\n            version++;\n            return of(void 0).pipe(\n              tap({\n                unsubscribe: () => {\n                  mapGroupIdToGroup.get(group_id)?.mapTerminalIdToInstance.delete(v.terminal_id);\n                  if (mapGroupIdToGroup.get(group_id)?.mapTerminalIdToInstance.size === 0) {\n                    mapGroupIdToGroup.delete(group_id);\n                  }\n                  version++;\n                },\n              }),\n            );\n          } catch {\n            console.info(\n              formatTime(Date.now()),\n              `[VEX][Quote]IgnoredGetQuotesProvider`,\n              `Ignored GetQuotes provider from terminal ${v.terminal_id}`,\n              `service ${v.serviceInfo.service_id} due to invalid schema.`,\n            );\n            return EMPTY;\n          }\n        },\n      ),\n    )\n    .subscribe();\n\n  const snapshot = () => {\n    const groups = Array.from(mapGroupIdToGroup.values());\n    if (!cachedIndices || cachedIndices.version !== version) {\n      cachedIndices = { version, indices: buildProviderIndices(groups) };\n    }\n    return { groups, indices: cachedIndices.indices };\n  };\n\n  const planOrThrow = (misses: IQuoteRequire[], updated_at: number): QuoteUpstreamPlan => {\n    const { groups, indices } = snapshot();\n    if (groups.length === 0) throw newError('VEX_QUOTE_PROVIDER_NOT_FOUND', { method: 'GetQuotes' });\n    return router.planOrThrow(misses, indices, updated_at);\n  };\n\n  const execute = async (requests: IPlannedRequestWithKey[]): Promise<IQuoteUpdateAction[]> =>\n    (await executor.execute(requests)) as any;\n\n  const fillQuoteStateFromUpstream: IQuoteProviderRegistry['fillQuoteStateFromUpstream'] = async (params) => {\n    const { quoteState, cacheMissed, updated_at } = params;\n    if (cacheMissed.length === 0) return;\n\n    const { groups } = snapshot();\n    console.info(\n      formatTime(Date.now()),\n      `[VEX][Quote]UpstreamProviderDiscovery`,\n      ` Discovered ${groups.length} GetQuotes provider groups from terminal infos.`,\n      JSON.stringify(groups),\n    );\n\n    const plan = planOrThrow(cacheMissed, updated_at);\n\n    const mapGroupIdToProductIds = new Map<string, Set<string>>();\n    for (const { planned } of plan.requests) {\n      let productIds = mapGroupIdToProductIds.get(planned.group_id);\n      if (!productIds) {\n        productIds = new Set<string>();\n        mapGroupIdToProductIds.set(planned.group_id, productIds);\n      }\n      for (const product_id of planned.req.product_ids) productIds.add(product_id);\n    }\n    console.info(\n      formatTime(Date.now()),\n      `[VEX][Quote]RouteDispatched`,\n      ` Routed ${cacheMissed.length} missed quotes to ${mapGroupIdToProductIds.size} provider groups.`,\n      JSON.stringify({\n        productsByGroupId: [...mapGroupIdToProductIds.entries()].map(([group_id, productIds]) => ({\n          group_id,\n          product_ids: [...productIds],\n        })),\n        default_action: plan.defaultAction,\n      }),\n    );\n\n    if (Object.keys(plan.defaultAction).length > 0) {\n      quoteState.update(plan.defaultAction);\n    }\n\n    console.info(\n      formatTime(Date.now()),\n      `[VEX][Quote]RequestPlanned`,\n      `Planned ${plan.requests.length} upstream GetQuotes requests.`,\n      JSON.stringify(\n        plan.requests.map(({ key, planned }) => ({\n          key,\n          group_id: planned.group_id,\n          product_ids: planned.req.product_ids,\n          fields: planned.req.fields,\n        })),\n      ),\n    );\n\n    const actions = await execute(plan.requests);\n    console.debug(\n      formatTime(Date.now()),\n      `[VEX][Quote]RequestReceived`,\n      `Received ${actions.length} upstream GetQuotes responses.`,\n      JSON.stringify(actions),\n    );\n\n    for (const action of actions) {\n      quoteState.update(action);\n    }\n  };\n\n  return { snapshot, planOrThrow, execute, fillQuoteStateFromUpstream };\n};\n"]}

package/dist/quote/upstream/router.js

@@ -0,0 +1,119 @@
+import { TextEncoder } from 'util';
+import { encodePath, fnv1a64Hex, newError } from '@yuants/utils';
+import { createSortedPrefixMatcher } from './prefix-matcher';
+const SEP_BYTE = new Uint8Array([0xff]);
+const encodeStrings = (parts) => {
+    const buffers = [];
+    for (const part of parts) {
+        buffers.push(new TextEncoder().encode(part));
+        buffers.push(SEP_BYTE);
+    }
+    const totalLength = buffers.reduce((sum, b) => sum + b.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const b of buffers) {
+        result.set(b, offset);
+        offset += b.length;
+    }
+    return result;
+};
+const fnv1a64HexFromStrings = (parts) => fnv1a64Hex(encodeStrings(parts));
+export const buildProviderIndices = (groups) => {
+    const mapGroupIdToGroup = new Map(groups.map((x) => [x.group_id, x]));
+    const prefixMatcher = createSortedPrefixMatcher(groups.map((group) => ({ prefix: group.meta.product_id_prefix, value: group.group_id })));
+    const mapFieldToGroupIds = new Map();
+    for (const group of groups) {
+        for (const field of group.meta.fields) {
+            let groupIds = mapFieldToGroupIds.get(field);
+            if (!groupIds) {
+                groupIds = new Set();
+                mapFieldToGroupIds.set(field, groupIds);
+            }
+            groupIds.add(group.group_id);
+        }
+    }
+    return { mapGroupIdToGroup, prefixMatcher, mapFieldToGroupIds };
+};
+const createRequestKey = (group_id, batchProductIds) => encodePath(group_id, fnv1a64HexFromStrings(batchProductIds));
+const planRequests = (params) => {
+    var _a;
+    const { productsByGroupId, mapGroupIdToGroup } = params;
+    const plannedRequests = [];
+    for (const [group_id, productIdSet] of productsByGroupId) {
+        const group = mapGroupIdToGroup.get(group_id);
+        if (!group)
+            continue;
+        const sortedProductIds = [...productIdSet].sort();
+        const max = (_a = group.meta.max_products_per_request) !== null && _a !== void 0 ? _a : sortedProductIds.length;
+        for (let i = 0; i < sortedProductIds.length; i += max) {
+            const batchProductIds = sortedProductIds.slice(i, i + max);
+            const key = createRequestKey(group_id, batchProductIds);
+            plannedRequests.push({
+                key,
+                planned: {
+                    group_id,
+                    instances: Array.from(group.mapTerminalIdToInstance.values()),
+                    req: { product_ids: batchProductIds, fields: group.meta.fields },
+                },
+            });
+        }
+    }
+    return plannedRequests;
+};
+export const createQuoteRouter = () => ({
+    planOrThrow: (misses, indices, updated_at) => {
+        const { prefixMatcher, mapFieldToGroupIds, mapGroupIdToGroup } = indices;
+        const productsByGroupId = new Map();
+        const defaultAction = {};
+        const unroutableProducts = new Set();
+        const mapProductIdToGroupIds = new Map();
+        for (const miss of misses) {
+            const { product_id, field } = miss;
+            let productGroupIds = mapProductIdToGroupIds.get(product_id);
+            if (!productGroupIds) {
+                productGroupIds = prefixMatcher.match(product_id);
+                mapProductIdToGroupIds.set(product_id, productGroupIds);
+            }
+            if (productGroupIds.length === 0) {
+                unroutableProducts.add(product_id);
+                continue;
+            }
+            const fieldGroupIds = mapFieldToGroupIds.get(field);
+            if (!fieldGroupIds) {
+                if (!defaultAction[product_id])
+                    defaultAction[product_id] = {};
+                defaultAction[product_id][field] = ['', updated_at];
+                continue;
+            }
+            let matched = false;
+            for (const group_id of productGroupIds) {
+                if (!fieldGroupIds.has(group_id))
+                    continue;
+                matched = true;
+                let productIds = productsByGroupId.get(group_id);
+                if (!productIds) {
+                    productIds = new Set();
+                    productsByGroupId.set(group_id, productIds);
+                }
+                productIds.add(product_id);
+            }
+            if (!matched) {
+                if (!defaultAction[product_id])
+                    defaultAction[product_id] = {};
+                defaultAction[product_id][field] = ['', updated_at];
+            }
+        }
+        if (unroutableProducts.size !== 0) {
+            throw newError('VEX_QUOTE_PRODUCT_UNROUTABLE', {
+                updated_at,
+                unroutable_products: [...unroutableProducts].slice(0, 200),
+                unroutable_products_total: unroutableProducts.size,
+            });
+        }
+        return {
+            defaultAction,
+            requests: planRequests({ productsByGroupId, mapGroupIdToGroup }),
+        };
+    },
+});
+//# sourceMappingURL=router.js.map