gloomberb 0.18.2 → 0.18.4

package/src/tickers/search/index.ts

@@ -34,11 +34,13 @@ import {
 } from "./result";
 
 import { typedPerpetualListing } from "../perpetual-listing";
-import { readTerminalListingCode } from "../terminal-codes";
+import { endsInLegalForm, readNearMissListing, readTerminalListingCode } from "../terminal-codes";
 import { isProviderBusy, ProviderBusyError, SEARCH_BUSY_MESSAGE } from "../../sources/provider-errors";
 import { settleWithin } from "../../utils/async-deadline";
 import { companyNameExtraWords } from "../company-name";
+import { withHomeListingLead } from "./home-listing";
 import { delistingLabel } from "../../market-data/quotes/delisting";
+import { hasNonLatinLetters } from "../../utils/typed-text";
 export type {
   ResolvedTickerSearch,
   TickerSearchCandidate,
@@ -49,6 +51,7 @@ export {
   rankTickerSearchItems,
 } from "./ranking";
 export { upsertTickerFromSearchResult } from "./upsert";
+export { candidateExchange, listingDistance } from "./home-listing";
 
 const MAX_NAMED_LISTINGS = 6;
 
@@ -120,10 +123,15 @@ interface TickerSearchCandidateOptions {
 
 /**
  * The ticker typed, upper case; a perpetual's market id (`binance:BZUSDT`) as its listing (`BZUSDT:BINANCE`),
- * and a terminal's code for a US listing (`WFC.N`, `JPM US Equity`) as that listing (`WFC:NYSE`, `JPM`).
+ * a terminal's code (`WFC.N`, `VOD LN`, `JPM US Equity`) as that listing (`WFC:NYSE`, `VOD:LSE`, `JPM`), and a
+ * symbol written nearly as one is (`700.HK`, `6758.JP`, `BRK/B`) as its working form (`0700.HK`, `6758.T`, `BRK-B`). A name
+ * before a legal form (`SAP SE`) stays the name, though SE is also a market code.
  */
 export function normalizeTickerInput(activeTicker: string | null, arg?: string): string | null {
-  const explicitTicker = arg ? (typedPerpetualListing(arg)?.key ?? readTerminalListingCode(arg)?.key ?? arg).trim().toUpperCase() : "";
+  const explicitTicker = arg
+    ? (typedPerpetualListing(arg)?.key ?? (endsInLegalForm(arg) ? null : readTerminalListingCode(arg)?.key) ?? readNearMissListing(arg)?.symbol ?? arg)
+      .trim().toUpperCase()
+    : "";
   if (explicitTicker) return explicitTicker;
   return activeTicker;
 }
@@ -294,7 +302,7 @@ export function buildTickerSearchCandidates({
     query,
   );
   const providerItems = createProviderTickerSearchCandidates(filteredProviderResults, tickers, candidateOptions);
-  const ranked = withSuccessorFirst(rankTickerSearchItems([...localItems, ...providerItems], query), query);
+  const ranked = withHomeListingLead(withSuccessorFirst(rankTickerSearchItems([...localItems, ...providerItems], query), query), query);
   return limitTickerSearchCandidates(assignTickerSearchCategories(ranked, query), totalLimit, localLimit);
 }
 
@@ -603,36 +611,63 @@ async function searchProviderResults(
 ): Promise<ProviderSearch> {
   const assetClass = parseAssetClassQuery(rawQuery);
   const query = assetClass?.symbolQuery ?? rawQuery;
-  // A Map rather than a list plus a seen set, because a later source can send
-  // back a richer version of a symbol already recorded. Overwriting a key keeps
-  // its original position, so an upgrade does not reorder the list.
+  // The variants are independent lookups of the same words, so they run
+  // together. Awaited in turn they multiplied every per-source timeout by the
+  // number of spellings tried. Their rows list in the order the spellings
+  // are, not as they arrive. Only keyword-tagged whole-query matches lead
+  // the spelling answers; a loose name match must not displace an exact listing.
+  const searchQueries = providerSearchQueries(rawQuery);
+  const answers: InstrumentSearchResult[][] = searchQueries.map(() => []);
+  const wholeIndex = assetClass && (assetClass.code === "ETF" || assetClass.code === "FUND")
+    ? searchQueries.findIndex((asked) => sameSearchText(asked, rawQuery)) : -1;
+  const taggedWhole = (result: InstrumentSearchResult) =>
+    searchResultKeywords(result)?.some((keyword) => sameSearchText(keyword, rawQuery)) ?? false;
+  // A later source can send back a richer version of a row already recorded;
+  // it replaces the row in its place, so an upgrade does not reorder the list.
+  const upgrades = new Map<string, InstrumentSearchResult>();
+  // A sector word's funds, which lead, and a market symbol verified by its quote, which follows (both below).
+  let leading: InstrumentSearchResult[] = [];
+  let verified: InstrumentSearchResult[] = [];
   const byKey = new Map<string, InstrumentSearchResult>();
-  const add = (results: InstrumentSearchResult[], upgrade = false) => {
-    for (const result of results) {
+  const merge = () => {
+    const named = new Map<string, InstrumentSearchResult>();
+    const whole = answers[wholeIndex] ?? [];
+    const ordered = [
+      ...whole.filter(taggedWhole),
+      ...answers.flatMap((rows, index) => index === wholeIndex ? [] : rows),
+      ...whole.filter((result) => !taggedWhole(result)),
+      ...verified,
+    ];
+    for (const result of ordered) {
       const key = buildProviderSearchResultKey(result);
-      if (!upgrade && byKey.has(key)) continue;
-      byKey.set(key, result);
+      if (!named.has(key)) named.set(key, upgrades.get(key) ?? result);
     }
+    byKey.clear();
+    for (const result of leading) {
+      const key = buildProviderSearchResultKey(result);
+      if (!byKey.has(key)) byKey.set(key, result);
+    }
+    // A row both found keeps the funds' place and the name search's richer version of it (its theme).
+    for (const [key, result] of named) byKey.set(key, result);
   };
-
-  // The variants are independent lookups of the same words, so they run
-  // together. Awaited in turn they multiplied every per-source timeout by the
-  // number of spellings tried.
   let busy = false;
-  await Promise.all(providerSearchQueries(rawQuery).map(async (searchQuery) => {
+  await Promise.all(searchQueries.map(async (searchQuery, index) => {
     try {
       const results = await dataProvider.search(searchQuery, {
         ...searchContext,
         ...(onPartial
           ? {
             onPartial: (upgraded: InstrumentSearchResult[]) => {
-              add(upgraded, true);
+              answers[index] = upgraded;
+              for (const result of upgraded) upgrades.set(buildProviderSearchResultKey(result), result);
+              merge();
               onPartial([...byKey.values()]);
             },
           }
           : {}),
       });
-      add(results);
+      answers[index] = results;
+      merge();
     } catch (error) {
       if (isProviderBusy(error)) busy = true;
       else logFailure("tickers-search", "Could not search ticker spelling")(error);
@@ -663,14 +698,15 @@ async function searchProviderResults(
           && (!exchange || canonicalExchange(quote.listingExchangeName || quote.exchangeName) === exchange)
           && (possibleCryptoPair ? normalizeTickerSymbol(quote.symbol) === symbol
             : findExactTickerSearchMatch([{ label: quote.symbol }], symbol))) {
-          add([{
+          verified = [{
             providerId: quote.providerId || dataProvider.id,
             symbol,
             name: quote.name || symbol,
             exchange: exchange || quote.listingExchangeName || quote.exchangeName || "",
             currency: quote.currency,
             type,
-          }]);
+          }];
+          merge();
         }
       } catch (error) {
         logFailure("tickers-search", "Could not verify market symbol")(error);
@@ -687,19 +723,26 @@ async function searchProviderResults(
     && (themeQueryWord([...byKey.values()], query) || (fundsAsked && /\s/.test(query.trim())));
   if (theme) {
     try {
+      // "spot bitcoin ETF" was asked whole above, and its answer is these funds: not asked twice.
+      const themeQuery = `${query.trim()} ETF`;
+      const askedAt = searchQueries.findIndex((asked) => sameSearchText(asked, themeQuery));
       const { onPartial: _ignored, ...context } = searchContext ?? {};
-      const funds = await dataProvider.search(`${query.trim()} ETF`, context);
-      const named = [...byKey];
-      byKey.clear();
-      add(funds);
-      // A row both found keeps the funds' place and the name search's richer version of it (its theme).
-      for (const [key, result] of named) byKey.set(key, result);
+      leading = askedAt >= 0 ? answers[askedAt]! : await dataProvider.search(themeQuery, context);
+      merge();
     } catch (error) {
       logFailure("tickers-search", "Could not search theme funds")(error);
     }
   }
 
-  return { results: [...byKey.values()], busy };
+  // Listings are named in Latin letters, and a source can fail on a name typed in another script (任天堂)
+  // every time it is asked: nothing found for it is no answer to wait for and ask again.
+  return { results: [...byKey.values()], busy: busy && !hasNonLatinLetters(query) };
+}
+
+/** Two search texts the server answers alike: its search ignores case and runs of spaces. */
+function sameSearchText(left: string, right: string): boolean {
+  const key = (text: string) => text.trim().replace(/\s+/g, " ").toLowerCase();
+  return key(left) === key(right);
 }
 
 /**
@@ -715,6 +758,12 @@ export function providerSearchQueries(rawQuery: string): string[] {
   const marketSymbol = assetClass ? assetClassMarketSymbol(assetClass) : null;
   const searchQueries = buildProviderSearchQueries(query);
   if (marketSymbol && !searchQueries.includes(marketSymbol)) searchQueries.push(marketSymbol);
+  // Words before ETF or FUND name a kind of fund as well as a name ("bitcoin ETF", "spot ether ETF"), which
+  // the server answers whole with the funds of that kind. merge() promotes its keyword-tagged rows.
+  if (assetClass && (assetClass.code === "ETF" || assetClass.code === "FUND") && /\p{L}{2}/u.test(query)) {
+    const whole = rawQuery.trim().replace(/\s+/g, " ");
+    if (!searchQueries.includes(whole)) searchQueries.unshift(whole);
+  }
   // A futures board code also finds its future, which the catalogue files under
   // the continuous symbol only: NG finds NovaGold and NG=F.
   const boardFuture = assetClass ? null : futuresContractByCode(query)?.symbol;

package/src/tickers/search/ranking.ts

@@ -132,7 +132,8 @@ export function isCryptoInstrumentType(type?: string): boolean {
   return ["CRYPTO", "CRYPTOCURRENCY", "DIGITALCURRENCY"].includes((type ?? "").toUpperCase().replace(/[\s_-]/g, ""));
 }
 
-function isQualifiedTickerQuery(query: string): boolean {
+/** A query that names its venue or market: SYM:EXCH, a listing suffix (7203.T), a future or a pair. */
+export function isQualifiedTickerQuery(query: string): boolean {
   const symbol = normalizeTickerSymbol(query);
   return isExplicitMarketSymbol(symbol) || !!parsePublicTickerKey(symbol).exchange || tickerHasListingSuffix(symbol);
 }
@@ -193,6 +194,9 @@ export function rankTickerSearchItems<T extends Pick<TickerSearchRankableItem, "
   // A sector word ("defense") is answered by the funds that track it before the small companies whose names
   // contain it; the server's own order of those funds is the best signal of their size.
   const themeWord = themeQueryWord(items, query);
+  // The rows the server sent for the whole query as a kind of fund ("spot bitcoin ETF" is IBIT, FBTC, ...)
+  // carry it as a keyword: they answer it more than a fund whose name happens to hold the words.
+  const categoryKeyword = assetClass ? normalizeSearchText(rawQuery) : "";
 
   const ranked = (assetClass
     ? items.filter((item) => assetClassKeeps(assetClass.code, instrumentClassCode({
@@ -261,6 +265,7 @@ export function rankTickerSearchItems<T extends Pick<TickerSearchRankableItem, "
           ? getIssuerGroupKey(item.detail)
           : null,
         explicitIntentScore,
+        categoryFund: categoryKeyword && item.searchKeywords?.some((keyword) => normalizeSearchText(keyword) === categoryKeyword) ? 1 : 0,
         themeFund: themeWord != null && isThemeFund(item, themeWord) ? 1 : 0,
         themeCompany: themeWord != null && isThemeCompany(item, themeWord) ? 1 : 0,
         normalizedSymbol: normalizeSearchText(item.symbol || item.label),
@@ -272,6 +277,17 @@ export function rankTickerSearchItems<T extends Pick<TickerSearchRankableItem, "
       };
     });
 
+  // A large company's name that is only a foreign line's ticker ("Visa" in Vienna, "Pfizer" for Pfizer Ltd. on
+  // NSE, "BofA" for a Canadian receipt) asks for the company, which the server marks with the name as typed: its
+  // US line leads, and those tickers rank by their names. A US line's ticker (SAP, GOLD) keeps every exact lead.
+  if (namesUsCompany(ranked, query)) {
+    for (const entry of ranked) {
+      if (entry.symbolMatchRank !== 3 && entry.symbolMatchRank !== 4) continue;
+      entry.symbolMatchRank = 0;
+      entry.nameOnly = entry.companyMatchRank > 0;
+    }
+  }
+
   // A company named exactly as the word is what was asked for ("tesla"), whatever else a theme tags.
   const namedCompany = ranked.some((entry) => entry.companyMatchRank === 3 && entry.item.instrumentClass === "equity");
   // The first few funds in the server's order lead; the others list as funds always have.
@@ -392,7 +408,7 @@ export function rankTickerSearchItems<T extends Pick<TickerSearchRankableItem, "
   }
 
   filtered.sort((a, b) => {
-    // Precedence is exact symbol, explicit query hints, a sector word's funds
+    // Precedence is exact symbol, the funds of a kind named in full, explicit query hints, a sector word's funds
     // and then the companies sent with it, each in the server's order, whole-word company relevance, issuer relevance, provider order within that issuer, partial
     // symbol, saved relevance, then stable source order.
     const aExactSymbolRank = a.symbolMatchRank >= 2 ? a.symbolMatchRank : 0;
@@ -403,6 +419,13 @@ export function rankTickerSearchItems<T extends Pick<TickerSearchRankableItem, "
       const bProviderRank = b.item.providerRank ?? Number.POSITIVE_INFINITY;
       if (aProviderRank !== bProviderRank) return aProviderRank - bProviderRank;
     }
+    // The funds of a kind named in full lead in the server's order, ahead of any venue or asset hint.
+    if (b.categoryFund !== a.categoryFund) return b.categoryFund - a.categoryFund;
+    if (a.categoryFund) {
+      const aProviderRank = a.item.providerRank ?? Number.POSITIVE_INFINITY;
+      const bProviderRank = b.item.providerRank ?? Number.POSITIVE_INFINITY;
+      if (aProviderRank !== bProviderRank) return aProviderRank - bProviderRank;
+    }
     if (hasExplicitHint && b.explicitIntentScore !== a.explicitIntentScore) {
       return b.explicitIntentScore - a.explicitIntentScore;
     }
@@ -482,12 +505,13 @@ function foldSearchText(text: string): string {
   return text.normalize("NFKD").replace(/\p{M}/gu, "").trim().toUpperCase();
 }
 
+// Letters of every script are kept: a name typed in kana or kanji (任天堂) is no empty query that matches every row.
 export function normalizeSearchText(text: string): string {
-  return foldSearchText(text).replace(/[^A-Z0-9]+/g, " ").trim();
+  return foldSearchText(text).replace(/[^\p{L}\p{N}]+/gu, " ").trim();
 }
 
 export function compactSearchText(text: string): string {
-  return foldSearchText(text).replace(/[^A-Z0-9]+/g, "");
+  return foldSearchText(text).replace(/[^\p{L}\p{N}]+/gu, "");
 }
 
 export function normalizeTickerSymbol(symbol: string): string {
@@ -813,6 +837,30 @@ function scoreExchangePreference(
   return matchesHint ? 2_000 : -800;
 }
 
+/** The venue a row lists on: a broker's SMART routing names none, its primary exchange does. */
+function listingVenue(item: Partial<TickerSearchRankableItem>): string {
+  return canonicalExchange(item.exchangeLabel === "SMART" ? item.primaryExchangeLabel : item.exchangeLabel || item.primaryExchangeLabel);
+}
+
+/**
+ * Whether the server answered a company's name with the company's US line,
+ * marked with the name as typed (`searchAliases`), while the rows whose ticker
+ * is the query all list abroad and none is saved. A ticker typed with its
+ * venue (VISA:VIE), or a US line that is the ticker, names that ticker instead.
+ */
+function namesUsCompany(
+  entries: ReadonlyArray<{ item: Partial<TickerSearchRankableItem>; symbolMatchRank: number }>,
+  query: string,
+): boolean {
+  if (isQualifiedTickerQuery(query) || !/\p{L}/u.test(query)) return false;
+  const exact = entries.filter((entry) => entry.symbolMatchRank === 3 || entry.symbolMatchRank === 4);
+  if (exact.length === 0 || exact.some(({ item }) => isUsListingExchange(listingVenue(item)) || item.saved || item.kind === "ticker")) return false;
+  const typed = compactSearchText(query);
+  return entries.some(({ item, symbolMatchRank }) => symbolMatchRank === 2 && item.kind === "search"
+    && item.instrumentClass === "equity" && isUsListingExchange(listingVenue(item))
+    && !!item.searchAliases?.some((alias) => compactSearchText(alias) === typed));
+}
+
 function isDepositaryReceiptType(type?: string): boolean {
   const normalized = normalizeSearchText(type || "");
   return normalized.includes("DEPOSITARY") || ["ADR", "ADS", "GDR", "CDR", "BDR"].includes(normalized);
@@ -885,6 +933,8 @@ export function getIssuerGroupKey(detail: string): string {
 
 function normalizeCompanyName(name: string): string {
   const tokens = normalizeSearchText(name).split(" ").filter(Boolean);
+  // "The TJX Companies, Inc." is TJX's name as much as Hanover's "TJX COMPANIES INC. R" is.
+  if (tokens.length > 1 && tokens[0] === "THE") tokens.shift();
   while (tokens.length > 1) {
     if (COMPANY_NAME_SUFFIXES.has(tokens.at(-1)!)) {
       tokens.pop();

package/src/tickers/search/upsert.ts

@@ -10,6 +10,7 @@ import {
 } from "./result";
 import { canonicalExchange, parsePublicTickerKey, publicTickerKey } from "../../utils/exchanges";
 import { scopedBrokerContractIdentityKey } from "../../utils/instrument-identity";
+import { listingSuffixVenue } from "../../sources/listing-symbols";
 
 /**
  * The key a result is saved under when its symbol is already saved for
@@ -36,11 +37,27 @@ export async function upsertTickerFromSearchResult(
   options: { tickerSymbol?: string } = {},
 ): Promise<{ ticker: TickerRecord; created: boolean }> {
   let symbol = options.tickerSymbol ?? getSearchResultSymbol(result);
+  // A vendor's suffixed symbol (GARAN.IS) is saved as the listing it is (GARAN on BIST), under the key
+  // GARAN:BIST is saved under, so one listing is one row. A key the caller names is kept as given.
+  const venue = options.tickerSymbol || result.brokerContract ? null : listingSuffixVenue(symbol, listingVenue(result));
+  if (venue) {
+    symbol = venue.symbol;
+    // A venue code the exchange table does not know (IST) is the one the suffix names.
+    const exchange = canonicalExchange(listingVenue(result)) === venue.exchange ? result.exchange : venue.exchange;
+    result = { ...result, symbol: venue.symbol, exchange };
+  }
   let ticker = await tickerRepository.loadTicker(symbol);
   const otherListingKey = otherListingTickerKey(result, symbol, ticker);
   if (otherListingKey) {
     symbol = otherListingKey;
     ticker = await tickerRepository.loadTicker(symbol);
+  } else if (!ticker && !options.tickerSymbol) {
+    // The listing's row saved under its venue's key while the bare key was free (a merged GARAN.IS, as GARAN:BIST).
+    const keyed = await venueKeyedRow(tickerRepository, result, symbol);
+    if (keyed) {
+      symbol = keyed.metadata.ticker;
+      ticker = keyed;
+    }
   }
   const created = !ticker;
 
@@ -83,6 +100,18 @@ export async function upsertTickerFromSearchResult(
   return { ticker, created };
 }
 
+/** The row saved under the result's venue key (GARAN:BIST) for a symbol the bare key does not hold, on that venue; null otherwise. */
+async function venueKeyedRow(tickerRepository: AppTickerRepositoryPort, result: InstrumentSearchResult, symbol: string): Promise<TickerRecord | null> {
+  const exchange = canonicalExchange(listingVenue(result));
+  if (!exchange || parsePublicTickerKey(symbol).exchange) return null;
+  const row = await tickerRepository.loadTicker(publicTickerKey(symbol, exchange));
+  return row && canonicalExchange(row.metadata.exchange) === exchange ? row : null;
+}
+
+function listingVenue(result: InstrumentSearchResult): string | undefined {
+  return result.exchange === "SMART" ? result.primaryExchange : result.exchange || result.primaryExchange;
+}
+
 function mergeTickerMetadataFromSearchResult(metadata: TickerMetadata, result: InstrumentSearchResult): boolean {
   let changed = false;
   const nextName = result.name?.trim();