@cyanheads/eurostat-mcp-server 0.11.2 → 0.11.3

package/dist/services/eurostat-catalogue/eurostat-catalogue-service.d.ts

@@ -7,6 +7,27 @@ import type { Context } from '@cyanheads/mcp-ts-core';
 import type { AppConfig } from '@cyanheads/mcp-ts-core/config';
 import type { StorageService } from '@cyanheads/mcp-ts-core/storage';
 import type { BrowseItem, DatasetResult } from './types.js';
+/** Most distinct words a search that matches no catalogue text looks for in dimension values. */
+export declare const MAX_VALUE_SEARCH_WORDS = 10;
+/** Most distinct words, in query order, a search that matches nothing names as held by no dataset. */
+export declare const MAX_UNMATCHED_WORD_CHECKS = 32;
+/**
+ * How a dataset matched a search, best first: 0, its label holds the words as a phrase
+ * starting at a word start; 1, its label or code holds every word; 2, some word matched
+ * only in its theme breadcrumb.
+ */
+export type SearchTier = 0 | 1 | 2;
+/**
+ * The tier test for one search over `words`, whitespace-free tokens as `search()` splits a
+ * query. The returned function ranks a dataset from its label, folded by
+ * {@link foldSearchText}, and its code, never its breadcrumb, so every TOC placement of a
+ * code ranks alike. The phrase is the words joined by single spaces and folded the same way;
+ * it must start where no letter or digit precedes it, so "employment rate" is not found
+ * inside "Unemployment rate". The word test is the substring test matching applies.
+ */
+export declare function searchTierOf(words: readonly string[]): (foldedLabel: string, code: string) => SearchTier;
+/** Why a search that matched no catalogue text did not read dimension values. */
+export type ValuesNotSearched = 'index_unavailable' | 'too_many_words';
 export declare class EurostatCatalogueService {
     private cache;
     /** Shared in-flight refresh so concurrent callers past the TTL trigger one fetch. */
@@ -19,6 +40,11 @@ export declare class EurostatCatalogueService {
      * past its TTL that the refresh could not replace.
      */
     private refreshRetryAt;
+    /**
+     * The dimension value index a search that matches no catalogue text reads. Undefined until
+     * a TOC load starts the first index load.
+     */
+    private valueIndex;
     constructor(_config: AppConfig, _storage: StorageService);
     /**
      * Ensure a usable TOC is loaded. A cache younger than the configured TTL is
@@ -43,16 +69,28 @@ export declare class EurostatCatalogueService {
      * Only the TOC is required. A Comext list that fails to load leaves the catalogue
      * answering from the TOC alone: the failure is logged, and the merge is retried
      * after `REFRESH_RETRY_COOLDOWN_MS` without re-fetching the TOC.
+     *
+     * Every TOC load also starts the dimension value index load, which nothing awaits.
      */
     private fetchAndParseToc;
     /** Retry a failed Comext merge onto the TOC already held; concurrent callers share it. */
     private retryComextMerge;
     private logComextFailure;
+    /**
+     * Start a dimension value index load unless one is in flight, a failed load's backoff has
+     * not elapsed, or the index in service is younger than the TOC's TTL. Catalogue calls never
+     * wait for it: the load runs detached from the calling request and its trace, logs to the
+     * server log, and settles into the slot. A failure keeps the previous index in service and
+     * holds off the next attempt for `REFRESH_RETRY_COOLDOWN_MS`, doubling with each failure in
+     * a row up to the TOC's TTL.
+     */
+    private loadValueIndex;
     /**
      * Assemble a catalogue snapshot. `flows` undefined means the Comext list failed to
      * load, which arms the merge retry. `loadedAt` is the TOC's load time — the TTL and
-     * the cursor generation both key on it. A later merge only appends entries, so a
-     * cursor paged over the TOC alone still addresses the same leading matches.
+     * the cursor generation both key on it. A later merge only appends entries, and
+     * `search()` pages entries appended after a walk began behind the ones it began over, so
+     * a cursor paged over the TOC alone still pages every match once.
      */
     private buildCache;
     /** The Comext host's dataflow list — the `DS-*` collections the TOC does not carry. */
@@ -134,12 +172,16 @@ export declare class EurostatCatalogueService {
      * Decode a search cursor and confirm it belongs to this search. Malformed,
      * cross-query and superseded-snapshot cursors are rejected identically —
      * the caller's recovery is the same in all three cases, and the cursor is
-     * opaque, so telling them apart would only describe its internal shape.
+     * opaque, so telling them apart would only describe its internal shape. `entryCount`,
+     * the snapshot's length when the walk began, can be no more than its length now.
+     * `valueIndex`, present on a walk over rows found through dimension values, is checked
+     * once `search()` knows which rows it pages.
      */
     private decodeSearchCursor;
+    /** One search result row for the TOC placement `entries[i]`. */
+    private toDatasetResult;
     /**
-     * Each dataset's search text — label, breadcrumb, and code, lowercased, with whitespace
-     * beside punctuation removed — indexed like `entries`. Folders, which search never
+     * Each dataset's search text, indexed like `entries`. Folders, which search never
      * returns, are undefined.
      */
     private buildSearchText;
@@ -151,8 +193,25 @@ export declare class EurostatCatalogueService {
      * typed. One pass: a placement missing exactly one word rescues its code for that
      * word, and a code counts once per word whichever placement rescued it — so each count
      * equals the `totalMatches` of the same search without that word.
+     *
+     * `unmatchedWords` is drawn from the first {@link MAX_UNMATCHED_WORD_CHECKS} distinct
+     * words. `unmatchedWordsPartial` is set when a word past them is held by no text the pass
+     * read, so it may be one no dataset carries.
      */
     private countWithoutEachWord;
+    /**
+     * The datasets a query that matched no catalogue text finds when a word may also occur
+     * at a word start of the English label of a value the metabase lists for the dataset.
+     * Every word must still be found, in the dataset's own text or in a value's label. Each
+     * code keeps its placement whose text holds the most words, the first such in TOC order.
+     *
+     * Ordered by fewer words found only through values; then rows where one value's label
+     * carries all of those words; then {@link searchTierOf} over the tokens the dataset's text
+     * holds, in query order; then TOC order. `marks` holds, for each word found through
+     * values, which labels carry it. `words` are the query's distinct tokens, at most
+     * {@link MAX_VALUE_SEARCH_WORDS}.
+     */
+    private searchValues;
     /**
      * Search datasets by keyword. The query is tokenized on whitespace and every
      * token must match (AND), case-insensitively, somewhere in a combined
@@ -167,18 +226,36 @@ export declare class EurostatCatalogueService {
      * several TOC branches keeps its first matching placement as the canonical
      * `themePath`, so totals and page slots count unique query targets.
      *
+     * Matches are ordered by {@link searchTierOf} over the tokens — label phrase, then
+     * every word in label or code, then breadcrumb-only — and by that first matching
+     * placement within a tier. A tier reads the label and code alone, so it is the same
+     * whichever placement supplied the `themePath`.
+     *
      * Results are paginated with an opaque cursor (`limit` = page size, capped at
      * 100) bound to the normalized query and the catalogue snapshot that produced
      * it. An unusable cursor — malformed, from another query, or from a catalogue
      * since refreshed — is rejected as `invalid_cursor` instead of silently paging
-     * a different result set.
+     * a different result set. A Comext merge that lands mid-walk keeps the snapshot,
+     * so the cursor also carries the snapshot's length when the walk began: the
+     * datasets the merge appended page after every match the walk began over, each
+     * match still once.
+     *
+     * When no dataset's text holds every token, {@link searchValues} looks for the tokens
+     * the text lacks in the English labels of the dataset's dimension values. Its rows
+     * carry `matchedValues` and the result `matchedThroughValues`, and their cursor also
+     * binds the value index snapshot (`valueIndex`), so an index refresh mid-walk rejects it
+     * like a catalogue refresh. No search waits for the index: until one has loaded,
+     * `valuesNotSearched` is `index_unavailable`; past {@link MAX_VALUE_SEARCH_WORDS}
+     * distinct tokens it is `too_many_words`.
      *
      * An empty match reports the `DS-*` codes the query names that neither source lists
      * (`undisseminatedCodes`), so the caller can be told the collection is not
      * disseminated rather than to broaden the search, and — for two or more distinct
      * words — each word whose removal alone finds matches (`wordRemovalCounts`) and each
      * word no dataset's search text contains (`unmatchedWords`, as first typed), with
-     * `allWordsUnmatched` when that is every word. `comextListMissing` flags that the
+     * `allWordsUnmatched` when that is every word. `unmatchedWords` reads the first
+     * {@link MAX_UNMATCHED_WORD_CHECKS} distinct words; `unmatchedWordsPartial` flags a query
+     * with a later word that may be one no dataset carries. `comextListMissing` flags that the
      * Comext list is not merged right now, so the `DS-*` collections were not searched and
      * absence from that list proves nothing; `catalogueLoadedAt` marks a catalogue a
      * failed refresh could not replace.
@@ -188,10 +265,13 @@ export declare class EurostatCatalogueService {
         catalogueLoadedAt?: string;
         comextListMissing?: boolean;
         datasets: DatasetResult[];
+        matchedThroughValues?: true;
         nextCursor?: string;
         totalMatches: number;
         undisseminatedCodes?: string[];
         unmatchedWords?: string[];
+        unmatchedWordsPartial?: true;
+        valuesNotSearched?: ValuesNotSearched;
         wordRemovalCounts?: Array<{
             totalMatches: number;
             word: string;

package/dist/services/eurostat-catalogue/eurostat-catalogue-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eurostat-catalogue-service.d.ts","sourceRoot":"","sources":["../../../src/services/eurostat-catalogue/eurostat-catalogue-service.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAO/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAgBrE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAY,MAAM,YAAY,CAAC;AA2EtE,qBAAa,wBAAwB;IACnC,OAAO,CAAC,KAAK,CAAuB;IACpC,qFAAqF;IACrF,OAAO,CAAC,OAAO,CAAgC;IAC/C,uDAAuD;IACvD,OAAO,CAAC,OAAO,CAAgC;IAC/C;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAK;IAK3B,YAAY,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAI;IAE5D;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,YAAY;YAIN,UAAU;IAyCxB;;;;;;OAMG;YACW,gBAAgB;IAY9B,0FAA0F;IAC1F,OAAO,CAAC,gBAAgB;IAoBxB,OAAO,CAAC,gBAAgB;IAUxB;;;;;OAKG;IACH,OAAO,CAAC,UAAU;IAclB,uFAAuF;YACzE,gBAAgB;IA4B9B;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;YA0CV,eAAe;IAmC7B;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;IAQtB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;IAwEhB,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,SAAS;IAMjB;;OAEG;IACH,OAAO,CAAC,SAAS;IAYjB,wDAAwD;IACxD,OAAO,CAAC,YAAY;IAOpB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAIjB;;;;;;;;;;;;;OAaG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC;QACT,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,KAAK,EAAE,UAAU,EAAE,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;QAC7B,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CA4ED;IAED,OAAO,CAAC,YAAY;IAepB;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAkB1B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;;;;;OAQG;IACH,OAAO,CAAC,oBAAoB;IAwC5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC;QACT,iBAAiB,CAAC,EAAE,IAAI,CAAC;QACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,QAAQ,EAAE,aAAa,EAAE,CAAC;QAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC/B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,iBAAiB,CAAC,EAAE,KAAK,CAAC;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACnE,CAAC,CAuED;CACF;AAgBD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAE7F;AAED,wBAAgB,2BAA2B,IAAI,wBAAwB,CAOtE"}
+{"version":3,"file":"eurostat-catalogue-service.d.ts","sourceRoot":"","sources":["../../../src/services/eurostat-catalogue/eurostat-catalogue-service.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAQ/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAqBrE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAY,MAAM,YAAY,CAAC;AAKtE,iGAAiG;AACjG,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAEzC,sGAAsG;AACtG,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAwC5C;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAEnC;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,SAAS,MAAM,EAAE,GACvB,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,UAAU,CAQnD;AAwDD,iFAAiF;AACjF,MAAM,MAAM,iBAAiB,GAAG,mBAAmB,GAAG,gBAAgB,CAAC;AASvE,qBAAa,wBAAwB;IACnC,OAAO,CAAC,KAAK,CAAuB;IACpC,qFAAqF;IACrF,OAAO,CAAC,OAAO,CAAgC;IAC/C,uDAAuD;IACvD,OAAO,CAAC,OAAO,CAAgC;IAC/C;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAK;IAC3B;;;OAGG;IACH,OAAO,CAAC,UAAU,CAA6B;IAK/C,YAAY,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAI;IAE5D;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,YAAY;YAIN,UAAU;IA0CxB;;;;;;;;OAQG;YACW,gBAAgB;IAc9B,0FAA0F;IAC1F,OAAO,CAAC,gBAAgB;IAoBxB,OAAO,CAAC,gBAAgB;IAUxB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAyCtB;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IAclB,uFAAuF;YACzE,gBAAgB;IA4B9B;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;YA0CV,eAAe;IAiD7B;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;IAQtB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;IAwEhB,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,SAAS;IAMjB;;OAEG;IACH,OAAO,CAAC,SAAS;IAYjB,wDAAwD;IACxD,OAAO,CAAC,YAAY;IAOpB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAIjB;;;;;;;;;;;;;OAaG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC;QACT,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,KAAK,EAAE,UAAU,EAAE,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;QAC7B,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CA4ED;IAED,OAAO,CAAC,YAAY;IAepB;;;;;;;;OAQG;IACH,OAAO,CAAC,kBAAkB;IA0B1B,gEAAgE;IAChE,OAAO,CAAC,eAAe;IAavB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAWvB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAiD5B;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,YAAY;IAoEpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+CG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC;QACT,iBAAiB,CAAC,EAAE,IAAI,CAAC;QACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,QAAQ,EAAE,aAAa,EAAE,CAAC;QAC1B,oBAAoB,CAAC,EAAE,IAAI,CAAC;QAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC/B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,qBAAqB,CAAC,EAAE,IAAI,CAAC;QAC7B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;QACtC,iBAAiB,CAAC,EAAE,KAAK,CAAC;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACnE,CAAC,CAkHD;CACF;AAwBD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAE7F;AAED,wBAAgB,2BAA2B,IAAI,wBAAwB,CAOtE"}