@cyanheads/eurostat-mcp-server 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -1
- package/README.md +29 -7
- package/dist/config/server-config.d.ts +2 -0
- package/dist/config/server-config.d.ts.map +1 -1
- package/dist/config/server-config.js +14 -0
- package/dist/config/server-config.js.map +1 -1
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-dataframe-describe.tool.js +2 -2
- package/dist/mcp-server/tools/definitions/eurostat-dataframe-describe.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-dataframe-query.tool.js +2 -2
- package/dist/mcp-server/tools/definitions/eurostat-dataframe-query.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-download-dataset.tool.d.ts +93 -0
- package/dist/mcp-server/tools/definitions/eurostat-download-dataset.tool.d.ts.map +1 -0
- package/dist/mcp-server/tools/definitions/eurostat-download-dataset.tool.js +362 -0
- package/dist/mcp-server/tools/definitions/eurostat-download-dataset.tool.js.map +1 -0
- package/dist/mcp-server/tools/definitions/eurostat-get-dataset-info.tool.js +2 -2
- package/dist/mcp-server/tools/definitions/eurostat-get-dataset-info.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-get-dimension-values.tool.js +2 -2
- package/dist/mcp-server/tools/definitions/eurostat-get-dimension-values.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-query-dataset.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-query-dataset.tool.js +6 -10
- package/dist/mcp-server/tools/definitions/eurostat-query-dataset.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/eurostat-search-datasets.tool.js +3 -3
- package/dist/mcp-server/tools/definitions/eurostat-search-datasets.tool.js.map +1 -1
- package/dist/services/canvas-accessor.d.ts +5 -0
- package/dist/services/canvas-accessor.d.ts.map +1 -1
- package/dist/services/canvas-accessor.js +7 -0
- package/dist/services/canvas-accessor.js.map +1 -1
- package/dist/services/eurostat-bulk/eurostat-bulk-service.d.ts +89 -0
- package/dist/services/eurostat-bulk/eurostat-bulk-service.d.ts.map +1 -0
- package/dist/services/eurostat-bulk/eurostat-bulk-service.js +501 -0
- package/dist/services/eurostat-bulk/eurostat-bulk-service.js.map +1 -0
- package/dist/services/eurostat-bulk/types.d.ts +98 -0
- package/dist/services/eurostat-bulk/types.d.ts.map +1 -0
- package/dist/services/eurostat-bulk/types.js +82 -0
- package/dist/services/eurostat-bulk/types.js.map +1 -0
- package/dist/services/eurostat-data/eurostat-data-service.d.ts +10 -0
- package/dist/services/eurostat-data/eurostat-data-service.d.ts.map +1 -1
- package/dist/services/eurostat-data/eurostat-data-service.js +17 -0
- package/dist/services/eurostat-data/eurostat-data-service.js.map +1 -1
- package/package.json +1 -1
- package/server.json +34 -6
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eurostat-search-datasets.tool.js","sourceRoot":"","sources":["../../../../src/mcp-server/tools/definitions/eurostat-search-datasets.tool.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAiB,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,MAAM,6DAA6D,CAAC;AAE1G,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC,0BAA0B,EAAE;IACrE,KAAK,EAAE,0BAA0B;IACjC,WAAW,EACT,
|
|
1
|
+
{"version":3,"file":"eurostat-search-datasets.tool.js","sourceRoot":"","sources":["../../../../src/mcp-server/tools/definitions/eurostat-search-datasets.tool.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAiB,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,MAAM,6DAA6D,CAAC;AAE1G,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC,0BAA0B,EAAE;IACrE,KAAK,EAAE,0BAA0B;IACjC,WAAW,EACT,0YAA0Y;IAC5Y,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE;IAC9E,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;QACd,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CAAC,IAAI,EAAE,6DAA6D,CAAC;aAC1E,QAAQ,CACP,sVAAsV,CACvV;QACH,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,GAAG,EAAE;aACL,GAAG,CAAC,CAAC,CAAC;aACN,GAAG,CAAC,GAAG,CAAC;aACR,OAAO,CAAC,EAAE,CAAC;aACX,QAAQ,CACP,qMAAqM,CACtM;QACH,MAAM,EAAE,CAAC;aACN,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,iaAAia,CACla;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,QAAQ,EAAE,CAAC;aACR,KAAK,CACJ,CAAC;aACE,MAAM,CAAC;YACN,IAAI,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,QAAQ,CACP,mIAAmI,CACpI;YACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;YAC3D,IAAI,EAAE,CAAC;iBACJ,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;iBAC1B,QAAQ,CACP,6EAA6E,CAC9E;YACH,SAAS,EAAE,CAAC;iBACT,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CACP,uFAAuF,CACxF;YACH,OAAO,EAAE,CAAC;iBACP,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CACP,0FAA0F,CAC3F;YACH,WAAW,EAAE,CAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CACP,2EAA2E,CAC5E;YACH,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CACP,4EAA4E,CAC7E;YACH,SAAS,EAAE,CAAC;iBACT,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CACP,2OAA2O,CAC5O;SACJ,CAAC;aACD,QAAQ,CAAC,0BAA0B,CAAC,CACxC;aACA,QAAQ,CAAC,oEAAoE,CAAC;QACjF,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,+FAA+F,CAChG;KACJ,CAAC;IACF,UAAU,EAAE;QACV,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;QACxD,YAAY,EAAE,CAAC;aACZ,MAAM,EAAE;aACR,QAAQ,CACP,0FAA0F,CAC3F;QACH,SAAS,EAAE,CAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,2FAA2F,CAC5F;QACH,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,kKAAkK,CACnK;KACJ;IAED,MAAM,EAAE;QACN;YACE,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,gBAAgB,CAAC,QAAQ;YAC/B,IAAI,EAAE,uCAAuC;YAC7C,QAAQ,EACN,2GAA2G;SAC9G;QACD;YACE,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE,gBAAgB,CAAC,aAAa;YACpC,IAAI,EAAE,mHAAmH;YACzH,QAAQ,EACN,uFAAuF;SAC1F;KACF;IAED,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG;QACtB,MAAM,GAAG,GAAG,2BAA2B,EAAE,CAAC;QAC1C,IAAI,KAA6C,CAAC;QAClD,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAAgB,CAAC,IAAI,EAAE,MAAM,KAAK,gBAAgB,EAAE,CAAC;gBACxD,MAAM,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAG,GAAa,CAAC,OAAO,EAAE;oBACvD,QAAQ,EAAE;wBACR,IAAI,EAAE,0BAA0B,KAAK,CAAC,KAAK,gEAAgE;qBAC5G;iBACF,CAAC,CAAC;YACL,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;QAErD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,wBAAwB,KAAK,CAAC,KAAK,IAAI,EAAE;gBAClE,QAAQ,EAAE;oBACR,IAAI,EAAE,yFAAyF;iBAChG;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,KAAK,SAAS,CAAC;QACzC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE;YACtC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,YAAY;YACZ,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,OAAO;SACR,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,CAAC;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,YAAY;YACZ,SAAS,EAAE,OAAO;YAClB,GAAG,CAAC,OAAO,IAAI,EAAE,UAAU,EAAE,CAAC;SAC/B,CAAC,CAAC;QACH,OAAO;YACL,QAAQ;YACR,QAAQ,EAAE,gLAAgL;SAC3L,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;QACjB,MAAM,KAAK,GAAa;YACtB,aAAa,MAAM,CAAC,QAAQ,CAAC,MAAM,UAAU,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM;SAC3F,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,SAAS,IAAI,GAAG,MAAM,CAAC,CAAC,OAAO,IAAI,GAAG,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YACvF,IAAI,CAAC,CAAC,WAAW;gBAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -13,6 +13,11 @@ export declare function setCanvas(canvas: DataCanvas | undefined): void;
|
|
|
13
13
|
* the server is a supported configuration with and without one.
|
|
14
14
|
*/
|
|
15
15
|
export declare function getCanvas(): DataCanvas | undefined;
|
|
16
|
+
/**
|
|
17
|
+
* A fresh handle for one staged table. Distinct per call — every staging call is
|
|
18
|
+
* its own result set, and a reused name would replace the previous table.
|
|
19
|
+
*/
|
|
20
|
+
export declare function newTableName(): string;
|
|
16
21
|
/**
|
|
17
22
|
* Acquire a canvas, translating a scratch-directory permission failure into an
|
|
18
23
|
* error that names the variable to change.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"canvas-accessor.d.ts","sourceRoot":"","sources":["../../src/services/canvas-accessor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAKhF,kFAAkF;AAClF,wBAAgB,SAAS,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE9D;AAED;;;;GAIG;AACH,wBAAgB,SAAS,IAAI,UAAU,GAAG,SAAS,CAElD;AAoBD;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,cAAc,CAAC,CAYzB"}
|
|
1
|
+
{"version":3,"file":"canvas-accessor.d.ts","sourceRoot":"","sources":["../../src/services/canvas-accessor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAKhF,kFAAkF;AAClF,wBAAgB,SAAS,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE9D;AAED;;;;GAIG;AACH,wBAAgB,SAAS,IAAI,UAAU,GAAG,SAAS,CAElD;AAED;;;GAGG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAoBD;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,cAAc,CAAC,CAYzB"}
|
|
@@ -17,6 +17,13 @@ export function setCanvas(canvas) {
|
|
|
17
17
|
export function getCanvas() {
|
|
18
18
|
return _canvas;
|
|
19
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* A fresh handle for one staged table. Distinct per call — every staging call is
|
|
22
|
+
* its own result set, and a reused name would replace the previous table.
|
|
23
|
+
*/
|
|
24
|
+
export function newTableName() {
|
|
25
|
+
return `df_${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
26
|
+
}
|
|
20
27
|
/**
|
|
21
28
|
* Errno codes meaning the configured scratch root cannot be written to.
|
|
22
29
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"canvas-accessor.js","sourceRoot":"","sources":["../../src/services/canvas-accessor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,IAAI,OAA+B,CAAC;AAEpC,kFAAkF;AAClF,MAAM,UAAU,SAAS,CAAC,MAA8B;IACtD,OAAO,GAAG,MAAM,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS;IACvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1E,6EAA6E;AAC7E,SAAS,OAAO,CAAC,GAAY;IAC3B,KAAK,IAAI,GAAG,GAAG,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,YAAY,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1E,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC;IAClB,CAAC;IACD,OAAO;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAkB,EAClB,QAA4B,EAC5B,GAAY;IAEZ,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,KAAK;YAAE,MAAM,GAAG,CAAC;QACtB,MAAM,kBAAkB,CACtB,sDAAsD,KAAK,gJAAgJ,EAC3M,EAAE,MAAM,EAAE,oBAAoB,EAAE,KAAK,EAAE,EACvC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAClD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"canvas-accessor.js","sourceRoot":"","sources":["../../src/services/canvas-accessor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,IAAI,OAA+B,CAAC;AAEpC,kFAAkF;AAClF,MAAM,UAAU,SAAS,CAAC,MAA8B;IACtD,OAAO,GAAG,MAAM,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS;IACvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY;IAC1B,OAAO,MAAM,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AAC5D,CAAC;AAED;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1E,6EAA6E;AAC7E,SAAS,OAAO,CAAC,GAAY;IAC3B,KAAK,IAAI,GAAG,GAAG,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,YAAY,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1E,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC;IAClB,CAAC;IACD,OAAO;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAkB,EAClB,QAA4B,EAC5B,GAAY;IAEZ,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,KAAK;YAAE,MAAM,GAAG,CAAC;QACtB,MAAM,kBAAkB,CACtB,sDAAsD,KAAK,gJAAgJ,EAC3M,EAAE,MAAM,EAAE,oBAAoB,EAAE,KAAK,EAAE,EACvC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAClD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Eurostat Bulk Service — streaming client for the SDMX 2.1 TSV
|
|
3
|
+
* dissemination endpoint. Builds the positional dimension key, sniffs gzip off
|
|
4
|
+
* the stream, enforces a byte budget mid-transfer, classifies XML SOAP faults
|
|
5
|
+
* and the asynchronous queue envelope, and expands the wide TSV layout into
|
|
6
|
+
* one row per observation.
|
|
7
|
+
* @module services/eurostat-bulk/eurostat-bulk-service
|
|
8
|
+
*/
|
|
9
|
+
import type { Context } from '@cyanheads/mcp-ts-core';
|
|
10
|
+
import type { ColumnSchema } from '@cyanheads/mcp-ts-core/canvas';
|
|
11
|
+
import type { AppConfig } from '@cyanheads/mcp-ts-core/config';
|
|
12
|
+
import type { StorageService } from '@cyanheads/mcp-ts-core/storage';
|
|
13
|
+
import { type BulkDownload, type TsvHeader } from './types.js';
|
|
14
|
+
/**
|
|
15
|
+
* Translate an XML body from the SDMX endpoint into the declared error contract.
|
|
16
|
+
*
|
|
17
|
+
* Two unrelated shapes arrive as XML and neither is data. A `S:Fault` envelope
|
|
18
|
+
* accompanies a 4xx and names the problem in a numeric `faultcode`. The
|
|
19
|
+
* `syncResponse` envelope arrives on **HTTP 200** carrying a queue ticket
|
|
20
|
+
* instead of the dataset — parsed as TSV it yields a header row of XML, so it is
|
|
21
|
+
* detected here rather than downstream.
|
|
22
|
+
*
|
|
23
|
+
* Always throws.
|
|
24
|
+
*/
|
|
25
|
+
export declare function classifyXmlBody(xml: string, datasetCode: string): never;
|
|
26
|
+
/**
|
|
27
|
+
* Split one wide-TSV cell into its value and flags.
|
|
28
|
+
*
|
|
29
|
+
* The grammar, read off live responses, is `<value>` `SPACE` `<flags>`, where
|
|
30
|
+
* flags is `<obs_flag>` optionally followed by `@<conf_status>` and either part
|
|
31
|
+
* may be empty: `429634.3 `, `3853937.0 p`, `: `, `: @C`. A cell that is empty
|
|
32
|
+
* on both sides of the split is not an observation at all and yields
|
|
33
|
+
* `undefined` — distinct from `: `, which is an observation Eurostat reports as
|
|
34
|
+
* unavailable.
|
|
35
|
+
*/
|
|
36
|
+
export declare function parseCell(cell: string): {
|
|
37
|
+
conf: string | null;
|
|
38
|
+
flag: string | null;
|
|
39
|
+
value: number | null;
|
|
40
|
+
} | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* Parse the header line of a wide TSV body.
|
|
43
|
+
*
|
|
44
|
+
* The first tab-separated field is the comma-joined dimension list with
|
|
45
|
+
* `\TIME_PERIOD` appended to the last name; the remaining fields are the period
|
|
46
|
+
* codes, each padded with a trailing space Eurostat does not strip.
|
|
47
|
+
*/
|
|
48
|
+
export declare function parseHeader(line: string): TsvHeader;
|
|
49
|
+
/**
|
|
50
|
+
* Build the positional dimension key the SDMX path takes.
|
|
51
|
+
*
|
|
52
|
+
* The key is a dot-separated segment carrying one position per dimension **in
|
|
53
|
+
* dataset order excluding time**; empty means wildcard and `+` means OR. The
|
|
54
|
+
* count must match exactly — a key with one position too few or too many is
|
|
55
|
+
* rejected upstream with faultcode 140 — so positions are emitted for every
|
|
56
|
+
* dimension in `dimensionOrder`, filtered or not, and never only for the
|
|
57
|
+
* dimensions the caller named.
|
|
58
|
+
*
|
|
59
|
+
* Returns `undefined` when no filter applies. An all-wildcard key is accepted
|
|
60
|
+
* by Eurostat, but omitting the segment asks the same question without staking
|
|
61
|
+
* the request on this server's copy of the dimension order being current.
|
|
62
|
+
*/
|
|
63
|
+
export declare function buildKeyPath(dimensionOrder: string[], filters: Record<string, string[]>): string | undefined;
|
|
64
|
+
/**
|
|
65
|
+
* Canvas column schema for {@link BulkRow}, declared rather than sniffed.
|
|
66
|
+
*
|
|
67
|
+
* Inference reads only the leading rows, which types an all-integer measure
|
|
68
|
+
* prefix as `BIGINT` — serialized back out as a string — and types an
|
|
69
|
+
* all-missing prefix from nulls alone. Every column is nullable: a measure is
|
|
70
|
+
* routinely unavailable, and flags are the exception rather than the rule.
|
|
71
|
+
*/
|
|
72
|
+
export declare function bulkRowSchema(dimensions: string[]): ColumnSchema[];
|
|
73
|
+
export declare class EurostatBulkService {
|
|
74
|
+
constructor(_config: AppConfig, _storage: StorageService);
|
|
75
|
+
private buildUrl;
|
|
76
|
+
/**
|
|
77
|
+
* Start a bulk download: issue the request, classify the response, parse the
|
|
78
|
+
* header, and hand back a generator over the body.
|
|
79
|
+
*
|
|
80
|
+
* The request is made once with no retry. A bulk body streams for up to
|
|
81
|
+
* `EUROSTAT_BULK_TIMEOUT_MS`, so re-running it on a transient failure costs
|
|
82
|
+
* the caller another full transfer for a request that is already the most
|
|
83
|
+
* expensive one this server makes; the failure is surfaced instead.
|
|
84
|
+
*/
|
|
85
|
+
startDownload(datasetCode: string, dimensionOrder: string[], filters: Record<string, string[]>, sinceP: string | undefined, untilP: string | undefined, ctx: Context): Promise<BulkDownload>;
|
|
86
|
+
}
|
|
87
|
+
export declare function initEurostatBulkService(config: AppConfig, storage: StorageService): void;
|
|
88
|
+
export declare function getEurostatBulkService(): EurostatBulkService;
|
|
89
|
+
//# sourceMappingURL=eurostat-bulk-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eurostat-bulk-service.d.ts","sourceRoot":"","sources":["../../../src/services/eurostat-bulk/eurostat-bulk-service.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAO/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAGrE,OAAO,EACL,KAAK,YAAY,EAWjB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AAmCpB;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,KAAK,CAmCvE;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,GACX;IAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,SAAS,CAchF;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAkBnD;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAChC,MAAM,GAAG,SAAS,CAapB;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,YAAY,EAAE,CAUlE;AAED,qBAAa,mBAAmB;IAI9B,YAAY,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAI;IAE5D,OAAO,CAAC,QAAQ;IAiBhB;;;;;;;;OAQG;IACG,aAAa,CACjB,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EACjC,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,YAAY,CAAC,CAyDvB;CACF;AA6RD,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAExF;AAED,wBAAgB,sBAAsB,IAAI,mBAAmB,CAO5D"}
|
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Eurostat Bulk Service — streaming client for the SDMX 2.1 TSV
|
|
3
|
+
* dissemination endpoint. Builds the positional dimension key, sniffs gzip off
|
|
4
|
+
* the stream, enforces a byte budget mid-transfer, classifies XML SOAP faults
|
|
5
|
+
* and the asynchronous queue envelope, and expands the wide TSV layout into
|
|
6
|
+
* one row per observation.
|
|
7
|
+
* @module services/eurostat-bulk/eurostat-bulk-service
|
|
8
|
+
*/
|
|
9
|
+
import { notFound, serviceUnavailable, validationError, } from '@cyanheads/mcp-ts-core/errors';
|
|
10
|
+
import { fetchWithTimeout } from '@cyanheads/mcp-ts-core/utils';
|
|
11
|
+
import { getServerConfig } from '../../config/server-config.js';
|
|
12
|
+
import { CONF_STATUS_COLUMN, CONF_STATUS_LABEL_COLUMN, CONF_STATUS_LABELS, OBS_FLAG_COLUMN, OBS_FLAG_LABEL_COLUMN, OBS_FLAG_LABELS, OBS_VALUE_COLUMN, TIME_COLUMN, } from './types.js';
|
|
13
|
+
// Context satisfies the runtime contract of RequestContext but lacks the index signature
|
|
14
|
+
// required by fetchWithTimeout.
|
|
15
|
+
const asReqCtx = (ctx) => ctx;
|
|
16
|
+
/**
|
|
17
|
+
* Decoded characters buffered before an XML body is classified.
|
|
18
|
+
*
|
|
19
|
+
* Both non-data shapes are small — the largest SOAP fault observed is under 400
|
|
20
|
+
* bytes and the queue envelope under 350 — so this only has to hold one of them.
|
|
21
|
+
* It deliberately does **not** bound the search for the TSV header's line break:
|
|
22
|
+
* that line carries one field per period and runs to 166 KB on a daily series.
|
|
23
|
+
*/
|
|
24
|
+
const XML_PEEK_CHARS = 8_192;
|
|
25
|
+
/** Separator between the observation flag and the confidentiality flag inside a TSV cell. */
|
|
26
|
+
const CONF_SEPARATOR = '@';
|
|
27
|
+
/** Eurostat's marker for an observation whose value is not available. */
|
|
28
|
+
const MISSING_VALUE = ':';
|
|
29
|
+
/** The header line's first field ends with this literal before the period columns begin. */
|
|
30
|
+
const TIME_HEADER_SUFFIX = '\\TIME_PERIOD';
|
|
31
|
+
/** Minimal XML entity decoding — the five predefined entities SOAP fault strings use. */
|
|
32
|
+
function decodeEntities(text) {
|
|
33
|
+
return text
|
|
34
|
+
.replaceAll('<', '<')
|
|
35
|
+
.replaceAll('>', '>')
|
|
36
|
+
.replaceAll('"', '"')
|
|
37
|
+
.replaceAll(''', "'")
|
|
38
|
+
.replaceAll('&', '&');
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Translate an XML body from the SDMX endpoint into the declared error contract.
|
|
42
|
+
*
|
|
43
|
+
* Two unrelated shapes arrive as XML and neither is data. A `S:Fault` envelope
|
|
44
|
+
* accompanies a 4xx and names the problem in a numeric `faultcode`. The
|
|
45
|
+
* `syncResponse` envelope arrives on **HTTP 200** carrying a queue ticket
|
|
46
|
+
* instead of the dataset — parsed as TSV it yields a header row of XML, so it is
|
|
47
|
+
* detected here rather than downstream.
|
|
48
|
+
*
|
|
49
|
+
* Always throws.
|
|
50
|
+
*/
|
|
51
|
+
export function classifyXmlBody(xml, datasetCode) {
|
|
52
|
+
if (xml.includes('syncResponse') || xml.includes('<status>SUBMITTED</status>')) {
|
|
53
|
+
throw serviceUnavailable(`Eurostat queued this extraction instead of returning data: the SDMX endpoint answered with an asynchronous "SUBMITTED" ticket for "${datasetCode}". The queued result is not retrievable through this server. Narrow the request with dimension filters or a period range so Eurostat serves it synchronously.`, { reason: 'async_queued', datasetCode, retryable: false });
|
|
54
|
+
}
|
|
55
|
+
const code = /<faultcode>\s*(\d+)\s*<\/faultcode>/.exec(xml)?.[1];
|
|
56
|
+
const detail = decodeEntities(/<faultstring>([\s\S]*?)<\/faultstring>/.exec(xml)?.[1]?.trim() ?? xml.slice(0, 300));
|
|
57
|
+
if (code === '100') {
|
|
58
|
+
throw notFound(`Dataset "${datasetCode}" is not available for dissemination. Eurostat fault ${code}: ${detail}`, { reason: 'not_found', datasetCode, faultcode: code });
|
|
59
|
+
}
|
|
60
|
+
if (code === '140') {
|
|
61
|
+
throw validationError(`Eurostat rejected the dimension key: the number of filter positions did not match the dataset's dimensions. Eurostat fault ${code}: ${detail}`, { reason: 'filter_arity', datasetCode, faultcode: code });
|
|
62
|
+
}
|
|
63
|
+
if (code === '150') {
|
|
64
|
+
throw validationError(`Eurostat rejected a filter value. This fault also covers a period range outside the dataset's coverage. Eurostat fault ${code}: ${detail}`, { reason: 'invalid_dimension', datasetCode, faultcode: code });
|
|
65
|
+
}
|
|
66
|
+
throw serviceUnavailable(`Eurostat SDMX endpoint returned a fault for "${datasetCode}"${code ? ` (faultcode ${code})` : ''}: ${detail}`, { reason: 'upstream_fault', datasetCode, ...(code && { faultcode: code }) });
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Split one wide-TSV cell into its value and flags.
|
|
70
|
+
*
|
|
71
|
+
* The grammar, read off live responses, is `<value>` `SPACE` `<flags>`, where
|
|
72
|
+
* flags is `<obs_flag>` optionally followed by `@<conf_status>` and either part
|
|
73
|
+
* may be empty: `429634.3 `, `3853937.0 p`, `: `, `: @C`. A cell that is empty
|
|
74
|
+
* on both sides of the split is not an observation at all and yields
|
|
75
|
+
* `undefined` — distinct from `: `, which is an observation Eurostat reports as
|
|
76
|
+
* unavailable.
|
|
77
|
+
*/
|
|
78
|
+
export function parseCell(cell) {
|
|
79
|
+
const sep = cell.indexOf(' ');
|
|
80
|
+
const rawValue = (sep === -1 ? cell : cell.slice(0, sep)).trim();
|
|
81
|
+
const rawFlags = (sep === -1 ? '' : cell.slice(sep + 1)).trim();
|
|
82
|
+
if (rawValue === '' && rawFlags === '')
|
|
83
|
+
return;
|
|
84
|
+
const atIdx = rawFlags.indexOf(CONF_SEPARATOR);
|
|
85
|
+
const flag = (atIdx === -1 ? rawFlags : rawFlags.slice(0, atIdx)) || null;
|
|
86
|
+
const conf = (atIdx === -1 ? '' : rawFlags.slice(atIdx + 1)) || null;
|
|
87
|
+
if (rawValue === '' || rawValue === MISSING_VALUE)
|
|
88
|
+
return { conf, flag, value: null };
|
|
89
|
+
const num = Number(rawValue);
|
|
90
|
+
return { conf, flag, value: Number.isFinite(num) ? num : null };
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Parse the header line of a wide TSV body.
|
|
94
|
+
*
|
|
95
|
+
* The first tab-separated field is the comma-joined dimension list with
|
|
96
|
+
* `\TIME_PERIOD` appended to the last name; the remaining fields are the period
|
|
97
|
+
* codes, each padded with a trailing space Eurostat does not strip.
|
|
98
|
+
*/
|
|
99
|
+
export function parseHeader(line) {
|
|
100
|
+
const fields = line.split('\t');
|
|
101
|
+
const keyField = (fields[0] ?? '').trim();
|
|
102
|
+
const dimensionField = keyField.endsWith(TIME_HEADER_SUFFIX)
|
|
103
|
+
? keyField.slice(0, -TIME_HEADER_SUFFIX.length)
|
|
104
|
+
: keyField;
|
|
105
|
+
const dimensions = dimensionField
|
|
106
|
+
.split(',')
|
|
107
|
+
.map((d) => d.trim())
|
|
108
|
+
.filter((d) => d !== '');
|
|
109
|
+
const periods = fields.slice(1).map((p) => p.trim());
|
|
110
|
+
if (dimensions.length === 0 || periods.length === 0) {
|
|
111
|
+
throw serviceUnavailable(`Eurostat returned a TSV body whose header names no ${dimensions.length === 0 ? 'dimensions' : 'periods'}. Header: ${line.slice(0, 200)}`, { reason: 'upstream_fault' });
|
|
112
|
+
}
|
|
113
|
+
return { dimensions, periods };
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Build the positional dimension key the SDMX path takes.
|
|
117
|
+
*
|
|
118
|
+
* The key is a dot-separated segment carrying one position per dimension **in
|
|
119
|
+
* dataset order excluding time**; empty means wildcard and `+` means OR. The
|
|
120
|
+
* count must match exactly — a key with one position too few or too many is
|
|
121
|
+
* rejected upstream with faultcode 140 — so positions are emitted for every
|
|
122
|
+
* dimension in `dimensionOrder`, filtered or not, and never only for the
|
|
123
|
+
* dimensions the caller named.
|
|
124
|
+
*
|
|
125
|
+
* Returns `undefined` when no filter applies. An all-wildcard key is accepted
|
|
126
|
+
* by Eurostat, but omitting the segment asks the same question without staking
|
|
127
|
+
* the request on this server's copy of the dimension order being current.
|
|
128
|
+
*/
|
|
129
|
+
export function buildKeyPath(dimensionOrder, filters) {
|
|
130
|
+
const applied = Object.entries(filters).filter(([, values]) => values.length > 0);
|
|
131
|
+
if (applied.length === 0)
|
|
132
|
+
return;
|
|
133
|
+
const unknown = applied.map(([dim]) => dim).filter((dim) => !dimensionOrder.includes(dim));
|
|
134
|
+
if (unknown.length > 0) {
|
|
135
|
+
throw validationError(`Filter names ${unknown.map((d) => `"${d}"`).join(', ')}, which ${unknown.length === 1 ? 'is not a dimension' : 'are not dimensions'} of this dataset. Filterable dimensions, in key order: ${dimensionOrder.join(', ')}. The time dimension is filtered with since_period/until_period instead.`, { reason: 'invalid_dimension', unknownDimensions: unknown, dimensionOrder });
|
|
136
|
+
}
|
|
137
|
+
return dimensionOrder.map((dim) => (filters[dim] ?? []).join('+')).join('.');
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Canvas column schema for {@link BulkRow}, declared rather than sniffed.
|
|
141
|
+
*
|
|
142
|
+
* Inference reads only the leading rows, which types an all-integer measure
|
|
143
|
+
* prefix as `BIGINT` — serialized back out as a string — and types an
|
|
144
|
+
* all-missing prefix from nulls alone. Every column is nullable: a measure is
|
|
145
|
+
* routinely unavailable, and flags are the exception rather than the rule.
|
|
146
|
+
*/
|
|
147
|
+
export function bulkRowSchema(dimensions) {
|
|
148
|
+
return [
|
|
149
|
+
...dimensions.map((dim) => ({ name: dim, type: 'VARCHAR', nullable: true })),
|
|
150
|
+
{ name: TIME_COLUMN, type: 'VARCHAR', nullable: true },
|
|
151
|
+
{ name: OBS_VALUE_COLUMN, type: 'DOUBLE', nullable: true },
|
|
152
|
+
{ name: OBS_FLAG_COLUMN, type: 'VARCHAR', nullable: true },
|
|
153
|
+
{ name: OBS_FLAG_LABEL_COLUMN, type: 'VARCHAR', nullable: true },
|
|
154
|
+
{ name: CONF_STATUS_COLUMN, type: 'VARCHAR', nullable: true },
|
|
155
|
+
{ name: CONF_STATUS_LABEL_COLUMN, type: 'VARCHAR', nullable: true },
|
|
156
|
+
];
|
|
157
|
+
}
|
|
158
|
+
export class EurostatBulkService {
|
|
159
|
+
// config and storage accepted to match the standard service init pattern;
|
|
160
|
+
// this service uses only the Eurostat public API and per-request config.
|
|
161
|
+
// biome-ignore lint/complexity/noUselessConstructor: standard init pattern
|
|
162
|
+
constructor(_config, _storage) { }
|
|
163
|
+
buildUrl(datasetCode, keyPath, sinceP, untilP) {
|
|
164
|
+
const { baseUrl } = getServerConfig();
|
|
165
|
+
const path = keyPath
|
|
166
|
+
? `${encodeURIComponent(datasetCode)}/${encodeURIComponent(keyPath)}`
|
|
167
|
+
: encodeURIComponent(datasetCode);
|
|
168
|
+
const url = new URL(`${baseUrl}/sdmx/2.1/data/${path}`);
|
|
169
|
+
url.searchParams.set('format', 'TSV');
|
|
170
|
+
if (sinceP)
|
|
171
|
+
url.searchParams.set('startPeriod', sinceP);
|
|
172
|
+
if (untilP)
|
|
173
|
+
url.searchParams.set('endPeriod', untilP);
|
|
174
|
+
return url;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Start a bulk download: issue the request, classify the response, parse the
|
|
178
|
+
* header, and hand back a generator over the body.
|
|
179
|
+
*
|
|
180
|
+
* The request is made once with no retry. A bulk body streams for up to
|
|
181
|
+
* `EUROSTAT_BULK_TIMEOUT_MS`, so re-running it on a transient failure costs
|
|
182
|
+
* the caller another full transfer for a request that is already the most
|
|
183
|
+
* expensive one this server makes; the failure is surfaced instead.
|
|
184
|
+
*/
|
|
185
|
+
async startDownload(datasetCode, dimensionOrder, filters, sinceP, untilP, ctx) {
|
|
186
|
+
const { bulkTimeoutMs, bulkMaxBytes } = getServerConfig();
|
|
187
|
+
const keyPath = buildKeyPath(dimensionOrder, filters);
|
|
188
|
+
const url = this.buildUrl(datasetCode, keyPath, sinceP, untilP);
|
|
189
|
+
ctx.log.info('Starting SDMX bulk download', {
|
|
190
|
+
datasetCode,
|
|
191
|
+
keyPath,
|
|
192
|
+
sinceP,
|
|
193
|
+
untilP,
|
|
194
|
+
maxBytes: bulkMaxBytes,
|
|
195
|
+
});
|
|
196
|
+
let response;
|
|
197
|
+
try {
|
|
198
|
+
response = await fetchWithTimeout(url.toString(), bulkTimeoutMs, asReqCtx(ctx), {
|
|
199
|
+
signal: ctx.signal,
|
|
200
|
+
// 400 (bad filter value or arity) and 404 (unknown dataset) are modeled outcomes
|
|
201
|
+
// reclassified below into the declared error contract, not service failures.
|
|
202
|
+
expectedStatuses: [400, 404],
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
// fetchWithTimeout throws for any non-2xx before the caller sees the body, so the
|
|
207
|
+
// SOAP fault is only reachable through the captured excerpt on err.data. Faults run
|
|
208
|
+
// well under the framework's 500-byte cap, so the excerpt carries the whole envelope.
|
|
209
|
+
const body = err.data;
|
|
210
|
+
if (typeof body?.body === 'string' && body.body.includes('<')) {
|
|
211
|
+
classifyXmlBody(body.body, datasetCode);
|
|
212
|
+
}
|
|
213
|
+
throw err;
|
|
214
|
+
}
|
|
215
|
+
if (!response.body) {
|
|
216
|
+
throw serviceUnavailable(`Eurostat SDMX endpoint returned an empty body for "${datasetCode}".`, { reason: 'upstream_fault', datasetCode });
|
|
217
|
+
}
|
|
218
|
+
const stats = {
|
|
219
|
+
bytesRead: 0,
|
|
220
|
+
budgetExceeded: false,
|
|
221
|
+
compressed: false,
|
|
222
|
+
rowCount: 0,
|
|
223
|
+
missingCount: 0,
|
|
224
|
+
periodsSeen: [],
|
|
225
|
+
};
|
|
226
|
+
const decoded = await decodeBody(response.body, stats, bulkMaxBytes);
|
|
227
|
+
const { header, remainder, iterator } = await readHeader(decoded, datasetCode);
|
|
228
|
+
return {
|
|
229
|
+
header,
|
|
230
|
+
stats,
|
|
231
|
+
url: url.toString(),
|
|
232
|
+
rows: () => expandRows(iterator, remainder, header, stats),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Wrap the response body in a byte-budgeted, gzip-aware text stream.
|
|
238
|
+
*
|
|
239
|
+
* Eurostat compresses large bodies **without** a `Content-Encoding` header — the
|
|
240
|
+
* only header-level tell is a `.tsv.gz` filename on `Content-Disposition`, and
|
|
241
|
+
* that is not what is trusted here. The first two bytes are read off the stream
|
|
242
|
+
* and checked for the gzip magic number, so the decision is made on what
|
|
243
|
+
* actually arrived. The switch is not monotonic in dataset size, so it cannot
|
|
244
|
+
* be predicted from the request either.
|
|
245
|
+
*
|
|
246
|
+
* The budget counts bytes *after* decompression. That is the uniform measure
|
|
247
|
+
* across both encodings, and since a gzip body is never larger than what it
|
|
248
|
+
* expands to, bounding the decoded size bounds the transfer as well. Hitting it
|
|
249
|
+
* cancels the underlying reader rather than draining the rest — the body is
|
|
250
|
+
* chunked with no `Content-Length`, so a budget checked after the fact would
|
|
251
|
+
* have already paid for the whole transfer.
|
|
252
|
+
*/
|
|
253
|
+
async function decodeBody(body, stats, maxBytes) {
|
|
254
|
+
const reader = body.getReader();
|
|
255
|
+
// Pull until at least the two magic bytes are in hand; a chunked body can deliver
|
|
256
|
+
// fewer on the first read, and an empty body delivers none.
|
|
257
|
+
const prefix = [];
|
|
258
|
+
let prefixBytes = 0;
|
|
259
|
+
while (prefixBytes < 2) {
|
|
260
|
+
const { done, value } = await reader.read();
|
|
261
|
+
if (done)
|
|
262
|
+
break;
|
|
263
|
+
if (value && value.length > 0) {
|
|
264
|
+
prefix.push(value);
|
|
265
|
+
prefixBytes += value.length;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
const head = concatChunks(prefix, prefixBytes);
|
|
269
|
+
stats.compressed = head.length >= 2 && head[0] === 0x1f && head[1] === 0x8b;
|
|
270
|
+
const replayed = new ReadableStream({
|
|
271
|
+
start(controller) {
|
|
272
|
+
if (head.length > 0)
|
|
273
|
+
controller.enqueue(head);
|
|
274
|
+
},
|
|
275
|
+
async pull(controller) {
|
|
276
|
+
const { done, value } = await reader.read();
|
|
277
|
+
if (done)
|
|
278
|
+
controller.close();
|
|
279
|
+
else
|
|
280
|
+
controller.enqueue(value);
|
|
281
|
+
},
|
|
282
|
+
cancel(reason) {
|
|
283
|
+
return reader.cancel(reason);
|
|
284
|
+
},
|
|
285
|
+
});
|
|
286
|
+
/**
|
|
287
|
+
* `DecompressionStream`'s writable side is typed `BufferSource`, wider than the
|
|
288
|
+
* `Uint8Array` chunks this pipeline carries, so `pipeThrough` will not accept it
|
|
289
|
+
* directly. The cast narrows the transform to the stream types actually in play
|
|
290
|
+
* rather than widening the pipeline to `BufferSource`.
|
|
291
|
+
*/
|
|
292
|
+
const byteStream = stats.compressed
|
|
293
|
+
? replayed.pipeThrough(new DecompressionStream('gzip'))
|
|
294
|
+
: replayed;
|
|
295
|
+
return budgetedText(byteStream, stats, maxBytes);
|
|
296
|
+
}
|
|
297
|
+
function concatChunks(chunks, total) {
|
|
298
|
+
if (chunks.length === 1)
|
|
299
|
+
return chunks[0];
|
|
300
|
+
const out = new Uint8Array(total);
|
|
301
|
+
let offset = 0;
|
|
302
|
+
for (const chunk of chunks) {
|
|
303
|
+
out.set(chunk, offset);
|
|
304
|
+
offset += chunk.length;
|
|
305
|
+
}
|
|
306
|
+
return out;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Decode a byte stream to text, stopping the moment the byte budget is spent.
|
|
310
|
+
*
|
|
311
|
+
* The generator ends cleanly on overspend rather than throwing: the caller has
|
|
312
|
+
* already paid for everything downloaded so far, and discarding it would leave
|
|
313
|
+
* the agent with an error where it could have had a prefix of the dataset plus
|
|
314
|
+
* `budgetExceeded` telling it exactly what happened.
|
|
315
|
+
*
|
|
316
|
+
* Every exit cancels the reader, so nothing further is transferred: overspend,
|
|
317
|
+
* a normal end, and abandonment alike. Abandonment is the case that needs it —
|
|
318
|
+
* a staging failure or an abort part-way through the canvas append runs the
|
|
319
|
+
* `finally`, and releasing the lock alone would leave the response body
|
|
320
|
+
* undrained and its connection held. Cancelling a stream that already ended is
|
|
321
|
+
* a no-op.
|
|
322
|
+
*/
|
|
323
|
+
async function* budgetedText(stream, stats, maxBytes) {
|
|
324
|
+
const reader = stream.getReader();
|
|
325
|
+
const decoder = new TextDecoder('utf-8');
|
|
326
|
+
try {
|
|
327
|
+
while (true) {
|
|
328
|
+
const { done, value } = await reader.read();
|
|
329
|
+
if (done)
|
|
330
|
+
break;
|
|
331
|
+
if (!value || value.length === 0)
|
|
332
|
+
continue;
|
|
333
|
+
stats.bytesRead += value.length;
|
|
334
|
+
if (stats.bytesRead > maxBytes) {
|
|
335
|
+
stats.budgetExceeded = true;
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
const text = decoder.decode(value, { stream: true });
|
|
339
|
+
if (text)
|
|
340
|
+
yield text;
|
|
341
|
+
}
|
|
342
|
+
const tail = decoder.decode();
|
|
343
|
+
if (tail)
|
|
344
|
+
yield tail;
|
|
345
|
+
}
|
|
346
|
+
finally {
|
|
347
|
+
await reader.cancel().catch(() => undefined);
|
|
348
|
+
reader.releaseLock();
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Consume enough of the text stream to tell data from XML, then take the header.
|
|
353
|
+
*
|
|
354
|
+
* Both non-data shapes announce themselves in the first character, so
|
|
355
|
+
* classification happens before a single row is parsed — a `syncResponse`
|
|
356
|
+
* envelope arrives on HTTP 200 and would otherwise be read as a TSV header of
|
|
357
|
+
* XML followed by no rows, which looks like a successful empty download.
|
|
358
|
+
*
|
|
359
|
+
* A body identified as XML is then buffered to the peek window before it is
|
|
360
|
+
* classified, rather than to the first line break. The XML declaration ends in a
|
|
361
|
+
* newline, so stopping there would hand the classifier the declaration alone and
|
|
362
|
+
* lose the `syncResponse` and `faultcode` markers that follow it.
|
|
363
|
+
*
|
|
364
|
+
* A TSV header, by contrast, is buffered to its line break with no length bound.
|
|
365
|
+
* Its length is proportional to the dataset's period count — 166 KB and 13,852
|
|
366
|
+
* fields on a daily series — so capping the search would parse a prefix of it as
|
|
367
|
+
* the whole header, silently drop every period column past the cap, and read the
|
|
368
|
+
* header's own tail as a data line. The byte budget is what bounds this loop.
|
|
369
|
+
*/
|
|
370
|
+
async function readHeader(chunks, datasetCode) {
|
|
371
|
+
let buf = '';
|
|
372
|
+
let exhausted = false;
|
|
373
|
+
const pull = async () => {
|
|
374
|
+
const next = await chunks.next();
|
|
375
|
+
if (next.done)
|
|
376
|
+
exhausted = true;
|
|
377
|
+
else
|
|
378
|
+
buf += next.value;
|
|
379
|
+
};
|
|
380
|
+
while (buf.trim() === '' && !exhausted)
|
|
381
|
+
await pull();
|
|
382
|
+
if (buf.trimStart().startsWith('<')) {
|
|
383
|
+
while (buf.length < XML_PEEK_CHARS && !exhausted)
|
|
384
|
+
await pull();
|
|
385
|
+
classifyXmlBody(buf, datasetCode);
|
|
386
|
+
}
|
|
387
|
+
while (!buf.includes('\n') && !exhausted)
|
|
388
|
+
await pull();
|
|
389
|
+
const newline = buf.indexOf('\n');
|
|
390
|
+
if (newline === -1) {
|
|
391
|
+
if (buf.trim() === '') {
|
|
392
|
+
throw serviceUnavailable(`Eurostat SDMX endpoint returned an empty body for "${datasetCode}".`, { reason: 'upstream_fault', datasetCode });
|
|
393
|
+
}
|
|
394
|
+
// A body that ended without a line break: one header line and no data rows.
|
|
395
|
+
return { header: parseHeader(stripCr(buf)), iterator: chunks, remainder: '' };
|
|
396
|
+
}
|
|
397
|
+
return {
|
|
398
|
+
header: parseHeader(stripCr(buf.slice(0, newline))),
|
|
399
|
+
iterator: chunks,
|
|
400
|
+
remainder: buf.slice(newline + 1),
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
function stripCr(line) {
|
|
404
|
+
return line.endsWith('\r') ? line.slice(0, -1) : line;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Expand the wide TSV body into one row per populated cell.
|
|
408
|
+
*
|
|
409
|
+
* Each data line carries a comma-joined dimension key and one cell per period
|
|
410
|
+
* column, so a line becomes as many rows as it has populated cells. Cells that
|
|
411
|
+
* are blank on both sides of the value/flag split carry no observation and are
|
|
412
|
+
* skipped; `:` is kept, because Eurostat reporting a value as unavailable is
|
|
413
|
+
* itself data.
|
|
414
|
+
*
|
|
415
|
+
* Only complete lines are parsed. When the byte budget cancels the transfer
|
|
416
|
+
* mid-line the trailing fragment is dropped rather than parsed into a row with
|
|
417
|
+
* silently truncated fields.
|
|
418
|
+
*/
|
|
419
|
+
async function* expandRows(chunks, remainder, header, stats) {
|
|
420
|
+
const seen = new Set();
|
|
421
|
+
let buf = remainder;
|
|
422
|
+
const emit = function* (line) {
|
|
423
|
+
const trimmed = stripCr(line);
|
|
424
|
+
if (trimmed === '')
|
|
425
|
+
return;
|
|
426
|
+
const fields = trimmed.split('\t');
|
|
427
|
+
const keyParts = (fields[0] ?? '').split(',');
|
|
428
|
+
const cellCount = Math.min(fields.length - 1, header.periods.length);
|
|
429
|
+
for (let i = 0; i < cellCount; i++) {
|
|
430
|
+
const parsed = parseCell(fields[i + 1] ?? '');
|
|
431
|
+
if (!parsed)
|
|
432
|
+
continue;
|
|
433
|
+
const period = header.periods[i] ?? '';
|
|
434
|
+
const row = {};
|
|
435
|
+
for (let d = 0; d < header.dimensions.length; d++) {
|
|
436
|
+
row[header.dimensions[d]] = keyParts[d]?.trim() ?? null;
|
|
437
|
+
}
|
|
438
|
+
row[TIME_COLUMN] = period;
|
|
439
|
+
row[OBS_VALUE_COLUMN] = parsed.value;
|
|
440
|
+
row[OBS_FLAG_COLUMN] = parsed.flag;
|
|
441
|
+
row[OBS_FLAG_LABEL_COLUMN] = parsed.flag ? (OBS_FLAG_LABELS[parsed.flag] ?? null) : null;
|
|
442
|
+
row[CONF_STATUS_COLUMN] = parsed.conf;
|
|
443
|
+
row[CONF_STATUS_LABEL_COLUMN] = parsed.conf
|
|
444
|
+
? (CONF_STATUS_LABELS[parsed.conf] ?? null)
|
|
445
|
+
: null;
|
|
446
|
+
stats.rowCount++;
|
|
447
|
+
if (parsed.value === null)
|
|
448
|
+
stats.missingCount++;
|
|
449
|
+
seen.add(period);
|
|
450
|
+
yield row;
|
|
451
|
+
}
|
|
452
|
+
};
|
|
453
|
+
/**
|
|
454
|
+
* Drain every complete line sitting in the buffer. Called before the read loop
|
|
455
|
+
* as well as inside it: a small body arrives whole while the header is being
|
|
456
|
+
* read, leaving every data line already buffered and no further chunk to
|
|
457
|
+
* trigger a split.
|
|
458
|
+
*/
|
|
459
|
+
const flush = function* () {
|
|
460
|
+
let newline = buf.indexOf('\n');
|
|
461
|
+
while (newline !== -1) {
|
|
462
|
+
yield* emit(buf.slice(0, newline));
|
|
463
|
+
buf = buf.slice(newline + 1);
|
|
464
|
+
newline = buf.indexOf('\n');
|
|
465
|
+
}
|
|
466
|
+
};
|
|
467
|
+
try {
|
|
468
|
+
yield* flush();
|
|
469
|
+
for await (const chunk of chunks) {
|
|
470
|
+
buf += chunk;
|
|
471
|
+
yield* flush();
|
|
472
|
+
}
|
|
473
|
+
// The final line of a complete body has no trailing newline. A budget-cancelled
|
|
474
|
+
// body's trailing fragment is not a final line — it is a line cut in half.
|
|
475
|
+
if (!stats.budgetExceeded)
|
|
476
|
+
yield* emit(buf);
|
|
477
|
+
stats.periodsSeen = [...seen].sort();
|
|
478
|
+
}
|
|
479
|
+
finally {
|
|
480
|
+
/**
|
|
481
|
+
* Close the text stream on every exit. A consumer that stops early — a canvas
|
|
482
|
+
* append that throws, an abort — abandons this generator, and a small body
|
|
483
|
+
* whose rows were all buffered while the header was read is abandoned before
|
|
484
|
+
* the `for await` above ever starts it. Neither path would otherwise reach the
|
|
485
|
+
* cancel that frees the response body.
|
|
486
|
+
*/
|
|
487
|
+
await chunks.return(undefined);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
// --- Init/accessor pattern ---
|
|
491
|
+
let _service;
|
|
492
|
+
export function initEurostatBulkService(config, storage) {
|
|
493
|
+
_service = new EurostatBulkService(config, storage);
|
|
494
|
+
}
|
|
495
|
+
export function getEurostatBulkService() {
|
|
496
|
+
if (!_service) {
|
|
497
|
+
throw new Error('EurostatBulkService not initialized — call initEurostatBulkService() in setup()');
|
|
498
|
+
}
|
|
499
|
+
return _service;
|
|
500
|
+
}
|
|
501
|
+
//# sourceMappingURL=eurostat-bulk-service.js.map
|