@aborruso/ckan-mcp-server 0.4.124 → 0.4.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/EXAMPLES.md +8 -8
- package/LOG.md +26 -0
- package/dist/index.js +317 -295
- package/package.json +1 -1
package/EXAMPLES.md
CHANGED
|
@@ -160,14 +160,14 @@ ckan_get_mqa_quality({
|
|
|
160
160
|
})
|
|
161
161
|
```
|
|
162
162
|
|
|
163
|
-
Returns quality score
|
|
164
|
-
-
|
|
165
|
-
-
|
|
166
|
-
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
**Note**: Only works with dati.gov.it datasets. Uses the `identifier` field (or falls back to `name`) to query the European MQA API.
|
|
163
|
+
Returns the quality score from the data.europa.eu MQA (Metadata Quality Assessment), [methodology v2](https://data.europa.eu/mqa/methodology):
|
|
164
|
+
- Final score on a 0-7.5 scale and its band (Sufficient < 2.5 ≤ Good < 5 ≤ Excellent)
|
|
165
|
+
- Dataset score, distribution average and data service average
|
|
166
|
+
- The failing metrics with the largest gain on the final score (DCAT-AP property, weight, how many distributions fail it)
|
|
167
|
+
|
|
168
|
+
`ckan_get_mqa_quality_details` lists every failing metric grouped by FAIR dimension (findability, accessibility, interoperability, reusability).
|
|
169
|
+
|
|
170
|
+
**Note**: Only works with dati.gov.it datasets. Uses the `identifier` field (or falls back to `name`) to query the European MQA API. Datasets that data.europa.eu has not re-evaluated with v2 yet are reported with the previous methodology (405-point scale), labelled as such.
|
|
171
171
|
|
|
172
172
|
## USA - data.gov
|
|
173
173
|
|
package/LOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# LOG
|
|
2
2
|
|
|
3
|
+
## 2026-09-23
|
|
4
|
+
|
|
5
|
+
### v0.4.126 - why MQA URL tests fail
|
|
6
|
+
|
|
7
|
+
Ships #552. For a caller: `ckan_get_mqa_quality_details` now says why URL tests fail (`HTTP test: 1100 timeout ×21, no status recorded ×3`). For the project: `npm run smoke` checks the MQA tool live against data.europa.eu, so a format change there fails the release gate.
|
|
8
|
+
|
|
9
|
+
### MQA: smoke gate and HTTP status of failing URL tests
|
|
10
|
+
|
|
11
|
+
- `npm run smoke` gains a live MQA case (Roma `czrm-…-2023`: `methodology` v2, `maxScore` 7.5, 0 ≤ `score` ≤ 7.5) and a generic `field_max`. v0.4.124 shipped broken MQA tools with green tests because the tests read stored fixtures; the gate now asks data.europa.eu. Negative check: expecting `v1` fails the case.
|
|
12
|
+
- `ckan_get_mqa_quality_details` says why URL tests fail: `HTTP test: 1100 timeout ×21, no status recorded ×3`. Codes come from the metrics graph (`http:statusCodeValue` in v2, `dqv:value` in v1; ≥1000 are piveau's own, 1100 = timeout), latest per distribution, 2xx excluded. Fetched only by the details tool and only when a URL test fails (the graph is ~570 KB); best effort, a failed fetch leaves the details intact.
|
|
13
|
+
- Bug caught by the tests: boolean result nodes share the metric id, and `Number(false)` read as status code 0.
|
|
14
|
+
|
|
15
|
+
### v0.4.125 - MQA scores on the new data.europa.eu methodology
|
|
16
|
+
|
|
17
|
+
Ships #549 (closes #548). For a caller: `ckan_get_mqa_quality` and `ckan_get_mqa_quality_details` report the MQA v2 score (0-7.5, band, failing metrics by gain) instead of printing v2 numbers on the old 405 scale or failing with "identifier not aligned"; datasets not yet re-evaluated get the previous-methodology score, labelled. The MQA tools work again on the Cloudflare Worker. **BREAKING** for JSON consumers of the two MQA tools: compact shape with `methodology`, `metricsVersion`, `score`, `maxScore`.
|
|
18
|
+
|
|
19
|
+
### MQA tools follow the data.europa.eu methodology v2 (#548)
|
|
20
|
+
|
|
21
|
+
- data.europa.eu switched the MQA to a new methodology (DCAT-AP 3, weights 1/0.5/0.25, 0-7.5 scale, four FAIR dimensions, no Contextuality, SHACL not scored). The dataset cache now serves only v2 and answers `404 No v2 metrics found` for datasets not yet re-evaluated: on day one, 1,051 of 61,897 dati.gov.it datasets.
|
|
22
|
+
- Both MQA tools were broken: re-evaluated datasets printed `6.5/405` and `Accessibility 2.25/100`, the JSON of a 24-distribution dataset exceeded 50k and came back `{_truncated}`; the others failed with a wrong "identifier not aligned" error.
|
|
23
|
+
- Now: v2 payload parsed directly — `datasetFinal`/7.5 with band, dataset/distribution/data service scores, failing metrics aggregated across distributions ("21 of 24") and ordered by their exact gain on the final score. Checked live: score + sum of gains = 7.5.
|
|
24
|
+
- `result: null` (test not run, e.g. no URL) counts as failing, as it does in the official score; found only on the live payload.
|
|
25
|
+
- No v2 yet → previous-methodology scores from the metrics endpoint, labelled; refused if the graph already carries v2-scale numbers.
|
|
26
|
+
- MQA tools were already broken on the Worker (`The 'cache' field on 'RequestInitializerDict' is not implemented`, axios fetch adapter): the cache call now goes through `safeFetch` like the metrics call. Verified with `wrangler dev` and the Node HTTP server.
|
|
27
|
+
- `DQV of dataset not found` and `No v2 metrics found` now give different errors. JSON output is compact (`methodology`, `metricsVersion`, `maxScore`), never the raw payload. OpenSpec change `update-mqa-methodology-v2`.
|
|
28
|
+
|
|
3
29
|
## 2026-09-06
|
|
4
30
|
|
|
5
31
|
### v0.4.124 - search that answers the question that was asked
|
package/dist/index.js
CHANGED
|
@@ -4061,7 +4061,6 @@ Typical workflow: ckan_group_search \u2192 ckan_group_show (get details) \u2192
|
|
|
4061
4061
|
|
|
4062
4062
|
// src/tools/quality.ts
|
|
4063
4063
|
import { z as z8 } from "zod";
|
|
4064
|
-
import axios2 from "axios";
|
|
4065
4064
|
var MQA_API_BASE = "https://data.europa.eu/api/mqa/cache/datasets";
|
|
4066
4065
|
var MQA_METRICS_BASE = "https://data.europa.eu/api/hub/repo/datasets";
|
|
4067
4066
|
var ALLOWED_MQA_HOSTS = /* @__PURE__ */ new Set(["dati.gov.it", "www.dati.gov.it"]);
|
|
@@ -4351,338 +4350,361 @@ function findNonMaxDimensions(scores) {
|
|
|
4351
4350
|
});
|
|
4352
4351
|
return nonMax;
|
|
4353
4352
|
}
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
const response = await axios2.get(mqaUrl, {
|
|
4370
|
-
timeout: 3e4,
|
|
4371
|
-
headers: {
|
|
4372
|
-
"User-Agent": "CKAN-MCP-Server/1.0"
|
|
4373
|
-
}
|
|
4374
|
-
});
|
|
4375
|
-
let metricsPayload;
|
|
4376
|
-
try {
|
|
4377
|
-
const metricsResponse = await safeFetch(metricsUrl, {
|
|
4378
|
-
headers: {
|
|
4379
|
-
"User-Agent": "CKAN-MCP-Server/1.0"
|
|
4380
|
-
}
|
|
4381
|
-
}, { httpsOnly: true });
|
|
4382
|
-
if (!metricsResponse.ok) {
|
|
4383
|
-
throw new Error(`MQA metrics error: ${metricsResponse.status} ${metricsResponse.statusText}`);
|
|
4384
|
-
}
|
|
4385
|
-
try {
|
|
4386
|
-
metricsPayload = await metricsResponse.json();
|
|
4387
|
-
} catch {
|
|
4388
|
-
metricsPayload = await metricsResponse.text();
|
|
4389
|
-
}
|
|
4390
|
-
} catch (metricsError) {
|
|
4391
|
-
if (metricsError instanceof Error) {
|
|
4392
|
-
throw new Error(`MQA metrics error: ${metricsError.message}`);
|
|
4393
|
-
}
|
|
4394
|
-
throw metricsError;
|
|
4395
|
-
}
|
|
4396
|
-
const scores = extractMetricsScores(metricsPayload);
|
|
4397
|
-
const resultEntry = response.data?.result?.results?.[0];
|
|
4398
|
-
const portalId = resultEntry?.info?.["dataset-id"] || europeanId;
|
|
4399
|
-
const breakdown = {
|
|
4400
|
-
scores,
|
|
4401
|
-
nonMaxDimensions: findNonMaxDimensions(scores),
|
|
4402
|
-
metricsUrl,
|
|
4403
|
-
mqaUrl,
|
|
4404
|
-
portalId
|
|
4405
|
-
};
|
|
4406
|
-
return {
|
|
4407
|
-
mqa: response.data,
|
|
4408
|
-
metrics: metricsPayload,
|
|
4409
|
-
breakdown
|
|
4410
|
-
};
|
|
4411
|
-
} catch (error) {
|
|
4412
|
-
if (axios2.isAxiosError(error)) {
|
|
4413
|
-
if (error.response?.status === 404) {
|
|
4414
|
-
continue;
|
|
4415
|
-
}
|
|
4416
|
-
throw new Error(`MQA API error: ${error.message}`);
|
|
4417
|
-
}
|
|
4418
|
-
throw error;
|
|
4353
|
+
var V2_MAX_SCORE = 7.5;
|
|
4354
|
+
var V1_MAX_SCORE = 405;
|
|
4355
|
+
var V2_DIMENSIONS = ["findability", "accessibility", "interoperability", "reusability"];
|
|
4356
|
+
var V1_NOTE = "Not yet re-evaluated with MQA methodology v2: scores come from the previous MQA methodology (405-point scale, five dimensions).";
|
|
4357
|
+
var PROPERTY_PREFIXES = [
|
|
4358
|
+
["http://purl.org/dc/terms/", "dct:"],
|
|
4359
|
+
["http://www.w3.org/ns/dcat#", "dcat:"],
|
|
4360
|
+
["http://www.w3.org/ns/adms#", "adms:"],
|
|
4361
|
+
["http://xmlns.com/foaf/0.1/", "foaf:"],
|
|
4362
|
+
["http://data.europa.eu/r5r/", "dcatap:"]
|
|
4363
|
+
];
|
|
4364
|
+
function shortProperty(uri) {
|
|
4365
|
+
for (const [namespace, prefix] of PROPERTY_PREFIXES) {
|
|
4366
|
+
if (uri.startsWith(namespace)) {
|
|
4367
|
+
return prefix + uri.slice(namespace.length);
|
|
4419
4368
|
}
|
|
4420
4369
|
}
|
|
4421
|
-
|
|
4422
|
-
`Quality metrics not found or identifier not aligned on data.europa.eu. Tried: ${candidates.join(", ")}. Check the dataset quality page on data.europa.eu to confirm the identifier (it may include a '~~1' suffix) or verify alignment on dati.gov.it (quality may be marked as 'Non disponibile o identificativo non allineato').`
|
|
4423
|
-
);
|
|
4370
|
+
return uri;
|
|
4424
4371
|
}
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
return
|
|
4428
|
-
|
|
4429
|
-
breakdown: result.breakdown
|
|
4430
|
-
};
|
|
4372
|
+
function mqaBand(score) {
|
|
4373
|
+
if (score >= 5) return "Excellent";
|
|
4374
|
+
if (score >= 2.5) return "Good";
|
|
4375
|
+
return "Sufficient";
|
|
4431
4376
|
}
|
|
4432
|
-
|
|
4433
|
-
const
|
|
4434
|
-
|
|
4435
|
-
return {
|
|
4436
|
-
breakdown: result.breakdown,
|
|
4437
|
-
details
|
|
4438
|
-
};
|
|
4377
|
+
function round(value, digits = 3) {
|
|
4378
|
+
const factor = 10 ** digits;
|
|
4379
|
+
return Math.round(value * factor) / factor;
|
|
4439
4380
|
}
|
|
4440
|
-
function
|
|
4441
|
-
if (
|
|
4442
|
-
return
|
|
4443
|
-
}
|
|
4444
|
-
for (const item of section) {
|
|
4445
|
-
if (item && typeof item === "object" && key in item) {
|
|
4446
|
-
return item[key];
|
|
4447
|
-
}
|
|
4381
|
+
function groupScore(entities) {
|
|
4382
|
+
if (entities.length === 0) {
|
|
4383
|
+
return null;
|
|
4448
4384
|
}
|
|
4449
|
-
|
|
4385
|
+
const average = entities.reduce((sum, entity) => sum + entity.score, 0) / entities.length;
|
|
4386
|
+
return { count: entities.length, average: round(average) };
|
|
4450
4387
|
}
|
|
4451
|
-
function
|
|
4452
|
-
|
|
4453
|
-
|
|
4388
|
+
function parseMqaV2(entry, links) {
|
|
4389
|
+
const dataset = entry.dataset;
|
|
4390
|
+
if (!dataset || typeof dataset.score !== "number") {
|
|
4391
|
+
throw new Error("Unexpected MQA payload: missing dataset score");
|
|
4454
4392
|
}
|
|
4455
|
-
const
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4393
|
+
const groups = [
|
|
4394
|
+
["dataset", [dataset]],
|
|
4395
|
+
["distribution", Array.isArray(entry.distributions) ? entry.distributions : []],
|
|
4396
|
+
["dataService", Array.isArray(entry.dataServices) ? entry.dataServices : []]
|
|
4397
|
+
];
|
|
4398
|
+
const groupsPresent = groups.filter(([, entities]) => entities.length > 0).length;
|
|
4399
|
+
const score = typeof entry.datasetFinal === "number" ? entry.datasetFinal : groupsPresent === 1 ? dataset.score : void 0;
|
|
4400
|
+
if (score === void 0) {
|
|
4401
|
+
throw new Error("Unexpected MQA payload: missing final dataset score");
|
|
4402
|
+
}
|
|
4403
|
+
const failing = [];
|
|
4404
|
+
for (const [entity, entities] of groups) {
|
|
4405
|
+
const byMetric = /* @__PURE__ */ new Map();
|
|
4406
|
+
for (const item of entities) {
|
|
4407
|
+
for (const metric of item.metrics ?? []) {
|
|
4408
|
+
if (metric.result === 1) continue;
|
|
4409
|
+
const current = byMetric.get(metric.metric);
|
|
4410
|
+
if (current) {
|
|
4411
|
+
current.failed += 1;
|
|
4412
|
+
} else {
|
|
4413
|
+
byMetric.set(metric.metric, { metric, failed: 1 });
|
|
4414
|
+
}
|
|
4473
4415
|
}
|
|
4474
4416
|
}
|
|
4475
|
-
|
|
4417
|
+
for (const { metric, failed } of byMetric.values()) {
|
|
4418
|
+
failing.push({
|
|
4419
|
+
metric: metric.metric,
|
|
4420
|
+
property: shortProperty(metric.property),
|
|
4421
|
+
dimension: metric.dimension,
|
|
4422
|
+
importance: metric.importance,
|
|
4423
|
+
weight: metric.weight,
|
|
4424
|
+
entity,
|
|
4425
|
+
failed,
|
|
4426
|
+
total: entities.length,
|
|
4427
|
+
// The group average spans all its entities, and the final score averages the groups
|
|
4428
|
+
gain: round(metric.weight * failed / entities.length / groupsPresent, 4)
|
|
4429
|
+
});
|
|
4430
|
+
}
|
|
4476
4431
|
}
|
|
4477
|
-
|
|
4432
|
+
failing.sort((a, b) => b.gain - a.gain || a.metric.localeCompare(b.metric));
|
|
4433
|
+
return {
|
|
4434
|
+
methodology: "v2",
|
|
4435
|
+
metricsVersion: typeof entry.metricsVersion === "string" ? entry.metricsVersion : "2",
|
|
4436
|
+
score: round(score, 4),
|
|
4437
|
+
maxScore: V2_MAX_SCORE,
|
|
4438
|
+
band: mqaBand(score),
|
|
4439
|
+
dataset: { score: dataset.score, maxScore: V2_MAX_SCORE },
|
|
4440
|
+
distributions: groupScore(groups[1][1]),
|
|
4441
|
+
dataServices: groupScore(groups[2][1]),
|
|
4442
|
+
failing,
|
|
4443
|
+
...links
|
|
4444
|
+
};
|
|
4478
4445
|
}
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4446
|
+
var MQA_TIMEOUT_MS = 3e4;
|
|
4447
|
+
function fetchMqa(url) {
|
|
4448
|
+
return safeFetch(url, {
|
|
4449
|
+
headers: { "User-Agent": "CKAN-MCP-Server/1.0" },
|
|
4450
|
+
signal: AbortSignal.timeout(MQA_TIMEOUT_MS)
|
|
4451
|
+
}, { httpsOnly: true });
|
|
4485
4452
|
}
|
|
4486
|
-
function
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
}
|
|
4492
|
-
if (statusKey) {
|
|
4493
|
-
const statusMetric = findSectionMetric(section, statusKey);
|
|
4494
|
-
const statusAvailable = metricArrayIsAvailable(statusMetric);
|
|
4495
|
-
if (statusAvailable !== void 0) {
|
|
4496
|
-
return { available: statusAvailable };
|
|
4453
|
+
async function fetchMetricsGraph(metricsUrl) {
|
|
4454
|
+
try {
|
|
4455
|
+
const response = await fetchMqa(metricsUrl);
|
|
4456
|
+
if (!response.ok) {
|
|
4457
|
+
throw new Error(`${response.status} ${response.statusText}`);
|
|
4497
4458
|
}
|
|
4459
|
+
try {
|
|
4460
|
+
return await response.json();
|
|
4461
|
+
} catch {
|
|
4462
|
+
return await response.text();
|
|
4463
|
+
}
|
|
4464
|
+
} catch (error) {
|
|
4465
|
+
throw new Error(`MQA metrics error: ${error instanceof Error ? error.message : String(error)}`);
|
|
4498
4466
|
}
|
|
4499
|
-
return void 0;
|
|
4500
4467
|
}
|
|
4501
|
-
function
|
|
4502
|
-
const
|
|
4503
|
-
const
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4468
|
+
async function fetchMqaV1(europeanId, links) {
|
|
4469
|
+
const metricsUrl = `${MQA_METRICS_BASE}/${europeanId}/metrics`;
|
|
4470
|
+
const notReEvaluated = `Dataset ${europeanId} is on data.europa.eu but not yet re-evaluated with MQA methodology v2, and previous-methodology metrics are not available`;
|
|
4471
|
+
let metrics;
|
|
4472
|
+
try {
|
|
4473
|
+
metrics = await fetchMetricsGraph(metricsUrl);
|
|
4474
|
+
} catch (error) {
|
|
4475
|
+
throw new Error(`${notReEvaluated} (${error instanceof Error ? error.message : String(error)}).`);
|
|
4476
|
+
}
|
|
4477
|
+
const scores = extractMetricsScores(metrics);
|
|
4478
|
+
const looksV1 = scores.contextuality !== void 0 || (scores.total ?? 0) > V2_MAX_SCORE;
|
|
4479
|
+
if (!looksV1) {
|
|
4480
|
+
throw new Error(`${notReEvaluated}. Try again after the next harvest.`);
|
|
4508
4481
|
}
|
|
4482
|
+
const nonMaxDimensions = findNonMaxDimensions(scores);
|
|
4509
4483
|
return {
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4484
|
+
methodology: "v1",
|
|
4485
|
+
metricsVersion: "1",
|
|
4486
|
+
note: V1_NOTE,
|
|
4487
|
+
score: scores.total ?? null,
|
|
4488
|
+
maxScore: V1_MAX_SCORE,
|
|
4489
|
+
breakdown: {
|
|
4490
|
+
scores,
|
|
4491
|
+
nonMaxDimensions,
|
|
4492
|
+
metricsUrl,
|
|
4493
|
+
mqaUrl: links.mqaUrl,
|
|
4494
|
+
portalId: europeanId
|
|
4515
4495
|
},
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
contactPoint: metricBoolean(resultEntry.reusability, "contactPointAvailability") !== void 0 ? { available: metricBoolean(resultEntry.reusability, "contactPointAvailability") } : void 0,
|
|
4519
|
-
publisher: metricBoolean(resultEntry.reusability, "publisherAvailability") !== void 0 ? { available: metricBoolean(resultEntry.reusability, "publisherAvailability") } : void 0
|
|
4520
|
-
},
|
|
4521
|
-
interoperability: {
|
|
4522
|
-
format: metricAvailability(resultEntry.interoperability, "formatAvailability"),
|
|
4523
|
-
mediaType: metricAvailability(resultEntry.interoperability, "mediaTypeAvailability")
|
|
4524
|
-
},
|
|
4525
|
-
findability: {
|
|
4526
|
-
keyword: metricBoolean(resultEntry.findability, "keywordAvailability") !== void 0 ? { available: metricBoolean(resultEntry.findability, "keywordAvailability") } : void 0,
|
|
4527
|
-
category: metricBoolean(resultEntry.findability, "categoryAvailability") !== void 0 ? { available: metricBoolean(resultEntry.findability, "categoryAvailability") } : void 0,
|
|
4528
|
-
spatial: metricBoolean(resultEntry.findability, "spatialAvailability") !== void 0 ? { available: metricBoolean(resultEntry.findability, "spatialAvailability") } : void 0,
|
|
4529
|
-
temporal: metricBoolean(resultEntry.findability, "temporalAvailability") !== void 0 ? { available: metricBoolean(resultEntry.findability, "temporalAvailability") } : void 0
|
|
4530
|
-
},
|
|
4531
|
-
contextuality: {
|
|
4532
|
-
byteSize: metricAvailability(resultEntry.contextuality, "byteSizeAvailability"),
|
|
4533
|
-
rights: metricAvailability(resultEntry.contextuality, "rightsAvailability")
|
|
4534
|
-
},
|
|
4535
|
-
breakdown
|
|
4496
|
+
details: extractMetricDetails(metrics, nonMaxDimensions),
|
|
4497
|
+
...links
|
|
4536
4498
|
};
|
|
4537
4499
|
}
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4500
|
+
var STATUS_METRICS = /* @__PURE__ */ new Set(["accessUrlStatusCode", "downloadUrlStatusCode"]);
|
|
4501
|
+
var HTTP_STATUS_CODE = "http://www.w3.org/2011/http#statusCodeValue";
|
|
4502
|
+
function extractStatusCodes(metricsData) {
|
|
4503
|
+
const latest = /* @__PURE__ */ new Map();
|
|
4504
|
+
const graph = decodeMetricsPayload(metricsData)?.["@graph"];
|
|
4505
|
+
for (const node of Array.isArray(graph) ? graph : []) {
|
|
4506
|
+
if (!node || typeof node !== "object") continue;
|
|
4507
|
+
const record = node;
|
|
4508
|
+
const metricRef = record["dqv:isMeasurementOf"];
|
|
4509
|
+
const metricId = typeof metricRef === "string" ? metricRef : metricRef?.["@id"];
|
|
4510
|
+
if (typeof metricId !== "string") continue;
|
|
4511
|
+
const metric = metricKeyFromId(metricId);
|
|
4512
|
+
if (!STATUS_METRICS.has(metric)) continue;
|
|
4513
|
+
const raw = record[HTTP_STATUS_CODE] !== void 0 ? Number(record[HTTP_STATUS_CODE]) : parseMetricValue(record["dqv:value"]);
|
|
4514
|
+
if (typeof raw !== "number" || !Number.isInteger(raw)) continue;
|
|
4515
|
+
const code = raw;
|
|
4516
|
+
const target = record["dqv:computedOn"]?.["@id"];
|
|
4517
|
+
const time = String(record["prov:generatedAtTime"]?.["@value"] ?? "");
|
|
4518
|
+
const key = `${metric} ${typeof target === "string" ? target : ""}`;
|
|
4519
|
+
const current = latest.get(key);
|
|
4520
|
+
if (current && current.time > time) continue;
|
|
4521
|
+
const description = typeof record["dct:description"] === "string" ? record["dct:description"] : "";
|
|
4522
|
+
const label = code >= 1e3 ? `${code} ${/timeout/i.test(description) ? "timeout" : "connection error"}` : String(code);
|
|
4523
|
+
latest.set(key, { metric, code, label, time });
|
|
4524
|
+
}
|
|
4525
|
+
const counts = /* @__PURE__ */ new Map();
|
|
4526
|
+
for (const { metric, code, label } of latest.values()) {
|
|
4527
|
+
if (code >= 200 && code < 300) continue;
|
|
4528
|
+
const perMetric = counts.get(metric) ?? {};
|
|
4529
|
+
perMetric[label] = (perMetric[label] ?? 0) + 1;
|
|
4530
|
+
counts.set(metric, perMetric);
|
|
4531
|
+
}
|
|
4532
|
+
return counts;
|
|
4533
|
+
}
|
|
4534
|
+
async function addHttpStatus(result) {
|
|
4535
|
+
const urlTests = result.failing.filter((item) => item.entity === "distribution" && STATUS_METRICS.has(item.metric));
|
|
4536
|
+
if (urlTests.length === 0) return;
|
|
4537
|
+
let counts;
|
|
4538
|
+
try {
|
|
4539
|
+
counts = extractStatusCodes(await fetchMetricsGraph(`${MQA_METRICS_BASE}/${result.portalId}/metrics`));
|
|
4540
|
+
} catch {
|
|
4541
|
+
return;
|
|
4546
4542
|
}
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
const order = [
|
|
4551
|
-
["accessibility", "Accessibility", DIMENSION_MAX.accessibility],
|
|
4552
|
-
["findability", "Findability", DIMENSION_MAX.findability],
|
|
4553
|
-
["interoperability", "Interoperability", DIMENSION_MAX.interoperability],
|
|
4554
|
-
["reusability", "Reusability", DIMENSION_MAX.reusability],
|
|
4555
|
-
["contextuality", "Contextuality", DIMENSION_MAX.contextuality]
|
|
4556
|
-
];
|
|
4557
|
-
for (const [key, label, max] of order) {
|
|
4558
|
-
const value = scores[key];
|
|
4559
|
-
if (typeof value === "number") {
|
|
4560
|
-
const isMax = value >= max;
|
|
4561
|
-
const status = isMax ? "\u2705" : "\u26A0\uFE0F";
|
|
4562
|
-
lines.push(`- ${label}: ${value}/${max} ${status}${isMax ? "" : ` (max ${max})`}`);
|
|
4563
|
-
}
|
|
4564
|
-
}
|
|
4565
|
-
if (normalized.breakdown.nonMaxDimensions.length > 0) {
|
|
4566
|
-
lines.push(`- Non-max dimension(s): ${normalized.breakdown.nonMaxDimensions.join(", ")}`);
|
|
4567
|
-
} else if (Object.keys(scores).length > 0) {
|
|
4568
|
-
lines.push("- Non-max dimension(s): none");
|
|
4569
|
-
}
|
|
4570
|
-
lines.push("");
|
|
4543
|
+
for (const item of urlTests) {
|
|
4544
|
+
const perCode = counts.get(item.metric);
|
|
4545
|
+
if (perCode) item.httpStatus = perCode;
|
|
4571
4546
|
}
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4547
|
+
}
|
|
4548
|
+
async function getMqaQuality(serverUrl, datasetId, opts = {}) {
|
|
4549
|
+
const dataset = await makeCkanRequest(
|
|
4550
|
+
serverUrl,
|
|
4551
|
+
"package_show",
|
|
4552
|
+
{ id: datasetId }
|
|
4553
|
+
);
|
|
4554
|
+
const baseIdentifier = dataset.identifier || dataset.name;
|
|
4555
|
+
const candidates = buildMqaIdCandidates(baseIdentifier);
|
|
4556
|
+
if (candidates.length === 0) {
|
|
4557
|
+
throw new Error("Dataset identifier is empty; cannot query MQA API");
|
|
4581
4558
|
}
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4559
|
+
for (const europeanId of candidates) {
|
|
4560
|
+
const links = {
|
|
4561
|
+
portalId: europeanId,
|
|
4562
|
+
portalUrl: `https://data.europa.eu/data/datasets/${europeanId}/quality?locale=it`,
|
|
4563
|
+
mqaUrl: `${MQA_API_BASE}/${europeanId}`
|
|
4564
|
+
};
|
|
4565
|
+
let response;
|
|
4566
|
+
try {
|
|
4567
|
+
response = await fetchMqa(links.mqaUrl);
|
|
4568
|
+
} catch (error) {
|
|
4569
|
+
throw new Error(`MQA API error: ${error instanceof Error ? error.message : String(error)}`);
|
|
4586
4570
|
}
|
|
4587
|
-
if (
|
|
4588
|
-
|
|
4571
|
+
if (response.status === 404) {
|
|
4572
|
+
if (/no v2 metrics/i.test(await response.text())) {
|
|
4573
|
+
return fetchMqaV1(europeanId, links);
|
|
4574
|
+
}
|
|
4575
|
+
continue;
|
|
4589
4576
|
}
|
|
4590
|
-
if (
|
|
4591
|
-
|
|
4577
|
+
if (!response.ok) {
|
|
4578
|
+
throw new Error(`MQA API error: ${response.status} ${response.statusText}`);
|
|
4592
4579
|
}
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
if (normalized.interoperability.format !== void 0) {
|
|
4598
|
-
lines.push(`- Format: ${normalized.interoperability.format.available ? "\u2713" : "\u2717"} Available`);
|
|
4580
|
+
const payload = await response.json();
|
|
4581
|
+
const entry = payload?.result?.results?.[0];
|
|
4582
|
+
if (!entry || typeof entry !== "object") {
|
|
4583
|
+
throw new Error("Unexpected MQA payload: no result");
|
|
4599
4584
|
}
|
|
4600
|
-
|
|
4601
|
-
|
|
4585
|
+
const result = parseMqaV2(entry, links);
|
|
4586
|
+
if (opts.httpStatus) {
|
|
4587
|
+
await addHttpStatus(result);
|
|
4602
4588
|
}
|
|
4603
|
-
|
|
4589
|
+
return result;
|
|
4604
4590
|
}
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
}
|
|
4619
|
-
|
|
4591
|
+
throw new Error(
|
|
4592
|
+
`No MQA record on data.europa.eu for this dataset (identifier may not be aligned). Tried: ${candidates.join(", ")}. Check the dataset quality page on data.europa.eu to confirm the identifier (it may include a '~~1' suffix) or verify alignment on dati.gov.it (quality may be marked as 'Non disponibile o identificativo non allineato').`
|
|
4593
|
+
);
|
|
4594
|
+
}
|
|
4595
|
+
function formatNumber(value) {
|
|
4596
|
+
return String(round(value, 2));
|
|
4597
|
+
}
|
|
4598
|
+
function describeFailing(item) {
|
|
4599
|
+
const scope = item.entity === "dataset" ? "dataset" : `${item.failed} of ${item.total} ${item.entity === "distribution" ? "distributions" : "data services"}`;
|
|
4600
|
+
let http = "";
|
|
4601
|
+
if (item.httpStatus) {
|
|
4602
|
+
const codes = Object.entries(item.httpStatus).sort((a, b) => b[1] - a[1]).map(([code, n]) => `${code} \xD7${n}`);
|
|
4603
|
+
const unexplained = item.failed - Object.values(item.httpStatus).reduce((sum, n) => sum + n, 0);
|
|
4604
|
+
if (unexplained > 0) codes.push(`no status recorded \xD7${unexplained}`);
|
|
4605
|
+
http = `; HTTP test: ${codes.join(", ")}`;
|
|
4620
4606
|
}
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4607
|
+
return `\`${item.property}\` (${item.metric}, ${item.importance.toLowerCase()} ${item.weight}) - ${scope}, +${formatNumber(item.gain)}${http}`;
|
|
4608
|
+
}
|
|
4609
|
+
function pushV2Scores(lines, result) {
|
|
4610
|
+
lines.push(`**Score**: ${formatNumber(result.score)}/${result.maxScore} (${result.band})`);
|
|
4611
|
+
lines.push(`Methodology: MQA v2 (metrics ${result.metricsVersion}). Bands: Sufficient < 2.5 \u2264 Good < 5 \u2264 Excellent.`);
|
|
4612
|
+
lines.push("");
|
|
4613
|
+
lines.push("## Scores");
|
|
4614
|
+
lines.push(`- Dataset: ${formatNumber(result.dataset.score)}/${result.dataset.maxScore}`);
|
|
4615
|
+
if (result.distributions) {
|
|
4616
|
+
lines.push(`- Distributions (${result.distributions.count}): average ${formatNumber(result.distributions.average)}/${result.maxScore}`);
|
|
4630
4617
|
}
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
lines.push(
|
|
4635
|
-
const metricsEndpoint = normalized.breakdown?.metricsUrl || `${MQA_METRICS_BASE}/${portalId}/metrics`;
|
|
4636
|
-
lines.push(`Metrics endpoint: ${metricsEndpoint}`);
|
|
4637
|
-
lines.push(`Tip: Use the metrics endpoint to explain score deductions (e.g., failing measurements such as knownLicence = false).`);
|
|
4638
|
-
return lines.join("\n");
|
|
4618
|
+
if (result.dataServices) {
|
|
4619
|
+
lines.push(`- Data services (${result.dataServices.count}): average ${formatNumber(result.dataServices.average)}/${result.maxScore}`);
|
|
4620
|
+
}
|
|
4621
|
+
lines.push("");
|
|
4639
4622
|
}
|
|
4640
|
-
function
|
|
4641
|
-
|
|
4642
|
-
const breakdown = data.breakdown;
|
|
4643
|
-
lines.push(`# Quality Details for Dataset: ${datasetId}`);
|
|
4623
|
+
function pushV1Scores(lines, result) {
|
|
4624
|
+
lines.push(`> ${result.note}`);
|
|
4644
4625
|
lines.push("");
|
|
4645
|
-
|
|
4646
|
-
|
|
4626
|
+
const scores = result.breakdown.scores;
|
|
4627
|
+
if (typeof scores.total === "number") {
|
|
4628
|
+
lines.push(`**Score**: ${scores.total}/${result.maxScore}`);
|
|
4647
4629
|
lines.push("");
|
|
4648
4630
|
}
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
const
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
4631
|
+
lines.push("## Dimension Scores");
|
|
4632
|
+
for (const dimension of Object.keys(DIMENSION_MAX)) {
|
|
4633
|
+
const value = scores[dimension];
|
|
4634
|
+
if (typeof value !== "number") continue;
|
|
4635
|
+
const max = DIMENSION_MAX[dimension];
|
|
4636
|
+
const isMax = value >= max;
|
|
4637
|
+
lines.push(`- ${DIMENSION_LABELS[dimension]}: ${value}/${max} ${isMax ? "\u2705" : "\u26A0\uFE0F"}`);
|
|
4638
|
+
}
|
|
4639
|
+
lines.push("");
|
|
4640
|
+
}
|
|
4641
|
+
function pushLinks(lines, result) {
|
|
4642
|
+
lines.push("---");
|
|
4643
|
+
lines.push(`Portal: ${result.portalUrl}`);
|
|
4644
|
+
lines.push(`MQA source: ${result.mqaUrl}`);
|
|
4645
|
+
if (result.methodology === "v1") {
|
|
4646
|
+
lines.push(`Metrics endpoint: ${result.breakdown.metricsUrl}`);
|
|
4647
|
+
}
|
|
4648
|
+
}
|
|
4649
|
+
var TOP_FIXES = 5;
|
|
4650
|
+
function formatQualityMarkdown(result, datasetId) {
|
|
4651
|
+
const lines = [`# Quality Metrics for Dataset: ${datasetId}`, ""];
|
|
4652
|
+
if (result.methodology === "v1") {
|
|
4653
|
+
pushV1Scores(lines, result);
|
|
4654
|
+
if (result.breakdown.nonMaxDimensions.length > 0) {
|
|
4655
|
+
lines.push(`Non-max dimension(s): ${result.breakdown.nonMaxDimensions.join(", ")}. Use ckan_get_mqa_quality_details for the reasons.`);
|
|
4656
|
+
lines.push("");
|
|
4657
|
+
}
|
|
4658
|
+
} else {
|
|
4659
|
+
pushV2Scores(lines, result);
|
|
4660
|
+
if (result.failing.length === 0) {
|
|
4661
|
+
lines.push("All evaluated metrics are fulfilled.");
|
|
4662
|
+
} else {
|
|
4663
|
+
lines.push(`## Top fixes (${Math.min(TOP_FIXES, result.failing.length)} of ${result.failing.length} failing metrics, by gain on the final score)`);
|
|
4664
|
+
for (const item of result.failing.slice(0, TOP_FIXES)) {
|
|
4665
|
+
lines.push(`- ${describeFailing(item)}`);
|
|
4666
|
+
}
|
|
4667
|
+
if (result.failing.length > TOP_FIXES) {
|
|
4668
|
+
lines.push("- Use ckan_get_mqa_quality_details for the full list.");
|
|
4663
4669
|
}
|
|
4664
4670
|
}
|
|
4665
4671
|
lines.push("");
|
|
4666
4672
|
}
|
|
4667
|
-
lines
|
|
4668
|
-
|
|
4669
|
-
|
|
4673
|
+
pushLinks(lines, result);
|
|
4674
|
+
return lines.join("\n");
|
|
4675
|
+
}
|
|
4676
|
+
function formatQualityDetailsMarkdown(result, datasetId) {
|
|
4677
|
+
const lines = [`# Quality Details for Dataset: ${datasetId}`, ""];
|
|
4678
|
+
if (result.methodology === "v1") {
|
|
4679
|
+
pushV1Scores(lines, result);
|
|
4680
|
+
lines.push("## Non-max Reasons");
|
|
4681
|
+
if (result.breakdown.nonMaxDimensions.length === 0) {
|
|
4682
|
+
lines.push("- All dimensions are at max score.");
|
|
4683
|
+
}
|
|
4684
|
+
for (const dimension of result.breakdown.nonMaxDimensions) {
|
|
4685
|
+
const reasons = result.details.reasons[dimension] || [];
|
|
4686
|
+
lines.push(`- ${DIMENSION_LABELS[dimension]}: ${reasons.length > 0 ? reasons.join("; ") : "no failing flags detected in metrics payload"}`);
|
|
4687
|
+
}
|
|
4688
|
+
lines.push("");
|
|
4670
4689
|
} else {
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4690
|
+
pushV2Scores(lines, result);
|
|
4691
|
+
if (result.failing.length === 0) {
|
|
4692
|
+
lines.push("All evaluated metrics are fulfilled.");
|
|
4693
|
+
lines.push("");
|
|
4694
|
+
}
|
|
4695
|
+
for (const dimension of V2_DIMENSIONS) {
|
|
4696
|
+
const items = result.failing.filter((item) => item.dimension === dimension);
|
|
4697
|
+
if (items.length === 0) continue;
|
|
4698
|
+
lines.push(`## ${DIMENSION_LABELS[dimension]}`);
|
|
4699
|
+
for (const item of items) {
|
|
4700
|
+
lines.push(`- ${describeFailing(item)}`);
|
|
4677
4701
|
}
|
|
4702
|
+
lines.push("");
|
|
4678
4703
|
}
|
|
4704
|
+
lines.push("Gain = increase of the final score if every listed entity fulfilled the metric.");
|
|
4705
|
+
lines.push("");
|
|
4679
4706
|
}
|
|
4680
|
-
lines
|
|
4681
|
-
lines.push("---");
|
|
4682
|
-
const portalId = breakdown.portalId || datasetId;
|
|
4683
|
-
lines.push(`Portal: https://data.europa.eu/data/datasets/${portalId}/quality?locale=it`);
|
|
4684
|
-
lines.push(`MQA source: ${MQA_API_BASE}/${portalId}`);
|
|
4685
|
-
lines.push(`Metrics endpoint: ${breakdown.metricsUrl || `${MQA_METRICS_BASE}/${portalId}/metrics`}`);
|
|
4707
|
+
pushLinks(lines, result);
|
|
4686
4708
|
return lines.join("\n");
|
|
4687
4709
|
}
|
|
4688
4710
|
function registerQualityTools(server) {
|
|
@@ -4690,7 +4712,7 @@ function registerQualityTools(server) {
|
|
|
4690
4712
|
"ckan_get_mqa_quality",
|
|
4691
4713
|
{
|
|
4692
4714
|
title: "Get MQA Quality Score",
|
|
4693
|
-
description: "Get MQA (Metadata Quality
|
|
4715
|
+
description: "Get MQA (Metadata Quality Assessment) quality score for a dataset on dati.gov.it from data.europa.eu. Returns the final score on the 0-7.5 scale of MQA methodology v2 with its band (Sufficient/Good/Excellent), dataset, distribution and data service scores, and the failing metrics with the largest gain. Datasets not yet re-evaluated fall back to the previous methodology (405 scale), labelled as such. Only works with dati.gov.it server. Typical workflow: ckan_package_show (get dataset ID) \u2192 ckan_get_mqa_quality \u2192 ckan_get_mqa_quality_details (full list of failing metrics)",
|
|
4694
4716
|
inputSchema: z8.object({
|
|
4695
4717
|
server_url: z8.string().url().describe("Base URL of dati.gov.it (e.g., https://www.dati.gov.it/opendata)"),
|
|
4696
4718
|
dataset_id: z8.string().describe("Dataset ID or name"),
|
|
@@ -4741,7 +4763,7 @@ The MQA (Metadata Quality Assurance) system is operated by data.europa.eu and on
|
|
|
4741
4763
|
"ckan_get_mqa_quality_details",
|
|
4742
4764
|
{
|
|
4743
4765
|
title: "Get MQA Quality Details",
|
|
4744
|
-
description: "Get detailed MQA (Metadata Quality
|
|
4766
|
+
description: "Get detailed MQA (Metadata Quality Assessment) quality reasons for a dataset on dati.gov.it. Lists every failing metric grouped by FAIR dimension, with DCAT-AP property, weight, how many distributions fail it and its gain on the final score, plus the HTTP status of failing URL tests (e.g. 1100 timeout, 404) (methodology v2); previous-methodology datasets get non-max reasons. Only works with dati.gov.it server. Typical workflow: ckan_get_mqa_quality (get overview scores) \u2192 ckan_get_mqa_quality_details (inspect failing metrics)",
|
|
4745
4767
|
inputSchema: z8.object({
|
|
4746
4768
|
server_url: z8.string().url().describe("Base URL of dati.gov.it (e.g., https://www.dati.gov.it/opendata)"),
|
|
4747
4769
|
dataset_id: z8.string().describe("Dataset ID or name"),
|
|
@@ -4770,7 +4792,7 @@ The MQA (Metadata Quality Assurance) system is operated by data.europa.eu and on
|
|
|
4770
4792
|
};
|
|
4771
4793
|
}
|
|
4772
4794
|
try {
|
|
4773
|
-
const details = await
|
|
4795
|
+
const details = await getMqaQuality(server_url, dataset_id, { httpStatus: true });
|
|
4774
4796
|
const format = response_format || "markdown" /* MARKDOWN */;
|
|
4775
4797
|
const output = format === "json" /* JSON */ ? truncateJson(details) : truncateText(addDemoFooter(formatQualityDetailsMarkdown(details, dataset_id)));
|
|
4776
4798
|
return {
|
|
@@ -5237,10 +5259,10 @@ ${error instanceof Error ? error.message : String(error)}`;
|
|
|
5237
5259
|
|
|
5238
5260
|
// src/tools/portal-discovery.ts
|
|
5239
5261
|
import { z as z11 } from "zod";
|
|
5240
|
-
import
|
|
5262
|
+
import axios2 from "axios";
|
|
5241
5263
|
var DATASHADES_URL = "https://datashades.info/api/portal/list";
|
|
5242
5264
|
async function fetchPortals() {
|
|
5243
|
-
const resp = await
|
|
5265
|
+
const resp = await axios2.get(DATASHADES_URL, {
|
|
5244
5266
|
timeout: 15e3,
|
|
5245
5267
|
headers: { "User-Agent": "CKAN-MCP-Server/1.0" }
|
|
5246
5268
|
});
|
|
@@ -5972,7 +5994,7 @@ var registerAllPrompts = (server) => {
|
|
|
5972
5994
|
function createServer() {
|
|
5973
5995
|
return new McpServer({
|
|
5974
5996
|
name: "ckan-mcp-server",
|
|
5975
|
-
version: "0.4.
|
|
5997
|
+
version: "0.4.126"
|
|
5976
5998
|
});
|
|
5977
5999
|
}
|
|
5978
6000
|
function registerAll(server) {
|