@cliwant/mcp-sam-gov 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.ja.md +248 -231
- package/README.ko.md +248 -231
- package/README.md +733 -714
- package/dist/errors.d.ts +10 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/feedback.d.ts +64 -0
- package/dist/feedback.d.ts.map +1 -0
- package/dist/feedback.js +131 -0
- package/dist/feedback.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +48 -2
- package/dist/server.js.map +1 -1
- package/dist/update-check.d.ts +38 -0
- package/dist/update-check.d.ts.map +1 -0
- package/dist/update-check.js +85 -0
- package/dist/update-check.js.map +1 -0
- package/package.json +111 -111
- package/src/attachments.ts +652 -652
- package/src/bea.ts +372 -372
- package/src/bls.ts +1943 -1943
- package/src/cache.ts +73 -73
- package/src/cbp-border.ts +177 -177
- package/src/census-economic.ts +431 -431
- package/src/census.ts +735 -735
- package/src/ckan.ts +495 -495
- package/src/clinicaltrials.ts +923 -923
- package/src/cms-facility.ts +379 -379
- package/src/cms-hospital.ts +344 -344
- package/src/cms-supplier.ts +527 -527
- package/src/cms-utilization.ts +389 -389
- package/src/cms.ts +634 -634
- package/src/coerce.ts +47 -47
- package/src/courtlistener.ts +465 -465
- package/src/cpsc.ts +333 -333
- package/src/datagov-catalog.ts +312 -312
- package/src/datagov.ts +907 -907
- package/src/datagovKey.ts +68 -68
- package/src/datasource.ts +721 -721
- package/src/disclosure.ts +61 -61
- package/src/dol.ts +515 -515
- package/src/ecfr.ts +248 -248
- package/src/echo.ts +496 -496
- package/src/edgar.ts +3046 -3046
- package/src/epa-envirofacts.ts +358 -358
- package/src/errors.ts +324 -314
- package/src/fac.ts +529 -529
- package/src/far.ts +1009 -1009
- package/src/fdic.ts +2052 -2052
- package/src/federal-register.ts +725 -725
- package/src/feedback.ts +160 -0
- package/src/fema.ts +680 -680
- package/src/fpds.ts +620 -620
- package/src/fred.ts +464 -464
- package/src/gao.ts +744 -744
- package/src/gov-domains.ts +237 -237
- package/src/govinfo.ts +497 -497
- package/src/grants.ts +290 -290
- package/src/gsa-csv.ts +992 -992
- package/src/gsa-perdiem.ts +361 -361
- package/src/integrity.ts +928 -928
- package/src/keys.ts +268 -268
- package/src/lda.ts +385 -385
- package/src/meta.ts +292 -292
- package/src/nhtsa.ts +352 -352
- package/src/nih.ts +375 -375
- package/src/nist-controls.ts +219 -219
- package/src/nonprofit.ts +460 -460
- package/src/nppes.ts +834 -834
- package/src/nsf.ts +706 -706
- package/src/nvd.ts +1124 -1124
- package/src/nws-weather.ts +167 -167
- package/src/ofac.ts +1166 -1166
- package/src/openfda-device.ts +356 -356
- package/src/openfda-drugsfda.ts +313 -313
- package/src/openfda.ts +518 -518
- package/src/pricing.ts +1075 -1075
- package/src/sam-gov/client.ts +774 -774
- package/src/sam-gov/index.ts +32 -32
- package/src/sam-gov/types.ts +152 -152
- package/src/sba.ts +357 -357
- package/src/server.ts +6692 -6639
- package/src/snapshot.ts +223 -223
- package/src/socrata.ts +532 -532
- package/src/treasury.ts +582 -582
- package/src/update-check.ts +88 -0
- package/src/usaspending.ts +2852 -2852
- package/src/usitc.ts +420 -420
package/src/meta.ts
CHANGED
|
@@ -1,292 +1,292 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @cliwant/mcp-sam-gov/meta — the `_meta` completeness/provenance convention.
|
|
3
|
-
*
|
|
4
|
-
* Why this exists
|
|
5
|
-
* ----------------
|
|
6
|
-
* These tools' outputs are consumed by an AI, not a human who can eyeball
|
|
7
|
-
* a table. For an AI consumer, silently-misleading data is strictly worse
|
|
8
|
-
* than missing data: the AI cannot tell a filtered-but-blank field from a
|
|
9
|
-
* genuinely-empty one, a hard cap from a complete list, or a swallowed
|
|
10
|
-
* error from a real zero — and will state the wrong answer confidently.
|
|
11
|
-
*
|
|
12
|
-
* Every tool's success payload therefore gains a sibling `_meta` object.
|
|
13
|
-
* `data` is UNCHANGED (backward compat): the envelope becomes
|
|
14
|
-
* `{ ok, data, _meta }` where existing consumers reading `ok`/`data.*`
|
|
15
|
-
* are 100% unaffected.
|
|
16
|
-
*
|
|
17
|
-
* See docs/research/02-truthful-outputs-spec.md §2 for the normative spec.
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
/** Two-phase (enrich) tools: rows lost to failed per-record enrichment. */
|
|
21
|
-
export type MetaDegraded = {
|
|
22
|
-
attempted: number;
|
|
23
|
-
succeeded: number;
|
|
24
|
-
failed: number;
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
/** Offset-based pagination descriptor for list/search/aggregate tools. */
|
|
28
|
-
export type MetaPagination = {
|
|
29
|
-
/**
|
|
30
|
-
* The row offset of this page. `number` for offset-paginated tools (unchanged —
|
|
31
|
-
* every existing tool sets an integer). ADR-0010: widened to `number | null` for
|
|
32
|
-
* OPAQUE-CURSOR tools (GovInfo), where no meaningful numeric offset exists — those
|
|
33
|
-
* tools set `offset:null` (the honest "not a numeric offset"; continuation is via
|
|
34
|
-
* `_meta.nextCursor`, not a numeric `nextOffset`). Additive: no existing tool changes.
|
|
35
|
-
*/
|
|
36
|
-
offset: number | null;
|
|
37
|
-
limit: number;
|
|
38
|
-
nextOffset: number | null;
|
|
39
|
-
hasMore: boolean;
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* The normative `_meta` shape (spec §2.1). Rides alongside `data` in the
|
|
44
|
-
* success envelope so the AI can branch on completeness/provenance
|
|
45
|
-
* deterministically instead of guessing from null fields.
|
|
46
|
-
*/
|
|
47
|
-
export type ResponseMeta = {
|
|
48
|
-
/** Human+machine label of the upstream + layer that served THIS response. */
|
|
49
|
-
source: string;
|
|
50
|
-
/** SAM tools: true when no SAM_GOV_API_KEY. Non-SAM tools: always true. */
|
|
51
|
-
keylessMode: boolean;
|
|
52
|
-
/** true iff this response contains the ENTIRE result set for the query. */
|
|
53
|
-
complete: boolean;
|
|
54
|
-
/** true iff a hard cap / top-N limited the rows. */
|
|
55
|
-
truncated: boolean;
|
|
56
|
-
/** Number of primary records in `data`. */
|
|
57
|
-
returned: number;
|
|
58
|
-
/** Upstream's total match count; null when the endpoint doesn't report it. */
|
|
59
|
-
totalAvailable: number | null;
|
|
60
|
-
/**
|
|
61
|
-
* Present (and true) when `totalAvailable` is a KNOWN LOWER BOUND rather than
|
|
62
|
-
* an exact count — i.e. the upstream reported the total as "≥ N" (e.g. SEC
|
|
63
|
-
* EDGAR full-text search returns `hits.total.relation: "gte"` with the value
|
|
64
|
-
* pinned at 10000). The true total is UNKNOWN and ≥ `totalAvailable`. Absent
|
|
65
|
-
* on endpoints that report an exact total (do NOT read absence as `false`
|
|
66
|
-
* meaning "exact" for tools that never set it — it is simply not applicable).
|
|
67
|
-
*/
|
|
68
|
-
totalIsLowerBound?: boolean;
|
|
69
|
-
/**
|
|
70
|
-
* Present (and true) when `totalAvailable` (or, for CKAN, the value disclosed
|
|
71
|
-
* in `notes`) is an upstream STATISTICAL ESTIMATE rather than an exact count —
|
|
72
|
-
* i.e. the source reported the total as an approximation that may be ABOVE OR
|
|
73
|
-
* BELOW the true count (CKAN `datastore_search` returns `total_was_estimated:
|
|
74
|
-
* true` for a PostgreSQL `reltuples`-style estimate, live-verified to overshoot
|
|
75
|
-
* — so it is NOT a lower bound, unlike `totalIsLowerBound`). CKAN sets this on
|
|
76
|
-
* the estimated path alongside `totalAvailable:null` (the estimate never drives
|
|
77
|
-
* pagination — ADR-0006 B1) + a disclosing note carrying the estimate value.
|
|
78
|
-
* Absent on endpoints that report an exact total (do NOT read absence as
|
|
79
|
-
* `false` meaning "exact" for tools that never set it — it is not applicable).
|
|
80
|
-
*/
|
|
81
|
-
totalIsEstimated?: boolean;
|
|
82
|
-
/**
|
|
83
|
-
* Present when this response comes from an OPAQUE-CURSOR-paginated source
|
|
84
|
-
* (ADR-0010 — GovInfo's `offsetMark`): the opaque continuation token to pass back
|
|
85
|
-
* to fetch the NEXT page (GovInfo: as the `pageMark` argument), or `null` on the
|
|
86
|
-
* last page (no further cursor). It is a source-minted token, NOT derived from any
|
|
87
|
-
* secret and NEVER the raw upstream `nextPage` URL (which embeds pageSize + the
|
|
88
|
-
* api_key). Cursor tools set `nextOffset:null`/`offset:null` (a numeric offset is
|
|
89
|
-
* meaningless for a cursor) and use THIS as the sole continuation surface. Absent
|
|
90
|
-
* on offset-paginated tools (do NOT read absence as "no more" for those). Added as
|
|
91
|
-
* a conditional passthrough exactly like totalIsLowerBound/totalIsEstimated —
|
|
92
|
-
* only surfaced when the tool provides it, so existing tools' `_meta` stays
|
|
93
|
-
* byte-identical and the tools/list snapshot is unaffected (it is a runtime `_meta`
|
|
94
|
-
* field, not part of any tool's input schema).
|
|
95
|
-
*/
|
|
96
|
-
nextCursor?: string | null;
|
|
97
|
-
/** Request filters the upstream verifiably honored. */
|
|
98
|
-
filtersApplied: string[];
|
|
99
|
-
/** Request filters sent but NOT honored (results are unfiltered on these). */
|
|
100
|
-
filtersDropped: string[];
|
|
101
|
-
/** Fields null/absent BY LIMITATION (keyless/endpoint), not "no data". */
|
|
102
|
-
fieldsUnavailable: string[];
|
|
103
|
-
/** Two-phase tools only: rows for which per-record detail was fetched. */
|
|
104
|
-
enrichedCount?: number;
|
|
105
|
-
/** Two-phase tools: enrichment accounting (see MetaDegraded). */
|
|
106
|
-
degraded?: MetaDegraded;
|
|
107
|
-
/** Present on list/search/aggregate tools. */
|
|
108
|
-
pagination?: MetaPagination;
|
|
109
|
-
/**
|
|
110
|
-
* P5 PROVENANCE (ADR-0045 B2/M1). The access path that served THIS response:
|
|
111
|
-
* `"live"` (the current, and in Phase 1 the ONLY, path) or `"snapshot"` (a
|
|
112
|
-
* self-hosted cache served when the live upstream was unreachable). Set by the
|
|
113
|
-
* resilience port's provenance-returning primitive, then threaded by the
|
|
114
|
-
* adapter ONLY when NON-live — so a live response OMITS this field entirely and
|
|
115
|
-
* stays byte-identical to pre-P5 output. Surfaced via the guarded
|
|
116
|
-
* `if (partial.dataPath !== undefined)` passthrough below (NO `??` default).
|
|
117
|
-
* Absent ⇒ live; do NOT read absence as any other value. Freshness enum, not a
|
|
118
|
-
* topology label (M2): an independent host serving live data is still `"live"`.
|
|
119
|
-
*/
|
|
120
|
-
dataPath?: "live" | "snapshot";
|
|
121
|
-
/**
|
|
122
|
-
* P5 FRESHNESS (ADR-0045 m3). ISO-8601 UTC timestamp of when a NON-live
|
|
123
|
-
* (`snapshot`) body was retrieved from the origin — the as-of instant for
|
|
124
|
-
* deterministic cross-tool freshness comparison. Present ONLY alongside a
|
|
125
|
-
* non-live `dataPath`; absent on a live response (byte-identical). When a
|
|
126
|
-
* snapshot carries a `totalAvailable`, it is qualified as an as-of figure via
|
|
127
|
-
* the existing `totalIsEstimated` flag (m1 forbids a dedicated `totalAsOf`
|
|
128
|
-
* field — the only NEW P5 fields are `dataPath` and `asOf`).
|
|
129
|
-
*/
|
|
130
|
-
asOf?: string;
|
|
131
|
-
/** Short, AI-actionable caveats (natural language). */
|
|
132
|
-
notes: string[];
|
|
133
|
-
};
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* Build a fully-populated, invariant-consistent ResponseMeta from a partial.
|
|
137
|
-
*
|
|
138
|
-
* Safe defaults (spec §2.1): a tool that supplies nothing gets a truthful,
|
|
139
|
-
* "single-record / known-complete" meta (`complete:true, truncated:false`).
|
|
140
|
-
*
|
|
141
|
-
* The server (or the caller) hands over whatever it knows; this helper fills
|
|
142
|
-
* the rest and then ENFORCES the §2.1 invariants so the flags the AI trusts
|
|
143
|
-
* are internally consistent:
|
|
144
|
-
* - `complete === true` ⟺ NOT truncated AND filtersDropped empty AND
|
|
145
|
-
* (no pagination OR hasMore===false) AND (no degraded OR failed===0).
|
|
146
|
-
* - If totalAvailable is known and returned < totalAvailable ⇒
|
|
147
|
-
* complete:false, truncated:true.
|
|
148
|
-
* - filtersDropped non-empty ⇒ at least one note is present.
|
|
149
|
-
*/
|
|
150
|
-
export function buildMeta(partial: Partial<ResponseMeta> = {}): ResponseMeta {
|
|
151
|
-
const source = partial.source ?? "unknown";
|
|
152
|
-
const keylessMode = partial.keylessMode ?? true;
|
|
153
|
-
const returned = partial.returned ?? 0;
|
|
154
|
-
const totalAvailable =
|
|
155
|
-
partial.totalAvailable === undefined ? null : partial.totalAvailable;
|
|
156
|
-
const filtersApplied = partial.filtersApplied ?? [];
|
|
157
|
-
const filtersDropped = partial.filtersDropped ?? [];
|
|
158
|
-
const fieldsUnavailable = partial.fieldsUnavailable ?? [];
|
|
159
|
-
const notes = partial.notes ? [...partial.notes] : [];
|
|
160
|
-
const pagination = partial.pagination;
|
|
161
|
-
const degraded = partial.degraded;
|
|
162
|
-
|
|
163
|
-
// --- Derive truncated -------------------------------------------------
|
|
164
|
-
// A known total that exceeds what we returned proves truncation.
|
|
165
|
-
const totalProvesTruncation =
|
|
166
|
-
totalAvailable !== null && returned < totalAvailable;
|
|
167
|
-
const paginationHasMore = pagination ? pagination.hasMore : false;
|
|
168
|
-
let truncated =
|
|
169
|
-
partial.truncated ?? (totalProvesTruncation || paginationHasMore);
|
|
170
|
-
if (totalProvesTruncation || paginationHasMore) truncated = true;
|
|
171
|
-
|
|
172
|
-
// --- P5 provenance (ADR-0045 B2/M1) -----------------------------------
|
|
173
|
-
// A NON-live response (a snapshot served because the live upstream was
|
|
174
|
-
// unreachable) is honesty-qualified below. For a live/absent dataPath this
|
|
175
|
-
// whole strand is INERT — `nonLiveProvenance` is false, so `complete`, the
|
|
176
|
-
// notes[], and totalIsEstimated are all derived EXACTLY as before P5 (Phase 1:
|
|
177
|
-
// no adapter threads a dataPath ⇒ byte-identical output on every one of the
|
|
178
|
-
// 110 tools).
|
|
179
|
-
const nonLiveProvenance =
|
|
180
|
-
partial.dataPath !== undefined && partial.dataPath !== "live";
|
|
181
|
-
|
|
182
|
-
// --- Derive complete (single source of truth = the §2.1 invariant) ----
|
|
183
|
-
const degradedLoss = degraded ? degraded.failed > 0 : false;
|
|
184
|
-
const derivedComplete =
|
|
185
|
-
!truncated &&
|
|
186
|
-
filtersDropped.length === 0 &&
|
|
187
|
-
!paginationHasMore &&
|
|
188
|
-
!degradedLoss &&
|
|
189
|
-
!totalProvesTruncation &&
|
|
190
|
-
// M1a: `complete` is defined against the LIVE result set (:53), so a snapshot
|
|
191
|
-
// can NEVER claim complete:true — gate the derivation behind a live/absent
|
|
192
|
-
// dataPath. (Inert for live: `!false` === true, no change.)
|
|
193
|
-
!nonLiveProvenance;
|
|
194
|
-
// Honor an explicit complete:false, but never let a caller claim
|
|
195
|
-
// complete:true when the invariant says otherwise.
|
|
196
|
-
const complete =
|
|
197
|
-
partial.complete === false ? false : derivedComplete;
|
|
198
|
-
|
|
199
|
-
// --- Invariant: filtersDropped non-empty ⇒ a note explaining it -------
|
|
200
|
-
if (filtersDropped.length > 0 && notes.length === 0) {
|
|
201
|
-
notes.push(
|
|
202
|
-
`The following requested filters were NOT applied by the upstream and the results are unfiltered on them: ${filtersDropped.join(", ")}. Treat these results as unfiltered on those facets.`,
|
|
203
|
-
);
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
// --- P5 (ADR-0045 M1c/m1): staleness note on a non-live response ------
|
|
207
|
-
// Push into the EXISTING notes[] (m1 killed the separate `stalenessNote`
|
|
208
|
-
// field). Inert for live/absent dataPath. Names the as-of instant when known.
|
|
209
|
-
if (nonLiveProvenance) {
|
|
210
|
-
const asOfPhrase = partial.asOf ? ` (as of ${partial.asOf})` : "";
|
|
211
|
-
notes.push(
|
|
212
|
-
`Live upstream was unreachable — this response is served from a ${partial.dataPath} snapshot${asOfPhrase}. Freshness is NOT guaranteed; treat completeness and totals as of the snapshot time, not live.`,
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
const meta: ResponseMeta = {
|
|
217
|
-
source,
|
|
218
|
-
keylessMode,
|
|
219
|
-
complete,
|
|
220
|
-
truncated,
|
|
221
|
-
returned,
|
|
222
|
-
totalAvailable,
|
|
223
|
-
filtersApplied,
|
|
224
|
-
filtersDropped,
|
|
225
|
-
fieldsUnavailable,
|
|
226
|
-
notes,
|
|
227
|
-
};
|
|
228
|
-
if (partial.enrichedCount !== undefined)
|
|
229
|
-
meta.enrichedCount = partial.enrichedCount;
|
|
230
|
-
if (degraded) meta.degraded = degraded;
|
|
231
|
-
if (pagination) meta.pagination = pagination;
|
|
232
|
-
// Conditional passthrough (like enrichedCount/pagination): only surfaced when
|
|
233
|
-
// the tool provides it, so existing tools' `_meta` output stays byte-identical.
|
|
234
|
-
if (partial.totalIsLowerBound !== undefined)
|
|
235
|
-
meta.totalIsLowerBound = partial.totalIsLowerBound;
|
|
236
|
-
// Conditional passthrough (identical shape to totalIsLowerBound above): only
|
|
237
|
-
// surfaced when the tool provides it (CKAN's estimated-total path), so every
|
|
238
|
-
// existing tool's `_meta` output stays byte-identical. NO new derivation logic.
|
|
239
|
-
if (partial.totalIsEstimated !== undefined)
|
|
240
|
-
meta.totalIsEstimated = partial.totalIsEstimated;
|
|
241
|
-
// Conditional passthrough (IDENTICAL shape to totalIsLowerBound/totalIsEstimated
|
|
242
|
-
// above): only surfaced when the tool provides it (GovInfo's opaque-cursor path),
|
|
243
|
-
// so every existing tool's `_meta` output stays byte-identical. NO new derivation
|
|
244
|
-
// logic — the value (the offsetMark token, or null on the last page) is set by the
|
|
245
|
-
// cursor tool and passed through verbatim.
|
|
246
|
-
if (partial.nextCursor !== undefined) meta.nextCursor = partial.nextCursor;
|
|
247
|
-
// P5 provenance passthrough (ADR-0045 B2) — IDENTICAL guarded idiom to
|
|
248
|
-
// totalIsLowerBound/nextCursor above: surfaced ONLY when the adapter provides
|
|
249
|
-
// it, so every existing (live, dataPath-absent) tool `_meta` stays
|
|
250
|
-
// byte-identical. NO `??` default. In Phase 1 no adapter threads these ⇒ inert.
|
|
251
|
-
if (partial.dataPath !== undefined) meta.dataPath = partial.dataPath;
|
|
252
|
-
if (partial.asOf !== undefined) meta.asOf = partial.asOf;
|
|
253
|
-
// M1b: a snapshot's totalAvailable is an as-of figure, NOT a live exact total —
|
|
254
|
-
// qualify it via the EXISTING totalIsEstimated flag (m1 forbids a new
|
|
255
|
-
// totalAsOf field) so an AI never reads it as a live count. Only when a total
|
|
256
|
-
// is actually present. Inert for live (nonLiveProvenance false).
|
|
257
|
-
if (nonLiveProvenance && totalAvailable !== null) meta.totalIsEstimated = true;
|
|
258
|
-
return meta;
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
/**
|
|
262
|
-
* Branded wrapper a tool handler returns to attach `_meta` to its payload.
|
|
263
|
-
*
|
|
264
|
-
* A handler may return either its raw domain object (server synthesizes a
|
|
265
|
-
* minimal truthful default) OR `withMeta(data, partialMeta)`. The brand lets
|
|
266
|
-
* the server distinguish "handler attached meta" from "domain object that
|
|
267
|
-
* happens to have `data`/`_meta` keys" with zero ambiguity.
|
|
268
|
-
*/
|
|
269
|
-
export class MetaBundle<T = unknown> {
|
|
270
|
-
readonly __isMetaBundle = true as const;
|
|
271
|
-
constructor(
|
|
272
|
-
readonly data: T,
|
|
273
|
-
readonly meta: Partial<ResponseMeta>,
|
|
274
|
-
) {}
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
/** Attach a partial `_meta` to a handler's `data`. Server finalizes it. */
|
|
278
|
-
export function withMeta<T>(
|
|
279
|
-
data: T,
|
|
280
|
-
meta: Partial<ResponseMeta>,
|
|
281
|
-
): MetaBundle<T> {
|
|
282
|
-
return new MetaBundle(data, meta);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/** Type guard: did the handler hand back a MetaBundle? */
|
|
286
|
-
export function isMetaBundle(v: unknown): v is MetaBundle {
|
|
287
|
-
return (
|
|
288
|
-
typeof v === "object" &&
|
|
289
|
-
v !== null &&
|
|
290
|
-
(v as { __isMetaBundle?: unknown }).__isMetaBundle === true
|
|
291
|
-
);
|
|
292
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* @cliwant/mcp-sam-gov/meta — the `_meta` completeness/provenance convention.
|
|
3
|
+
*
|
|
4
|
+
* Why this exists
|
|
5
|
+
* ----------------
|
|
6
|
+
* These tools' outputs are consumed by an AI, not a human who can eyeball
|
|
7
|
+
* a table. For an AI consumer, silently-misleading data is strictly worse
|
|
8
|
+
* than missing data: the AI cannot tell a filtered-but-blank field from a
|
|
9
|
+
* genuinely-empty one, a hard cap from a complete list, or a swallowed
|
|
10
|
+
* error from a real zero — and will state the wrong answer confidently.
|
|
11
|
+
*
|
|
12
|
+
* Every tool's success payload therefore gains a sibling `_meta` object.
|
|
13
|
+
* `data` is UNCHANGED (backward compat): the envelope becomes
|
|
14
|
+
* `{ ok, data, _meta }` where existing consumers reading `ok`/`data.*`
|
|
15
|
+
* are 100% unaffected.
|
|
16
|
+
*
|
|
17
|
+
* See docs/research/02-truthful-outputs-spec.md §2 for the normative spec.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** Two-phase (enrich) tools: rows lost to failed per-record enrichment. */
|
|
21
|
+
export type MetaDegraded = {
|
|
22
|
+
attempted: number;
|
|
23
|
+
succeeded: number;
|
|
24
|
+
failed: number;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** Offset-based pagination descriptor for list/search/aggregate tools. */
|
|
28
|
+
export type MetaPagination = {
|
|
29
|
+
/**
|
|
30
|
+
* The row offset of this page. `number` for offset-paginated tools (unchanged —
|
|
31
|
+
* every existing tool sets an integer). ADR-0010: widened to `number | null` for
|
|
32
|
+
* OPAQUE-CURSOR tools (GovInfo), where no meaningful numeric offset exists — those
|
|
33
|
+
* tools set `offset:null` (the honest "not a numeric offset"; continuation is via
|
|
34
|
+
* `_meta.nextCursor`, not a numeric `nextOffset`). Additive: no existing tool changes.
|
|
35
|
+
*/
|
|
36
|
+
offset: number | null;
|
|
37
|
+
limit: number;
|
|
38
|
+
nextOffset: number | null;
|
|
39
|
+
hasMore: boolean;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The normative `_meta` shape (spec §2.1). Rides alongside `data` in the
|
|
44
|
+
* success envelope so the AI can branch on completeness/provenance
|
|
45
|
+
* deterministically instead of guessing from null fields.
|
|
46
|
+
*/
|
|
47
|
+
export type ResponseMeta = {
|
|
48
|
+
/** Human+machine label of the upstream + layer that served THIS response. */
|
|
49
|
+
source: string;
|
|
50
|
+
/** SAM tools: true when no SAM_GOV_API_KEY. Non-SAM tools: always true. */
|
|
51
|
+
keylessMode: boolean;
|
|
52
|
+
/** true iff this response contains the ENTIRE result set for the query. */
|
|
53
|
+
complete: boolean;
|
|
54
|
+
/** true iff a hard cap / top-N limited the rows. */
|
|
55
|
+
truncated: boolean;
|
|
56
|
+
/** Number of primary records in `data`. */
|
|
57
|
+
returned: number;
|
|
58
|
+
/** Upstream's total match count; null when the endpoint doesn't report it. */
|
|
59
|
+
totalAvailable: number | null;
|
|
60
|
+
/**
|
|
61
|
+
* Present (and true) when `totalAvailable` is a KNOWN LOWER BOUND rather than
|
|
62
|
+
* an exact count — i.e. the upstream reported the total as "≥ N" (e.g. SEC
|
|
63
|
+
* EDGAR full-text search returns `hits.total.relation: "gte"` with the value
|
|
64
|
+
* pinned at 10000). The true total is UNKNOWN and ≥ `totalAvailable`. Absent
|
|
65
|
+
* on endpoints that report an exact total (do NOT read absence as `false`
|
|
66
|
+
* meaning "exact" for tools that never set it — it is simply not applicable).
|
|
67
|
+
*/
|
|
68
|
+
totalIsLowerBound?: boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Present (and true) when `totalAvailable` (or, for CKAN, the value disclosed
|
|
71
|
+
* in `notes`) is an upstream STATISTICAL ESTIMATE rather than an exact count —
|
|
72
|
+
* i.e. the source reported the total as an approximation that may be ABOVE OR
|
|
73
|
+
* BELOW the true count (CKAN `datastore_search` returns `total_was_estimated:
|
|
74
|
+
* true` for a PostgreSQL `reltuples`-style estimate, live-verified to overshoot
|
|
75
|
+
* — so it is NOT a lower bound, unlike `totalIsLowerBound`). CKAN sets this on
|
|
76
|
+
* the estimated path alongside `totalAvailable:null` (the estimate never drives
|
|
77
|
+
* pagination — ADR-0006 B1) + a disclosing note carrying the estimate value.
|
|
78
|
+
* Absent on endpoints that report an exact total (do NOT read absence as
|
|
79
|
+
* `false` meaning "exact" for tools that never set it — it is not applicable).
|
|
80
|
+
*/
|
|
81
|
+
totalIsEstimated?: boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Present when this response comes from an OPAQUE-CURSOR-paginated source
|
|
84
|
+
* (ADR-0010 — GovInfo's `offsetMark`): the opaque continuation token to pass back
|
|
85
|
+
* to fetch the NEXT page (GovInfo: as the `pageMark` argument), or `null` on the
|
|
86
|
+
* last page (no further cursor). It is a source-minted token, NOT derived from any
|
|
87
|
+
* secret and NEVER the raw upstream `nextPage` URL (which embeds pageSize + the
|
|
88
|
+
* api_key). Cursor tools set `nextOffset:null`/`offset:null` (a numeric offset is
|
|
89
|
+
* meaningless for a cursor) and use THIS as the sole continuation surface. Absent
|
|
90
|
+
* on offset-paginated tools (do NOT read absence as "no more" for those). Added as
|
|
91
|
+
* a conditional passthrough exactly like totalIsLowerBound/totalIsEstimated —
|
|
92
|
+
* only surfaced when the tool provides it, so existing tools' `_meta` stays
|
|
93
|
+
* byte-identical and the tools/list snapshot is unaffected (it is a runtime `_meta`
|
|
94
|
+
* field, not part of any tool's input schema).
|
|
95
|
+
*/
|
|
96
|
+
nextCursor?: string | null;
|
|
97
|
+
/** Request filters the upstream verifiably honored. */
|
|
98
|
+
filtersApplied: string[];
|
|
99
|
+
/** Request filters sent but NOT honored (results are unfiltered on these). */
|
|
100
|
+
filtersDropped: string[];
|
|
101
|
+
/** Fields null/absent BY LIMITATION (keyless/endpoint), not "no data". */
|
|
102
|
+
fieldsUnavailable: string[];
|
|
103
|
+
/** Two-phase tools only: rows for which per-record detail was fetched. */
|
|
104
|
+
enrichedCount?: number;
|
|
105
|
+
/** Two-phase tools: enrichment accounting (see MetaDegraded). */
|
|
106
|
+
degraded?: MetaDegraded;
|
|
107
|
+
/** Present on list/search/aggregate tools. */
|
|
108
|
+
pagination?: MetaPagination;
|
|
109
|
+
/**
|
|
110
|
+
* P5 PROVENANCE (ADR-0045 B2/M1). The access path that served THIS response:
|
|
111
|
+
* `"live"` (the current, and in Phase 1 the ONLY, path) or `"snapshot"` (a
|
|
112
|
+
* self-hosted cache served when the live upstream was unreachable). Set by the
|
|
113
|
+
* resilience port's provenance-returning primitive, then threaded by the
|
|
114
|
+
* adapter ONLY when NON-live — so a live response OMITS this field entirely and
|
|
115
|
+
* stays byte-identical to pre-P5 output. Surfaced via the guarded
|
|
116
|
+
* `if (partial.dataPath !== undefined)` passthrough below (NO `??` default).
|
|
117
|
+
* Absent ⇒ live; do NOT read absence as any other value. Freshness enum, not a
|
|
118
|
+
* topology label (M2): an independent host serving live data is still `"live"`.
|
|
119
|
+
*/
|
|
120
|
+
dataPath?: "live" | "snapshot";
|
|
121
|
+
/**
|
|
122
|
+
* P5 FRESHNESS (ADR-0045 m3). ISO-8601 UTC timestamp of when a NON-live
|
|
123
|
+
* (`snapshot`) body was retrieved from the origin — the as-of instant for
|
|
124
|
+
* deterministic cross-tool freshness comparison. Present ONLY alongside a
|
|
125
|
+
* non-live `dataPath`; absent on a live response (byte-identical). When a
|
|
126
|
+
* snapshot carries a `totalAvailable`, it is qualified as an as-of figure via
|
|
127
|
+
* the existing `totalIsEstimated` flag (m1 forbids a dedicated `totalAsOf`
|
|
128
|
+
* field — the only NEW P5 fields are `dataPath` and `asOf`).
|
|
129
|
+
*/
|
|
130
|
+
asOf?: string;
|
|
131
|
+
/** Short, AI-actionable caveats (natural language). */
|
|
132
|
+
notes: string[];
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Build a fully-populated, invariant-consistent ResponseMeta from a partial.
|
|
137
|
+
*
|
|
138
|
+
* Safe defaults (spec §2.1): a tool that supplies nothing gets a truthful,
|
|
139
|
+
* "single-record / known-complete" meta (`complete:true, truncated:false`).
|
|
140
|
+
*
|
|
141
|
+
* The server (or the caller) hands over whatever it knows; this helper fills
|
|
142
|
+
* the rest and then ENFORCES the §2.1 invariants so the flags the AI trusts
|
|
143
|
+
* are internally consistent:
|
|
144
|
+
* - `complete === true` ⟺ NOT truncated AND filtersDropped empty AND
|
|
145
|
+
* (no pagination OR hasMore===false) AND (no degraded OR failed===0).
|
|
146
|
+
* - If totalAvailable is known and returned < totalAvailable ⇒
|
|
147
|
+
* complete:false, truncated:true.
|
|
148
|
+
* - filtersDropped non-empty ⇒ at least one note is present.
|
|
149
|
+
*/
|
|
150
|
+
export function buildMeta(partial: Partial<ResponseMeta> = {}): ResponseMeta {
|
|
151
|
+
const source = partial.source ?? "unknown";
|
|
152
|
+
const keylessMode = partial.keylessMode ?? true;
|
|
153
|
+
const returned = partial.returned ?? 0;
|
|
154
|
+
const totalAvailable =
|
|
155
|
+
partial.totalAvailable === undefined ? null : partial.totalAvailable;
|
|
156
|
+
const filtersApplied = partial.filtersApplied ?? [];
|
|
157
|
+
const filtersDropped = partial.filtersDropped ?? [];
|
|
158
|
+
const fieldsUnavailable = partial.fieldsUnavailable ?? [];
|
|
159
|
+
const notes = partial.notes ? [...partial.notes] : [];
|
|
160
|
+
const pagination = partial.pagination;
|
|
161
|
+
const degraded = partial.degraded;
|
|
162
|
+
|
|
163
|
+
// --- Derive truncated -------------------------------------------------
|
|
164
|
+
// A known total that exceeds what we returned proves truncation.
|
|
165
|
+
const totalProvesTruncation =
|
|
166
|
+
totalAvailable !== null && returned < totalAvailable;
|
|
167
|
+
const paginationHasMore = pagination ? pagination.hasMore : false;
|
|
168
|
+
let truncated =
|
|
169
|
+
partial.truncated ?? (totalProvesTruncation || paginationHasMore);
|
|
170
|
+
if (totalProvesTruncation || paginationHasMore) truncated = true;
|
|
171
|
+
|
|
172
|
+
// --- P5 provenance (ADR-0045 B2/M1) -----------------------------------
|
|
173
|
+
// A NON-live response (a snapshot served because the live upstream was
|
|
174
|
+
// unreachable) is honesty-qualified below. For a live/absent dataPath this
|
|
175
|
+
// whole strand is INERT — `nonLiveProvenance` is false, so `complete`, the
|
|
176
|
+
// notes[], and totalIsEstimated are all derived EXACTLY as before P5 (Phase 1:
|
|
177
|
+
// no adapter threads a dataPath ⇒ byte-identical output on every one of the
|
|
178
|
+
// 110 tools).
|
|
179
|
+
const nonLiveProvenance =
|
|
180
|
+
partial.dataPath !== undefined && partial.dataPath !== "live";
|
|
181
|
+
|
|
182
|
+
// --- Derive complete (single source of truth = the §2.1 invariant) ----
|
|
183
|
+
const degradedLoss = degraded ? degraded.failed > 0 : false;
|
|
184
|
+
const derivedComplete =
|
|
185
|
+
!truncated &&
|
|
186
|
+
filtersDropped.length === 0 &&
|
|
187
|
+
!paginationHasMore &&
|
|
188
|
+
!degradedLoss &&
|
|
189
|
+
!totalProvesTruncation &&
|
|
190
|
+
// M1a: `complete` is defined against the LIVE result set (:53), so a snapshot
|
|
191
|
+
// can NEVER claim complete:true — gate the derivation behind a live/absent
|
|
192
|
+
// dataPath. (Inert for live: `!false` === true, no change.)
|
|
193
|
+
!nonLiveProvenance;
|
|
194
|
+
// Honor an explicit complete:false, but never let a caller claim
|
|
195
|
+
// complete:true when the invariant says otherwise.
|
|
196
|
+
const complete =
|
|
197
|
+
partial.complete === false ? false : derivedComplete;
|
|
198
|
+
|
|
199
|
+
// --- Invariant: filtersDropped non-empty ⇒ a note explaining it -------
|
|
200
|
+
if (filtersDropped.length > 0 && notes.length === 0) {
|
|
201
|
+
notes.push(
|
|
202
|
+
`The following requested filters were NOT applied by the upstream and the results are unfiltered on them: ${filtersDropped.join(", ")}. Treat these results as unfiltered on those facets.`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// --- P5 (ADR-0045 M1c/m1): staleness note on a non-live response ------
|
|
207
|
+
// Push into the EXISTING notes[] (m1 killed the separate `stalenessNote`
|
|
208
|
+
// field). Inert for live/absent dataPath. Names the as-of instant when known.
|
|
209
|
+
if (nonLiveProvenance) {
|
|
210
|
+
const asOfPhrase = partial.asOf ? ` (as of ${partial.asOf})` : "";
|
|
211
|
+
notes.push(
|
|
212
|
+
`Live upstream was unreachable — this response is served from a ${partial.dataPath} snapshot${asOfPhrase}. Freshness is NOT guaranteed; treat completeness and totals as of the snapshot time, not live.`,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const meta: ResponseMeta = {
|
|
217
|
+
source,
|
|
218
|
+
keylessMode,
|
|
219
|
+
complete,
|
|
220
|
+
truncated,
|
|
221
|
+
returned,
|
|
222
|
+
totalAvailable,
|
|
223
|
+
filtersApplied,
|
|
224
|
+
filtersDropped,
|
|
225
|
+
fieldsUnavailable,
|
|
226
|
+
notes,
|
|
227
|
+
};
|
|
228
|
+
if (partial.enrichedCount !== undefined)
|
|
229
|
+
meta.enrichedCount = partial.enrichedCount;
|
|
230
|
+
if (degraded) meta.degraded = degraded;
|
|
231
|
+
if (pagination) meta.pagination = pagination;
|
|
232
|
+
// Conditional passthrough (like enrichedCount/pagination): only surfaced when
|
|
233
|
+
// the tool provides it, so existing tools' `_meta` output stays byte-identical.
|
|
234
|
+
if (partial.totalIsLowerBound !== undefined)
|
|
235
|
+
meta.totalIsLowerBound = partial.totalIsLowerBound;
|
|
236
|
+
// Conditional passthrough (identical shape to totalIsLowerBound above): only
|
|
237
|
+
// surfaced when the tool provides it (CKAN's estimated-total path), so every
|
|
238
|
+
// existing tool's `_meta` output stays byte-identical. NO new derivation logic.
|
|
239
|
+
if (partial.totalIsEstimated !== undefined)
|
|
240
|
+
meta.totalIsEstimated = partial.totalIsEstimated;
|
|
241
|
+
// Conditional passthrough (IDENTICAL shape to totalIsLowerBound/totalIsEstimated
|
|
242
|
+
// above): only surfaced when the tool provides it (GovInfo's opaque-cursor path),
|
|
243
|
+
// so every existing tool's `_meta` output stays byte-identical. NO new derivation
|
|
244
|
+
// logic — the value (the offsetMark token, or null on the last page) is set by the
|
|
245
|
+
// cursor tool and passed through verbatim.
|
|
246
|
+
if (partial.nextCursor !== undefined) meta.nextCursor = partial.nextCursor;
|
|
247
|
+
// P5 provenance passthrough (ADR-0045 B2) — IDENTICAL guarded idiom to
|
|
248
|
+
// totalIsLowerBound/nextCursor above: surfaced ONLY when the adapter provides
|
|
249
|
+
// it, so every existing (live, dataPath-absent) tool `_meta` stays
|
|
250
|
+
// byte-identical. NO `??` default. In Phase 1 no adapter threads these ⇒ inert.
|
|
251
|
+
if (partial.dataPath !== undefined) meta.dataPath = partial.dataPath;
|
|
252
|
+
if (partial.asOf !== undefined) meta.asOf = partial.asOf;
|
|
253
|
+
// M1b: a snapshot's totalAvailable is an as-of figure, NOT a live exact total —
|
|
254
|
+
// qualify it via the EXISTING totalIsEstimated flag (m1 forbids a new
|
|
255
|
+
// totalAsOf field) so an AI never reads it as a live count. Only when a total
|
|
256
|
+
// is actually present. Inert for live (nonLiveProvenance false).
|
|
257
|
+
if (nonLiveProvenance && totalAvailable !== null) meta.totalIsEstimated = true;
|
|
258
|
+
return meta;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Branded wrapper a tool handler returns to attach `_meta` to its payload.
|
|
263
|
+
*
|
|
264
|
+
* A handler may return either its raw domain object (server synthesizes a
|
|
265
|
+
* minimal truthful default) OR `withMeta(data, partialMeta)`. The brand lets
|
|
266
|
+
* the server distinguish "handler attached meta" from "domain object that
|
|
267
|
+
* happens to have `data`/`_meta` keys" with zero ambiguity.
|
|
268
|
+
*/
|
|
269
|
+
export class MetaBundle<T = unknown> {
|
|
270
|
+
readonly __isMetaBundle = true as const;
|
|
271
|
+
constructor(
|
|
272
|
+
readonly data: T,
|
|
273
|
+
readonly meta: Partial<ResponseMeta>,
|
|
274
|
+
) {}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Attach a partial `_meta` to a handler's `data`. Server finalizes it. */
|
|
278
|
+
export function withMeta<T>(
|
|
279
|
+
data: T,
|
|
280
|
+
meta: Partial<ResponseMeta>,
|
|
281
|
+
): MetaBundle<T> {
|
|
282
|
+
return new MetaBundle(data, meta);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Type guard: did the handler hand back a MetaBundle? */
|
|
286
|
+
export function isMetaBundle(v: unknown): v is MetaBundle {
|
|
287
|
+
return (
|
|
288
|
+
typeof v === "object" &&
|
|
289
|
+
v !== null &&
|
|
290
|
+
(v as { __isMetaBundle?: unknown }).__isMetaBundle === true
|
|
291
|
+
);
|
|
292
|
+
}
|