@cliwant/mcp-sam-gov 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.ja.md +48 -16
- package/README.ko.md +48 -16
- package/README.md +226 -64
- package/dist/attachments.d.ts +63 -0
- package/dist/attachments.d.ts.map +1 -0
- package/dist/attachments.js +576 -0
- package/dist/attachments.js.map +1 -0
- package/dist/bls.d.ts +267 -0
- package/dist/bls.d.ts.map +1 -0
- package/dist/bls.js +1549 -0
- package/dist/bls.js.map +1 -0
- package/dist/cache.d.ts +2 -0
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +4 -0
- package/dist/cache.js.map +1 -1
- package/dist/census.d.ts +182 -0
- package/dist/census.d.ts.map +1 -0
- package/dist/census.js +576 -0
- package/dist/census.js.map +1 -0
- package/dist/ckan.d.ts +141 -0
- package/dist/ckan.d.ts.map +1 -0
- package/dist/ckan.js +393 -0
- package/dist/ckan.js.map +1 -0
- package/dist/clinicaltrials.d.ts +180 -0
- package/dist/clinicaltrials.d.ts.map +1 -0
- package/dist/clinicaltrials.js +730 -0
- package/dist/clinicaltrials.js.map +1 -0
- package/dist/cms.d.ts +140 -0
- package/dist/cms.d.ts.map +1 -0
- package/dist/cms.js +482 -0
- package/dist/cms.js.map +1 -0
- package/dist/coerce.d.ts +32 -0
- package/dist/coerce.d.ts.map +1 -0
- package/dist/coerce.js +50 -0
- package/dist/coerce.js.map +1 -0
- package/dist/datagov-catalog.d.ts +84 -0
- package/dist/datagov-catalog.d.ts.map +1 -0
- package/dist/datagov-catalog.js +233 -0
- package/dist/datagov-catalog.js.map +1 -0
- package/dist/datagov.d.ts +146 -0
- package/dist/datagov.d.ts.map +1 -0
- package/dist/datagov.js +689 -0
- package/dist/datagov.js.map +1 -0
- package/dist/datagovKey.d.ts +36 -0
- package/dist/datagovKey.d.ts.map +1 -0
- package/dist/datagovKey.js +60 -0
- package/dist/datagovKey.js.map +1 -0
- package/dist/datasource.d.ts +313 -0
- package/dist/datasource.d.ts.map +1 -0
- package/dist/datasource.js +551 -0
- package/dist/datasource.js.map +1 -0
- package/dist/disclosure.d.ts +55 -0
- package/dist/disclosure.d.ts.map +1 -0
- package/dist/disclosure.js +57 -0
- package/dist/disclosure.js.map +1 -0
- package/dist/ecfr.d.ts +4 -2
- package/dist/ecfr.d.ts.map +1 -1
- package/dist/ecfr.js +74 -1
- package/dist/ecfr.js.map +1 -1
- package/dist/echo.d.ts +143 -0
- package/dist/echo.d.ts.map +1 -0
- package/dist/echo.js +424 -0
- package/dist/echo.js.map +1 -0
- package/dist/edgar.d.ts +377 -0
- package/dist/edgar.d.ts.map +1 -0
- package/dist/edgar.js +2418 -0
- package/dist/edgar.js.map +1 -0
- package/dist/errors.d.ts +26 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +69 -1
- package/dist/errors.js.map +1 -1
- package/dist/fac.d.ts +180 -0
- package/dist/fac.d.ts.map +1 -0
- package/dist/fac.js +416 -0
- package/dist/fac.js.map +1 -0
- package/dist/far.d.ts +170 -0
- package/dist/far.d.ts.map +1 -0
- package/dist/far.js +804 -0
- package/dist/far.js.map +1 -0
- package/dist/fdic.d.ts +599 -0
- package/dist/fdic.d.ts.map +1 -0
- package/dist/fdic.js +1624 -0
- package/dist/fdic.js.map +1 -0
- package/dist/federal-register.d.ts +139 -2
- package/dist/federal-register.d.ts.map +1 -1
- package/dist/federal-register.js +419 -3
- package/dist/federal-register.js.map +1 -1
- package/dist/fema.d.ts +181 -0
- package/dist/fema.d.ts.map +1 -0
- package/dist/fema.js +436 -0
- package/dist/fema.js.map +1 -0
- package/dist/fpds.d.ts +108 -0
- package/dist/fpds.d.ts.map +1 -0
- package/dist/fpds.js +519 -0
- package/dist/fpds.js.map +1 -0
- package/dist/gao.d.ts +64 -0
- package/dist/gao.d.ts.map +1 -0
- package/dist/gao.js +640 -0
- package/dist/gao.js.map +1 -0
- package/dist/govinfo.d.ts +111 -0
- package/dist/govinfo.d.ts.map +1 -0
- package/dist/govinfo.js +422 -0
- package/dist/govinfo.js.map +1 -0
- package/dist/grants.d.ts +27 -4
- package/dist/grants.d.ts.map +1 -1
- package/dist/grants.js +111 -6
- package/dist/grants.js.map +1 -1
- package/dist/gsa-csv.d.ts +249 -0
- package/dist/gsa-csv.d.ts.map +1 -0
- package/dist/gsa-csv.js +784 -0
- package/dist/gsa-csv.js.map +1 -0
- package/dist/integrity.d.ts +212 -0
- package/dist/integrity.d.ts.map +1 -0
- package/dist/integrity.js +707 -0
- package/dist/integrity.js.map +1 -0
- package/dist/meta.d.ts +165 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +162 -0
- package/dist/meta.js.map +1 -0
- package/dist/nih.d.ts +117 -0
- package/dist/nih.d.ts.map +1 -0
- package/dist/nih.js +291 -0
- package/dist/nih.js.map +1 -0
- package/dist/nppes.d.ts +157 -0
- package/dist/nppes.d.ts.map +1 -0
- package/dist/nppes.js +648 -0
- package/dist/nppes.js.map +1 -0
- package/dist/nsf.d.ts +176 -0
- package/dist/nsf.d.ts.map +1 -0
- package/dist/nsf.js +554 -0
- package/dist/nsf.js.map +1 -0
- package/dist/nvd.d.ts +176 -0
- package/dist/nvd.d.ts.map +1 -0
- package/dist/nvd.js +912 -0
- package/dist/nvd.js.map +1 -0
- package/dist/ofac.d.ts +205 -0
- package/dist/ofac.d.ts.map +1 -0
- package/dist/ofac.js +919 -0
- package/dist/ofac.js.map +1 -0
- package/dist/pricing.d.ts +110 -0
- package/dist/pricing.d.ts.map +1 -0
- package/dist/pricing.js +843 -0
- package/dist/pricing.js.map +1 -0
- package/dist/sam-gov/client.d.ts +60 -2
- package/dist/sam-gov/client.d.ts.map +1 -1
- package/dist/sam-gov/client.js +320 -54
- package/dist/sam-gov/client.js.map +1 -1
- package/dist/sam-gov/index.d.ts +1 -1
- package/dist/sam-gov/index.d.ts.map +1 -1
- package/dist/sam-gov/index.js +1 -1
- package/dist/sam-gov/index.js.map +1 -1
- package/dist/sam-gov/types.d.ts +24 -0
- package/dist/sam-gov/types.d.ts.map +1 -1
- package/dist/sba.d.ts +72 -0
- package/dist/sba.d.ts.map +1 -0
- package/dist/sba.js +281 -0
- package/dist/sba.js.map +1 -0
- package/dist/server.d.ts +14 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3889 -296
- package/dist/server.js.map +1 -1
- package/dist/snapshot.d.ts +98 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +146 -0
- package/dist/snapshot.js.map +1 -0
- package/dist/socrata.d.ts +157 -0
- package/dist/socrata.d.ts.map +1 -0
- package/dist/socrata.js +448 -0
- package/dist/socrata.js.map +1 -0
- package/dist/treasury.d.ts +143 -0
- package/dist/treasury.d.ts.map +1 -0
- package/dist/treasury.js +436 -0
- package/dist/treasury.js.map +1 -0
- package/dist/usaspending.d.ts +260 -65
- package/dist/usaspending.d.ts.map +1 -1
- package/dist/usaspending.js +1629 -201
- package/dist/usaspending.js.map +1 -1
- package/dist/usitc.d.ts +142 -0
- package/dist/usitc.d.ts.map +1 -0
- package/dist/usitc.js +339 -0
- package/dist/usitc.js.map +1 -0
- package/package.json +24 -2
- package/src/attachments.ts +652 -0
- package/src/bls.ts +1943 -0
- package/src/cache.ts +5 -0
- package/src/census.ts +735 -0
- package/src/ckan.ts +495 -0
- package/src/clinicaltrials.ts +923 -0
- package/src/cms.ts +634 -0
- package/src/coerce.ts +47 -0
- package/src/datagov-catalog.ts +296 -0
- package/src/datagov.ts +907 -0
- package/src/datagovKey.ts +68 -0
- package/src/datasource.ts +721 -0
- package/src/disclosure.ts +61 -0
- package/src/ecfr.ts +98 -1
- package/src/echo.ts +496 -0
- package/src/edgar.ts +3014 -0
- package/src/errors.ts +84 -1
- package/src/fac.ts +529 -0
- package/src/far.ts +1007 -0
- package/src/fdic.ts +2052 -0
- package/src/federal-register.ts +512 -3
- package/src/fema.ts +541 -0
- package/src/fpds.ts +620 -0
- package/src/gao.ts +744 -0
- package/src/govinfo.ts +497 -0
- package/src/grants.ts +139 -7
- package/src/gsa-csv.ts +992 -0
- package/src/integrity.ts +928 -0
- package/src/meta.ts +292 -0
- package/src/nih.ts +375 -0
- package/src/nppes.ts +834 -0
- package/src/nsf.ts +706 -0
- package/src/nvd.ts +1124 -0
- package/src/ofac.ts +1166 -0
- package/src/pricing.ts +1075 -0
- package/src/sam-gov/client.ts +345 -63
- package/src/sam-gov/index.ts +5 -1
- package/src/sam-gov/types.ts +22 -0
- package/src/sba.ts +357 -0
- package/src/server.ts +4551 -328
- package/src/snapshot.ts +192 -0
- package/src/socrata.ts +532 -0
- package/src/treasury.ts +575 -0
- package/src/usaspending.ts +1946 -214
- package/src/usitc.ts +420 -0
package/dist/bls.js
ADDED
|
@@ -0,0 +1,1549 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLS Public Data API v1/v2 — US labor-economics & price-index time series
|
|
3
|
+
* (keyless v1 default; an OPTIONAL free BLS_API_KEY lifts to v2). ADR-0032.
|
|
4
|
+
*
|
|
5
|
+
* A NEW capability axis for the server — the PRICING / ESCALATION layer: CPI-U &
|
|
6
|
+
* ECI drive federal contract escalation / economic-price-adjustment (EPA) clauses;
|
|
7
|
+
* PPI benchmarks materials pricing; CES employment/wages give labor-rate context
|
|
8
|
+
* (next to GSA CALC + SAM wage determinations). Consumer of the R2 `getJson`/
|
|
9
|
+
* `throughGate` port; the SECOND POST-batch consumer (after NIH RePORTER).
|
|
10
|
+
*
|
|
11
|
+
* ON-DOMAIN HONESTY: these are PUBLIC AGGREGATE STATISTICS (no PII). The tool
|
|
12
|
+
* LABELS units per series (an ECI "…A" 3.4 is a 12-month PERCENT CHANGE, not an
|
|
13
|
+
* index level) and NEVER fabricates a value — the BLS "-" unavailable marker maps
|
|
14
|
+
* to null (NEVER 0) with the footnote reason surfaced, so a data gap (e.g. the
|
|
15
|
+
* 2025 lapse-in-appropriations) is DISCLOSED, never a silent null and never a 0.
|
|
16
|
+
*
|
|
17
|
+
* The module writes ZERO fetch/coercion/error/meta code — it REUSES `getJson`
|
|
18
|
+
* (method:"POST" + body + redirect:"error"), `driftError`, `throughGate`,
|
|
19
|
+
* `num`·`str` (coerce.ts, null-never-0), `withMeta`·`buildMeta`, and MIRRORS the
|
|
20
|
+
* fixed-host SSRF idiom from NIH/FDIC + the body-carried optional-key discipline
|
|
21
|
+
* of datagovKey.ts (but the key rides in the POST BODY, never a header/URL/log).
|
|
22
|
+
*
|
|
23
|
+
* ★ SSRF GUARD (policy① — the smallest surface): host+path are compile-time
|
|
24
|
+
* CONSTANTS; NO caller value touches them. seriesids ride in the MODULE-BUILT
|
|
25
|
+
* POST body (JSON.stringify'd typed payload — no raw passthrough), so a caller
|
|
26
|
+
* value has NO host-alteration surface. Each raw seriesId is charclass-validated
|
|
27
|
+
* `^[A-Z0-9]{1,20}$` (rejects `../`, encoded traversal, `@host`, `;`, a trailing
|
|
28
|
+
* `\n`). Curated keys resolve through the FROZEN catalog map (never user text). A
|
|
29
|
+
* post-construction hostname/protocol assertion + `redirect:"error"` lock it.
|
|
30
|
+
*
|
|
31
|
+
* ★ THE HONESTY FRONTIER (P3): each observation's `value` is a STRING → `num()`.
|
|
32
|
+
* `num("-")` / any non-numeric → **null NEVER 0**; a genuine `"0"` → 0. The row
|
|
33
|
+
* carries `valueUnavailable` (true iff the raw value was present but unparseable)
|
|
34
|
+
* + its `footnotes[]`, and the footnote text is LIFTED into `_meta.notes` so the
|
|
35
|
+
* absence is disclosed. `value` is ALWAYS the parsed number, never the raw string.
|
|
36
|
+
*
|
|
37
|
+
* ★ STATUS GATE (P2): `status !== "REQUEST_SUCCEEDED"` THROWS (never a fake-empty):
|
|
38
|
+
* `REQUEST_NOT_PROCESSED` (the v1 daily-limit case) → `rate_limited` (retryable)
|
|
39
|
+
* with the tier disclosure; `REQUEST_FAILED`/other → `upstream_unavailable` (or
|
|
40
|
+
* `invalid_input` when `message[]` reads like a bad request). A non-JSON 200, or
|
|
41
|
+
* a SUCCESS body missing `Results.series` (non-array), → `driftError`.
|
|
42
|
+
*
|
|
43
|
+
* ★ OPTIONAL BLS_API_KEY (v2) — body-carried, NEVER leaked: read from env
|
|
44
|
+
* (trimmed; empty ⇒ keyless v1). When present → the `…/v2/…` path + a
|
|
45
|
+
* `registrationkey` field in the POST BODY, and NOWHERE else (never the URL, a
|
|
46
|
+
* header, the label, the `source`, `_meta`, or a log). When keyless,
|
|
47
|
+
* `registrationkey` is ABSENT from the body. Only the MODE is ever disclosed.
|
|
48
|
+
*/
|
|
49
|
+
import { ToolErrorCarrier, errorFromResponse } from "./errors.js";
|
|
50
|
+
import { getJson, driftError, throughGate, isRedirectError } from "./datasource.js";
|
|
51
|
+
import { parseRecordFields } from "./gsa-csv.js";
|
|
52
|
+
import { num, str } from "./coerce.js";
|
|
53
|
+
import { withMeta } from "./meta.js";
|
|
54
|
+
// Re-export the shared honesty coercion (single audited copy in ./coerce.js —
|
|
55
|
+
// ADR-0005 v2 FIX-C) so the fault suite's num-parity guard resolves the SAME
|
|
56
|
+
// `num` (bls.num === coerce.num — a num regression fails together across sources).
|
|
57
|
+
export { num };
|
|
58
|
+
// ─── Fixed endpoint (SSRF core — compile-time CONSTANTS) ──────────
|
|
59
|
+
const BLS_HOST = "api.bls.gov";
|
|
60
|
+
const BLS_PATH_V1 = "/publicAPI/v1/timeseries/data/";
|
|
61
|
+
const BLS_PATH_V2 = "/publicAPI/v2/timeseries/data/";
|
|
62
|
+
// The self-throttle gate key + the min interval (one host = one shared budget —
|
|
63
|
+
// the FDIC pattern; the gate serializes bursts but CANNOT enforce the per-IP/day
|
|
64
|
+
// cap, so the honest surface for the daily limit is the disclosure + the throw).
|
|
65
|
+
const BLS_GATE_KEY = "bls";
|
|
66
|
+
const BLS_MIN_INTERVAL_MS = 250;
|
|
67
|
+
// ─── The FROZEN curated catalog (ADR §catalog) ────────────────────
|
|
68
|
+
// The SINGLE source of truth for: the `series` enum's value list, the resolver
|
|
69
|
+
// (key → seriesId), and the per-series `units` label. ALL seriesIDs live-verified
|
|
70
|
+
// (ADR §Context 2026-07-13). The ECI "…A" series carry "12-mo % change" units
|
|
71
|
+
// (a consumer misreads 3.4 as an index level otherwise).
|
|
72
|
+
export const BLS_SERIES_KEYS = [
|
|
73
|
+
"cpi_u_all",
|
|
74
|
+
"cpi_u_core",
|
|
75
|
+
"ppi_final_demand",
|
|
76
|
+
"eci_total_comp",
|
|
77
|
+
"eci_wages",
|
|
78
|
+
"unemployment_rate",
|
|
79
|
+
"labor_force_participation",
|
|
80
|
+
"employment_total_nonfarm",
|
|
81
|
+
"avg_hourly_earnings",
|
|
82
|
+
];
|
|
83
|
+
export const BLS_CATALOG = {
|
|
84
|
+
cpi_u_all: {
|
|
85
|
+
seriesId: "CUUR0000SA0",
|
|
86
|
+
meaning: "CPI-U, all items, US city avg (NSA)",
|
|
87
|
+
units: "index 1982-84=100",
|
|
88
|
+
},
|
|
89
|
+
cpi_u_core: {
|
|
90
|
+
seriesId: "CUUR0000SA0L1E",
|
|
91
|
+
meaning: "CPI-U, all items less food & energy — core (NSA)",
|
|
92
|
+
units: "index 1982-84=100",
|
|
93
|
+
},
|
|
94
|
+
ppi_final_demand: {
|
|
95
|
+
seriesId: "WPUFD4",
|
|
96
|
+
meaning: "PPI, final demand (NSA)",
|
|
97
|
+
units: "index Nov-2009=100",
|
|
98
|
+
},
|
|
99
|
+
eci_total_comp: {
|
|
100
|
+
seriesId: "CIU1010000000000A",
|
|
101
|
+
meaning: "ECI, total compensation, all civilian",
|
|
102
|
+
units: "12-mo % change",
|
|
103
|
+
},
|
|
104
|
+
eci_wages: {
|
|
105
|
+
seriesId: "CIU2020000000000A",
|
|
106
|
+
meaning: "ECI, wages & salaries, all civilian",
|
|
107
|
+
units: "12-mo % change",
|
|
108
|
+
},
|
|
109
|
+
unemployment_rate: {
|
|
110
|
+
seriesId: "LNS14000000",
|
|
111
|
+
meaning: "Unemployment rate (SA)",
|
|
112
|
+
units: "percent",
|
|
113
|
+
},
|
|
114
|
+
labor_force_participation: {
|
|
115
|
+
seriesId: "LNS11300000",
|
|
116
|
+
meaning: "Labor force participation rate (SA)",
|
|
117
|
+
units: "percent",
|
|
118
|
+
},
|
|
119
|
+
employment_total_nonfarm: {
|
|
120
|
+
seriesId: "CES0000000001",
|
|
121
|
+
meaning: "Total nonfarm employment (SA)",
|
|
122
|
+
units: "thousands of persons",
|
|
123
|
+
},
|
|
124
|
+
avg_hourly_earnings: {
|
|
125
|
+
seriesId: "CES0500000003",
|
|
126
|
+
meaning: "Avg hourly earnings, total private (SA)",
|
|
127
|
+
units: "dollars/hour",
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
// Reverse lookup seriesId → {key, entry}. A RAW seriesId that HAPPENS to equal a
|
|
131
|
+
// curated ID is then also labeled with its units (known-valid) — the honest,
|
|
132
|
+
// helpful behavior.
|
|
133
|
+
const BLS_ID_TO_KEY = new Map(BLS_SERIES_KEYS.map((k) => [BLS_CATALOG[k].seriesId, k]));
|
|
134
|
+
// ─── Raw-seriesId charclass (SSRF + "verify the ID" honesty) ──────
|
|
135
|
+
// `^[A-Z0-9]{1,20}$` — uppercase alnum, explicit length bound. In JS (no `m`
|
|
136
|
+
// flag) `$` matches only end-of-input, and `[A-Z0-9]` cannot include `\n`, so a
|
|
137
|
+
// trailing newline is rejected. Rejects `../`, encoded traversal, `@host`, `;`.
|
|
138
|
+
const SERIES_ID_RE = /^[A-Z0-9]{1,20}$/;
|
|
139
|
+
// ─── Year bounds ──────────────────────────────────────────────────
|
|
140
|
+
const YEAR_MIN = 1900;
|
|
141
|
+
const CURRENT_YEAR = new Date().getUTCFullYear();
|
|
142
|
+
export const YEAR_MAX = CURRENT_YEAR + 1;
|
|
143
|
+
const V1_CAPS = { seriesCap: 25, spanCap: 10, dailyLabel: "~25 queries/day/IP" };
|
|
144
|
+
const V2_CAPS = { seriesCap: 50, spanCap: 20, dailyLabel: "~500 queries/day" };
|
|
145
|
+
// ─── Optional BLS_API_KEY seam (body-carried, NEVER leaked) ───────
|
|
146
|
+
// Mirror datagovKey.ts, but the value goes into the POST BODY as
|
|
147
|
+
// `registrationkey` and NOWHERE else. `resolvedBlsKey()` is the ONLY reader of
|
|
148
|
+
// the value; every other helper exposes only the MODE.
|
|
149
|
+
function resolvedBlsKey() {
|
|
150
|
+
const raw = process.env.BLS_API_KEY;
|
|
151
|
+
const trimmed = typeof raw === "string" ? raw.trim() : "";
|
|
152
|
+
return trimmed;
|
|
153
|
+
}
|
|
154
|
+
/** true when no BLS_API_KEY is configured (keyless v1). Drives mode + caps + path. */
|
|
155
|
+
export function usingKeylessV1() {
|
|
156
|
+
return resolvedBlsKey() === "";
|
|
157
|
+
}
|
|
158
|
+
function apiPath() {
|
|
159
|
+
return usingKeylessV1() ? BLS_PATH_V1 : BLS_PATH_V2;
|
|
160
|
+
}
|
|
161
|
+
/** The MODE label — NEVER the value. */
|
|
162
|
+
function keyModeLabel() {
|
|
163
|
+
return usingKeylessV1() ? "v1 keyless" : "v2 (BLS_API_KEY)";
|
|
164
|
+
}
|
|
165
|
+
function tierCaps() {
|
|
166
|
+
return usingKeylessV1() ? V1_CAPS : V2_CAPS;
|
|
167
|
+
}
|
|
168
|
+
/** Host+path only label (surfaces in ToolError.upstreamEndpoint; no token). */
|
|
169
|
+
function blsLabel() {
|
|
170
|
+
return `bls:${apiPath()}`;
|
|
171
|
+
}
|
|
172
|
+
function blsSource() {
|
|
173
|
+
const version = usingKeylessV1() ? "v1" : "v2";
|
|
174
|
+
const mode = usingKeylessV1() ? "keyless v1" : "BLS_API_KEY v2";
|
|
175
|
+
return `api.bls.gov /publicAPI/${version}/timeseries/data/ (${mode})`;
|
|
176
|
+
}
|
|
177
|
+
// ─── Disclosure text ──────────────────────────────────────────────
|
|
178
|
+
/** The mandatory ALWAYS-ON tier disclosure (P1). Reports the ACTIVE tier + caps
|
|
179
|
+
* + the as-of/footnote caveat. */
|
|
180
|
+
function tierNote() {
|
|
181
|
+
const c = tierCaps();
|
|
182
|
+
const escapeHatch = usingKeylessV1()
|
|
183
|
+
? " Set a free BLS_API_KEY (https://data.bls.gov/registrationEngine/) for v2 (~500 queries/day, 50 series/query, ~20-year span) — the key is sent only in the POST request body, never logged."
|
|
184
|
+
: "";
|
|
185
|
+
return `Active BLS tier: ${keyModeLabel()} — approximately ${c.dailyLabel}, ${c.seriesCap} series/query, ~${c.spanCap}-year span/query.${escapeHatch} Values are as-published by BLS and may be preliminary or revised (see per-observation footnote codes, e.g. r=revised, p=preliminary, X=unavailable).`;
|
|
186
|
+
}
|
|
187
|
+
/** The mandatory per-series UNITS caveat. */
|
|
188
|
+
const UNITS_NOTE = 'Each returned series carries its own units label — an ECI "…A" series is a 12-month PERCENT CHANGE (e.g. 3.4 means 3.4%), NOT an index level; CPI/PPI are index levels; CES nonfarm employment is thousands of persons; a raw seriesId has units:null (consult BLS). Do NOT compare values across series without reading each units label.';
|
|
189
|
+
/** The value-gap disclosure preamble (lifted alongside the verbatim footnote texts). */
|
|
190
|
+
const GAP_NOTE_PREFIX = 'One or more observations are UNAVAILABLE (BLS "-" marker): value is null with valueUnavailable:true and the footnote reason on the observation — NEVER a fabricated 0. Disclosed reason(s): ';
|
|
191
|
+
/** The value-gap disclosure when every gap is TEXTLESS (footnotes:[{}] / no text). */
|
|
192
|
+
const GAP_NOTE_TEXTLESS = 'One or more observations are unavailable (BLS "-" marker) with no footnote reason supplied — surfaced as value:null / valueUnavailable:true.';
|
|
193
|
+
function clampNote(requestedStart, sentStart, endYear, spanCap) {
|
|
194
|
+
return `The requested span ${requestedStart}–${endYear} (${endYear - requestedStart + 1} years) exceeds the active BLS tier's ~${spanCap}-year/query cap; startYear was clamped to ${sentStart} BEFORE the request (sent span ${sentStart}–${endYear}). Widen with a smaller window or a BLS_API_KEY (v2, ~${V2_CAPS.spanCap}-year span).`;
|
|
195
|
+
}
|
|
196
|
+
/** Map a footnotes[] array → non-empty {code,text} entries (drops the `[{}]` noise). */
|
|
197
|
+
function mapFootnotes(x) {
|
|
198
|
+
if (!Array.isArray(x))
|
|
199
|
+
return [];
|
|
200
|
+
return x
|
|
201
|
+
.map((f) => {
|
|
202
|
+
const o = (f ?? {});
|
|
203
|
+
return { code: str(o.code), text: str(o.text) };
|
|
204
|
+
})
|
|
205
|
+
.filter((f) => f.code !== null || f.text !== null);
|
|
206
|
+
}
|
|
207
|
+
/** Map ONE BLS data[] row → the curated observation (P3 — null-never-0 + footnotes). */
|
|
208
|
+
function mapObservation(raw) {
|
|
209
|
+
const r = (raw ?? {});
|
|
210
|
+
const rawValue = r.value;
|
|
211
|
+
const value = num(rawValue); // "-"/non-numeric → null; "0" → 0
|
|
212
|
+
// valueUnavailable: the raw value was PRESENT but did not parse (a disclosed
|
|
213
|
+
// gap), not merely absent. `!= null` catches both null and undefined.
|
|
214
|
+
const valueUnavailable = value === null && rawValue != null;
|
|
215
|
+
return {
|
|
216
|
+
year: str(r.year),
|
|
217
|
+
period: str(r.period),
|
|
218
|
+
periodName: str(r.periodName),
|
|
219
|
+
value,
|
|
220
|
+
valueUnavailable,
|
|
221
|
+
footnotes: mapFootnotes(r.footnotes),
|
|
222
|
+
latest: r.latest === "true" || r.latest === true,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
/** Coerce a `message` field (BLS returns an array of strings) → string[]. */
|
|
226
|
+
function messageArray(x) {
|
|
227
|
+
if (!Array.isArray(x))
|
|
228
|
+
return [];
|
|
229
|
+
return x.map((m) => str(m)).filter((m) => m !== null);
|
|
230
|
+
}
|
|
231
|
+
/** Heuristic: does a non-success message read like a bad request (→ invalid_input)? */
|
|
232
|
+
function looksLikeBadRequest(messages) {
|
|
233
|
+
return messages.some((m) => /invalid|not a valid|must be|malformed|bad request|unable to parse|parameter|exceeds the maximum|too many/i.test(m));
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Parse the HTTP-200 body: THROW on any non-SUCCESS status (P2, never a
|
|
237
|
+
* fake-empty), else extract the per-series map + the top-level message[].
|
|
238
|
+
* A body that is not an object, or a SUCCESS body whose `Results.series` is not
|
|
239
|
+
* an array, → driftError.
|
|
240
|
+
*/
|
|
241
|
+
function parseBlsBody(body, label) {
|
|
242
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
243
|
+
throw driftError(label, "BLS returned a 200 body that is not an object (an array or scalar) — refusing to report it as an empty result.");
|
|
244
|
+
}
|
|
245
|
+
const b = body;
|
|
246
|
+
const status = str(b.status);
|
|
247
|
+
const messages = messageArray(b.message);
|
|
248
|
+
if (status !== "REQUEST_SUCCEEDED") {
|
|
249
|
+
const detail = messages.length > 0 ? ` Upstream message(s): ${messages.join(" | ")}.` : "";
|
|
250
|
+
if (status === "REQUEST_NOT_PROCESSED") {
|
|
251
|
+
// The v1 daily-limit / threshold case — an honest retryable rate_limited
|
|
252
|
+
// carrying the tier disclosure (never a fake-empty).
|
|
253
|
+
throw new ToolErrorCarrier({
|
|
254
|
+
kind: "rate_limited",
|
|
255
|
+
message: `BLS did not process the request (status REQUEST_NOT_PROCESSED). The BLS free v1 tier allows ~25 queries/day/IP; this limit was likely hit. Set a free BLS_API_KEY (https://data.bls.gov/registrationEngine/) to use the v2 tier (~500/day) — the key is sent only in the request body, never logged.${detail}`,
|
|
256
|
+
retryable: true,
|
|
257
|
+
retryAfterSeconds: 60,
|
|
258
|
+
upstreamEndpoint: label,
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
// REQUEST_FAILED / any other non-SUCCEEDED status.
|
|
262
|
+
const kind = looksLikeBadRequest(messages) ? "invalid_input" : "upstream_unavailable";
|
|
263
|
+
throw new ToolErrorCarrier({
|
|
264
|
+
kind,
|
|
265
|
+
message: `BLS did not succeed (status ${JSON.stringify(status)}).${detail}`,
|
|
266
|
+
retryable: kind === "upstream_unavailable",
|
|
267
|
+
...(kind === "upstream_unavailable" ? { retryAfterSeconds: 30 } : {}),
|
|
268
|
+
upstreamEndpoint: label,
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
// SUCCESS — the Results.series array must be present (else drift; never a fake empty).
|
|
272
|
+
const results = (b.Results ?? {});
|
|
273
|
+
if (!Array.isArray(results.series)) {
|
|
274
|
+
throw driftError(label, "BLS returned REQUEST_SUCCEEDED but Results.series is missing/non-array — treating as schema drift (never a fake empty).");
|
|
275
|
+
}
|
|
276
|
+
const byId = new Map();
|
|
277
|
+
for (const s of results.series) {
|
|
278
|
+
const so = (s ?? {});
|
|
279
|
+
const id = str(so.seriesID);
|
|
280
|
+
if (id !== null)
|
|
281
|
+
byId.set(id, so);
|
|
282
|
+
}
|
|
283
|
+
return { byId, messages };
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Fetch one or more BLS time-series over a year range → normalized observations
|
|
287
|
+
* (null-never-0 values + footnotes + per-series units label) + honest `_meta`.
|
|
288
|
+
* At least one of `series`/`seriesId` is required; both may be combined. The
|
|
289
|
+
* resolved+deduped series set is refused over the active tier's series cap
|
|
290
|
+
* (never a silent drop). The span is CLAMPED to the tier cap BEFORE the fetch
|
|
291
|
+
* and disclosed. seriesids ride in the module-built POST body (SSRF: no raw
|
|
292
|
+
* host/path passthrough); the OPTIONAL BLS_API_KEY rides ONLY in the body.
|
|
293
|
+
*/
|
|
294
|
+
export async function timeseries(args) {
|
|
295
|
+
const label = blsLabel();
|
|
296
|
+
const caps = tierCaps();
|
|
297
|
+
// ── Resolve + dedup the series set (curated keys → seriesId via the frozen
|
|
298
|
+
// catalog; raw seriesIds charclass-validated). Preserve request order. ──
|
|
299
|
+
const resolved = [];
|
|
300
|
+
const seen = new Set();
|
|
301
|
+
const pushResolved = (seriesId, key) => {
|
|
302
|
+
if (seen.has(seriesId))
|
|
303
|
+
return;
|
|
304
|
+
seen.add(seriesId);
|
|
305
|
+
// A raw ID that equals a known catalog ID is labeled curated (known-valid).
|
|
306
|
+
const catalogKey = key ?? BLS_ID_TO_KEY.get(seriesId) ?? null;
|
|
307
|
+
resolved.push({ seriesId, key: catalogKey, fromCurated: catalogKey !== null });
|
|
308
|
+
};
|
|
309
|
+
for (const k of args.series ?? []) {
|
|
310
|
+
const entry = BLS_CATALOG[k];
|
|
311
|
+
if (!entry) {
|
|
312
|
+
// Belt-and-suspenders behind the server's Zod enum.
|
|
313
|
+
throw new ToolErrorCarrier({
|
|
314
|
+
kind: "invalid_input",
|
|
315
|
+
message: `Unknown curated series key ${JSON.stringify(k)} — expected one of: ${BLS_SERIES_KEYS.join(", ")}.`,
|
|
316
|
+
retryable: false,
|
|
317
|
+
upstreamEndpoint: label,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
pushResolved(entry.seriesId, k);
|
|
321
|
+
}
|
|
322
|
+
for (const raw of args.seriesId ?? []) {
|
|
323
|
+
if (typeof raw !== "string" || !SERIES_ID_RE.test(raw)) {
|
|
324
|
+
throw new ToolErrorCarrier({
|
|
325
|
+
kind: "invalid_input",
|
|
326
|
+
message: `Invalid seriesId ${JSON.stringify(raw)} — expected 1..20 uppercase alphanumeric characters (^[A-Z0-9]{1,20}$). A BLS series ID like 'CUUR0000SA0'; punctuation/whitespace/lowercase are rejected (SSRF + "verify the ID" honesty).`,
|
|
327
|
+
retryable: false,
|
|
328
|
+
upstreamEndpoint: label,
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
pushResolved(raw, null);
|
|
332
|
+
}
|
|
333
|
+
// ── At least one of series/seriesId is required (P4 — never a silent no-op). ──
|
|
334
|
+
if (resolved.length === 0) {
|
|
335
|
+
throw new ToolErrorCarrier({
|
|
336
|
+
kind: "invalid_input",
|
|
337
|
+
message: "At least one of `series` (a curated enum key) or `seriesId` (a raw BLS series ID) is required — nothing to query.",
|
|
338
|
+
retryable: false,
|
|
339
|
+
upstreamEndpoint: label,
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
// ── Refuse over the tier's series cap — NEVER a silent drop of the overflow (P4). ──
|
|
343
|
+
if (resolved.length > caps.seriesCap) {
|
|
344
|
+
throw new ToolErrorCarrier({
|
|
345
|
+
kind: "invalid_input",
|
|
346
|
+
message: `Requested ${resolved.length} distinct series, over the active BLS ${keyModeLabel()} tier cap of ${caps.seriesCap} series/query. Reduce the request (or set a BLS_API_KEY for the v2 tier, ${V2_CAPS.seriesCap} series/query) — the overflow is NOT silently dropped.`,
|
|
347
|
+
retryable: false,
|
|
348
|
+
upstreamEndpoint: label,
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
// ── Resolve the year window + defaults; enforce startYear ≤ endYear. ──
|
|
352
|
+
const endYear = args.endYear ?? CURRENT_YEAR;
|
|
353
|
+
// The AUTO-derived default is CLAMPED to the 1900 floor (Math.max) so an early
|
|
354
|
+
// endYear (e.g. 1905 ⇒ 1896) never self-inflicts an out-of-range startYear and
|
|
355
|
+
// false-rejects an in-range endYear. An EXPLICIT caller startYear is NOT clamped
|
|
356
|
+
// — an explicit 1850 is still an honest invalid_input at the floor guard below.
|
|
357
|
+
const requestedStart = args.startYear ?? Math.max(YEAR_MIN, endYear - (caps.spanCap - 1));
|
|
358
|
+
if (requestedStart < YEAR_MIN || endYear > YEAR_MAX) {
|
|
359
|
+
throw new ToolErrorCarrier({
|
|
360
|
+
kind: "invalid_input",
|
|
361
|
+
message: `Year out of range — startYear/endYear must be within ${YEAR_MIN}..${YEAR_MAX}. Got startYear ${requestedStart}, endYear ${endYear}.`,
|
|
362
|
+
retryable: false,
|
|
363
|
+
upstreamEndpoint: label,
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
if (requestedStart > endYear) {
|
|
367
|
+
throw new ToolErrorCarrier({
|
|
368
|
+
kind: "invalid_input",
|
|
369
|
+
message: `startYear (${requestedStart}) must be ≤ endYear (${endYear}).`,
|
|
370
|
+
retryable: false,
|
|
371
|
+
upstreamEndpoint: label,
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
// ── Span clamp (P1) — clamp startYear BEFORE the fetch (avoid burning a query
|
|
375
|
+
// on a guaranteed REQUEST_NOT_PROCESSED) + disclose requested-vs-sent + cap. ──
|
|
376
|
+
let sentStart = requestedStart;
|
|
377
|
+
let clamped = false;
|
|
378
|
+
if (endYear - requestedStart + 1 > caps.spanCap) {
|
|
379
|
+
sentStart = endYear - caps.spanCap + 1;
|
|
380
|
+
clamped = true;
|
|
381
|
+
}
|
|
382
|
+
// ── Build the module-built typed payload (SSRF: no raw host/path passthrough;
|
|
383
|
+
// seriesids + years + the OPTIONAL key ride in the JSON body). ──
|
|
384
|
+
const seriesIds = resolved.map((r) => r.seriesId);
|
|
385
|
+
const payload = {
|
|
386
|
+
seriesid: seriesIds,
|
|
387
|
+
startyear: String(sentStart),
|
|
388
|
+
endyear: String(endYear),
|
|
389
|
+
};
|
|
390
|
+
// The OPTIONAL BLS_API_KEY — injected ONLY here, in the body, and NOWHERE else.
|
|
391
|
+
// When keyless, `registrationkey` is ABSENT from the body entirely.
|
|
392
|
+
const key = resolvedBlsKey();
|
|
393
|
+
if (key !== "")
|
|
394
|
+
payload.registrationkey = key;
|
|
395
|
+
// ── SSRF belt-and-suspenders: the URL is a compile-time constant, but assert
|
|
396
|
+
// it cannot have drifted (a future typo / downgrade). ──
|
|
397
|
+
const url = `https://${BLS_HOST}${apiPath()}`;
|
|
398
|
+
const built = new URL(url);
|
|
399
|
+
if (built.hostname !== BLS_HOST || built.protocol !== "https:") {
|
|
400
|
+
throw new ToolErrorCarrier({
|
|
401
|
+
kind: "invalid_input",
|
|
402
|
+
message: `Constructed BLS URL host ${JSON.stringify(built.hostname)} (${built.protocol}) is not ${BLS_HOST} over https — refusing to fetch (SSRF safety).`,
|
|
403
|
+
retryable: false,
|
|
404
|
+
upstreamEndpoint: label,
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
// ── The POST-batch fetch through the shared self-throttle gate. A non-JSON 200
|
|
408
|
+
// body makes getJson's r.json() throw a SyntaxError → reclassify to driftError
|
|
409
|
+
// at THIS call site (the FDIC pattern); the 429/5xx/404/400 taxonomy propagates
|
|
410
|
+
// unchanged; redirect:"error" fails closed on any off-host 3xx. ──
|
|
411
|
+
let body;
|
|
412
|
+
try {
|
|
413
|
+
body = await throughGate(BLS_GATE_KEY, BLS_MIN_INTERVAL_MS, () => getJson(built.toString(), {
|
|
414
|
+
label,
|
|
415
|
+
method: "POST",
|
|
416
|
+
headers: { "Content-Type": "application/json" },
|
|
417
|
+
body: JSON.stringify(payload),
|
|
418
|
+
redirect: "error",
|
|
419
|
+
}));
|
|
420
|
+
}
|
|
421
|
+
catch (e) {
|
|
422
|
+
if (e instanceof ToolErrorCarrier)
|
|
423
|
+
throw e;
|
|
424
|
+
if (e instanceof SyntaxError) {
|
|
425
|
+
throw driftError(label, "BLS returned a non-JSON body at HTTP 200 (an HTML error page) — treating as schema drift (never read as an empty result).");
|
|
426
|
+
}
|
|
427
|
+
throw e;
|
|
428
|
+
}
|
|
429
|
+
const parsed = parseBlsBody(body, label);
|
|
430
|
+
// ── Assemble the per-series output; account for EVERY requested series (P4). ──
|
|
431
|
+
const seriesOut = [];
|
|
432
|
+
const fieldsUnavailable = [];
|
|
433
|
+
const notes = [];
|
|
434
|
+
const gapTexts = new Set();
|
|
435
|
+
// Tracks whether ANY returned observation is unavailable — so a textless "-" gap
|
|
436
|
+
// (footnotes:[{}] / a code with no text) still contributes ONE aggregate note,
|
|
437
|
+
// not just the per-observation valueUnavailable flag.
|
|
438
|
+
let anyUnavailable = false;
|
|
439
|
+
const emptyNotes = [];
|
|
440
|
+
const absentNotes = [];
|
|
441
|
+
for (const rs of resolved) {
|
|
442
|
+
const entry = rs.key ? BLS_CATALOG[rs.key] : null;
|
|
443
|
+
const units = entry ? entry.units : null;
|
|
444
|
+
const meaning = entry ? entry.meaning : null;
|
|
445
|
+
const rawSeries = parsed.byId.get(rs.seriesId);
|
|
446
|
+
if (rawSeries === undefined) {
|
|
447
|
+
// Requested but ENTIRELY ABSENT from Results.series (BLS omitted it) — P4:
|
|
448
|
+
// disclosed, never silently dropped.
|
|
449
|
+
fieldsUnavailable.push(`${rs.seriesId} (not returned by upstream)`);
|
|
450
|
+
absentNotes.push(`Series ${rs.seriesId}${rs.key ? ` (${rs.key})` : ""} was requested but NOT returned by BLS (absent from Results.series) — disclosed as unavailable, never silently dropped.`);
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
const rawData = Array.isArray(rawSeries.data) ? rawSeries.data : [];
|
|
454
|
+
const observations = rawData.map(mapObservation);
|
|
455
|
+
const years = observations
|
|
456
|
+
.map((o) => (o.year !== null ? Number(o.year) : NaN))
|
|
457
|
+
.filter((y) => Number.isFinite(y));
|
|
458
|
+
const coveredRange = years.length > 0
|
|
459
|
+
? { from: Math.min(...years), to: Math.max(...years) }
|
|
460
|
+
: { from: null, to: null };
|
|
461
|
+
// Lift each disclosed data-gap footnote text into _meta.notes (P3 disclosure).
|
|
462
|
+
for (const o of observations) {
|
|
463
|
+
if (o.valueUnavailable) {
|
|
464
|
+
anyUnavailable = true;
|
|
465
|
+
for (const f of o.footnotes)
|
|
466
|
+
if (f.text)
|
|
467
|
+
gapTexts.add(f.text);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
// Empty-data ambiguity disclosure (P2 refinement).
|
|
471
|
+
if (observations.length === 0) {
|
|
472
|
+
if (rs.fromCurated) {
|
|
473
|
+
emptyNotes.push(`No observations for ${rs.key ?? rs.seriesId} (${rs.seriesId}) over ${sentStart}–${endYear} — a genuine empty range for a valid series.`);
|
|
474
|
+
}
|
|
475
|
+
else {
|
|
476
|
+
emptyNotes.push(`No observations for ${rs.seriesId} over ${sentStart}–${endYear} — this is EITHER a genuine empty range OR a nonexistent/mistyped series ID (BLS returns success + empty data for both); verify the ID.`);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
seriesOut.push({
|
|
480
|
+
seriesId: rs.seriesId,
|
|
481
|
+
key: rs.key,
|
|
482
|
+
meaning,
|
|
483
|
+
units,
|
|
484
|
+
observations,
|
|
485
|
+
observationCount: observations.length,
|
|
486
|
+
coveredRange,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
// ── Notes (always: the tier disclosure + the units caveat). ──
|
|
490
|
+
notes.push(tierNote());
|
|
491
|
+
notes.push(UNITS_NOTE);
|
|
492
|
+
if (clamped)
|
|
493
|
+
notes.push(clampNote(requestedStart, sentStart, endYear, caps.spanCap));
|
|
494
|
+
if (gapTexts.size > 0) {
|
|
495
|
+
notes.push(GAP_NOTE_PREFIX + [...gapTexts].map((t) => `"${t}"`).join("; ") + ".");
|
|
496
|
+
}
|
|
497
|
+
else if (anyUnavailable) {
|
|
498
|
+
// All gaps were textless (footnotes:[{}] / no text) — the per-observation flag
|
|
499
|
+
// is honest, but the aggregate summary must reflect the gap too (never omitted).
|
|
500
|
+
notes.push(GAP_NOTE_TEXTLESS);
|
|
501
|
+
}
|
|
502
|
+
notes.push(...emptyNotes);
|
|
503
|
+
notes.push(...absentNotes);
|
|
504
|
+
if (parsed.messages.length > 0) {
|
|
505
|
+
notes.push(`BLS returned top-level message(s): ${parsed.messages.join(" | ")}.`);
|
|
506
|
+
}
|
|
507
|
+
// filtersApplied = the resolved series descriptors + the (possibly clamped) span (P4).
|
|
508
|
+
const filtersApplied = [
|
|
509
|
+
...resolved.map((r) => (r.key ? `${r.key}=${r.seriesId}` : r.seriesId)),
|
|
510
|
+
`years:${sentStart}-${endYear}`,
|
|
511
|
+
];
|
|
512
|
+
const metaOut = {
|
|
513
|
+
source: blsSource(),
|
|
514
|
+
keylessMode: usingKeylessV1(),
|
|
515
|
+
returned: seriesOut.length,
|
|
516
|
+
totalAvailable: resolved.length,
|
|
517
|
+
filtersApplied,
|
|
518
|
+
filtersDropped: [],
|
|
519
|
+
fieldsUnavailable,
|
|
520
|
+
notes,
|
|
521
|
+
};
|
|
522
|
+
return withMeta({ series: seriesOut }, metaOut);
|
|
523
|
+
}
|
|
524
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
525
|
+
// OEWS — Occupational Employment & Wage Statistics (the 2nd tool, ADR-0033)
|
|
526
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
527
|
+
// A SECOND tool on this keyless BLS source: occupational LABOR-RATE benchmarking
|
|
528
|
+
// (mean/median annual & hourly wages + employment by SOC occupation × geography)
|
|
529
|
+
// — the LEVEL layer next to bls_timeseries's ESCALATION layer, GSA CALC, and SAM
|
|
530
|
+
// wage determinations. OEWS series IDs are 25 chars — they EXCEED the
|
|
531
|
+
// bls_timeseries raw-seriesId cap (^[A-Z0-9]{1,20}$), so OEWS is NOT reachable
|
|
532
|
+
// there; this tool BUILDS the 25-char ID INTERNALLY from validated structured
|
|
533
|
+
// inputs (the built ID is never caller-supplied raw text). The endpoint + shape
|
|
534
|
+
// are IDENTICAL to bls_timeseries (same POST api.bls.gov, same
|
|
535
|
+
// {status, Results.series[].data[]} envelope), so the fetch/parse/honesty layer
|
|
536
|
+
// is REUSED byte-identical (getJson-POST, throughGate, parseBlsBody status-throw,
|
|
537
|
+
// mapObservation "-"→null-never-0 + footnotes, the resolvedBlsKey/blsSource/tier
|
|
538
|
+
// key seam, the SSRF host assert, num/str, withMeta) — only the ID builder +
|
|
539
|
+
// curated maps + output shaping are new.
|
|
540
|
+
//
|
|
541
|
+
// OEWS-specific honesty ON TOP of the reused adapter:
|
|
542
|
+
// (H1) OEWS is an ANNUAL point-in-time snapshot (reference May <year>, period
|
|
543
|
+
// A01); the API serves ONLY the latest release (may lag ~1yr) — NOT
|
|
544
|
+
// monthly/current-quarter; historical needs the downloadable tables.
|
|
545
|
+
// (H2) a built-ID that returns empty data[] (or is absent from Results.series)
|
|
546
|
+
// ⇒ value:null, valueUnavailable:FALSE (it is ABSENT, not a "-" gap) + the
|
|
547
|
+
// not-published note + the surfaced upstream "Series does not exist…"
|
|
548
|
+
// message + the ID in fieldsUnavailable — NEVER a fabricated 0. (Contrast:
|
|
549
|
+
// a present-with-data row goes through mapObservation; a "-" in-band value
|
|
550
|
+
// ⇒ the adapter's null + valueUnavailable:true + footnote.)
|
|
551
|
+
// (H3) each row's measure.units from the datatype map (dollars/year |
|
|
552
|
+
// dollars/hour | count (jobs)) — never mislabel.
|
|
553
|
+
// (H4) the API returns real numerics (no "#"); very-high percentile/median
|
|
554
|
+
// wages are estimates (published-tables boundary ≥$115/hr or ≥$239,200/yr).
|
|
555
|
+
// ─── Curated occupation catalog (SOC codes LIVE-verified, ADR §catalog) ──────
|
|
556
|
+
// The FROZEN single source of truth for the `occupation` enum, the resolver, and
|
|
557
|
+
// the output label (mirrors BLS_CATALOG). The long tail (~830 SOC detailed
|
|
558
|
+
// occupations) is reachable via the raw `soc` passthrough (^\d{6}$).
|
|
559
|
+
export const BLS_OEWS_OCCUPATION_KEYS = [
|
|
560
|
+
"all_occupations",
|
|
561
|
+
"software_developer",
|
|
562
|
+
"computer_systems_analyst",
|
|
563
|
+
"info_security_analyst",
|
|
564
|
+
"management_analyst",
|
|
565
|
+
"project_mgmt_specialist",
|
|
566
|
+
"logistician",
|
|
567
|
+
"accountant_auditor",
|
|
568
|
+
"general_ops_manager",
|
|
569
|
+
"civil_engineer",
|
|
570
|
+
"electrical_engineer",
|
|
571
|
+
"mechanical_engineer",
|
|
572
|
+
"industrial_engineer",
|
|
573
|
+
"lawyer",
|
|
574
|
+
"technical_writer",
|
|
575
|
+
"admin_assistant",
|
|
576
|
+
];
|
|
577
|
+
export const BLS_OEWS_OCCUPATIONS = {
|
|
578
|
+
all_occupations: { soc: "000000", label: "All Occupations" },
|
|
579
|
+
software_developer: { soc: "151252", label: "Software Developers" },
|
|
580
|
+
computer_systems_analyst: { soc: "151211", label: "Computer Systems Analysts" },
|
|
581
|
+
info_security_analyst: { soc: "151212", label: "Information Security Analysts" },
|
|
582
|
+
management_analyst: { soc: "131111", label: "Management Analysts" },
|
|
583
|
+
project_mgmt_specialist: { soc: "131082", label: "Project Management Specialists" },
|
|
584
|
+
logistician: { soc: "131081", label: "Logisticians" },
|
|
585
|
+
accountant_auditor: { soc: "132011", label: "Accountants and Auditors" },
|
|
586
|
+
general_ops_manager: { soc: "111021", label: "General and Operations Managers" },
|
|
587
|
+
civil_engineer: { soc: "172051", label: "Civil Engineers" },
|
|
588
|
+
electrical_engineer: { soc: "172071", label: "Electrical Engineers" },
|
|
589
|
+
mechanical_engineer: { soc: "172141", label: "Mechanical Engineers" },
|
|
590
|
+
industrial_engineer: { soc: "172112", label: "Industrial Engineers" },
|
|
591
|
+
lawyer: { soc: "231011", label: "Lawyers" },
|
|
592
|
+
technical_writer: { soc: "273042", label: "Technical Writers" },
|
|
593
|
+
admin_assistant: {
|
|
594
|
+
soc: "436014",
|
|
595
|
+
label: "Secretaries and Administrative Assistants, Except Legal, Medical, and Executive",
|
|
596
|
+
},
|
|
597
|
+
};
|
|
598
|
+
// Reverse SOC → curated key: a RAW soc that HAPPENS to equal a curated one is then
|
|
599
|
+
// also labeled (known-valid) — the honest, helpful behavior (mirrors BLS_ID_TO_KEY).
|
|
600
|
+
const BLS_SOC_TO_KEY = new Map(BLS_OEWS_OCCUPATION_KEYS.map((k) => [BLS_OEWS_OCCUPATIONS[k].soc, k]));
|
|
601
|
+
// ─── State USPS → 2-digit FIPS map (ADR §area formation) ─────────────────────
|
|
602
|
+
// The curated `area` state enum. State areaCode = 2-digit FIPS + "00000" (CA
|
|
603
|
+
// 06 → "0600000"). The 5 unused FIPS (03/07/14/43/52) are simply absent.
|
|
604
|
+
export const BLS_STATE_FIPS = {
|
|
605
|
+
AL: "01", AK: "02", AZ: "04", AR: "05", CA: "06", CO: "08", CT: "09",
|
|
606
|
+
DE: "10", DC: "11", FL: "12", GA: "13", HI: "15", ID: "16", IL: "17",
|
|
607
|
+
IN: "18", IA: "19", KS: "20", KY: "21", LA: "22", ME: "23", MD: "24",
|
|
608
|
+
MA: "25", MI: "26", MN: "27", MS: "28", MO: "29", MT: "30", NE: "31",
|
|
609
|
+
NV: "32", NH: "33", NJ: "34", NM: "35", NY: "36", NC: "37", ND: "38",
|
|
610
|
+
OH: "39", OK: "40", OR: "41", PA: "42", RI: "44", SC: "45", SD: "46",
|
|
611
|
+
TN: "47", TX: "48", UT: "49", VT: "50", VA: "51", WA: "53", WV: "54",
|
|
612
|
+
WI: "55", WY: "56", PR: "72",
|
|
613
|
+
};
|
|
614
|
+
/** The curated `area` state enum values (the USPS 2-letter codes). */
|
|
615
|
+
export const BLS_STATE_KEYS = Object.keys(BLS_STATE_FIPS);
|
|
616
|
+
// ─── Datatype map (LIVE-verified; friendly enum → 2-digit code + units) ──────
|
|
617
|
+
// The FROZEN source of truth for the `datatype` enum, the series-ID datatype
|
|
618
|
+
// component, and the per-row measure.units label (H3). Default = annual_mean.
|
|
619
|
+
export const BLS_OEWS_DATATYPE_KEYS = [
|
|
620
|
+
"annual_mean",
|
|
621
|
+
"annual_median",
|
|
622
|
+
"hourly_mean",
|
|
623
|
+
"hourly_median",
|
|
624
|
+
"employment",
|
|
625
|
+
];
|
|
626
|
+
export const BLS_OEWS_DATATYPES = {
|
|
627
|
+
annual_mean: { code: "04", units: "dollars/year" },
|
|
628
|
+
annual_median: { code: "13", units: "dollars/year" },
|
|
629
|
+
hourly_mean: { code: "03", units: "dollars/hour" },
|
|
630
|
+
hourly_median: { code: "08", units: "dollars/hour" },
|
|
631
|
+
employment: { code: "01", units: "count (jobs)" },
|
|
632
|
+
};
|
|
633
|
+
// ─── The OEWS series-ID builder (the crux — ADR §crux, PINNED) ───────────────
|
|
634
|
+
// A 25-char ID: OE + U + areatype(N/S/M) + area(7) + industry(000000) +
|
|
635
|
+
// occupation(6) + datatype(2). Each component is validated BEFORE concat, then
|
|
636
|
+
// the assembled ID is re-asserted ^OEU[NSM][0-9]{21}$ — a wrong-width component
|
|
637
|
+
// can NEVER silently produce a DIFFERENT valid series. Pure fn (no fetch).
|
|
638
|
+
const OEWS_AREATYPES = new Set(["N", "S", "M"]);
|
|
639
|
+
const OEWS_AREA_RE = /^[0-9]{7}$/;
|
|
640
|
+
const OEWS_OCC_RE = /^[0-9]{6}$/;
|
|
641
|
+
const OEWS_CBSA_RE = /^[0-9]{5}$/;
|
|
642
|
+
const OEWS_INDUSTRY = "000000"; // cross-industry / all industries (the B2G default)
|
|
643
|
+
const OEWS_DATATYPE_CODES = new Set(["01", "03", "04", "08", "13"]);
|
|
644
|
+
const OEWS_ID_RE = /^OEU[NSM][0-9]{21}$/;
|
|
645
|
+
/** Throw a uniform invalid_input (host+path-only label — no token). */
|
|
646
|
+
function oewsInvalid(message) {
|
|
647
|
+
throw new ToolErrorCarrier({
|
|
648
|
+
kind: "invalid_input",
|
|
649
|
+
message,
|
|
650
|
+
retryable: false,
|
|
651
|
+
upstreamEndpoint: blsLabel(),
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
export function buildOewsSeriesId(parts) {
|
|
655
|
+
const { areatype, areaCode, occupation, datatype } = parts;
|
|
656
|
+
if (!OEWS_AREATYPES.has(areatype)) {
|
|
657
|
+
oewsInvalid(`Cannot build OEWS series ID: areatype ${JSON.stringify(areatype)} must be one of N (national), S (state), M (metro).`);
|
|
658
|
+
}
|
|
659
|
+
if (!OEWS_AREA_RE.test(areaCode)) {
|
|
660
|
+
oewsInvalid(`Cannot build OEWS series ID: areaCode ${JSON.stringify(areaCode)} must be exactly 7 digits (^[0-9]{7}$).`);
|
|
661
|
+
}
|
|
662
|
+
if (!OEWS_OCC_RE.test(occupation)) {
|
|
663
|
+
oewsInvalid(`Cannot build OEWS series ID: occupation ${JSON.stringify(occupation)} must be exactly 6 digits — a HYPHENLESS SOC (e.g. 151252, not 15-1252).`);
|
|
664
|
+
}
|
|
665
|
+
if (!OEWS_DATATYPE_CODES.has(datatype)) {
|
|
666
|
+
oewsInvalid(`Cannot build OEWS series ID: datatype code ${JSON.stringify(datatype)} must be one of 01/03/04/08/13.`);
|
|
667
|
+
}
|
|
668
|
+
const id = "OE" + "U" + areatype + areaCode + OEWS_INDUSTRY + occupation + datatype;
|
|
669
|
+
if (!OEWS_ID_RE.test(id)) {
|
|
670
|
+
oewsInvalid(`Cannot build OEWS series ID: assembled ID ${JSON.stringify(id)} failed the ^OEU[NSM][0-9]{21}$ assertion (defense-in-depth — a wrong-width component must never silently produce a different valid series).`);
|
|
671
|
+
}
|
|
672
|
+
return id;
|
|
673
|
+
}
|
|
674
|
+
/** Resolve one area token → areatype + zero-padded areaCode + output descriptor. */
|
|
675
|
+
function resolveOewsArea(token) {
|
|
676
|
+
if (token === "national") {
|
|
677
|
+
return { areatype: "N", areaCode: "0000000", type: "national", code: "0000000", label: "United States" };
|
|
678
|
+
}
|
|
679
|
+
const fips = BLS_STATE_FIPS[token];
|
|
680
|
+
if (fips !== undefined) {
|
|
681
|
+
const areaCode = fips + "00000"; // 2-digit FIPS + 5 zeros = 7 digits
|
|
682
|
+
return { areatype: "S", areaCode, type: "state", code: areaCode, label: token };
|
|
683
|
+
}
|
|
684
|
+
if (OEWS_CBSA_RE.test(token)) {
|
|
685
|
+
const areaCode = "00" + token; // "00" + 5-digit CBSA = 7 digits
|
|
686
|
+
return { areatype: "M", areaCode, type: "metro", code: areaCode, label: `CBSA ${token}` };
|
|
687
|
+
}
|
|
688
|
+
return oewsInvalid(`Unknown area ${JSON.stringify(token)} — expected "national", a 2-letter USPS state code (e.g. CA, TX, DC), or a 5-digit CBSA metro code (e.g. 19100). Punctuation/mistyped codes are rejected (SSRF + verify-the-input honesty).`);
|
|
689
|
+
}
|
|
690
|
+
// ─── OEWS honesty notes (H1/H3/H4 always-on; H2 per empty combo) ─────────────
|
|
691
|
+
/** (H1) The always-on annual / latest-year-only cadence disclosure. */
|
|
692
|
+
function oewsAnnualNote(refYear) {
|
|
693
|
+
const ref = refYear
|
|
694
|
+
? `reference May ${refYear}, period A01`
|
|
695
|
+
: "reference May of the latest release year, period A01";
|
|
696
|
+
return `OEWS is an ANNUAL point-in-time snapshot (${ref}); the BLS API serves ONLY the most recent release and may lag ~1 year. These are NOT monthly time-series values and NOT current-quarter figures — do not read a wage as "this month's". Historical OEWS is not in the API; it requires the downloadable OEWS tables (https://www.bls.gov/oes/tables.htm).`;
|
|
697
|
+
}
|
|
698
|
+
/** (H3) The always-on units-per-datatype caveat. */
|
|
699
|
+
const OEWS_UNITS_NOTE = "Each row's measure.units is set from its datatype: annual_mean/annual_median = dollars/year, hourly_mean/hourly_median = dollars/hour, employment = count (jobs). Never read an employment count as a wage, or an hourly rate as an annual salary — check measure.units on every row.";
|
|
700
|
+
/** (H4) The always-on top-coding informational note. */
|
|
701
|
+
const OEWS_TOPCODE_NOTE = "The BLS API returns the actual numeric estimate (no '#' top-code); very-high percentile/median wages are BLS estimates, and the published OEWS tables show values >= $115.00/hr or >= $239,200/yr as a boundary. If a '#' ever appeared it is non-numeric and maps to value:null + valueUnavailable + a footnote (never a fabricated number).";
|
|
702
|
+
/** (H2) The per-combo not-published disclosure (empty/absent built ID). */
|
|
703
|
+
function oewsNotPublishedNote(occLabel, areaLabel, measureLabel, seriesId) {
|
|
704
|
+
return `OEWS publishes no estimate for ${occLabel} in ${areaLabel} (${measureLabel}) — the occupation may not be surveyed/estimated in that area, OR the cell was suppressed for confidentiality/reliability. This is NOT a tool error; the built series ${seriesId} simply has no published value (value:null, not a fabricated 0).`;
|
|
705
|
+
}
|
|
706
|
+
/** Pick the latest observation (explicit `latest` flag, else max year). */
|
|
707
|
+
function pickLatestOewsObservation(observations) {
|
|
708
|
+
let best = null;
|
|
709
|
+
for (const o of observations) {
|
|
710
|
+
if (best === null) {
|
|
711
|
+
best = o;
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
const oy = o.year !== null ? Number(o.year) : -Infinity;
|
|
715
|
+
const by = best.year !== null ? Number(best.year) : -Infinity;
|
|
716
|
+
if (o.latest && !best.latest)
|
|
717
|
+
best = o;
|
|
718
|
+
else if (o.latest === best.latest && oy > by)
|
|
719
|
+
best = o;
|
|
720
|
+
}
|
|
721
|
+
return best;
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* OEWS occupational wage benchmarking — build validated 25-char series IDs from
|
|
725
|
+
* structured (area × occupation × datatype) inputs, batch them into ONE POST
|
|
726
|
+
* (REUSING the bls_timeseries transport/parse/honesty layer), and return one
|
|
727
|
+
* normalized wage/employment row per resolved combo + honest _meta. NO year
|
|
728
|
+
* input (OEWS serves only the latest release; the tool requests a small recent
|
|
729
|
+
* window internally and discloses the reference year from the returned A01 row).
|
|
730
|
+
*/
|
|
731
|
+
export async function oewsWages(args) {
|
|
732
|
+
const label = blsLabel();
|
|
733
|
+
const caps = tierCaps();
|
|
734
|
+
// ── Resolve occupations (curated keys + raw socs), preserving order. ──
|
|
735
|
+
const occupations = [];
|
|
736
|
+
for (const k of args.occupation ?? []) {
|
|
737
|
+
const entry = BLS_OEWS_OCCUPATIONS[k];
|
|
738
|
+
if (!entry) {
|
|
739
|
+
oewsInvalid(`Unknown occupation key ${JSON.stringify(k)} — expected one of: ${BLS_OEWS_OCCUPATION_KEYS.join(", ")}, or pass a raw 6-digit SOC via 'soc'.`);
|
|
740
|
+
}
|
|
741
|
+
occupations.push({ soc: entry.soc, key: k, label: entry.label });
|
|
742
|
+
}
|
|
743
|
+
for (const raw of args.soc ?? []) {
|
|
744
|
+
if (typeof raw !== "string" || !OEWS_OCC_RE.test(raw)) {
|
|
745
|
+
oewsInvalid(`Invalid soc ${JSON.stringify(raw)} — expected exactly 6 digits with NO hyphen (use 151252, not 15-1252). The curated 'occupation' enum is typo-proof; 'soc' is the long-tail passthrough.`);
|
|
746
|
+
}
|
|
747
|
+
const key = BLS_SOC_TO_KEY.get(raw) ?? null;
|
|
748
|
+
occupations.push({ soc: raw, key, label: key ? BLS_OEWS_OCCUPATIONS[key].label : null });
|
|
749
|
+
}
|
|
750
|
+
// ── At least one of occupation/soc is required (P4 — never a silent no-op). ──
|
|
751
|
+
if (occupations.length === 0) {
|
|
752
|
+
oewsInvalid("At least one of `occupation` (a curated enum key) or `soc` (a raw 6-digit SOC) is required — nothing to query.");
|
|
753
|
+
}
|
|
754
|
+
// ── Resolve areas (default national) + datatypes (default annual_mean). An
|
|
755
|
+
// empty array falls back to the default (never a silent 0-row no-op). ──
|
|
756
|
+
const areaTokens = args.area && args.area.length > 0 ? args.area : ["national"];
|
|
757
|
+
const areas = areaTokens.map((t) => resolveOewsArea(String(t)));
|
|
758
|
+
const datatypeKeys = args.datatype && args.datatype.length > 0 ? args.datatype : ["annual_mean"];
|
|
759
|
+
const datatypes = datatypeKeys.map((k) => {
|
|
760
|
+
const entry = BLS_OEWS_DATATYPES[k];
|
|
761
|
+
if (!entry) {
|
|
762
|
+
oewsInvalid(`Unknown datatype ${JSON.stringify(k)} — expected one of: ${BLS_OEWS_DATATYPE_KEYS.join(", ")}.`);
|
|
763
|
+
}
|
|
764
|
+
return { key: k, code: entry.code, units: entry.units };
|
|
765
|
+
});
|
|
766
|
+
// ── Refuse over the tier's series cap — NEVER a silent drop of the overflow
|
|
767
|
+
// (P4). The count is the cartesian product size, named explicitly. ──
|
|
768
|
+
const product = areas.length * occupations.length * datatypes.length;
|
|
769
|
+
if (product > caps.seriesCap) {
|
|
770
|
+
oewsInvalid(`Requested ${areas.length} area(s) × ${occupations.length} occupation(s) × ${datatypes.length} datatype(s) = ${product} series, over the active BLS ${keyModeLabel()} tier cap of ${caps.seriesCap} series/query. Reduce the request (or set a free BLS_API_KEY for the v2 tier, ${V2_CAPS.seriesCap} series/query) — the overflow is NOT silently dropped.`);
|
|
771
|
+
}
|
|
772
|
+
const planned = [];
|
|
773
|
+
const seen = new Set();
|
|
774
|
+
for (const area of areas) {
|
|
775
|
+
for (const occ of occupations) {
|
|
776
|
+
for (const dt of datatypes) {
|
|
777
|
+
const seriesId = buildOewsSeriesId({
|
|
778
|
+
areatype: area.areatype,
|
|
779
|
+
areaCode: area.areaCode,
|
|
780
|
+
occupation: occ.soc,
|
|
781
|
+
datatype: dt.code,
|
|
782
|
+
});
|
|
783
|
+
if (seen.has(seriesId))
|
|
784
|
+
continue;
|
|
785
|
+
seen.add(seriesId);
|
|
786
|
+
planned.push({ area, occ, dt, seriesId });
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
// ── The recent-window years (NO caller year input; OEWS serves only the
|
|
791
|
+
// latest release — a 3-year window covers the ~1-year lag). ──
|
|
792
|
+
const endYear = CURRENT_YEAR;
|
|
793
|
+
const startYear = CURRENT_YEAR - 2;
|
|
794
|
+
// ── Module-built typed payload (SSRF: no raw host/path passthrough; the built
|
|
795
|
+
// seriesIDs + years + the OPTIONAL key ride in the JSON body). ──
|
|
796
|
+
const payload = {
|
|
797
|
+
seriesid: planned.map((p) => p.seriesId),
|
|
798
|
+
startyear: String(startYear),
|
|
799
|
+
endyear: String(endYear),
|
|
800
|
+
};
|
|
801
|
+
// The OPTIONAL BLS_API_KEY — injected ONLY here, in the body, and NOWHERE else.
|
|
802
|
+
const key = resolvedBlsKey();
|
|
803
|
+
if (key !== "")
|
|
804
|
+
payload.registrationkey = key;
|
|
805
|
+
// ── SSRF belt-and-suspenders: the URL is a compile-time constant; assert it
|
|
806
|
+
// cannot have drifted (a future typo / downgrade). ──
|
|
807
|
+
const url = `https://${BLS_HOST}${apiPath()}`;
|
|
808
|
+
const built = new URL(url);
|
|
809
|
+
if (built.hostname !== BLS_HOST || built.protocol !== "https:") {
|
|
810
|
+
oewsInvalid(`Constructed BLS URL host ${JSON.stringify(built.hostname)} (${built.protocol}) is not ${BLS_HOST} over https — refusing to fetch (SSRF safety).`);
|
|
811
|
+
}
|
|
812
|
+
// ── The POST-batch fetch through the shared self-throttle gate (the SAME idiom
|
|
813
|
+
// as timeseries). A non-JSON 200 → driftError; the status/error taxonomy +
|
|
814
|
+
// redirect:"error" propagate unchanged. ──
|
|
815
|
+
let body;
|
|
816
|
+
try {
|
|
817
|
+
body = await throughGate(BLS_GATE_KEY, BLS_MIN_INTERVAL_MS, () => getJson(built.toString(), {
|
|
818
|
+
label,
|
|
819
|
+
method: "POST",
|
|
820
|
+
headers: { "Content-Type": "application/json" },
|
|
821
|
+
body: JSON.stringify(payload),
|
|
822
|
+
redirect: "error",
|
|
823
|
+
}));
|
|
824
|
+
}
|
|
825
|
+
catch (e) {
|
|
826
|
+
if (e instanceof ToolErrorCarrier)
|
|
827
|
+
throw e;
|
|
828
|
+
if (e instanceof SyntaxError) {
|
|
829
|
+
throw driftError(label, "BLS returned a non-JSON body at HTTP 200 (an HTML error page) — treating as schema drift (never read as an empty result).");
|
|
830
|
+
}
|
|
831
|
+
throw e;
|
|
832
|
+
}
|
|
833
|
+
const parsed = parseBlsBody(body, label);
|
|
834
|
+
// ── Assemble one row per planned combo; account for EVERY combo (P4). ──
|
|
835
|
+
const rows = [];
|
|
836
|
+
const fieldsUnavailable = [];
|
|
837
|
+
const notPublishedNotes = [];
|
|
838
|
+
let refYear = null;
|
|
839
|
+
for (const p of planned) {
|
|
840
|
+
const rawSeries = parsed.byId.get(p.seriesId);
|
|
841
|
+
const rawData = rawSeries && Array.isArray(rawSeries.data) ? rawSeries.data : [];
|
|
842
|
+
const observations = rawData.map(mapObservation);
|
|
843
|
+
const latest = pickLatestOewsObservation(observations);
|
|
844
|
+
const areaOut = { type: p.area.type, code: p.area.code, label: p.area.label };
|
|
845
|
+
const occOut = { soc: p.occ.soc, key: p.occ.key, label: p.occ.label };
|
|
846
|
+
const measureOut = { key: p.dt.key, code: p.dt.code, units: p.dt.units };
|
|
847
|
+
if (latest === null) {
|
|
848
|
+
// (H2) empty data[] OR absent from Results.series → NOT-PUBLISHED (absent,
|
|
849
|
+
// not a "-" gap): value:null, valueUnavailable:FALSE, never a fabricated 0.
|
|
850
|
+
fieldsUnavailable.push(`${p.seriesId} (OEWS publishes no estimate)`);
|
|
851
|
+
notPublishedNotes.push(oewsNotPublishedNote(p.occ.label ?? p.occ.soc, p.area.label, `${p.dt.key} (${p.dt.units})`, p.seriesId));
|
|
852
|
+
rows.push({
|
|
853
|
+
area: areaOut,
|
|
854
|
+
occupation: occOut,
|
|
855
|
+
measure: measureOut,
|
|
856
|
+
value: null,
|
|
857
|
+
valueUnavailable: false,
|
|
858
|
+
referenceYear: null,
|
|
859
|
+
referencePeriod: null,
|
|
860
|
+
footnotes: [],
|
|
861
|
+
seriesId: p.seriesId,
|
|
862
|
+
});
|
|
863
|
+
continue;
|
|
864
|
+
}
|
|
865
|
+
// Present-with-data → the reused mapObservation path (null-never-0 +
|
|
866
|
+
// valueUnavailable + footnotes for an in-band "-"; a real number otherwise).
|
|
867
|
+
if (latest.year !== null && (refYear === null || Number(latest.year) > Number(refYear))) {
|
|
868
|
+
refYear = latest.year;
|
|
869
|
+
}
|
|
870
|
+
rows.push({
|
|
871
|
+
area: areaOut,
|
|
872
|
+
occupation: occOut,
|
|
873
|
+
measure: measureOut,
|
|
874
|
+
value: latest.value,
|
|
875
|
+
valueUnavailable: latest.valueUnavailable,
|
|
876
|
+
referenceYear: latest.year,
|
|
877
|
+
referencePeriod: latest.period,
|
|
878
|
+
footnotes: latest.footnotes,
|
|
879
|
+
seriesId: p.seriesId,
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
// ── Notes: H1 + H3 + H4 always; the tier disclosure (reused, P1); H2 per empty
|
|
883
|
+
// combo; any surfaced top-level BLS message. ──
|
|
884
|
+
const notes = [];
|
|
885
|
+
notes.push(oewsAnnualNote(refYear));
|
|
886
|
+
notes.push(OEWS_UNITS_NOTE);
|
|
887
|
+
notes.push(OEWS_TOPCODE_NOTE);
|
|
888
|
+
notes.push(tierNote());
|
|
889
|
+
notes.push(...notPublishedNotes);
|
|
890
|
+
if (parsed.messages.length > 0) {
|
|
891
|
+
notes.push(`BLS returned top-level message(s): ${parsed.messages.join(" | ")}.`);
|
|
892
|
+
}
|
|
893
|
+
const filtersApplied = [
|
|
894
|
+
`areas:${areas.map((a) => a.code).join(",")}`,
|
|
895
|
+
`occupations:${occupations.map((o) => o.soc).join(",")}`,
|
|
896
|
+
`datatypes:${datatypes.map((d) => `${d.key}=${d.code}`).join(",")}`,
|
|
897
|
+
`years:${startYear}-${endYear}`,
|
|
898
|
+
];
|
|
899
|
+
const metaOut = {
|
|
900
|
+
source: blsSource(),
|
|
901
|
+
keylessMode: usingKeylessV1(),
|
|
902
|
+
returned: rows.length,
|
|
903
|
+
totalAvailable: rows.length,
|
|
904
|
+
filtersApplied,
|
|
905
|
+
filtersDropped: [],
|
|
906
|
+
fieldsUnavailable,
|
|
907
|
+
notes,
|
|
908
|
+
};
|
|
909
|
+
return withMeta({ results: rows }, metaOut);
|
|
910
|
+
}
|
|
911
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
912
|
+
// QCEW — Quarterly Census of Employment & Wages (the 3rd tool, ADR-0042)
|
|
913
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
914
|
+
// A THIRD tool on the BLS provider — but a SECOND, DIFFERENT, keyless,
|
|
915
|
+
// un-rate-limited BLS DOMAIN: the QCEW Open Data Access CSV files on
|
|
916
|
+
// data.bls.gov/cew (NOT the rate-limited api.bls.gov/publicAPI timeseries API the
|
|
917
|
+
// two tools above share). It answers the market-size / competition-density
|
|
918
|
+
// question no existing tool can: for a county (area_fips) or a NAICS industry ×
|
|
919
|
+
// quarter — establishment COUNT (market size / competitor density), county×NAICS
|
|
920
|
+
// employment, average weekly wage (labor cost), and the LOCATION QUOTIENT
|
|
921
|
+
// (concentration vs the national average = competition density).
|
|
922
|
+
//
|
|
923
|
+
// This path DELIBERATELY does NOT touch the api.bls.gov key seam
|
|
924
|
+
// (resolvedBlsKey/apiPath/blsSource/BLS_GATE_KEY="bls"): QCEW is keyless, a NEW
|
|
925
|
+
// host, and uses a NEW self-throttle gate key ("bls_qcew") so it never serializes
|
|
926
|
+
// behind — or shares the ~25/day budget of — the timeseries tools. NO BLS_API_KEY
|
|
927
|
+
// is read here.
|
|
928
|
+
//
|
|
929
|
+
// ★ THE DISCLOSURE-SUPPRESSION HONESTY CRUX (P3). Each row carries THREE
|
|
930
|
+
// disclosure codes governing THREE blocks: `disclosure_code` (base),
|
|
931
|
+
// `lq_disclosure_code` (lq), `oty_disclosure_code` (oty). QCEW encodes a
|
|
932
|
+
// SUPPRESSED (confidential) employment/wage value as a literal `0`; because
|
|
933
|
+
// num("0") === 0, a naive map would surface a withheld field as a real "$0 wage /
|
|
934
|
+
// 0 employment" — the exact data-absence-as-zero masquerade the project forbids
|
|
935
|
+
// (the FDIC-CBLR-sentinel lesson applied to a CSV). The fix is a BLOCK-scoped,
|
|
936
|
+
// CODE-scoped, FIELD-specific mapper (NEVER a blanket 0→null): under 'N' the
|
|
937
|
+
// confidential emplvl/wage/avg-wkly fields → null while the establishment COUNT
|
|
938
|
+
// (qtrly_estabs / lq_qtrly_estabs) — AND the over-the-year establishment CHANGE
|
|
939
|
+
// (oty_qtrly_estabs_chg / _pct_chg) — stay DISCLOSED (real, live-confirmed real in
|
|
940
|
+
// 167/526 'N' rows); under '-' (or any other non-blank) the WHOLE block incl. the
|
|
941
|
+
// estabs field(s) → null; under blank a genuine reported/negative `0` SURVIVES
|
|
942
|
+
// (the disclosed federal taxable=0/contrib=0 and the oty_*_chg=0 "no change").
|
|
943
|
+
// ─── Fixed endpoint + transport constants (SSRF core — compile-time CONSTANTS) ──
|
|
944
|
+
const QCEW_HOST = "data.bls.gov";
|
|
945
|
+
/** A NEW self-throttle gate key — DELIBERATELY NOT "bls" (a different, un-rate-
|
|
946
|
+
* limited host; QCEW must not share the api.bls.gov ~25/day budget or serialize
|
|
947
|
+
* behind the timeseries tools). */
|
|
948
|
+
const QCEW_GATE_KEY = "bls_qcew";
|
|
949
|
+
const QCEW_MIN_INTERVAL_MS = 250;
|
|
950
|
+
/** Host-only error/endpoint label (never a token — QCEW is keyless anyway). */
|
|
951
|
+
const QCEW_LABEL = "data.bls.gov/cew";
|
|
952
|
+
/** Browser-ish UA — data.bls.gov/cew serves the keyless CSV to a normal client. */
|
|
953
|
+
const QCEW_UA = "Mozilla/5.0 (compatible; @cliwant/mcp-sam-gov; +https://github.com/cliwant/mcp-sam-gov)";
|
|
954
|
+
const QCEW_FETCH_TIMEOUT_MS = 30_000;
|
|
955
|
+
/**
|
|
956
|
+
* Hard streamed read cap. The largest single slice observed is 3.84 MB
|
|
957
|
+
* (industry/10.csv, all industries × all ~3,800 areas); a detailed-NAICS slice is
|
|
958
|
+
* ~664 KB, an area slice ~100 KB–a few MB. 16 MB clears the max with ~4× headroom
|
|
959
|
+
* AND catches a drifted giant. Applied to BOTH the declared content-length
|
|
960
|
+
* (pre-check) AND the streamed read (abort past this bound) — never content-length
|
|
961
|
+
* alone (ADR-0042 §S3, the OFAC readCappedBody replication).
|
|
962
|
+
*/
|
|
963
|
+
const MAX_QCEW_BYTES = 16 * 1024 * 1024;
|
|
964
|
+
/** QCEW Open Data coverage floor (a pre-coverage year is an honest per-tuple 404). */
|
|
965
|
+
const QCEW_YEAR_MIN = 1990;
|
|
966
|
+
/** The 42 PINNED column names, in exact order (ADR-0042 fact 2, live-verified). */
|
|
967
|
+
export const QCEW_COLUMNS = [
|
|
968
|
+
"area_fips",
|
|
969
|
+
"own_code",
|
|
970
|
+
"industry_code",
|
|
971
|
+
"agglvl_code",
|
|
972
|
+
"size_code",
|
|
973
|
+
"year",
|
|
974
|
+
"qtr",
|
|
975
|
+
"disclosure_code",
|
|
976
|
+
"qtrly_estabs",
|
|
977
|
+
"month1_emplvl",
|
|
978
|
+
"month2_emplvl",
|
|
979
|
+
"month3_emplvl",
|
|
980
|
+
"total_qtrly_wages",
|
|
981
|
+
"taxable_qtrly_wages",
|
|
982
|
+
"qtrly_contributions",
|
|
983
|
+
"avg_wkly_wage",
|
|
984
|
+
"lq_disclosure_code",
|
|
985
|
+
"lq_qtrly_estabs",
|
|
986
|
+
"lq_month1_emplvl",
|
|
987
|
+
"lq_month2_emplvl",
|
|
988
|
+
"lq_month3_emplvl",
|
|
989
|
+
"lq_total_qtrly_wages",
|
|
990
|
+
"lq_taxable_qtrly_wages",
|
|
991
|
+
"lq_qtrly_contributions",
|
|
992
|
+
"lq_avg_wkly_wage",
|
|
993
|
+
"oty_disclosure_code",
|
|
994
|
+
"oty_qtrly_estabs_chg",
|
|
995
|
+
"oty_qtrly_estabs_pct_chg",
|
|
996
|
+
"oty_month1_emplvl_chg",
|
|
997
|
+
"oty_month1_emplvl_pct_chg",
|
|
998
|
+
"oty_month2_emplvl_chg",
|
|
999
|
+
"oty_month2_emplvl_pct_chg",
|
|
1000
|
+
"oty_month3_emplvl_chg",
|
|
1001
|
+
"oty_month3_emplvl_pct_chg",
|
|
1002
|
+
"oty_total_qtrly_wages_chg",
|
|
1003
|
+
"oty_total_qtrly_wages_pct_chg",
|
|
1004
|
+
"oty_taxable_qtrly_wages_chg",
|
|
1005
|
+
"oty_taxable_qtrly_wages_pct_chg",
|
|
1006
|
+
"oty_qtrly_contributions_chg",
|
|
1007
|
+
"oty_qtrly_contributions_pct_chg",
|
|
1008
|
+
"oty_avg_wkly_wage_chg",
|
|
1009
|
+
"oty_avg_wkly_wage_pct_chg",
|
|
1010
|
+
];
|
|
1011
|
+
/** The expected column count. Passed as `maxCol` to parseRecordFields so a
|
|
1012
|
+
* too-MANY-columns row materializes 43 fields and trips the drift check (the
|
|
1013
|
+
* OFAC `maxCol=cols` SYMMETRY — `cols-1` would silently cap a column ADDITION). */
|
|
1014
|
+
const QCEW_COLS = QCEW_COLUMNS.length; // 42
|
|
1015
|
+
// ─── SSRF path-segment charclasses (validate PRE-interpolation) ──────────────
|
|
1016
|
+
const QCEW_MODES = new Set(["area", "industry"]);
|
|
1017
|
+
/** area_fips: county 01005, statewide 01000, national US000, MSA C1018, CSA
|
|
1018
|
+
* CS122 — letter prefixes exist, so alphanumeric. Rejects `/` `.` `..` `%2F`
|
|
1019
|
+
* `%2E` `%00` `@host` whitespace newline. */
|
|
1020
|
+
const QCEW_AREA_RE = /^[0-9A-Za-z]{1,6}$/;
|
|
1021
|
+
/** industry NAICS: STRICTLY digit-only. A hyphenated NAICS supersector
|
|
1022
|
+
* (Manufacturing 31-33, Retail 44-45) 404s live (`industry/31-33.csv` → HTTP
|
|
1023
|
+
* 404) — a hyphen never resolves AND widens the SSRF charclass with a
|
|
1024
|
+
* non-alphanumeric, so it is REJECTED (invalid_input pointing at the digit
|
|
1025
|
+
* aggregate code), never silently stripped. */
|
|
1026
|
+
const QCEW_INDUSTRY_RE = /^[0-9]{1,6}$/;
|
|
1027
|
+
const QCEW_YEAR_RE = /^\d{4}$/;
|
|
1028
|
+
/** quarter: ship 1-4 (all live-confirmed). The annual `a` is UNVERIFIED this
|
|
1029
|
+
* cycle — NOT enabled (a live `…/a/…` 200 HEAD + a <16 MB size check must land
|
|
1030
|
+
* first); the charclass would permit it safely via `^([1-4]|a)$` when enabled. */
|
|
1031
|
+
const QCEW_QUARTER_RE = /^[1-4]$/;
|
|
1032
|
+
/** Uniform invalid_input (host-only label, no token — QCEW is keyless). */
|
|
1033
|
+
function qcewInvalid(message) {
|
|
1034
|
+
throw new ToolErrorCarrier({
|
|
1035
|
+
kind: "invalid_input",
|
|
1036
|
+
message,
|
|
1037
|
+
retryable: false,
|
|
1038
|
+
upstreamEndpoint: QCEW_LABEL,
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* ★ Build + validate the QCEW slice URL. Each caller-influenced path segment is
|
|
1043
|
+
* charclass-validated BEFORE interpolation; the host is a compile-time constant;
|
|
1044
|
+
* a post-construction `new URL` host/protocol assert locks it (the per-segment
|
|
1045
|
+
* charclass is the real guard — the hostname check alone does not stop a same-host
|
|
1046
|
+
* `../`). The client-side filters (ownership/aggregationLevel/sizeCode/narrow)
|
|
1047
|
+
* NEVER touch the URL — no SSRF surface. Pure fn (no fetch).
|
|
1048
|
+
*/
|
|
1049
|
+
export function buildQcewUrl(mode, year, quarter, code) {
|
|
1050
|
+
if (!QCEW_MODES.has(mode)) {
|
|
1051
|
+
qcewInvalid(`Cannot build QCEW URL: mode ${JSON.stringify(mode)} must be one of area, industry.`);
|
|
1052
|
+
}
|
|
1053
|
+
const y = String(year);
|
|
1054
|
+
if (!QCEW_YEAR_RE.test(y)) {
|
|
1055
|
+
qcewInvalid(`Cannot build QCEW URL: year ${JSON.stringify(y)} must be exactly 4 digits (^\\d{4}$).`);
|
|
1056
|
+
}
|
|
1057
|
+
const yr = Number(y);
|
|
1058
|
+
if (yr < QCEW_YEAR_MIN || yr > CURRENT_YEAR) {
|
|
1059
|
+
qcewInvalid(`Cannot build QCEW URL: year ${yr} is out of range ${QCEW_YEAR_MIN}..${CURRENT_YEAR} (a pre-coverage or future year is an absent slice; QCEW Open Data begins ~${QCEW_YEAR_MIN}).`);
|
|
1060
|
+
}
|
|
1061
|
+
const q = String(quarter);
|
|
1062
|
+
if (!QCEW_QUARTER_RE.test(q)) {
|
|
1063
|
+
qcewInvalid(`Cannot build QCEW URL: quarter ${JSON.stringify(q)} must be one of 1, 2, 3, 4 (the annual 'a' is not enabled this build).`);
|
|
1064
|
+
}
|
|
1065
|
+
if (mode === "area") {
|
|
1066
|
+
if (!QCEW_AREA_RE.test(code)) {
|
|
1067
|
+
qcewInvalid(`Cannot build QCEW URL: area ${JSON.stringify(code)} must be 1..6 alphanumeric chars (^[0-9A-Za-z]{1,6}$) — an area_fips like 01005 (county), 01000 (statewide), US000 (national), C1018 (MSA). Slashes/dots/encoded traversal are rejected (SSRF).`);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
else {
|
|
1071
|
+
if (!QCEW_INDUSTRY_RE.test(code)) {
|
|
1072
|
+
// Digit-only: a hyphenated NAICS supersector (e.g. 31-33, 44-45) 404s live
|
|
1073
|
+
// (industry/31-33.csv → HTTP 404) — pass the digit aggregate code instead.
|
|
1074
|
+
qcewInvalid(`Cannot build QCEW URL: industry ${JSON.stringify(code)} must be 1..6 DIGITS (^[0-9]{1,6}$) — a NAICS code like 5415 or the aggregate 10. A hyphenated NAICS supersector (31-33, 44-45) 404s on QCEW; use its digit aggregate code, not the hyphenated form.`);
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
const url = `https://${QCEW_HOST}/cew/data/api/${y}/${q}/${mode}/${code}.csv`;
|
|
1078
|
+
const built = new URL(url);
|
|
1079
|
+
if (built.hostname !== QCEW_HOST || built.protocol !== "https:") {
|
|
1080
|
+
qcewInvalid(`Constructed QCEW URL host ${JSON.stringify(built.hostname)} (${built.protocol}) is not ${QCEW_HOST} over https — refusing to fetch (SSRF safety).`);
|
|
1081
|
+
}
|
|
1082
|
+
return built.toString();
|
|
1083
|
+
}
|
|
1084
|
+
// ─── Bounded streamed read (OFAC readCappedBody replication — ADR-0042 §S3) ──
|
|
1085
|
+
/** Concatenate streamed chunks into one Uint8Array. */
|
|
1086
|
+
function concatQcewChunks(chunks, total) {
|
|
1087
|
+
const out = new Uint8Array(total);
|
|
1088
|
+
let off = 0;
|
|
1089
|
+
for (const c of chunks) {
|
|
1090
|
+
out.set(c, off);
|
|
1091
|
+
off += c.byteLength;
|
|
1092
|
+
}
|
|
1093
|
+
return out;
|
|
1094
|
+
}
|
|
1095
|
+
/**
|
|
1096
|
+
* Read the response body with a HARD byte cap — abort past `maxBytes` rather than
|
|
1097
|
+
* trusting content-length alone. Streams via `res.body` when available (real
|
|
1098
|
+
* data.bls.gov), else falls back to `arrayBuffer()` + a post-read cap (the offline
|
|
1099
|
+
* fetch-mock, which exposes no stream). An over-cap read is a distinct honest
|
|
1100
|
+
* THROW, never a truncated body handed to the parser. Byte-for-byte the OFAC
|
|
1101
|
+
* idiom (replicated here — NO ofac.ts edit).
|
|
1102
|
+
*/
|
|
1103
|
+
async function readCappedQcewBody(res, maxBytes, label) {
|
|
1104
|
+
const tooBig = () => new ToolErrorCarrier({
|
|
1105
|
+
kind: "invalid_input",
|
|
1106
|
+
message: `QCEW ${label} body exceeded the ${Math.round(maxBytes / 1048576)} MB read cap — refusing to buffer it (a drifted giant, not the ≤~3.84 MB slices). Narrow by area/agglvl or verify the pinned endpoint.`,
|
|
1107
|
+
retryable: false,
|
|
1108
|
+
upstreamEndpoint: label,
|
|
1109
|
+
});
|
|
1110
|
+
const body = res.body;
|
|
1111
|
+
if (body && typeof body.getReader === "function") {
|
|
1112
|
+
const reader = body.getReader();
|
|
1113
|
+
const chunks = [];
|
|
1114
|
+
let total = 0;
|
|
1115
|
+
for (;;) {
|
|
1116
|
+
const { done, value } = await reader.read();
|
|
1117
|
+
if (done)
|
|
1118
|
+
break;
|
|
1119
|
+
if (value) {
|
|
1120
|
+
total += value.byteLength;
|
|
1121
|
+
if (total > maxBytes) {
|
|
1122
|
+
try {
|
|
1123
|
+
await reader.cancel();
|
|
1124
|
+
}
|
|
1125
|
+
catch {
|
|
1126
|
+
/* ignore */
|
|
1127
|
+
}
|
|
1128
|
+
throw tooBig();
|
|
1129
|
+
}
|
|
1130
|
+
chunks.push(value);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
return concatQcewChunks(chunks, total);
|
|
1134
|
+
}
|
|
1135
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
1136
|
+
if (buf.byteLength > maxBytes)
|
|
1137
|
+
throw tooBig();
|
|
1138
|
+
return buf;
|
|
1139
|
+
}
|
|
1140
|
+
// ─── RFC-4180 record assembler (replicated from ofac.ts/gsa-csv.ts — no edit) ──
|
|
1141
|
+
/** Strip a trailing SUB (0x1A) + terminal newlines BEFORE record assembly. */
|
|
1142
|
+
function stripTrailingSubQcew(body) {
|
|
1143
|
+
return body.replace(/[\r\n\x1a]+$/g, "");
|
|
1144
|
+
}
|
|
1145
|
+
/** Is a parsed record an empty (whitespace-only) non-content row? */
|
|
1146
|
+
function isEmptyQcewRecord(fields) {
|
|
1147
|
+
return fields.every((f) => f.trim() === "");
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* Assemble physical lines into LOGICAL CSV records via `parseRecordFields`
|
|
1151
|
+
* (gsa-csv), correctly re-joining a record whose quoted field contains a newline
|
|
1152
|
+
* (defense-in-depth — QCEW rows carry no embedded newlines observed). `maxCol =
|
|
1153
|
+
* QCEW_COLS` so a too-many-columns row materializes 43 fields and trips the drift
|
|
1154
|
+
* check (the OFAC symmetry). Empty rows (the trailing newline) are skipped.
|
|
1155
|
+
*/
|
|
1156
|
+
function assembleQcewRecords(body) {
|
|
1157
|
+
const text = stripTrailingSubQcew(body);
|
|
1158
|
+
const lines = text.split("\n");
|
|
1159
|
+
const records = [];
|
|
1160
|
+
let pending = null;
|
|
1161
|
+
for (const line of lines) {
|
|
1162
|
+
const candidate = pending === null ? line : pending + "\n" + line;
|
|
1163
|
+
const res = parseRecordFields(candidate, QCEW_COLS);
|
|
1164
|
+
if (res.inQuotes) {
|
|
1165
|
+
pending = candidate;
|
|
1166
|
+
continue;
|
|
1167
|
+
}
|
|
1168
|
+
pending = null;
|
|
1169
|
+
if (isEmptyQcewRecord(res.fields))
|
|
1170
|
+
continue;
|
|
1171
|
+
records.push(res.fields);
|
|
1172
|
+
}
|
|
1173
|
+
if (pending !== null) {
|
|
1174
|
+
const res = parseRecordFields(pending, QCEW_COLS);
|
|
1175
|
+
if (!isEmptyQcewRecord(res.fields))
|
|
1176
|
+
records.push(res.fields);
|
|
1177
|
+
}
|
|
1178
|
+
return records;
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* ★ M2 — the header drift assertion runs POST-parse (on the quote-STRIPPED
|
|
1182
|
+
* record). The live QCEW header is FULLY double-quoted (`"area_fips","own_code",…`);
|
|
1183
|
+
* `parseRecordFields` strips the RFC-4180 quotes, so the parsed first record is the
|
|
1184
|
+
* 42 UNQUOTED names — compared here against the pinned array, NEVER the raw
|
|
1185
|
+
* pre-parse line / split(","). A missing / renamed / added / removed / reordered
|
|
1186
|
+
* header → schema_drift (a shifted schema must NEVER be read positionally — that
|
|
1187
|
+
* would map avg_wkly_wage values under total_qtrly_wages).
|
|
1188
|
+
*/
|
|
1189
|
+
function assertQcewHeader(headerFields, label) {
|
|
1190
|
+
const drift = headerFields.length !== QCEW_COLS ||
|
|
1191
|
+
QCEW_COLUMNS.some((name, i) => headerFields[i] !== name);
|
|
1192
|
+
if (drift) {
|
|
1193
|
+
throw driftError(label, `QCEW header drifted — expected the pinned ${QCEW_COLS} columns in order but got ${headerFields.length} column(s): [${headerFields.slice(0, 45).join(", ")}]. Refusing to read a renamed/added/removed/reordered schema positionally (a shifted header would map wage values under the wrong field).`);
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* BASE block (rec cols 7–15), governed by `disclosure_code` (rec[7]).
|
|
1198
|
+
* - blank (disclosed) → EVERY field via num (a genuine reported 0 SURVIVES).
|
|
1199
|
+
* - 'N' → qtrly_estabs REAL (via num); the 6 emplvl/wage/avg-wkly fields → null.
|
|
1200
|
+
* - '-' or any OTHER non-blank → the WHOLE base block incl qtrly_estabs → null.
|
|
1201
|
+
*/
|
|
1202
|
+
function mapBaseBlock(rec) {
|
|
1203
|
+
const disc = str(rec[7]);
|
|
1204
|
+
const disclosed = disc === null;
|
|
1205
|
+
// The establishment COUNT is non-confidential: disclosed under blank OR 'N',
|
|
1206
|
+
// but withheld under '-'/other (the whole block goes null).
|
|
1207
|
+
const estabsDisclosed = disclosed || disc === "N";
|
|
1208
|
+
return {
|
|
1209
|
+
disclosed,
|
|
1210
|
+
disclosureCode: disc,
|
|
1211
|
+
qtrly_estabs: estabsDisclosed ? num(rec[8]) : null,
|
|
1212
|
+
month1_emplvl: disclosed ? num(rec[9]) : null,
|
|
1213
|
+
month2_emplvl: disclosed ? num(rec[10]) : null,
|
|
1214
|
+
month3_emplvl: disclosed ? num(rec[11]) : null,
|
|
1215
|
+
total_qtrly_wages: disclosed ? num(rec[12]) : null,
|
|
1216
|
+
taxable_qtrly_wages: disclosed ? num(rec[13]) : null,
|
|
1217
|
+
qtrly_contributions: disclosed ? num(rec[14]) : null,
|
|
1218
|
+
avg_wkly_wage: disclosed ? num(rec[15]) : null,
|
|
1219
|
+
};
|
|
1220
|
+
}
|
|
1221
|
+
/**
|
|
1222
|
+
* LQ block (rec cols 16–24, ratios), governed by `lq_disclosure_code` (rec[16]).
|
|
1223
|
+
* IDENTICAL field-specific rule as the base block: under 'N' lq_qtrly_estabs is
|
|
1224
|
+
* DISCLOSED (a real ratio) and the rest → null; under '-'/other → whole block
|
|
1225
|
+
* null; under blank → all via num.
|
|
1226
|
+
*/
|
|
1227
|
+
function mapLqBlock(rec) {
|
|
1228
|
+
const disc = str(rec[16]);
|
|
1229
|
+
const disclosed = disc === null;
|
|
1230
|
+
const estabsDisclosed = disclosed || disc === "N";
|
|
1231
|
+
return {
|
|
1232
|
+
disclosed,
|
|
1233
|
+
disclosureCode: disc,
|
|
1234
|
+
lq_qtrly_estabs: estabsDisclosed ? num(rec[17]) : null,
|
|
1235
|
+
lq_month1_emplvl: disclosed ? num(rec[18]) : null,
|
|
1236
|
+
lq_month2_emplvl: disclosed ? num(rec[19]) : null,
|
|
1237
|
+
lq_month3_emplvl: disclosed ? num(rec[20]) : null,
|
|
1238
|
+
lq_total_qtrly_wages: disclosed ? num(rec[21]) : null,
|
|
1239
|
+
lq_taxable_qtrly_wages: disclosed ? num(rec[22]) : null,
|
|
1240
|
+
lq_qtrly_contributions: disclosed ? num(rec[23]) : null,
|
|
1241
|
+
lq_avg_wkly_wage: disclosed ? num(rec[24]) : null,
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
/**
|
|
1245
|
+
* ★ M1 — OTY block (rec cols 25–41, over-the-year changes/pct-changes), governed
|
|
1246
|
+
* by `oty_disclosure_code` (rec[25]). The OTY block has the SAME establishment
|
|
1247
|
+
* exception as base/lq: under 'N', BOTH oty_qtrly_estabs_chg AND
|
|
1248
|
+
* oty_qtrly_estabs_pct_chg are DISCLOSED via num (a real value / a negative / a
|
|
1249
|
+
* genuine-0 "no change" SURVIVES — live-confirmed real in 167/526 'N' rows), while
|
|
1250
|
+
* the 14 employment/wage oty fields → null. Under '-' or any OTHER non-blank → the
|
|
1251
|
+
* WHOLE oty block INCLUDING the estabs-change pair → null (conservative; OTY '-'
|
|
1252
|
+
* was not observed — the exception is NOT extended to '-'). Under blank → all via
|
|
1253
|
+
* num so a genuine 0/negative survives.
|
|
1254
|
+
*/
|
|
1255
|
+
function mapOtyBlock(rec) {
|
|
1256
|
+
const disc = str(rec[25]);
|
|
1257
|
+
const disclosed = disc === null;
|
|
1258
|
+
const estabsChgDisclosed = disclosed || disc === "N";
|
|
1259
|
+
return {
|
|
1260
|
+
disclosed,
|
|
1261
|
+
disclosureCode: disc,
|
|
1262
|
+
oty_qtrly_estabs_chg: estabsChgDisclosed ? num(rec[26]) : null,
|
|
1263
|
+
oty_qtrly_estabs_pct_chg: estabsChgDisclosed ? num(rec[27]) : null,
|
|
1264
|
+
oty_month1_emplvl_chg: disclosed ? num(rec[28]) : null,
|
|
1265
|
+
oty_month1_emplvl_pct_chg: disclosed ? num(rec[29]) : null,
|
|
1266
|
+
oty_month2_emplvl_chg: disclosed ? num(rec[30]) : null,
|
|
1267
|
+
oty_month2_emplvl_pct_chg: disclosed ? num(rec[31]) : null,
|
|
1268
|
+
oty_month3_emplvl_chg: disclosed ? num(rec[32]) : null,
|
|
1269
|
+
oty_month3_emplvl_pct_chg: disclosed ? num(rec[33]) : null,
|
|
1270
|
+
oty_total_qtrly_wages_chg: disclosed ? num(rec[34]) : null,
|
|
1271
|
+
oty_total_qtrly_wages_pct_chg: disclosed ? num(rec[35]) : null,
|
|
1272
|
+
oty_taxable_qtrly_wages_chg: disclosed ? num(rec[36]) : null,
|
|
1273
|
+
oty_taxable_qtrly_wages_pct_chg: disclosed ? num(rec[37]) : null,
|
|
1274
|
+
oty_qtrly_contributions_chg: disclosed ? num(rec[38]) : null,
|
|
1275
|
+
oty_qtrly_contributions_pct_chg: disclosed ? num(rec[39]) : null,
|
|
1276
|
+
oty_avg_wkly_wage_chg: disclosed ? num(rec[40]) : null,
|
|
1277
|
+
oty_avg_wkly_wage_pct_chg: disclosed ? num(rec[41]) : null,
|
|
1278
|
+
};
|
|
1279
|
+
}
|
|
1280
|
+
/** Map ONE parsed 42-field record → a disclosure-aware output row. */
|
|
1281
|
+
export function mapQcewRow(rec) {
|
|
1282
|
+
return {
|
|
1283
|
+
area_fips: str(rec[0]),
|
|
1284
|
+
own_code: str(rec[1]),
|
|
1285
|
+
industry_code: str(rec[2]),
|
|
1286
|
+
agglvl_code: str(rec[3]),
|
|
1287
|
+
size_code: str(rec[4]),
|
|
1288
|
+
base: mapBaseBlock(rec),
|
|
1289
|
+
locationQuotient: mapLqBlock(rec),
|
|
1290
|
+
overTheYear: mapOtyBlock(rec),
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
// ─── QCEW honesty note constants ─────────────────────────────────────────────
|
|
1294
|
+
const QCEW_SUPPRESSION_NOTE = "BLS QCEW WITHHOLDS employment/wage values for confidentiality when too few establishments would be identifiable. A suppressed field is null with disclosed:false and its disclosureCode ('N' = not disclosable / confidential; '-' = not available) — it is WITHHELD, NOT zero. qtrly_estabs / lq_qtrly_estabs and oty_qtrly_estabs_chg / oty_qtrly_estabs_pct_chg remain DISCLOSED under 'N' (establishment counts and their change are non-confidential) but are withheld under '-'. Do NOT read a null as 0, and do NOT sum/average across rows treating suppressed cells as zero.";
|
|
1295
|
+
const QCEW_MIXED_AGGLVL_NOTE = "This slice MIXES aggregation levels (agglvl_code, e.g. 70=total-all-industries down to 78=6-digit-NAICS-by-ownership) and ownerships (own_code, incl. 0=Total, 1=Federal, 2=State, 3=Local, 5=Private). Do NOT sum qtrly_estabs/employment/wages across different agglvl_code, or across own_code=0 plus its parts — that double-counts. Filter to ONE agglvl_code (and one ownership) for a coherent total.";
|
|
1296
|
+
const QCEW_LQ_RATIO_NOTE = "Location-quotient (lq_*) fields are a RATIO vs the national average (1.00 = same concentration as the nation; >1.00 = MORE concentrated here = higher specialization / competition density; <1.00 = less), each governed by its own lq_disclosure_code.";
|
|
1297
|
+
/** Uniform base for a client-side filter descriptor. */
|
|
1298
|
+
function qcewTrimEq(cell, want) {
|
|
1299
|
+
return (cell ?? "").trim() === want.trim();
|
|
1300
|
+
}
|
|
1301
|
+
/**
|
|
1302
|
+
* ★ `bls_qcew` — keyless QCEW county×NAICS market-size / wages / location-quotient,
|
|
1303
|
+
* disclosure-aware. Fetches ONE slice CSV (fetch-once — QCEW does not paginate),
|
|
1304
|
+
* parses ALL rows through the symmetric column-drift guard + the POST-parse header
|
|
1305
|
+
* assertion, applies the CLIENT-SIDE filters (ownership/aggregationLevel/sizeCode +
|
|
1306
|
+
* the narrow industry/area), windows with limit/offset, and maps each page row
|
|
1307
|
+
* through the block/code/field-scoped disclosure mapper (suppressed → null NEVER 0;
|
|
1308
|
+
* a genuine 0 survives). A per-tuple HTTP 404 → an honest empty (found:false), a
|
|
1309
|
+
* 5xx/timeout → THROW, a 200 non-CSV / drifted header → schema_drift THROW. No
|
|
1310
|
+
* BLS_API_KEY is read on this keyless path.
|
|
1311
|
+
*/
|
|
1312
|
+
export async function qcew(args) {
|
|
1313
|
+
const label = QCEW_LABEL;
|
|
1314
|
+
const mode = String(args.mode ?? "");
|
|
1315
|
+
if (!QCEW_MODES.has(mode)) {
|
|
1316
|
+
qcewInvalid(`mode ${JSON.stringify(args.mode)} must be one of area, industry.`);
|
|
1317
|
+
}
|
|
1318
|
+
// The path code + the OPTIONAL client-side narrow (the OTHER field). In area
|
|
1319
|
+
// mode: `area` is the required path segment, `industry` is an optional narrow
|
|
1320
|
+
// filter; in industry mode the roles swap.
|
|
1321
|
+
let pathCode;
|
|
1322
|
+
let narrowField = null;
|
|
1323
|
+
let narrowValue = null;
|
|
1324
|
+
if (mode === "area") {
|
|
1325
|
+
pathCode = String(args.area ?? "");
|
|
1326
|
+
if (!QCEW_AREA_RE.test(pathCode)) {
|
|
1327
|
+
qcewInvalid(`area ${JSON.stringify(args.area)} is required for mode=area and must be 1..6 alphanumeric chars (^[0-9A-Za-z]{1,6}$) — an area_fips like 01005.`);
|
|
1328
|
+
}
|
|
1329
|
+
if (args.industry !== undefined && args.industry !== null && String(args.industry) !== "") {
|
|
1330
|
+
const narrow = String(args.industry);
|
|
1331
|
+
if (!QCEW_INDUSTRY_RE.test(narrow)) {
|
|
1332
|
+
qcewInvalid(`industry (client-side narrow) ${JSON.stringify(args.industry)} must be 1..6 DIGITS (^[0-9]{1,6}$) — a NAICS code like 5415. A hyphenated NAICS is rejected.`);
|
|
1333
|
+
}
|
|
1334
|
+
narrowField = "industry_code";
|
|
1335
|
+
narrowValue = narrow;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
else {
|
|
1339
|
+
pathCode = String(args.industry ?? "");
|
|
1340
|
+
if (!QCEW_INDUSTRY_RE.test(pathCode)) {
|
|
1341
|
+
qcewInvalid(`industry ${JSON.stringify(args.industry)} is required for mode=industry and must be 1..6 DIGITS (^[0-9]{1,6}$) — a NAICS code like 5415 or the aggregate 10. A hyphenated NAICS supersector (31-33) 404s; use its digit aggregate code.`);
|
|
1342
|
+
}
|
|
1343
|
+
if (args.area !== undefined && args.area !== null && String(args.area) !== "") {
|
|
1344
|
+
const narrow = String(args.area);
|
|
1345
|
+
if (!QCEW_AREA_RE.test(narrow)) {
|
|
1346
|
+
qcewInvalid(`area (client-side narrow) ${JSON.stringify(args.area)} must be 1..6 alphanumeric chars (^[0-9A-Za-z]{1,6}$) — an area_fips like 01005.`);
|
|
1347
|
+
}
|
|
1348
|
+
narrowField = "area_fips";
|
|
1349
|
+
narrowValue = narrow;
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
const year = args.year;
|
|
1353
|
+
if (typeof year !== "number" || !Number.isFinite(year)) {
|
|
1354
|
+
qcewInvalid("year is required and must be a 4-digit integer (e.g. 2023).");
|
|
1355
|
+
}
|
|
1356
|
+
const quarter = String(args.quarter ?? "");
|
|
1357
|
+
// buildQcewUrl re-validates every path segment (year range / quarter / code /
|
|
1358
|
+
// mode) + the fixed-host assert (belt-and-suspenders behind the checks above).
|
|
1359
|
+
const url = buildQcewUrl(mode, year, quarter, pathCode);
|
|
1360
|
+
const ownership = str(args.ownership);
|
|
1361
|
+
const aggregationLevel = str(args.aggregationLevel);
|
|
1362
|
+
const sizeCode = str(args.sizeCode);
|
|
1363
|
+
const limit = Math.min(1000, Math.max(1, Math.floor(args.limit ?? 50)));
|
|
1364
|
+
const offset = Math.max(0, Math.floor(args.offset ?? 0));
|
|
1365
|
+
// filtersApplied — mode/year/quarter always; + client-side filters when set.
|
|
1366
|
+
const filtersApplied = [
|
|
1367
|
+
`mode:${mode}`,
|
|
1368
|
+
`${mode}:${pathCode}`,
|
|
1369
|
+
`year:${year}`,
|
|
1370
|
+
`quarter:${quarter}`,
|
|
1371
|
+
];
|
|
1372
|
+
if (ownership !== null)
|
|
1373
|
+
filtersApplied.push(`ownership:${ownership}`);
|
|
1374
|
+
if (aggregationLevel !== null)
|
|
1375
|
+
filtersApplied.push(`aggregationLevel:${aggregationLevel}`);
|
|
1376
|
+
if (sizeCode !== null)
|
|
1377
|
+
filtersApplied.push(`sizeCode:${sizeCode}`);
|
|
1378
|
+
if (narrowField && narrowValue !== null)
|
|
1379
|
+
filtersApplied.push(`narrow ${narrowField}:${narrowValue}`);
|
|
1380
|
+
// ── Fetch the slice through the NEW self-throttle gate (keyless; NO
|
|
1381
|
+
// BLS_API_KEY). A per-tuple 404 (not_found) → honest empty; every other
|
|
1382
|
+
// failure THROWS (never a fake empty). ──
|
|
1383
|
+
let body;
|
|
1384
|
+
try {
|
|
1385
|
+
body = await throughGate(QCEW_GATE_KEY, QCEW_MIN_INTERVAL_MS, () => fetchQcewCsv(url, label));
|
|
1386
|
+
}
|
|
1387
|
+
catch (e) {
|
|
1388
|
+
if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
|
|
1389
|
+
// The slice file does not exist (nonexistent area/naics, an unpublished /
|
|
1390
|
+
// pre-coverage quarter) — an ABSENT slice, NOT zero establishments. The HTML
|
|
1391
|
+
// 404 body was NEVER parsed (classified on status before any read).
|
|
1392
|
+
return withMeta({ found: false, mode, code: pathCode, [mode]: pathCode, year, quarter, rows: [] }, {
|
|
1393
|
+
source: qcewSource(),
|
|
1394
|
+
keylessMode: true,
|
|
1395
|
+
complete: true,
|
|
1396
|
+
returned: 0,
|
|
1397
|
+
totalAvailable: 0,
|
|
1398
|
+
filtersApplied,
|
|
1399
|
+
filtersDropped: [],
|
|
1400
|
+
fieldsUnavailable: [],
|
|
1401
|
+
notes: [
|
|
1402
|
+
`No QCEW slice exists for ${mode} ${pathCode}, ${year} Q${quarter} (HTTP 404). The ${mode === "area" ? "area_fips" : "NAICS"} code may not exist, or the quarter may be unpublished / before coverage — this is an ABSENT slice, NOT zero establishments. Verify the code and that the quarter is published.`,
|
|
1403
|
+
],
|
|
1404
|
+
});
|
|
1405
|
+
}
|
|
1406
|
+
throw e;
|
|
1407
|
+
}
|
|
1408
|
+
// ── Parse: assemble records → assert the header (POST-parse, quote-stripped) →
|
|
1409
|
+
// symmetric per-row field-count guard. ──
|
|
1410
|
+
const records = assembleQcewRecords(body);
|
|
1411
|
+
if (records.length === 0) {
|
|
1412
|
+
throw driftError(label, "QCEW returned a 200 text/csv body with no parseable records (no header) — treating as schema drift (never a fake empty).");
|
|
1413
|
+
}
|
|
1414
|
+
assertQcewHeader(records[0] ?? [], label);
|
|
1415
|
+
const contentRecords = records.slice(1);
|
|
1416
|
+
let rowNo = 0;
|
|
1417
|
+
for (const rec of contentRecords) {
|
|
1418
|
+
rowNo++;
|
|
1419
|
+
if (rec.length !== QCEW_COLS) {
|
|
1420
|
+
throw driftError(label, `QCEW content row ${rowNo} has ${rec.length} column(s), expected exactly ${QCEW_COLS} — the download was truncated (too few) or the file schema drifted (a column added/removed, too many). Refusing to read a truncated OR column-shifted slice.`);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
// ── Client-side filters (fetch-once → filter → EXACT filtered total). ──
|
|
1424
|
+
const rawCount = contentRecords.length;
|
|
1425
|
+
const filtered = contentRecords.filter((rec) => {
|
|
1426
|
+
if (ownership !== null && !qcewTrimEq(rec[1], ownership))
|
|
1427
|
+
return false;
|
|
1428
|
+
if (aggregationLevel !== null && !qcewTrimEq(rec[3], aggregationLevel))
|
|
1429
|
+
return false;
|
|
1430
|
+
if (sizeCode !== null && !qcewTrimEq(rec[4], sizeCode))
|
|
1431
|
+
return false;
|
|
1432
|
+
if (narrowField === "industry_code" && narrowValue !== null && !qcewTrimEq(rec[2], narrowValue))
|
|
1433
|
+
return false;
|
|
1434
|
+
if (narrowField === "area_fips" && narrowValue !== null && !qcewTrimEq(rec[0], narrowValue))
|
|
1435
|
+
return false;
|
|
1436
|
+
return true;
|
|
1437
|
+
});
|
|
1438
|
+
const totalAvailable = filtered.length;
|
|
1439
|
+
// ── Client-side window (preserve upstream order) → the page rows. ──
|
|
1440
|
+
const pageRecords = filtered.slice(offset, offset + limit);
|
|
1441
|
+
const rows = pageRecords.map(mapQcewRow);
|
|
1442
|
+
const returned = rows.length;
|
|
1443
|
+
const hasMore = offset + returned < totalAvailable;
|
|
1444
|
+
const nextOffset = hasMore ? offset + returned : null;
|
|
1445
|
+
// ── Honesty surface: suppressed field names + the conditional notes. ──
|
|
1446
|
+
const suppressedFields = new Set();
|
|
1447
|
+
let anySuppressed = false;
|
|
1448
|
+
const distinctAgglvl = new Set();
|
|
1449
|
+
const distinctOwn = new Set();
|
|
1450
|
+
for (const r of rows) {
|
|
1451
|
+
if (r.agglvl_code !== null)
|
|
1452
|
+
distinctAgglvl.add(r.agglvl_code);
|
|
1453
|
+
if (r.own_code !== null)
|
|
1454
|
+
distinctOwn.add(r.own_code);
|
|
1455
|
+
for (const [block, keys] of [
|
|
1456
|
+
[r.base, ["qtrly_estabs", "month1_emplvl", "month2_emplvl", "month3_emplvl", "total_qtrly_wages", "taxable_qtrly_wages", "qtrly_contributions", "avg_wkly_wage"]],
|
|
1457
|
+
[r.locationQuotient, ["lq_qtrly_estabs", "lq_month1_emplvl", "lq_month2_emplvl", "lq_month3_emplvl", "lq_total_qtrly_wages", "lq_taxable_qtrly_wages", "lq_qtrly_contributions", "lq_avg_wkly_wage"]],
|
|
1458
|
+
[r.overTheYear, ["oty_qtrly_estabs_chg", "oty_qtrly_estabs_pct_chg", "oty_month1_emplvl_chg", "oty_month1_emplvl_pct_chg", "oty_month2_emplvl_chg", "oty_month2_emplvl_pct_chg", "oty_month3_emplvl_chg", "oty_month3_emplvl_pct_chg", "oty_total_qtrly_wages_chg", "oty_total_qtrly_wages_pct_chg", "oty_taxable_qtrly_wages_chg", "oty_taxable_qtrly_wages_pct_chg", "oty_qtrly_contributions_chg", "oty_qtrly_contributions_pct_chg", "oty_avg_wkly_wage_chg", "oty_avg_wkly_wage_pct_chg"]],
|
|
1459
|
+
]) {
|
|
1460
|
+
if (block.disclosed === false) {
|
|
1461
|
+
anySuppressed = true;
|
|
1462
|
+
for (const k of keys)
|
|
1463
|
+
if (block[k] === null)
|
|
1464
|
+
suppressedFields.add(k);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
const notes = [];
|
|
1469
|
+
if (anySuppressed)
|
|
1470
|
+
notes.push(QCEW_SUPPRESSION_NOTE);
|
|
1471
|
+
if (distinctAgglvl.size >= 2 || distinctOwn.size >= 2)
|
|
1472
|
+
notes.push(QCEW_MIXED_AGGLVL_NOTE);
|
|
1473
|
+
notes.push(QCEW_LQ_RATIO_NOTE);
|
|
1474
|
+
notes.push(`Parsed ${rawCount} content row(s) for this ${mode} slice; after filters → ${totalAvailable} pageable; totalAvailable reflects the FILTERED set (not the page length). The full slice was fetched in ONE request (QCEW does not paginate); limit/offset is a client-side window.`);
|
|
1475
|
+
const metaOut = {
|
|
1476
|
+
source: qcewSource(),
|
|
1477
|
+
keylessMode: true,
|
|
1478
|
+
returned,
|
|
1479
|
+
totalAvailable,
|
|
1480
|
+
filtersApplied,
|
|
1481
|
+
filtersDropped: [],
|
|
1482
|
+
fieldsUnavailable: [...suppressedFields],
|
|
1483
|
+
pagination: { offset, limit, nextOffset, hasMore },
|
|
1484
|
+
notes,
|
|
1485
|
+
};
|
|
1486
|
+
return withMeta({ found: true, mode, code: pathCode, [mode]: pathCode, year, quarter, rows }, metaOut);
|
|
1487
|
+
}
|
|
1488
|
+
/** The QCEW source label (mirrors blsSource(); keyless CSV domain). */
|
|
1489
|
+
function qcewSource() {
|
|
1490
|
+
return "data.bls.gov/cew QCEW Open Data Access (keyless CSV)";
|
|
1491
|
+
}
|
|
1492
|
+
/**
|
|
1493
|
+
* Fetch ONE QCEW slice CSV. Order: fetch(redirect:"error") → check status
|
|
1494
|
+
* (404 → not_found kind, caught by the caller for an honest empty; 5xx/429 →
|
|
1495
|
+
* throw; off-host redirect → schema_drift) → assert text/csv → content-length
|
|
1496
|
+
* pre-check → readCappedBody → decode. Every failure THROWS a classified error
|
|
1497
|
+
* (never a fake empty); the HTML 404 body is NEVER read (classified on status).
|
|
1498
|
+
*/
|
|
1499
|
+
async function fetchQcewCsv(url, label) {
|
|
1500
|
+
let res;
|
|
1501
|
+
try {
|
|
1502
|
+
res = await fetch(url, {
|
|
1503
|
+
headers: { "User-Agent": QCEW_UA, Accept: "text/csv, */*" },
|
|
1504
|
+
redirect: "error",
|
|
1505
|
+
signal: AbortSignal.timeout(QCEW_FETCH_TIMEOUT_MS),
|
|
1506
|
+
});
|
|
1507
|
+
}
|
|
1508
|
+
catch (e) {
|
|
1509
|
+
if (isRedirectError(e)) {
|
|
1510
|
+
// Fail closed — never follow an off-host redirect, never read its body.
|
|
1511
|
+
throw driftError(label, `QCEW ${label} refused an off-host redirect (redirect:"error") — the fixed host ${QCEW_HOST} must serve the CSV directly (SSRF safety).`);
|
|
1512
|
+
}
|
|
1513
|
+
// timeout / abort / network — retryable OUTAGE (THROW — never a fake empty).
|
|
1514
|
+
throw new ToolErrorCarrier({
|
|
1515
|
+
kind: "upstream_unavailable",
|
|
1516
|
+
message: `Network error fetching QCEW ${label}: ${e.message}. The service is unavailable, NOT an empty slice — retry.`,
|
|
1517
|
+
retryable: true,
|
|
1518
|
+
retryAfterSeconds: 30,
|
|
1519
|
+
upstreamEndpoint: label,
|
|
1520
|
+
});
|
|
1521
|
+
}
|
|
1522
|
+
if (!res.ok) {
|
|
1523
|
+
// 404 → not_found (the caller renders an honest empty); 429 → rate_limited;
|
|
1524
|
+
// 5xx → upstream_unavailable; 4xx → invalid_input. A DOWN endpoint NEVER
|
|
1525
|
+
// reads empty; the HTML 404 body is never read (classified on status here).
|
|
1526
|
+
throw new ToolErrorCarrier(errorFromResponse(res, label));
|
|
1527
|
+
}
|
|
1528
|
+
// ★ M2 — the ONLY pre-parse content check is the text/csv Content-Type metadata
|
|
1529
|
+
// guard; the header-name assertion runs POST-parse (assertQcewHeader). A 200
|
|
1530
|
+
// whose Content-Type is not text/csv (an HTML interstitial slipping through at
|
|
1531
|
+
// 200) → schema_drift (never read as data).
|
|
1532
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
1533
|
+
if (!/text\/csv/i.test(ct)) {
|
|
1534
|
+
throw driftError(label, `QCEW ${label} returned HTTP 200 with Content-Type ${JSON.stringify(ct)} (not text/csv) — refusing to read a non-CSV 200 body as data (schema drift).`);
|
|
1535
|
+
}
|
|
1536
|
+
// Size guard (content-length) BEFORE buffering — belt to the streamed cap.
|
|
1537
|
+
const declaredLen = Number(res.headers.get("content-length") ?? "");
|
|
1538
|
+
if (Number.isFinite(declaredLen) && declaredLen > MAX_QCEW_BYTES) {
|
|
1539
|
+
throw new ToolErrorCarrier({
|
|
1540
|
+
kind: "invalid_input",
|
|
1541
|
+
message: `QCEW ${label} declares ${Math.round(declaredLen / 1048576)} MB, over this tool's ${Math.round(MAX_QCEW_BYTES / 1048576)} MB per-slice cap — refusing (a drifted giant, not the ≤~3.84 MB slices). Narrow by area/agglvl.`,
|
|
1542
|
+
retryable: false,
|
|
1543
|
+
upstreamEndpoint: label,
|
|
1544
|
+
});
|
|
1545
|
+
}
|
|
1546
|
+
const bytes = await readCappedQcewBody(res, MAX_QCEW_BYTES, label);
|
|
1547
|
+
return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
|
1548
|
+
}
|
|
1549
|
+
//# sourceMappingURL=bls.js.map
|