data-analyst-mcp 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/README.md +286 -262
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +10 -1
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.d.ts +4 -1
  91. package/dist/report/scan_secrets.js +38 -18
  92. package/dist/report/scan_secrets.js.map +1 -1
  93. package/dist/server.d.ts +3 -1
  94. package/dist/server.js +46 -13
  95. package/dist/server.js.map +1 -1
  96. package/dist/tools.js +27 -6
  97. package/dist/tools.js.map +1 -1
  98. package/guide/WORKFLOW.md +134 -134
  99. package/guide/assets/disclaimer.txt +1 -1
  100. package/guide/assets/download_data.ps1 +164 -162
  101. package/guide/assets/download_data.sh +8 -2
  102. package/guide/assets/report_template.html +597 -597
  103. package/guide/references/routing.md +198 -171
  104. package/guide/references/style_guide.md +398 -398
  105. package/package.json +56 -56
  106. package/packs/acf/endpoints.json +114 -114
  107. package/packs/acf/policy.json +104 -104
  108. package/packs/actuaries/endpoints.json +82 -82
  109. package/packs/actuaries/policy.json +109 -109
  110. package/packs/bea/assets/download_bea.ps1 +182 -182
  111. package/packs/bea/docs/bea_api.md +461 -461
  112. package/packs/bea/docs/datasets.md +157 -157
  113. package/packs/bea/docs/report_patterns.md +112 -112
  114. package/packs/bea/policy.json +170 -170
  115. package/packs/bea/source.md +152 -152
  116. package/packs/bjs/endpoints.json +150 -0
  117. package/packs/bjs/policy.json +114 -0
  118. package/packs/bjs/source.md +98 -0
  119. package/packs/bls/assets/download_bls.ps1 +152 -152
  120. package/packs/bls/docs/analysis_patterns.md +108 -108
  121. package/packs/bls/docs/bls_api.md +73 -73
  122. package/packs/bls/docs/series_ids.md +149 -149
  123. package/packs/bls/policy.json +238 -238
  124. package/packs/bts/endpoints.json +346 -346
  125. package/packs/bts/policy.json +94 -94
  126. package/packs/cbp/endpoints.json +31 -0
  127. package/packs/cbp/policy.json +85 -0
  128. package/packs/cbp/source.md +57 -0
  129. package/packs/cdc/endpoints.json +197 -197
  130. package/packs/cdc/policy.json +83 -83
  131. package/packs/cdc/source.md +1 -1
  132. package/packs/cdc-wonder/endpoints.json +26 -0
  133. package/packs/cdc-wonder/policy.json +84 -0
  134. package/packs/cdc-wonder/source.md +53 -0
  135. package/packs/census/endpoints.json +99 -99
  136. package/packs/cfpb/endpoints.json +326 -326
  137. package/packs/cfpb/policy.json +119 -119
  138. package/packs/clinical-trials/endpoints.json +166 -166
  139. package/packs/clinical-trials/policy.json +84 -84
  140. package/packs/cms/endpoints.json +69 -69
  141. package/packs/cms/policy.json +96 -96
  142. package/packs/college-scorecard/endpoints.json +252 -252
  143. package/packs/college-scorecard/policy.json +106 -106
  144. package/packs/college-scorecard/source.md +1 -1
  145. package/packs/congress/endpoints.json +1701 -1701
  146. package/packs/congress/policy.json +117 -117
  147. package/packs/cpsc/endpoints.json +91 -0
  148. package/packs/cpsc/policy.json +91 -0
  149. package/packs/cpsc/source.md +76 -0
  150. package/packs/dhs-ohss/endpoints.json +66 -66
  151. package/packs/dhs-ohss/policy.json +107 -107
  152. package/packs/dhs-ohss/source.md +1 -1
  153. package/packs/doj-news/endpoints.json +59 -59
  154. package/packs/doj-news/policy.json +96 -96
  155. package/packs/doj-news/source.md +1 -1
  156. package/packs/dol/endpoints.json +817 -817
  157. package/packs/dol/policy.json +135 -135
  158. package/packs/eia/endpoints.json +717 -717
  159. package/packs/eia/policy.json +112 -112
  160. package/packs/epa/endpoints.json +291 -291
  161. package/packs/epa/policy.json +99 -99
  162. package/packs/epa-aqs/endpoints.json +173 -173
  163. package/packs/epa-aqs/policy.json +96 -96
  164. package/packs/fbi/endpoints.json +1841 -1841
  165. package/packs/fbi/policy.json +118 -118
  166. package/packs/fbi/source.md +3 -3
  167. package/packs/fda/endpoints.json +237 -237
  168. package/packs/fda/policy.json +121 -121
  169. package/packs/fdic/endpoints.json +416 -416
  170. package/packs/fdic/policy.json +83 -83
  171. package/packs/fec/endpoints.json +707 -707
  172. package/packs/fec/policy.json +93 -93
  173. package/packs/fed-board/endpoints.json +89 -89
  174. package/packs/fed-board/policy.json +136 -136
  175. package/packs/federal-register/endpoints.json +143 -143
  176. package/packs/federal-register/policy.json +86 -86
  177. package/packs/fema/endpoints.json +645 -645
  178. package/packs/fema/policy.json +103 -103
  179. package/packs/fhfa/endpoints.json +105 -105
  180. package/packs/fhfa/policy.json +109 -109
  181. package/packs/fhwa/endpoints.json +48 -0
  182. package/packs/fhwa/policy.json +91 -0
  183. package/packs/fhwa/source.md +59 -0
  184. package/packs/foreign-assistance/endpoints.json +166 -0
  185. package/packs/foreign-assistance/policy.json +80 -0
  186. package/packs/foreign-assistance/source.md +76 -0
  187. package/packs/fred/assets/download_fred.ps1 +122 -122
  188. package/packs/fred/docs/analysis_patterns.md +124 -124
  189. package/packs/fred/docs/fred_api.md +94 -94
  190. package/packs/fred/docs/series.md +188 -188
  191. package/packs/fred/policy.json +253 -253
  192. package/packs/fsa/endpoints.json +150 -150
  193. package/packs/fsa/policy.json +99 -99
  194. package/packs/fsa/source.md +1 -1
  195. package/packs/govinfo/endpoints.json +519 -519
  196. package/packs/gsa-calc/endpoints.json +93 -93
  197. package/packs/gsa-calc/policy.json +97 -97
  198. package/packs/hud/endpoints.json +146 -146
  199. package/packs/hud/policy.json +84 -84
  200. package/packs/hud/source.md +1 -1
  201. package/packs/hud-ahar/endpoints.json +92 -0
  202. package/packs/hud-ahar/policy.json +88 -0
  203. package/packs/hud-ahar/source.md +72 -0
  204. package/packs/ipums/docs/report_patterns.md +117 -117
  205. package/packs/irs-soi/endpoints.json +114 -114
  206. package/packs/irs-soi/policy.json +107 -107
  207. package/packs/medicaid/endpoints.json +412 -412
  208. package/packs/medicaid/policy.json +108 -108
  209. package/packs/naep/endpoints.json +217 -217
  210. package/packs/naep/policy.json +100 -100
  211. package/packs/naep/source.md +1 -1
  212. package/packs/nces/endpoints.json +101 -0
  213. package/packs/nces/policy.json +99 -0
  214. package/packs/nces/source.md +75 -0
  215. package/packs/nhtsa/endpoints.json +132 -132
  216. package/packs/nhtsa/policy.json +93 -93
  217. package/packs/nhtsa/source.md +1 -1
  218. package/packs/nih/endpoints.json +452 -452
  219. package/packs/nih/policy.json +86 -86
  220. package/packs/noaa/endpoints.json +431 -431
  221. package/packs/noaa/policy.json +95 -95
  222. package/packs/nrel/endpoints.json +327 -327
  223. package/packs/nrel/policy.json +96 -96
  224. package/packs/nsf/endpoints.json +33 -0
  225. package/packs/nsf/policy.json +83 -0
  226. package/packs/nsf/source.md +59 -0
  227. package/packs/nws/endpoints.json +271 -271
  228. package/packs/nws/policy.json +82 -82
  229. package/packs/nyfed/endpoints.json +80 -80
  230. package/packs/nyfed/policy.json +92 -92
  231. package/packs/omb/endpoints.json +460 -460
  232. package/packs/omb/policy.json +106 -106
  233. package/packs/onrr/endpoints.json +198 -0
  234. package/packs/onrr/policy.json +91 -0
  235. package/packs/onrr/source.md +55 -0
  236. package/packs/open-payments/endpoints.json +297 -297
  237. package/packs/open-payments/policy.json +101 -101
  238. package/packs/opm/assets/download_opm.ps1 +125 -125
  239. package/packs/opm/docs/fields.md +184 -184
  240. package/packs/opm/docs/opm_api.md +71 -71
  241. package/packs/opm/docs/report_patterns.md +83 -83
  242. package/packs/opm/lib/data_dictionary.csv +73 -73
  243. package/packs/pbgc/endpoints.json +483 -483
  244. package/packs/pbgc/policy.json +94 -94
  245. package/packs/philfed/endpoints.json +85 -85
  246. package/packs/philfed/policy.json +94 -94
  247. package/packs/port.json +65 -0
  248. package/packs/regulations/endpoints.json +287 -287
  249. package/packs/regulations/policy.json +86 -86
  250. package/packs/sba/endpoints.json +34 -0
  251. package/packs/sba/policy.json +86 -0
  252. package/packs/sba/source.md +60 -0
  253. package/packs/sec/endpoints.json +236 -236
  254. package/packs/sec/policy.json +86 -86
  255. package/packs/senate-lobbying/endpoints.json +553 -553
  256. package/packs/senate-lobbying/policy.json +99 -99
  257. package/packs/snapqc/endpoints.json +1426 -1426
  258. package/packs/snapqc/policy.json +92 -92
  259. package/packs/treasury/catalog.json +2267 -2267
  260. package/packs/treasury/docs/api_guide.md +245 -245
  261. package/packs/treasury/docs/datasets.md +56 -56
  262. package/packs/treasury/docs/report_patterns.md +126 -126
  263. package/packs/treasury/policy.json +159 -159
  264. package/packs/treasury/source.md +170 -165
  265. package/packs/treasury-tic/endpoints.json +77 -0
  266. package/packs/treasury-tic/policy.json +89 -0
  267. package/packs/treasury-tic/source.md +80 -0
  268. package/packs/treasury-yields/endpoints.json +226 -226
  269. package/packs/treasury-yields/policy.json +86 -86
  270. package/packs/usaspending/endpoints.json +1315 -1315
  271. package/packs/usaspending/policy.json +109 -109
  272. package/packs/uscourts/endpoints.json +22 -0
  273. package/packs/uscourts/policy.json +87 -0
  274. package/packs/uscourts/source.md +56 -0
  275. package/packs/usda-fna/policy.json +133 -133
  276. package/packs/usda-fooddata/endpoints.json +180 -180
  277. package/packs/usda-fooddata/policy.json +94 -94
  278. package/packs/usda-nass/endpoints.json +323 -323
  279. package/packs/usda-nass/policy.json +95 -95
  280. package/packs/usda-outlook/endpoints.json +86 -86
  281. package/packs/usda-outlook/policy.json +109 -109
  282. package/packs/usgs/endpoints.json +313 -313
  283. package/packs/usgs/policy.json +95 -95
  284. package/packs/usitc/endpoints.json +74 -74
  285. package/packs/usitc/policy.json +111 -111
  286. package/packs/uspto/endpoints.json +141 -141
  287. package/packs/uspto/policy.json +95 -95
  288. package/packs/uspto/source.md +1 -1
  289. package/packs/va/endpoints.json +414 -414
  290. package/packs/va/policy.json +101 -101
  291. package/packs/world-bank/endpoints.json +122 -122
  292. package/packs/world-bank/policy.json +101 -101
@@ -1,184 +1,184 @@
1
- # OPM workforce data: fields, codes and traps
2
-
3
- Read §1 and §2 before choosing data for any analysis; they decide whether a question can be answered
4
- at all. `data_dictionary.csv` (shipped with the pack, and copied into every `data/opm/` folder) has the
5
- official definition of all 72 fields and which datasets carry each one.
6
-
7
- ## Contents
8
- 1. What the data are (and are not)
9
- 2. Traps that silently produce wrong numbers
10
- 3. Key fields and codes
11
- 4. Standard definitions
12
- 5. Columns to request for common questions
13
-
14
- ## 1. What the data are (and are not)
15
- - **Source:** OPM's Enterprise Human Resources Integration (EHRI) system, fed by agencies' core HR
16
- systems. **Administrative records for the whole covered workforce, not a survey**: no weights, no
17
- sampling error. A difference of 50 employees is a real difference in the records; whether it is
18
- meaningful depends on reporting quality, not on a confidence interval.
19
- - **One row = one employee** (employment: on board on the last day of the month) or **one personnel
20
- action** (accessions, separations). `count` is always 1. There is **no person identifier**, so you
21
- cannot follow individuals across months, link a separation to the employee's earlier record, or
22
- distinguish one person's two actions.
23
- - **Coverage:** civilian employees of the executive branch, **except** USPS, the Postal Regulatory
24
- Commission, CIA, NSA, DIA, NGA, ODNI, the White House Office, the Office and Residence of the Vice
25
- President, Census Bureau temporary workers, Foreign Service personnel at State, TVA, the Federal
26
- Reserve Board of Governors (CFPB *is* included, under the Federal Reserve), the PHS Commissioned
27
- Corps, nonappropriated-fund employees, foreign nationals overseas, and most of the legislative and
28
- judicial branches (a few bodies do report: the Government Publishing Office `LP` and some judicial-branch
29
- records `JL` appear; drop them by `agency_code` for an executive-branch count and say so). Uniformed
30
- military are not civilians and are not in the data. Totals are therefore below
31
- "all federal employees" figures that include USPS (about 600,000) or the intelligence community.
32
- - **Not in the public files:** sex, race, ethnicity, disability, citizenship, exact age (only 5-year
33
- `age_bracket`), names, performance ratings, awards, telework, leave, bonuses, total compensation,
34
- benefits, retirement eligibility dates, adverse-action detail (removals, probationary terminations
35
- and suspensions are withheld by OPM policy). Say so plainly when a question needs them, and offer
36
- what the data can show instead (for example age bracket and length of service as a proxy for
37
- retirement eligibility).
38
-
39
- ## 2. Traps that silently produce wrong numbers
40
- 1. **Redacted pay and location (the big one).** `annualized_adjusted_basic_pay`, all `duty_station_*`
41
- fields, CBSA/CSA, `locality_pay_area*`, `bargaining_unit*` and `personnel_office_identifier_code`
42
- are the string `REDACTED` for:
43
- - **every Department of Defense employee** (Army, Navy, Air Force, and defense agencies: about
44
- 680,000 people, a third of the workforce), in every year;
45
- - security/sensitive agencies: FBI, DEA, ATF, TTB, Secret Service, CBP, ICE, Bureau of Prisons, U.S.
46
- Attorneys' offices, DHS intelligence (HSOS04) — outside the Washington area;
47
- - sensitive occupations in any agency: series 0007, 0082, 0083, 0084, 0132, 0134, 0401 (DHS), 0436,
48
- 0512, 0840, 0930, 1169, 1171 (IRS), 1801, 1802, 1805, 1810, 1811, 1812, 1816, 1854, 1881, 1884,
49
- 1890, 1895, 1896.
50
- Result: about 45 to 48 percent of employment records lack pay and location (DHS about 76 percent,
51
- Justice about 66 percent, Treasury and Energy about 17 percent, most others under 5 percent).
52
- **Consequences:** a "governmentwide average salary" from these files is the average of non-DoD,
53
- non-security employees, and state counts undercount every state with a big DoD presence. Always count
54
- redacted records by the groups reported (`aggregate` gives `pay_redacted`), save that coverage table as an
55
- exhibit, report the coverage next to every pay or location
56
- figure, and restrict the stated universe ("employees whose pay is published, which excludes DoD …").
57
- OPM's own "Average salary" (Table Builder, dashboards) is the mean over every record of the group whose
58
- pay is published, on any work schedule; the default universe here (full-time, per annum) differs by up
59
- to $10,000 a department: match OPM's definition when benchmarking against it, and show the other as a
60
- sensitivity.
61
- For full state counts (DoD included), OPM's Federal Workforce Data location dashboard
62
- (data.opm.gov/explore-data/analytics/location) publishes them; FedScope's old state tables also included
63
- DoD. Read them into `published/` and keep the file-based count as the covered-part check.
64
- OPM's Table Builder and dashboards count separations and accessions by effective date: count the same
65
- way to match them. Processing month differs a lot for FY2026, because about 11,000 Deferred Resignation
66
- separations effective before October 2025 were processed in FY2026 files (it flips the sign of a
67
- year-over-year comparison).
68
- Read the map's state values: the dashboard's state table puts the D.C. commuting area in its own "D.C.
69
- AREA" row, so its MD, VA and WV rows leave out their D.C.-area employees (Aug 2026: MD 54,097 in the
70
- table, 121,612 on the map). The map has a "table view" (states 25 to a page) that lists the map's values;
71
- the dashboard is a live app with no JSON to fetch, so read that view in a browser. The location map counts employees in pay status, 1 to 2 percent below the
72
- files' on-board count; the Demographics dashboard (age, education, veteran status by department) and
73
- Table Builder match the files exactly (Aug 2026), so they benchmark shares and headcounts by department.
74
- Headcounts by department, occupation, grade, age, education and so on are complete — only pay and
75
- location are withheld.
76
- 2. **Counting accessions and separations.** A file holds actions *processed* that month; 5 to 10
77
- percent are effective in earlier months (some a year or more back). Two defensible bases:
78
- - **effective month** (`personnel_action_effective_date_yyyymm`): what happened when; needs the files for later
79
- months too, and the most recent two or three months are incomplete;
80
- - **processing month** (`file_yyyymm`): what was recorded each month; complete as soon as the file
81
- is out.
82
- Pick one, state it, and show the other in a sensitivity check when they differ; comparing the two shows
83
- how much arrives late.
84
- 3. **Versions and revisions.** Use `current` versions; numbers in an earlier report can differ because
85
- OPM has since reissued the file. Pin versions (the spec's `pin`) to reproduce exactly.
86
- 4. **Pay plans change too.** Wildland firefighters moved from GS to their own pay plan GW in 2025 (about
87
- 16,000 to 17,000 employees at Agriculture and Interior): a GS-only series breaks there; count GS+GW for
88
- a like-for-like trend. OPM's "pay plan and grade" dashboard table counts permanent full-time employees only
89
- (`nsftp_indicator` = Y): filter the same way to match it.
90
- 5. **Agency names change; codes are stable.** For example, "DEPARTMENT OF DEFENSE" also appears as
91
- "DEPARTMENT OF WAR" on OPM pages from 2025. Group by `department_code`, `agency_code`,
92
- `agency_subelement_code`, `occupational_series_code`, and display the latest label
93
- of each code. Reorganizations (new subelements, transfers between agencies) show up
94
- as jumps: check before calling them hiring or losses. Occupational series move too: wildland firefighters
95
- were reclassified from 0462 (Forestry Technician) into 0456 (Wildland Fire Management, created 2022), and
96
- the 0x99 student-trainee series swing with internship cycles; check the receiving series before calling
97
- a drop a loss.
98
- 6. **Snapshot frequency.** Employment files are quarterly before 2011. A fiscal-year average headcount
99
- uses 12 monthly snapshots from FY2012 on and four before: flag fiscal years with fewer snapshots.
100
- 7. **Missing agency submissions.** An agency that misses a month shows a dip and a rebound. Check the
101
- FWD Data Quality page ("Missing EHRI files") before reporting a one-month change.
102
- 8. **Sentinel values.** `NO DATA REPORTED`, `INVALID`, `UNSPECIFIED`, `*` appear in coded fields;
103
- the data keep them as text: treat them as missing. Report how many records are missing for the fields you use.
104
- A department code reported as missing on records with a valid agency code belongs to that agency's department
105
- (each agency code belongs to one department): fill it from the agency, so department totals add up to the
106
- governmentwide total, and say so.
107
- 9. **Pay oddities.** Pay is annualized basic pay plus locality or special-rate supplement, after the
108
- pay cap — not total compensation (no overtime, awards, benefits). Records with pay 0 (without
109
- compensation), hourly or per-diem pay bases, intermittent and seasonal schedules distort means.
110
- Default pay universe: full-time (`work_schedule_code` F), per annum (`pay_basis_code` PA), pay > 0.
111
- Physicians (VM/VN pay plans), some financial regulators and SES earn above
112
- the GS cap legitimately; screen pay for implausible values but do not drop them without a reason.
113
- 10. **Strings everywhere.** Codes keep leading zeros (`occupational_series_code` "0346", `grade` "08").
114
- Do not convert codes to integers.
115
- 11. **Separations on the last day of a month show up in the next snapshot.** An employee whose
116
- separation takes effect on the last day of a month is still on board in that month's snapshot. A
117
- wave of end-of-month separations (retirements on September 30 or December 31, or the 2025 Deferred
118
- Resignation Program, effective September 30) therefore shows up in the following month's
119
- headcount. In 2025: counted across the files available in October 2026, 132,949 separations were
120
- effective in September (106,883 of them deferred resignations; earlier files showed fewer because
121
- actions keep arriving late, see §2.2), but headcount fell 28,342 from August to September
122
- and 92,685 from September to October. A September-to-September change misses such a wave. When separations bunch at the end of
123
- the period's last month, also show the change to the next month's snapshot, or use December
124
- snapshots, and say which you used.
125
-
126
- ## 3. Key fields and codes
127
- | Field | Datasets | Notes |
128
- |---|---|---|
129
- | `snapshot_yyyymm` | E | month of the status snapshot |
130
- | `personnel_action_effective_date_yyyymm` | A, S | effective month of the action |
131
- | `department_code` / `department` | all | the 15 executive departments: AG Agriculture, CM Commerce, DOD Defense, ED Education, DN Energy, HE HHS, HS Homeland Security, HU HUD, DJ Justice, DL Labor, ST State, IN Interior, TR Treasury, TD Transportation, VA Veterans Affairs; independent agencies have their own codes (SZ SSA, EP EPA, ...) |
132
- | `agency_code` / `agency` | all | e.g. AR Army, NV Navy, AF Air Force, VA, HS |
133
- | `agency_subelement_code` | all | bureau or component, e.g. IN05 Bureau of Land Management, TR93 IRS |
134
- | `cfo_act_agency_indicator` | all | CFO ACT AGENCY (24 large agencies) / OTHER AGENCY |
135
- | `separation_category_code` | S | SA transfer out (individual), SB transfer out (mass), SC quit, SD retirement voluntary, SE retirement early out, SG retirement other, SH reduction in force (RIF), SJ termination (expired appointment/other), SL other separation |
136
- | `drp_indicator` | S | Y = separated under the 2025 Deferred Resignation Program (about 96,000 processed in the September 2025 file alone; about 107,000 effective in September 2025 across all files as of October 2026; coded as quit or retirement) |
137
- | `accession_category_code` | A | AA transfer in (individual), AB transfer in (mass), AC new hire competitive service, AD new hire excepted service, AE new hire SES |
138
- | `work_schedule_code` | all | F full-time, P part-time, I intermittent, G full-time seasonal, Q part-time seasonal, J intermittent seasonal; the files also hold B, R, S, T and blank: read their labels with `--values work_schedule` and state the grouping (part-time = P, Q, S, T, R; intermittent = I, J) |
139
- | `pay_plan_code` | all | GS General Schedule (~63 percent), WG/WL/WS wage grade, VN/VM VA medical, NH/ND DoD demo, AD administratively determined, SV TSA, ES Senior Executive Service |
140
- | `grade`, `step_or_rate_type_code` | all | strings; GS grades "01" to "15" |
141
- | `pay_basis_code` | all | PA per annum, PH per hour, SY school year, PD per diem, WC without compensation |
142
- | `annualized_adjusted_basic_pay` | all | dollars as a string, or REDACTED |
143
- | `occupational_series_code` | all | 4-digit OPM series ("2210" IT management); `occupational_group_code` 2-digit family; `occupational_category_code` P/A/T/C/O/B (professional, administrative, technical, clerical, other white collar, blue collar) |
144
- | `stem_occupation_type` | all | science, technology, engineering, mathematics, health, all other |
145
- | `supervisory_status_code` | all | 2 supervisor or manager, 4 supervisor (CSRA), 5 management official, 6 leader, 7 team leader, 8 all other |
146
- | `appointment_type_code`, `tenure_code`, `position_occupied_code` | all | permanence and service: 10 career, 15 career-conditional, 30/32/35/38 excepted permanent, 50/55 SES; tenure 1 career, 2 conditional (probationary), 3 term; position 1 competitive, 2 excepted, 3/4 SES |
147
- | `nsftp_indicator` | all | Y = nonseasonal full-time permanent (OPM's standard "permanent workforce" universe) |
148
- | `age_bracket` | all | LESS THAN 20, 20-24, …, 60-64, 65 OR MORE |
149
- | `education_level_bracket` | all | HIGH SCHOOL OR LESS, SOME COLLEGE OR ASSOCIATES DEGREE, BACHELORS DEGREE, MASTERS OR PROFESSIONAL DEGREE, DOCTORATE DEGREE |
150
- | `length_of_service_years` | all | federal civilian plus creditable military service, years with one decimal. **Definition break in March 2026**: from the 2026-03 file it equals years since the leave service computation date for every record; before, for only 72-79 percent (a quarter of records ran about 4 years shorter). A change across March 2026 (+1.6 years Sep 2024 to Aug 2026 as published) is mostly this break: compare on one basis (years since `service_computation_date_leave`) and say so. OPM's Table Builder metric "Average length of service" follows the published field |
151
- | `veteran_indicator` | all | Y / N, derived by OPM from several veteran fields |
152
- | `bargaining_unit_status` | all | ELIGIBLE_IN_BU, ELIGIBLE_NOT_IN_BU, NOT_ELIGIBLE (or REDACTED) |
153
- | `duty_station_state_abbreviation` | all | state of the worksite (REDACTED as in §2) |
154
-
155
- ## 4. Standard definitions
156
- - **Headcount (on-board employment):** employees in the status file for the month. Annual series use
157
- the **September** snapshot (end of the fiscal year; FedScope and OPM convention) unless the user
158
- wants calendar-year (December) or another month. State the month. End-of-month separations leave
159
- the headcount only in the next month's snapshot (§2.10).
160
- - **Permanent workforce:** `nsftp_indicator` = Y (nonseasonal, full-time, permanent). Common in OPM
161
- and CRS work when temporary and seasonal swings would mislead.
162
- - **Separation rate:** separations in the period ÷ average headcount in the period (mean of the monthly
163
- snapshots). Report quit, retirement, RIF and transfer rates separately when they matter; transfers
164
- between agencies are separations for the losing agency but not losses to the government.
165
- - **Accession (hiring) rate:** accessions ÷ average headcount. New hires = AC, AD, AE; transfers in are
166
- not new to government.
167
- - **Net change:** headcount at the end minus headcount at the start (from status files). Accessions
168
- minus separations will not match it exactly (conversions, late actions, coverage changes); say so if
169
- both are shown.
170
- - **Average and median pay:** of full-time, per-annum employees whose pay is published (§2.1).
171
- Prefer the median for comparisons across groups; quote coverage.
172
- - **Retirement-eligible proxy:** no birth date or retirement plan is public. Age 60 or older with 20 or
173
- more years of service, or age 62+ with 5+ years, approximates FERS eligibility; label it a proxy.
174
- - **Fiscal year:** October through September (in `compute`, make `yyyymm` a `YYYY-MM` date with `substr`, then `fiscal_year`; FY2025 = October 2024 to
175
- September 2025). Say "fiscal year 2025" in prose.
176
-
177
- ## 5. Columns to request for common questions
178
- Keep the spec lean: ten September snapshots with all 64 columns are ~20 million rows.
179
- - Headcount trends by agency: `department_code, department, agency_code, agency, nsftp_indicator, work_schedule_code`
180
- - Composition (age, tenure, education): add `age_bracket, length_of_service_years, education_level_bracket, supervisory_status_code, veteran_indicator`
181
- - Occupations: `occupational_series_code, occupational_series, occupational_category_code, stem_occupation_type, grade, pay_plan_code`
182
- - Pay: `annualized_adjusted_basic_pay, pay_plan_code, grade, pay_basis_code, work_schedule_code, occupational_series_code, duty_station_state_abbreviation` (+ the department/agency codes to measure redaction)
183
- - Turnover: separations `separation_category_code, separation_category, drp_indicator, agency_code, department_code, age_bracket, length_of_service_years`; accessions `accession_category_code, …`; and the employment snapshots of the same months for the denominator
184
- - Location: `duty_station_state_abbreviation, duty_station_state, core_based_statistical_area` (redaction!)
1
+ # OPM workforce data: fields, codes and traps
2
+
3
+ Read §1 and §2 before choosing data for any analysis; they decide whether a question can be answered
4
+ at all. `data_dictionary.csv` (shipped with the pack, and copied into every `data/opm/` folder) has the
5
+ official definition of all 72 fields and which datasets carry each one.
6
+
7
+ ## Contents
8
+ 1. What the data are (and are not)
9
+ 2. Traps that silently produce wrong numbers
10
+ 3. Key fields and codes
11
+ 4. Standard definitions
12
+ 5. Columns to request for common questions
13
+
14
+ ## 1. What the data are (and are not)
15
+ - **Source:** OPM's Enterprise Human Resources Integration (EHRI) system, fed by agencies' core HR
16
+ systems. **Administrative records for the whole covered workforce, not a survey**: no weights, no
17
+ sampling error. A difference of 50 employees is a real difference in the records; whether it is
18
+ meaningful depends on reporting quality, not on a confidence interval.
19
+ - **One row = one employee** (employment: on board on the last day of the month) or **one personnel
20
+ action** (accessions, separations). `count` is always 1. There is **no person identifier**, so you
21
+ cannot follow individuals across months, link a separation to the employee's earlier record, or
22
+ distinguish one person's two actions.
23
+ - **Coverage:** civilian employees of the executive branch, **except** USPS, the Postal Regulatory
24
+ Commission, CIA, NSA, DIA, NGA, ODNI, the White House Office, the Office and Residence of the Vice
25
+ President, Census Bureau temporary workers, Foreign Service personnel at State, TVA, the Federal
26
+ Reserve Board of Governors (CFPB *is* included, under the Federal Reserve), the PHS Commissioned
27
+ Corps, nonappropriated-fund employees, foreign nationals overseas, and most of the legislative and
28
+ judicial branches (a few bodies do report: the Government Publishing Office `LP` and some judicial-branch
29
+ records `JL` appear; drop them by `agency_code` for an executive-branch count and say so). Uniformed
30
+ military are not civilians and are not in the data. Totals are therefore below
31
+ "all federal employees" figures that include USPS (about 600,000) or the intelligence community.
32
+ - **Not in the public files:** sex, race, ethnicity, disability, citizenship, exact age (only 5-year
33
+ `age_bracket`), names, performance ratings, awards, telework, leave, bonuses, total compensation,
34
+ benefits, retirement eligibility dates, adverse-action detail (removals, probationary terminations
35
+ and suspensions are withheld by OPM policy). Say so plainly when a question needs them, and offer
36
+ what the data can show instead (for example age bracket and length of service as a proxy for
37
+ retirement eligibility).
38
+
39
+ ## 2. Traps that silently produce wrong numbers
40
+ 1. **Redacted pay and location (the big one).** `annualized_adjusted_basic_pay`, all `duty_station_*`
41
+ fields, CBSA/CSA, `locality_pay_area*`, `bargaining_unit*` and `personnel_office_identifier_code`
42
+ are the string `REDACTED` for:
43
+ - **every Department of Defense employee** (Army, Navy, Air Force, and defense agencies: about
44
+ 680,000 people, a third of the workforce), in every year;
45
+ - security/sensitive agencies: FBI, DEA, ATF, TTB, Secret Service, CBP, ICE, Bureau of Prisons, U.S.
46
+ Attorneys' offices, DHS intelligence (HSOS04) — outside the Washington area;
47
+ - sensitive occupations in any agency: series 0007, 0082, 0083, 0084, 0132, 0134, 0401 (DHS), 0436,
48
+ 0512, 0840, 0930, 1169, 1171 (IRS), 1801, 1802, 1805, 1810, 1811, 1812, 1816, 1854, 1881, 1884,
49
+ 1890, 1895, 1896.
50
+ Result: about 45 to 48 percent of employment records lack pay and location (DHS about 76 percent,
51
+ Justice about 66 percent, Treasury and Energy about 17 percent, most others under 5 percent).
52
+ **Consequences:** a "governmentwide average salary" from these files is the average of non-DoD,
53
+ non-security employees, and state counts undercount every state with a big DoD presence. Always count
54
+ redacted records by the groups reported (`aggregate` gives `pay_redacted`), save that coverage table as an
55
+ exhibit, report the coverage next to every pay or location
56
+ figure, and restrict the stated universe ("employees whose pay is published, which excludes DoD …").
57
+ OPM's own "Average salary" (Table Builder, dashboards) is the mean over every record of the group whose
58
+ pay is published, on any work schedule; the default universe here (full-time, per annum) differs by up
59
+ to $10,000 a department: match OPM's definition when benchmarking against it, and show the other as a
60
+ sensitivity.
61
+ For full state counts (DoD included), OPM's Federal Workforce Data location dashboard
62
+ (data.opm.gov/explore-data/analytics/location) publishes them; FedScope's old state tables also included
63
+ DoD. Read them into `published/` and keep the file-based count as the covered-part check.
64
+ OPM's Table Builder and dashboards count separations and accessions by effective date: count the same
65
+ way to match them. Processing month differs a lot for FY2026, because about 11,000 Deferred Resignation
66
+ separations effective before October 2025 were processed in FY2026 files (it flips the sign of a
67
+ year-over-year comparison).
68
+ Read the map's state values: the dashboard's state table puts the D.C. commuting area in its own "D.C.
69
+ AREA" row, so its MD, VA and WV rows leave out their D.C.-area employees (Aug 2026: MD 54,097 in the
70
+ table, 121,612 on the map). The map has a "table view" (states 25 to a page) that lists the map's values;
71
+ the dashboard is a live app with no JSON to fetch, so read that view in a browser. The location map counts employees in pay status, 1 to 2 percent below the
72
+ files' on-board count; the Demographics dashboard (age, education, veteran status by department) and
73
+ Table Builder match the files exactly (Aug 2026), so they benchmark shares and headcounts by department.
74
+ Headcounts by department, occupation, grade, age, education and so on are complete — only pay and
75
+ location are withheld.
76
+ 2. **Counting accessions and separations.** A file holds actions *processed* that month; 5 to 10
77
+ percent are effective in earlier months (some a year or more back). Two defensible bases:
78
+ - **effective month** (`personnel_action_effective_date_yyyymm`): what happened when; needs the files for later
79
+ months too, and the most recent two or three months are incomplete;
80
+ - **processing month** (`file_yyyymm`): what was recorded each month; complete as soon as the file
81
+ is out.
82
+ Pick one, state it, and show the other in a sensitivity check when they differ; comparing the two shows
83
+ how much arrives late.
84
+ 3. **Versions and revisions.** Use `current` versions; numbers in an earlier report can differ because
85
+ OPM has since reissued the file. Pin versions (the spec's `pin`) to reproduce exactly.
86
+ 4. **Pay plans change too.** Wildland firefighters moved from GS to their own pay plan GW in 2025 (about
87
+ 16,000 to 17,000 employees at Agriculture and Interior): a GS-only series breaks there; count GS+GW for
88
+ a like-for-like trend. OPM's "pay plan and grade" dashboard table counts permanent full-time employees only
89
+ (`nsftp_indicator` = Y): filter the same way to match it.
90
+ 5. **Agency names change; codes are stable.** For example, "DEPARTMENT OF DEFENSE" also appears as
91
+ "DEPARTMENT OF WAR" on OPM pages from 2025. Group by `department_code`, `agency_code`,
92
+ `agency_subelement_code`, `occupational_series_code`, and display the latest label
93
+ of each code. Reorganizations (new subelements, transfers between agencies) show up
94
+ as jumps: check before calling them hiring or losses. Occupational series move too: wildland firefighters
95
+ were reclassified from 0462 (Forestry Technician) into 0456 (Wildland Fire Management, created 2022), and
96
+ the 0x99 student-trainee series swing with internship cycles; check the receiving series before calling
97
+ a drop a loss.
98
+ 6. **Snapshot frequency.** Employment files are quarterly before 2011. A fiscal-year average headcount
99
+ uses 12 monthly snapshots from FY2012 on and four before: flag fiscal years with fewer snapshots.
100
+ 7. **Missing agency submissions.** An agency that misses a month shows a dip and a rebound. Check the
101
+ FWD Data Quality page ("Missing EHRI files") before reporting a one-month change.
102
+ 8. **Sentinel values.** `NO DATA REPORTED`, `INVALID`, `UNSPECIFIED`, `*` appear in coded fields;
103
+ the data keep them as text: treat them as missing. Report how many records are missing for the fields you use.
104
+ A department code reported as missing on records with a valid agency code belongs to that agency's department
105
+ (each agency code belongs to one department): fill it from the agency, so department totals add up to the
106
+ governmentwide total, and say so.
107
+ 9. **Pay oddities.** Pay is annualized basic pay plus locality or special-rate supplement, after the
108
+ pay cap — not total compensation (no overtime, awards, benefits). Records with pay 0 (without
109
+ compensation), hourly or per-diem pay bases, intermittent and seasonal schedules distort means.
110
+ Default pay universe: full-time (`work_schedule_code` F), per annum (`pay_basis_code` PA), pay > 0.
111
+ Physicians (VM/VN pay plans), some financial regulators and SES earn above
112
+ the GS cap legitimately; screen pay for implausible values but do not drop them without a reason.
113
+ 10. **Strings everywhere.** Codes keep leading zeros (`occupational_series_code` "0346", `grade` "08").
114
+ Do not convert codes to integers.
115
+ 11. **Separations on the last day of a month show up in the next snapshot.** An employee whose
116
+ separation takes effect on the last day of a month is still on board in that month's snapshot. A
117
+ wave of end-of-month separations (retirements on September 30 or December 31, or the 2025 Deferred
118
+ Resignation Program, effective September 30) therefore shows up in the following month's
119
+ headcount. In 2025: counted across the files available in October 2026, 132,949 separations were
120
+ effective in September (106,883 of them deferred resignations; earlier files showed fewer because
121
+ actions keep arriving late, see §2.2), but headcount fell 28,342 from August to September
122
+ and 92,685 from September to October. A September-to-September change misses such a wave. When separations bunch at the end of
123
+ the period's last month, also show the change to the next month's snapshot, or use December
124
+ snapshots, and say which you used.
125
+
126
+ ## 3. Key fields and codes
127
+ | Field | Datasets | Notes |
128
+ |---|---|---|
129
+ | `snapshot_yyyymm` | E | month of the status snapshot |
130
+ | `personnel_action_effective_date_yyyymm` | A, S | effective month of the action |
131
+ | `department_code` / `department` | all | the 15 executive departments: AG Agriculture, CM Commerce, DOD Defense, ED Education, DN Energy, HE HHS, HS Homeland Security, HU HUD, DJ Justice, DL Labor, ST State, IN Interior, TR Treasury, TD Transportation, VA Veterans Affairs; independent agencies have their own codes (SZ SSA, EP EPA, ...) |
132
+ | `agency_code` / `agency` | all | e.g. AR Army, NV Navy, AF Air Force, VA, HS |
133
+ | `agency_subelement_code` | all | bureau or component, e.g. IN05 Bureau of Land Management, TR93 IRS |
134
+ | `cfo_act_agency_indicator` | all | CFO ACT AGENCY (24 large agencies) / OTHER AGENCY |
135
+ | `separation_category_code` | S | SA transfer out (individual), SB transfer out (mass), SC quit, SD retirement voluntary, SE retirement early out, SG retirement other, SH reduction in force (RIF), SJ termination (expired appointment/other), SL other separation |
136
+ | `drp_indicator` | S | Y = separated under the 2025 Deferred Resignation Program (about 96,000 processed in the September 2025 file alone; about 107,000 effective in September 2025 across all files as of October 2026; coded as quit or retirement) |
137
+ | `accession_category_code` | A | AA transfer in (individual), AB transfer in (mass), AC new hire competitive service, AD new hire excepted service, AE new hire SES |
138
+ | `work_schedule_code` | all | F full-time, P part-time, I intermittent, G full-time seasonal, Q part-time seasonal, J intermittent seasonal; the files also hold B, R, S, T and blank: read their labels with `--values work_schedule` and state the grouping (part-time = P, Q, S, T, R; intermittent = I, J) |
139
+ | `pay_plan_code` | all | GS General Schedule (~63 percent), WG/WL/WS wage grade, VN/VM VA medical, NH/ND DoD demo, AD administratively determined, SV TSA, ES Senior Executive Service |
140
+ | `grade`, `step_or_rate_type_code` | all | strings; GS grades "01" to "15" |
141
+ | `pay_basis_code` | all | PA per annum, PH per hour, SY school year, PD per diem, WC without compensation |
142
+ | `annualized_adjusted_basic_pay` | all | dollars as a string, or REDACTED |
143
+ | `occupational_series_code` | all | 4-digit OPM series ("2210" IT management); `occupational_group_code` 2-digit family; `occupational_category_code` P/A/T/C/O/B (professional, administrative, technical, clerical, other white collar, blue collar) |
144
+ | `stem_occupation_type` | all | science, technology, engineering, mathematics, health, all other |
145
+ | `supervisory_status_code` | all | 2 supervisor or manager, 4 supervisor (CSRA), 5 management official, 6 leader, 7 team leader, 8 all other |
146
+ | `appointment_type_code`, `tenure_code`, `position_occupied_code` | all | permanence and service: 10 career, 15 career-conditional, 30/32/35/38 excepted permanent, 50/55 SES; tenure 1 career, 2 conditional (probationary), 3 term; position 1 competitive, 2 excepted, 3/4 SES |
147
+ | `nsftp_indicator` | all | Y = nonseasonal full-time permanent (OPM's standard "permanent workforce" universe) |
148
+ | `age_bracket` | all | LESS THAN 20, 20-24, …, 60-64, 65 OR MORE |
149
+ | `education_level_bracket` | all | HIGH SCHOOL OR LESS, SOME COLLEGE OR ASSOCIATES DEGREE, BACHELORS DEGREE, MASTERS OR PROFESSIONAL DEGREE, DOCTORATE DEGREE |
150
+ | `length_of_service_years` | all | federal civilian plus creditable military service, years with one decimal. **Definition break in March 2026**: from the 2026-03 file it equals years since the leave service computation date for every record; before, for only 72-79 percent (a quarter of records ran about 4 years shorter). A change across March 2026 (+1.6 years Sep 2024 to Aug 2026 as published) is mostly this break: compare on one basis (years since `service_computation_date_leave`) and say so. OPM's Table Builder metric "Average length of service" follows the published field |
151
+ | `veteran_indicator` | all | Y / N, derived by OPM from several veteran fields |
152
+ | `bargaining_unit_status` | all | ELIGIBLE_IN_BU, ELIGIBLE_NOT_IN_BU, NOT_ELIGIBLE (or REDACTED) |
153
+ | `duty_station_state_abbreviation` | all | state of the worksite (REDACTED as in §2) |
154
+
155
+ ## 4. Standard definitions
156
+ - **Headcount (on-board employment):** employees in the status file for the month. Annual series use
157
+ the **September** snapshot (end of the fiscal year; FedScope and OPM convention) unless the user
158
+ wants calendar-year (December) or another month. State the month. End-of-month separations leave
159
+ the headcount only in the next month's snapshot (§2.10).
160
+ - **Permanent workforce:** `nsftp_indicator` = Y (nonseasonal, full-time, permanent). Common in OPM
161
+ and CRS work when temporary and seasonal swings would mislead.
162
+ - **Separation rate:** separations in the period ÷ average headcount in the period (mean of the monthly
163
+ snapshots). Report quit, retirement, RIF and transfer rates separately when they matter; transfers
164
+ between agencies are separations for the losing agency but not losses to the government.
165
+ - **Accession (hiring) rate:** accessions ÷ average headcount. New hires = AC, AD, AE; transfers in are
166
+ not new to government.
167
+ - **Net change:** headcount at the end minus headcount at the start (from status files). Accessions
168
+ minus separations will not match it exactly (conversions, late actions, coverage changes); say so if
169
+ both are shown.
170
+ - **Average and median pay:** of full-time, per-annum employees whose pay is published (§2.1).
171
+ Prefer the median for comparisons across groups; quote coverage.
172
+ - **Retirement-eligible proxy:** no birth date or retirement plan is public. Age 60 or older with 20 or
173
+ more years of service, or age 62+ with 5+ years, approximates FERS eligibility; label it a proxy.
174
+ - **Fiscal year:** October through September (in `compute`, make `yyyymm` a `YYYY-MM` date with `substr`, then `fiscal_year`; FY2025 = October 2024 to
175
+ September 2025). Say "fiscal year 2025" in prose.
176
+
177
+ ## 5. Columns to request for common questions
178
+ Keep the spec lean: ten September snapshots with all 64 columns are ~20 million rows.
179
+ - Headcount trends by agency: `department_code, department, agency_code, agency, nsftp_indicator, work_schedule_code`
180
+ - Composition (age, tenure, education): add `age_bracket, length_of_service_years, education_level_bracket, supervisory_status_code, veteran_indicator`
181
+ - Occupations: `occupational_series_code, occupational_series, occupational_category_code, stem_occupation_type, grade, pay_plan_code`
182
+ - Pay: `annualized_adjusted_basic_pay, pay_plan_code, grade, pay_basis_code, work_schedule_code, occupational_series_code, duty_station_state_abbreviation` (+ the department/agency codes to measure redaction)
183
+ - Turnover: separations `separation_category_code, separation_category, drp_indicator, agency_code, department_code, age_bracket, length_of_service_years`; accessions `accession_category_code, …`; and the employment snapshots of the same months for the denominator
184
+ - Location: `duty_station_state_abbreviation, duty_station_state, core_based_statistical_area` (redaction!)
@@ -1,71 +1,71 @@
1
- # OPM Federal Workforce Data API cheat sheet
2
-
3
- This server's `fetch` for opm implements all of this. Read this file when debugging or
4
- when you need something the spec does not expose. Source: https://data.opm.gov/get-data/api-access
5
- (checked October 2026; the page is a JavaScript app, so read it in a browser, not with curl).
6
-
7
- ## Basics
8
- - Base URL `https://data.opm.gov/api/v1/files`. **No authentication, no key, no account.**
9
- - Data are public (OPM releases them under its Public Data Release Policy), so the filtered data may ship
10
- with a report and its replication package.
11
- - OPM releases each month's data about five weeks after the month ends (for example, August data on
12
- October 7). The FWD Data Sources page shows the next release date.
13
-
14
- ## Endpoints
15
- | What | Call | Returns |
16
- |---|---|---|
17
- | List files | `GET /api/v1/files/{dataset}` | JSON list: `{filename, publishDate, version, current, month, year}` (~75 KiB) |
18
- | Download | `GET /api/v1/files/{dataset}/{year}/{month}/{version}/download` | one Parquet file |
19
-
20
- `dataset` is `employment`, `accessions` or `separations`. `month` is two digits (`08`).
21
-
22
- List filters (query string): `current=true|false`, `version`, `month` (`09`, `Sep`, `September`),
23
- `year` (YYYY), `dataCapturedThrough` (MM/DD/YYYY), `publishedBefore` / `publishedAfter` (MM/DD/YYYY, inclusive).
24
-
25
- Response codes: 200 ok; 204 no content; 400 malformed or over the 25 MB request limit; 404 file not
26
- found (a wrong version returns "The specified blob does not exist."); 429 rate limit; 500 server error.
27
- The client retries 429 and 5xx with backoff.
28
-
29
- ## Files
30
- | Dataset | Content | Rows per file | Size | Periods |
31
- |---|---|---|---|---|
32
- | employment | status snapshot: every employee on board on the last day of the month | ~2.0-2.3 million | 26-75 MB | quarterly (Mar/Jun/Sep/Dec) 2005 to March 2011, monthly from June 2011 (no Jan, Feb, Apr, May 2011), latest about two months back |
33
- | accessions | personnel actions that brought people in, **processed** in the month | 10k-60k | 0.06-1 MB | monthly from 2005-01 |
34
- | separations | personnel actions that took people out, processed in the month | 10k-120k | 0.1-4 MB | monthly from 2005-01 |
35
-
36
- - **Versions:** OPM reissues files when late or corrected agency submissions arrive (versions 1 to 5
37
- seen). Exactly one version per month has `current: true`; use it unless reproducing earlier work, in
38
- which case pin the versions from `data/opm/manifest.json` (the spec's `pin`, e.g. `{"employment_202509": 4}`).
39
- - **Schema changes:** current versions have 64 (employment), 68 (accessions) and 69 (separations)
40
- columns. Some older, superseded versions had only 31 columns, so always use current versions, and
41
- check `missing_columns` in the manifest.
42
- - Every column is a string in Parquet, including numbers (`count`, `annualized_adjusted_basic_pay`,
43
- `length_of_service_years`). The CSV keeps them as published; in `compute`, a column that mixes numbers and
44
- text (pay with `REDACTED`) is read as text: convert it with `number()`.
45
- - Download speed seen: a 57 MB employment file in about 4 seconds. Ten years of September snapshots
46
- (11 files, ~750 MB raw) take one to three minutes plus processing. Raw files are cached in
47
- `<cache root>/opm-fwd` (`~/.cache/data-analyst/opm-fwd`; `$OPM_CACHE_DIR` overrides).
48
-
49
- ## Formats
50
- - **API:** Parquet only. Query parameters such as `?format=txt` are ignored (checked October 2026).
51
- Parquet is columnar and compressed: the August 2026 employment month is 57 MB for 2.0 million rows ×
52
- 64 columns.
53
- - **Data Downloads web page** (https://data.opm.gov/get-data/data-downloads): each file's **Download**
54
- menu offers **.TXT** and **.JSON** only. The download runs through the page's live server connection
55
- (no plain URL to script), so these suit users clicking in a browser. Expect them to be several
56
- times larger than Parquet. OPM warns that employment files exceed Excel's row limit. The exact TXT
57
- delimiter was not confirmed; `ingest` sniffs it (comma, pipe, tab, semicolon, caret) and expects
58
- field names in the first row.
59
- - **Data dictionary:** an Excel file (`FWD Data Dictionary.xlsx`); this pack ships it as
60
- `data_dictionary.csv` (copied into every `data/opm/` folder; `request_schema` lists its columns), so it
61
- never needs downloading.
62
- - **Reading Parquet:** `fetch` and `ingest` read the Parquet files themselves; no other software is needed.
63
-
64
- ## Other OPM tools (no API)
65
- - **Table Builder** (https://data.opm.gov/explore-data/data/table-builder): aggregate tables in a browser;
66
- the fallback when a user cannot download files at all.
67
- - **Dashboards** ("Workforce Insights" pages): published headline figures to benchmark against.
68
- - **FedScope** (fedscope.opm.gov) was the predecessor; FWD replaced it. Older reports cite FedScope's
69
- September snapshots, which is why September is the conventional annual snapshot.
70
- - **Data Quality page** (https://data.opm.gov/resources/data-quality): validation errors by agency and a
71
- "Missing EHRI files" report. Check it when an agency's count jumps or drops.
1
+ # OPM Federal Workforce Data API cheat sheet
2
+
3
+ This server's `fetch` for opm implements all of this. Read this file when debugging or
4
+ when you need something the spec does not expose. Source: https://data.opm.gov/get-data/api-access
5
+ (checked October 2026; the page is a JavaScript app, so read it in a browser, not with curl).
6
+
7
+ ## Basics
8
+ - Base URL `https://data.opm.gov/api/v1/files`. **No authentication, no key, no account.**
9
+ - Data are public (OPM releases them under its Public Data Release Policy), so the filtered data may ship
10
+ with a report and its replication package.
11
+ - OPM releases each month's data about five weeks after the month ends (for example, August data on
12
+ October 7). The FWD Data Sources page shows the next release date.
13
+
14
+ ## Endpoints
15
+ | What | Call | Returns |
16
+ |---|---|---|
17
+ | List files | `GET /api/v1/files/{dataset}` | JSON list: `{filename, publishDate, version, current, month, year}` (~75 KiB) |
18
+ | Download | `GET /api/v1/files/{dataset}/{year}/{month}/{version}/download` | one Parquet file |
19
+
20
+ `dataset` is `employment`, `accessions` or `separations`. `month` is two digits (`08`).
21
+
22
+ List filters (query string): `current=true|false`, `version`, `month` (`09`, `Sep`, `September`),
23
+ `year` (YYYY), `dataCapturedThrough` (MM/DD/YYYY), `publishedBefore` / `publishedAfter` (MM/DD/YYYY, inclusive).
24
+
25
+ Response codes: 200 ok; 204 no content; 400 malformed or over the 25 MB request limit; 404 file not
26
+ found (a wrong version returns "The specified blob does not exist."); 429 rate limit; 500 server error.
27
+ The client retries 429 and 5xx with backoff.
28
+
29
+ ## Files
30
+ | Dataset | Content | Rows per file | Size | Periods |
31
+ |---|---|---|---|---|
32
+ | employment | status snapshot: every employee on board on the last day of the month | ~2.0-2.3 million | 26-75 MB | quarterly (Mar/Jun/Sep/Dec) 2005 to March 2011, monthly from June 2011 (no Jan, Feb, Apr, May 2011), latest about two months back |
33
+ | accessions | personnel actions that brought people in, **processed** in the month | 10k-60k | 0.06-1 MB | monthly from 2005-01 |
34
+ | separations | personnel actions that took people out, processed in the month | 10k-120k | 0.1-4 MB | monthly from 2005-01 |
35
+
36
+ - **Versions:** OPM reissues files when late or corrected agency submissions arrive (versions 1 to 5
37
+ seen). Exactly one version per month has `current: true`; use it unless reproducing earlier work, in
38
+ which case pin the versions from `data/opm/manifest.json` (the spec's `pin`, e.g. `{"employment_202509": 4}`).
39
+ - **Schema changes:** current versions have 64 (employment), 68 (accessions) and 69 (separations)
40
+ columns. Some older, superseded versions had only 31 columns, so always use current versions, and
41
+ check `missing_columns` in the manifest.
42
+ - Every column is a string in Parquet, including numbers (`count`, `annualized_adjusted_basic_pay`,
43
+ `length_of_service_years`). The CSV keeps them as published; in `compute`, a column that mixes numbers and
44
+ text (pay with `REDACTED`) is read as text: convert it with `number()`.
45
+ - Download speed seen: a 57 MB employment file in about 4 seconds. Ten years of September snapshots
46
+ (11 files, ~750 MB raw) take one to three minutes plus processing. Raw files are cached in
47
+ `<cache root>/opm-fwd` (`~/.cache/data-analyst/opm-fwd`; `$OPM_CACHE_DIR` overrides).
48
+
49
+ ## Formats
50
+ - **API:** Parquet only. Query parameters such as `?format=txt` are ignored (checked October 2026).
51
+ Parquet is columnar and compressed: the August 2026 employment month is 57 MB for 2.0 million rows ×
52
+ 64 columns.
53
+ - **Data Downloads web page** (https://data.opm.gov/get-data/data-downloads): each file's **Download**
54
+ menu offers **.TXT** and **.JSON** only. The download runs through the page's live server connection
55
+ (no plain URL to script), so these suit users clicking in a browser. Expect them to be several
56
+ times larger than Parquet. OPM warns that employment files exceed Excel's row limit. The exact TXT
57
+ delimiter was not confirmed; `ingest` sniffs it (comma, pipe, tab, semicolon, caret) and expects
58
+ field names in the first row.
59
+ - **Data dictionary:** an Excel file (`FWD Data Dictionary.xlsx`); this pack ships it as
60
+ `data_dictionary.csv` (copied into every `data/opm/` folder; `request_schema` lists its columns), so it
61
+ never needs downloading.
62
+ - **Reading Parquet:** `fetch` and `ingest` read the Parquet files themselves; no other software is needed.
63
+
64
+ ## Other OPM tools (no API)
65
+ - **Table Builder** (https://data.opm.gov/explore-data/data/table-builder): aggregate tables in a browser;
66
+ the fallback when a user cannot download files at all.
67
+ - **Dashboards** ("Workforce Insights" pages): published headline figures to benchmark against.
68
+ - **FedScope** (fedscope.opm.gov) was the predecessor; FWD replaced it. Older reports cite FedScope's
69
+ September snapshots, which is why September is the conventional annual snapshot.
70
+ - **Data Quality page** (https://data.opm.gov/resources/data-quality): validation errors by agency and a
71
+ "Missing EHRI files" report. Check it when an agency's count jumps or drops.