data-analyst-mcp 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/adapters/_census_micro.d.ts +2 -0
- package/dist/adapters/_census_micro.js +24 -1
- package/dist/adapters/_census_micro.js.map +1 -1
- package/dist/analysis/combine.js +3 -2
- package/dist/analysis/combine.js.map +1 -1
- package/dist/analysis/compute.d.ts +30 -7
- package/dist/analysis/compute.js +414 -162
- package/dist/analysis/compute.js.map +1 -1
- package/dist/analysis/expr.js +6 -2
- package/dist/analysis/expr.js.map +1 -1
- package/dist/analysis/handoff.d.ts +1 -1
- package/dist/analysis/handoff.js +44 -3
- package/dist/analysis/handoff.js.map +1 -1
- package/dist/analysis/rerun.js +2 -1
- package/dist/analysis/rerun.js.map +1 -1
- package/dist/analysis/rerun_job.d.ts +30 -0
- package/dist/analysis/rerun_job.js +173 -0
- package/dist/analysis/rerun_job.js.map +1 -0
- package/dist/analysis/rerun_worker.d.ts +1 -0
- package/dist/analysis/rerun_worker.js +13 -0
- package/dist/analysis/rerun_worker.js.map +1 -0
- package/dist/analysis/table.d.ts +7 -0
- package/dist/analysis/table.js +30 -1
- package/dist/analysis/table.js.map +1 -1
- package/dist/analysis/tools.d.ts +10 -7
- package/dist/analysis/tools.js +43 -45
- package/dist/analysis/tools.js.map +1 -1
- package/dist/core/compact.d.ts +15 -0
- package/dist/core/compact.js +98 -0
- package/dist/core/compact.js.map +1 -0
- package/dist/core/package.d.ts +8 -0
- package/dist/core/package.js +30 -1
- package/dist/core/package.js.map +1 -1
- package/dist/core/packs.d.ts +4 -2
- package/dist/core/sections.d.ts +21 -0
- package/dist/core/sections.js +53 -0
- package/dist/core/sections.js.map +1 -0
- package/dist/deep/opm.js +10 -2
- package/dist/deep/opm.js.map +1 -1
- package/dist/deep/treasury.js +7 -3
- package/dist/deep/treasury.js.map +1 -1
- package/dist/envelope.d.ts +1 -1
- package/dist/envelope.js +1 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +57 -4
- package/dist/report/audit_answer.d.ts +1 -1
- package/dist/report/audit_answer.js +2 -2
- package/dist/report/audit_answer.js.map +1 -1
- package/dist/report/audit_report.d.ts +1 -1
- package/dist/report/audit_report.js +2 -2
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.d.ts +18 -0
- package/dist/report/build_report.js +52 -7
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.d.ts +4 -3
- package/dist/report/method_audit.js +8 -5
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/tools.d.ts +31 -1
- package/dist/report/tools.js +101 -23
- package/dist/report/tools.js.map +1 -1
- package/dist/server.d.ts +2 -2
- package/dist/server.js +54 -41
- package/dist/server.js.map +1 -1
- package/dist/session.d.ts +8 -2
- package/dist/session.js +9 -0
- package/dist/session.js.map +1 -1
- package/dist/tools.d.ts +98 -6
- package/dist/tools.js +154 -17
- package/dist/tools.js.map +1 -1
- package/dist/workspace_tool.d.ts +47 -0
- package/dist/workspace_tool.js +94 -0
- package/dist/workspace_tool.js.map +1 -0
- package/guide/WORKFLOW.md +7 -4
- package/guide/references/compute.md +51 -0
- package/guide/references/report_quick.md +60 -0
- package/package.json +1 -1
- package/packs/census/docs/microdata.md +77 -0
- package/packs/census/docs/timeseries.md +71 -0
- package/packs/census/source.md +10 -140
- package/packs/congress/docs/nominations_crs.md +26 -0
- package/packs/congress/docs/votes.md +30 -0
- package/packs/congress/source.md +4 -48
- package/packs/govinfo/docs/collections.md +88 -0
- package/packs/govinfo/source.md +6 -86
- package/packs/irs-soi/docs/eo.md +15 -0
- package/packs/irs-soi/docs/tables.md +50 -0
- package/packs/irs-soi/source.md +6 -57
package/dist/analysis/compute.js
CHANGED
|
@@ -15,11 +15,12 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import fs from "node:fs";
|
|
17
17
|
import path from "node:path";
|
|
18
|
+
import { namesHint } from "../core/compact.js";
|
|
18
19
|
import { addStateCodes, alignCheck, CombineError, deflate, fiscalYear, toFiscal } from "./combine.js";
|
|
19
|
-
import { compare, compile, evaluate, ExprError, isTruthy } from "./expr.js";
|
|
20
|
+
import { compare, compile, evaluate, ExprError, inspect, isTruthy, parse as parseExpr } from "./expr.js";
|
|
20
21
|
import { pyRepr } from "./pyfmt.js";
|
|
21
|
-
import { replicateSe } from "./replicate.js";
|
|
22
|
-
import { cellText, newRow,
|
|
22
|
+
import { replicateColumns, replicateSe } from "./replicate.js";
|
|
23
|
+
import { cellText, csvGrid, headerOf, newRow, PARQUET_MAX_CELLS, parquetColumns, ParquetInputError, readParquetTable, sha256, tableCsv, tableFromGrid } from "./table.js";
|
|
23
24
|
export class ComputeError extends Error {
|
|
24
25
|
}
|
|
25
26
|
export const LINEAGE = "exhibits/lineage.json";
|
|
@@ -60,8 +61,82 @@ export function normInput(alias, v) {
|
|
|
60
61
|
}
|
|
61
62
|
throw new ComputeError(`input ${alias}: a workspace path, or {path, columns: [names]}, not ${pyRepr(v)}`);
|
|
62
63
|
}
|
|
64
|
+
/** The Parquet copy fetch writes beside a large data/<id>/<table>.csv, when it is there and not older than the CSV. */
|
|
65
|
+
function parquetCopy(csvAbs) {
|
|
66
|
+
if (!/[\\/]data[\\/][^\\/]+[\\/][^\\/]+\.csv$/i.test(csvAbs))
|
|
67
|
+
return null;
|
|
68
|
+
const pq = csvAbs.replace(/\.csv$/i, ".parquet");
|
|
69
|
+
try {
|
|
70
|
+
return fs.statSync(pq).mtimeMs >= fs.statSync(csvAbs).mtimeMs ? pq : null;
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** An input's column names, read before the table (the header line of a CSV, the schema of a Parquet file). */
|
|
77
|
+
function inputHeader(alias, rel, abs, buf, parquet) {
|
|
78
|
+
if (parquet) {
|
|
79
|
+
try {
|
|
80
|
+
return parquetColumns(buf, abs);
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
throw new ComputeError(`input ${alias}: ${relPath(rel)} could not be read as Parquet: ${e?.message || e}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return headerOf(buf.subarray(0, Math.min(buf.length, 1 << 20)).toString("utf8").replace(/^\uFEFF/, ""));
|
|
87
|
+
}
|
|
88
|
+
/** The most cells of a CSV parsed at once (every column of every row, before any column is dropped). A 37 MB ACS PUMS
|
|
89
|
+
* file of 62,777 rows x 175 columns (11M cells) took 1.5 s and about 360 MB to parse; 60M is about 2 GB. */
|
|
90
|
+
export const CSV_MAX_RAW_CELLS = { value: 60_000_000 };
|
|
91
|
+
/** Parsed CSV grids kept across calls, keyed by path and content hash: compute and the rerun read the same large
|
|
92
|
+
* microdata file again and again. Only files of at least CACHE_MIN_BYTES; at most the budget in total (about ten
|
|
93
|
+
* times the file size each), least recently used dropped first. */
|
|
94
|
+
const GRID_CACHE = new Map();
|
|
95
|
+
const CACHE_MIN_BYTES = 5_000_000;
|
|
96
|
+
function cacheBudget() {
|
|
97
|
+
const mb = Number(process.env.DATA_ANALYST_PARSE_CACHE_MB);
|
|
98
|
+
return (Number.isFinite(mb) && mb >= 0 ? mb : 512) * 1024 * 1024;
|
|
99
|
+
}
|
|
100
|
+
function cachedGrid(abs, sha, buf) {
|
|
101
|
+
const key = `${abs}\u0000${sha}`;
|
|
102
|
+
const hit = GRID_CACHE.get(key);
|
|
103
|
+
if (hit) {
|
|
104
|
+
GRID_CACHE.delete(key); // most recently used last
|
|
105
|
+
GRID_CACHE.set(key, hit);
|
|
106
|
+
return hit.grid;
|
|
107
|
+
}
|
|
108
|
+
const grid = csvGrid(buf.toString("utf8"));
|
|
109
|
+
const bytes = buf.length * 10;
|
|
110
|
+
const budget = cacheBudget();
|
|
111
|
+
if (buf.length >= CACHE_MIN_BYTES && bytes <= budget) {
|
|
112
|
+
let used = [...GRID_CACHE.values()].reduce((a, v) => a + v.bytes, 0);
|
|
113
|
+
for (const [k, v] of GRID_CACHE) {
|
|
114
|
+
if (used + bytes <= budget)
|
|
115
|
+
break;
|
|
116
|
+
GRID_CACHE.delete(k);
|
|
117
|
+
used -= v.bytes;
|
|
118
|
+
}
|
|
119
|
+
GRID_CACHE.set(key, { grid, bytes });
|
|
120
|
+
}
|
|
121
|
+
return grid;
|
|
122
|
+
}
|
|
123
|
+
/** Drop every cached grid (tests). */
|
|
124
|
+
export function clearParseCache() {
|
|
125
|
+
GRID_CACHE.clear();
|
|
126
|
+
}
|
|
127
|
+
/** How many parsed grids are cached (tests). */
|
|
128
|
+
export function parseCacheSize() {
|
|
129
|
+
return GRID_CACHE.size;
|
|
130
|
+
}
|
|
131
|
+
/** Line ends in a buffer (an upper bound on a CSV's rows, for the cell caps). */
|
|
132
|
+
function lineEnds(buf) {
|
|
133
|
+
let n = 0;
|
|
134
|
+
for (let i = buf.indexOf(10); i !== -1; i = buf.indexOf(10, i + 1))
|
|
135
|
+
n++;
|
|
136
|
+
return n;
|
|
137
|
+
}
|
|
63
138
|
/** Read one input as a typed table (only its named columns, when given). */
|
|
64
|
-
function readInput(alias, abs, inp, buf) {
|
|
139
|
+
function readInput(alias, abs, inp, buf, sha, header) {
|
|
65
140
|
if (relPath(inp.path).endsWith(".parquet")) {
|
|
66
141
|
try {
|
|
67
142
|
return readParquetTable(buf, abs, inp.columns);
|
|
@@ -72,19 +147,104 @@ function readInput(alias, abs, inp, buf) {
|
|
|
72
147
|
throw new ComputeError(`input ${alias}: ${relPath(inp.path)} could not be read as Parquet: ${e?.message || e}`);
|
|
73
148
|
}
|
|
74
149
|
}
|
|
75
|
-
const t = parseTable(buf.toString("utf8"));
|
|
76
150
|
const want = inp.columns;
|
|
77
|
-
if (!
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
151
|
+
if (!GRID_CACHE.has(`${abs}\u0000${sha}`)) {
|
|
152
|
+
const rows = lineEnds(buf);
|
|
153
|
+
const raw = rows * header.length, typed = rows * (want ? want.length : header.length);
|
|
154
|
+
if (raw > CSV_MAX_RAW_CELLS.value || typed > PARQUET_MAX_CELLS.value) {
|
|
155
|
+
const n = (x) => x.toLocaleString("en-US");
|
|
156
|
+
throw new ComputeError(`input ${alias}: ${relPath(inp.path)} has about ${n(rows)} rows x ${n(header.length)} columns, ` +
|
|
157
|
+
`more than compute reads at once (${n(CSV_MAX_RAW_CELLS.value)} cells parsed, ${n(PARQUET_MAX_CELLS.value)} kept). ` +
|
|
158
|
+
(raw > CSV_MAX_RAW_CELLS.value
|
|
159
|
+
? "Read the .parquet file beside it when the fetch wrote one ({path, columns}), or fetch fewer rows or variables."
|
|
160
|
+
: `Name only the columns the steps need: {"${alias}": {"path": "${relPath(inp.path)}", "columns": [...]}}.`));
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return tableFromGrid(header, cachedGrid(abs, sha, buf), want);
|
|
164
|
+
}
|
|
165
|
+
/** The input columns a single-input recipe reads, when its steps say so: every column the steps name up to the first
|
|
166
|
+
* step that keeps only named columns (select, rollup, replicate_se with estimates, pivot_wider with id_cols). Null when
|
|
167
|
+
* any step could read columns it does not name (dedupe: true, other tables, ...) or no step narrows the table, so
|
|
168
|
+
* every column reaches the output: then the whole table is read. */
|
|
169
|
+
export function neededColumns(p, header) {
|
|
170
|
+
try {
|
|
171
|
+
const need = new Set();
|
|
172
|
+
const add = (c) => { if (typeof c === "string")
|
|
173
|
+
need.add(c); };
|
|
174
|
+
const ref = (src) => { for (const c of inspect(parseExpr(src)).cols)
|
|
175
|
+
need.add(c); };
|
|
176
|
+
const done = () => {
|
|
177
|
+
const out = header.filter((c) => need.has(c));
|
|
178
|
+
return out.length && out.length < header.length ? out : null;
|
|
179
|
+
};
|
|
180
|
+
for (const step of p.steps || []) {
|
|
181
|
+
const verbs = Object.keys(step || {}).filter((k) => VERBS.includes(k));
|
|
182
|
+
if (verbs.length !== 1)
|
|
183
|
+
return null;
|
|
184
|
+
const verb = verbs[0], arg = step[verb];
|
|
185
|
+
if (step.groupby !== undefined)
|
|
186
|
+
strList(step.groupby, "groupby").forEach(add);
|
|
187
|
+
switch (verb) {
|
|
188
|
+
case "filter":
|
|
189
|
+
ref(arg);
|
|
190
|
+
break;
|
|
191
|
+
case "derive":
|
|
192
|
+
for (const [, e] of exprs(arg, "derive"))
|
|
193
|
+
ref(e);
|
|
194
|
+
break;
|
|
195
|
+
case "rename":
|
|
196
|
+
for (const [a] of exprs(arg, "rename"))
|
|
197
|
+
add(a);
|
|
198
|
+
break;
|
|
199
|
+
case "orderby":
|
|
200
|
+
strList(arg, "orderby").forEach((c) => add(c.replace(/^-/, "")));
|
|
201
|
+
break;
|
|
202
|
+
case "slice": break;
|
|
203
|
+
case "dedupe":
|
|
204
|
+
if (arg === true)
|
|
205
|
+
return null;
|
|
206
|
+
strList(arg, "dedupe").forEach(add);
|
|
207
|
+
break;
|
|
208
|
+
case "fiscal_year":
|
|
209
|
+
add(typeof arg === "string" ? arg : arg?.date);
|
|
210
|
+
break;
|
|
211
|
+
case "state_codes":
|
|
212
|
+
add(typeof arg === "string" ? arg : arg?.col);
|
|
213
|
+
break;
|
|
214
|
+
case "select":
|
|
215
|
+
strList(arg, "select").forEach(add);
|
|
216
|
+
return done();
|
|
217
|
+
case "rollup":
|
|
218
|
+
for (const [, e] of exprs(arg, "rollup"))
|
|
219
|
+
ref(e);
|
|
220
|
+
return done();
|
|
221
|
+
case "replicate_se": {
|
|
222
|
+
if (!arg || typeof arg !== "object" || !arg.estimates || typeof arg.estimates !== "object")
|
|
223
|
+
return null;
|
|
224
|
+
add(arg.weight);
|
|
225
|
+
replicateColumns(arg.replicates).forEach(add);
|
|
226
|
+
for (const e of Object.values(arg.estimates))
|
|
227
|
+
ref(e);
|
|
228
|
+
for (const e of Object.values(arg.also || {}))
|
|
229
|
+
ref(e);
|
|
230
|
+
return done();
|
|
231
|
+
}
|
|
232
|
+
case "pivot_wider":
|
|
233
|
+
if (!arg?.id_cols)
|
|
234
|
+
return null;
|
|
235
|
+
strList(arg.id_cols, "id_cols").forEach(add);
|
|
236
|
+
add(arg.names_from);
|
|
237
|
+
add(arg.values_from);
|
|
238
|
+
return done();
|
|
239
|
+
default:
|
|
240
|
+
return null; // other tables, value-dependent columns
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
return null; // a malformed step: the full read reports it
|
|
247
|
+
}
|
|
88
248
|
}
|
|
89
249
|
export function readLineage(workspace) {
|
|
90
250
|
try {
|
|
@@ -126,7 +286,7 @@ const EXTRA = { filter: ["groupby"], derive: ["groupby"], rollup: ["groupby"], r
|
|
|
126
286
|
export function need(t, cols, what) {
|
|
127
287
|
const miss = cols.filter((c) => !t.cols.includes(c));
|
|
128
288
|
if (miss.length)
|
|
129
|
-
throw new ComputeError(`${what}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")};
|
|
289
|
+
throw new ComputeError(`${what}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; ${namesHint(miss, t.cols, "columns", pyRepr)}`);
|
|
130
290
|
}
|
|
131
291
|
export function strList(v, what) {
|
|
132
292
|
const xs = typeof v === "string" ? [v] : v;
|
|
@@ -166,14 +326,18 @@ function exprs(v, what) {
|
|
|
166
326
|
throw new ComputeError(`${what}: give at least one {name: expression}`);
|
|
167
327
|
return out;
|
|
168
328
|
}
|
|
329
|
+
/** Row lists this pipeline made itself and nothing else holds: derive may add columns to their rows in place instead
|
|
330
|
+
* of copying every row (an input table, a parse-cache entry or an "as" table is never in it). */
|
|
331
|
+
const OWNED = new WeakSet();
|
|
169
332
|
function derive(t, spec, by) {
|
|
170
333
|
let cols = [...t.cols];
|
|
171
|
-
const rows = t.rows.map((r) => Object.assign(newRow(), r));
|
|
334
|
+
const rows = OWNED.has(t.rows) ? t.rows : t.rows.map((r) => Object.assign(newRow(), r));
|
|
335
|
+
OWNED.add(rows);
|
|
172
336
|
for (const [name, src] of exprs(spec, "derive")) {
|
|
173
337
|
const node = compile(src, cols, { what: "derive" });
|
|
174
338
|
for (const g of groupRows({ cols, rows }, by, false)) {
|
|
175
|
-
const ctx = { i: 0, group: g, cache: new Map() };
|
|
176
|
-
const vals = g.map((r, i) =>
|
|
339
|
+
const ctx = { i: 0, group: g, cache: new Map() }; // moved row by row: evaluate keeps no reference to it
|
|
340
|
+
const vals = g.map((r, i) => ((ctx.row = r), (ctx.i = i), evaluate(node, ctx)));
|
|
177
341
|
g.forEach((r, i) => (r[name] = vals[i]));
|
|
178
342
|
}
|
|
179
343
|
if (!cols.includes(name))
|
|
@@ -413,14 +577,16 @@ function wrap(i, verb, fn) {
|
|
|
413
577
|
throw e;
|
|
414
578
|
}
|
|
415
579
|
}
|
|
416
|
-
/** Read the inputs and run the steps (no file is written).
|
|
417
|
-
|
|
580
|
+
/** Read the inputs and run the steps (no file is written). `parsed` keeps each input read once across calls (the
|
|
581
|
+
* rerun replays many recipes over one large microdata file); it is keyed by path, content hash and columns, and is
|
|
582
|
+
* safe to share because no step changes its input table in place (every verb builds new rows). */
|
|
583
|
+
export function runPipeline(workspace, p, parsed) {
|
|
418
584
|
if (!p.inputs || typeof p.inputs !== "object" || !Object.keys(p.inputs).length) {
|
|
419
585
|
throw new ComputeError('inputs: {alias: "data/<id>/<table>.csv" | "data/<id>/<table>.parquet" | "exhibits/<x>.csv" | ' +
|
|
420
586
|
"{path, columns}}, at least one");
|
|
421
587
|
}
|
|
422
|
-
const env = new Map();
|
|
423
588
|
const inputs = {};
|
|
589
|
+
const prepared = [];
|
|
424
590
|
for (const [alias, raw] of Object.entries(p.inputs)) {
|
|
425
591
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(alias) || alias === "this" || alias.startsWith("__"))
|
|
426
592
|
throw new ComputeError(`input alias ${pyRepr(alias)}: letters, digits and _ only`);
|
|
@@ -430,9 +596,48 @@ export function runPipeline(workspace, p) {
|
|
|
430
596
|
if (!fs.existsSync(abs))
|
|
431
597
|
throw new ComputeError(`input ${alias}: no file ${relPath(rel)} in the workspace (fetch it first, or check the path)`);
|
|
432
598
|
const buf = fs.readFileSync(abs);
|
|
433
|
-
|
|
434
|
-
|
|
599
|
+
const sha = sha256(buf);
|
|
600
|
+
inputs[alias] = { path: relPath(rel), sha256: sha, ...(inp.columns ? { columns: inp.columns } : {}) };
|
|
601
|
+
const parquet = relPath(rel).endsWith(".parquet");
|
|
602
|
+
prepared.push({ alias, inp, abs, buf, sha, parquet, header: inputHeader(alias, rel, abs, buf, parquet) });
|
|
603
|
+
}
|
|
604
|
+
for (const x of prepared) {
|
|
605
|
+
const miss = (x.inp.columns || []).filter((c) => !x.header.includes(c));
|
|
606
|
+
if (!miss.length)
|
|
607
|
+
continue;
|
|
608
|
+
const what = `no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}`;
|
|
609
|
+
throw new ComputeError(x.parquet // the messages the readers give (the Parquet one names the file)
|
|
610
|
+
? `input ${x.alias} (${relPath(x.inp.path)}): ${what}; ${namesHint(miss, x.header, "columns", (v) => v.join(", "))}`
|
|
611
|
+
: `input ${x.alias}: ${what}; ${namesHint(miss, x.header, "columns", pyRepr)}`);
|
|
612
|
+
}
|
|
613
|
+
// a dry run of the steps on empty tables with the inputs' headers: a misspelt column fails here, before a large
|
|
614
|
+
// file is parsed (only what the data cannot change is reported; data-dependent steps end the dry run)
|
|
615
|
+
runSteps(p, new Map(prepared.map((x) => [x.alias, { cols: x.inp.columns ?? x.header, rows: [] }])), true);
|
|
616
|
+
// one input and no columns named: read only the columns the steps use, when the steps say which those are
|
|
617
|
+
const auto = prepared.length === 1 && !prepared[0].inp.columns ? neededColumns(p, prepared[0].header) : null;
|
|
618
|
+
const env = new Map();
|
|
619
|
+
for (const x of prepared) {
|
|
620
|
+
const want = x.inp.columns ?? auto ?? undefined;
|
|
621
|
+
const key = `${x.abs}\u0000${x.sha}\u0000${want ? JSON.stringify(want) : ""}`;
|
|
622
|
+
let t = parsed?.get(key);
|
|
623
|
+
if (!t) {
|
|
624
|
+
// only some columns of a fetched CSV that has a Parquet copy: decode those columns from the copy (same table)
|
|
625
|
+
const pq = want && !x.parquet ? parquetCopy(x.abs) : null;
|
|
626
|
+
t = pq ? readInput(x.alias, pq, { path: relPath(x.inp.path).replace(/\.csv$/, ".parquet"), columns: want }, fs.readFileSync(pq), x.sha, x.header)
|
|
627
|
+
: readInput(x.alias, x.abs, { path: x.inp.path, columns: want }, x.buf, x.sha, x.header);
|
|
628
|
+
parsed?.set(key, t);
|
|
629
|
+
}
|
|
630
|
+
env.set(x.alias, t);
|
|
435
631
|
}
|
|
632
|
+
const r = runSteps(p, env, false);
|
|
633
|
+
return { table: r.table, notes: r.notes, inputs };
|
|
634
|
+
}
|
|
635
|
+
/** Verbs whose result depends on the data's values (new columns from values, lookups, checks): a dry run stops there. */
|
|
636
|
+
const DRY_STOP = new Set(["pivot_wider", "pivot_longer", "to_fiscal", "deflate", "replicate_se", "align_check"]);
|
|
637
|
+
/** Errors a dry run reports: a column or table that does not exist. */
|
|
638
|
+
const STRUCTURAL = /unknown column|no column|is not a table here|is not an input alias/;
|
|
639
|
+
/** Run the steps over the tables in env (`dry`: on empty tables, as a check before the inputs are read). */
|
|
640
|
+
function runSteps(p, env, dry) {
|
|
436
641
|
const start = p.from ?? Object.keys(p.inputs)[0];
|
|
437
642
|
if (!env.has(start))
|
|
438
643
|
throw new ComputeError(`from: ${pyRepr(start)} is not an input alias (${Object.keys(p.inputs).join(", ")})`);
|
|
@@ -448,7 +653,8 @@ export function runPipeline(workspace, p) {
|
|
|
448
653
|
};
|
|
449
654
|
if (!Array.isArray(p.steps))
|
|
450
655
|
throw new ComputeError("steps: a list of steps");
|
|
451
|
-
p.steps.
|
|
656
|
+
for (let i = 0; i < p.steps.length; i++) {
|
|
657
|
+
const step = p.steps[i];
|
|
452
658
|
if (!step || typeof step !== "object" || Array.isArray(step))
|
|
453
659
|
throw new ComputeError(`step ${i + 1}: an object such as {"filter": "year >= 2020"}`);
|
|
454
660
|
const verbs = Object.keys(step).filter((k) => VERBS.includes(k));
|
|
@@ -461,144 +667,161 @@ export function runPipeline(workspace, p) {
|
|
|
461
667
|
if (extra.length)
|
|
462
668
|
throw new ComputeError(`step ${i + 1} (${verb}): unknown key(s) ${pyRepr(extra)}; allowed with ${verb}: ${pyRepr(EXTRA[verb] || [])}`);
|
|
463
669
|
const arg = step[verb];
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
const map = new Map(pairs);
|
|
488
|
-
const cols = cur.cols.map((c) => map.get(c) ?? c);
|
|
489
|
-
const dup = cols.filter((c, j) => cols.indexOf(c) !== j);
|
|
490
|
-
if (dup.length)
|
|
491
|
-
throw new ComputeError(`rename: ${pyRepr([...new Set(dup)])} would name two columns`);
|
|
492
|
-
cur = { cols, rows: cur.rows.map((r) => {
|
|
493
|
-
const o = newRow();
|
|
494
|
-
cur.cols.forEach((c, j) => (o[cols[j]] = r[c]));
|
|
495
|
-
return o;
|
|
496
|
-
}) };
|
|
497
|
-
break;
|
|
498
|
-
}
|
|
499
|
-
case "join": {
|
|
500
|
-
const r = join(cur, table(arg, "join"), step);
|
|
501
|
-
cur = r.table;
|
|
502
|
-
notes.push(...r.notes);
|
|
503
|
-
break;
|
|
504
|
-
}
|
|
505
|
-
case "pivot_wider":
|
|
506
|
-
cur = pivotWider(cur, arg);
|
|
507
|
-
break;
|
|
508
|
-
case "pivot_longer":
|
|
509
|
-
cur = pivotLonger(cur, arg);
|
|
510
|
-
break;
|
|
511
|
-
case "orderby":
|
|
512
|
-
cur = orderby(cur, arg);
|
|
513
|
-
break;
|
|
514
|
-
case "slice":
|
|
515
|
-
cur = slice(cur, arg);
|
|
516
|
-
break;
|
|
517
|
-
case "dedupe":
|
|
518
|
-
cur = dedupe(cur, arg);
|
|
519
|
-
break;
|
|
520
|
-
case "concat":
|
|
521
|
-
cur = concat([cur, ...(Array.isArray(arg) ? arg : [arg]).map((a) => table(a, "concat"))]);
|
|
522
|
-
break;
|
|
523
|
-
case "fiscal_year": {
|
|
524
|
-
const o = typeof arg === "string" ? { date: arg } : arg || {};
|
|
525
|
-
const as = String(o.as ?? "fiscal_year");
|
|
526
|
-
need(cur, [String(o.date)], "fiscal_year");
|
|
527
|
-
cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r) => {
|
|
528
|
-
const x = Object.assign(newRow(), r);
|
|
529
|
-
x[as] = r[o.date] === null ? null : fiscalYear(r[o.date], o.date);
|
|
530
|
-
return x;
|
|
531
|
-
}) };
|
|
532
|
-
break;
|
|
533
|
-
}
|
|
534
|
-
case "to_fiscal": {
|
|
535
|
-
const o = arg || {};
|
|
536
|
-
const r = toFiscal(cur, String(o.date), String(o.value), o.how ?? "sum", o.freq ?? "M");
|
|
537
|
-
cur = r.table;
|
|
538
|
-
notes.push(...r.notes);
|
|
539
|
-
break;
|
|
540
|
-
}
|
|
541
|
-
case "state_codes":
|
|
542
|
-
cur = addStateCodes(cur, typeof arg === "string" ? arg : String(arg?.col));
|
|
543
|
-
break;
|
|
544
|
-
case "deflate": {
|
|
545
|
-
const o = arg || {};
|
|
546
|
-
if (!o.name || typeof o.name !== "string") {
|
|
547
|
-
throw new ComputeError('deflate: name the price index and its source in "name" (e.g. "CPI-U, BLS CUUR0000SA0"), so the ' +
|
|
548
|
-
"answer can say which dollars these are");
|
|
670
|
+
if (dry && DRY_STOP.has(verb))
|
|
671
|
+
break;
|
|
672
|
+
const before = cur.rows;
|
|
673
|
+
try {
|
|
674
|
+
wrap(i, verb, () => {
|
|
675
|
+
const by = step.groupby === undefined ? [] : strList(step.groupby, "groupby");
|
|
676
|
+
if (by.length)
|
|
677
|
+
need(cur, by, "groupby");
|
|
678
|
+
switch (verb) {
|
|
679
|
+
case "filter":
|
|
680
|
+
cur = filter(cur, arg, by);
|
|
681
|
+
break;
|
|
682
|
+
case "derive":
|
|
683
|
+
cur = derive(cur, arg, by);
|
|
684
|
+
break;
|
|
685
|
+
case "rollup":
|
|
686
|
+
cur = rollup(cur, arg, by);
|
|
687
|
+
break;
|
|
688
|
+
case "select": {
|
|
689
|
+
const cols = strList(arg, "select");
|
|
690
|
+
need(cur, cols, "select");
|
|
691
|
+
cur = { cols: [...new Set(cols)], rows: cur.rows };
|
|
692
|
+
break;
|
|
549
693
|
}
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
694
|
+
case "rename": {
|
|
695
|
+
const pairs = exprs(arg, "rename").map(([a, b]) => [a, String(b)]);
|
|
696
|
+
need(cur, pairs.map(([a]) => a), "rename");
|
|
697
|
+
const map = new Map(pairs);
|
|
698
|
+
const cols = cur.cols.map((c) => map.get(c) ?? c);
|
|
699
|
+
const dup = cols.filter((c, j) => cols.indexOf(c) !== j);
|
|
700
|
+
if (dup.length)
|
|
701
|
+
throw new ComputeError(`rename: ${pyRepr([...new Set(dup)])} would name two columns`);
|
|
702
|
+
cur = { cols, rows: cur.rows.map((r) => {
|
|
703
|
+
const o = newRow();
|
|
704
|
+
cur.cols.forEach((c, j) => (o[cols[j]] = r[c]));
|
|
705
|
+
return o;
|
|
706
|
+
}) };
|
|
707
|
+
break;
|
|
564
708
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
709
|
+
case "join": {
|
|
710
|
+
const r = join(cur, table(arg, "join"), step);
|
|
711
|
+
cur = r.table;
|
|
712
|
+
notes.push(...r.notes);
|
|
713
|
+
break;
|
|
714
|
+
}
|
|
715
|
+
case "pivot_wider":
|
|
716
|
+
cur = pivotWider(cur, arg);
|
|
717
|
+
break;
|
|
718
|
+
case "pivot_longer":
|
|
719
|
+
cur = pivotLonger(cur, arg);
|
|
720
|
+
break;
|
|
721
|
+
case "orderby":
|
|
722
|
+
cur = orderby(cur, arg);
|
|
723
|
+
break;
|
|
724
|
+
case "slice":
|
|
725
|
+
cur = slice(cur, arg);
|
|
726
|
+
break;
|
|
727
|
+
case "dedupe":
|
|
728
|
+
cur = dedupe(cur, arg);
|
|
729
|
+
break;
|
|
730
|
+
case "concat":
|
|
731
|
+
cur = concat([cur, ...(Array.isArray(arg) ? arg : [arg]).map((a) => table(a, "concat"))]);
|
|
732
|
+
break;
|
|
733
|
+
case "fiscal_year": {
|
|
734
|
+
const o = typeof arg === "string" ? { date: arg } : arg || {};
|
|
735
|
+
const as = String(o.as ?? "fiscal_year");
|
|
736
|
+
need(cur, [String(o.date)], "fiscal_year");
|
|
737
|
+
cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r) => {
|
|
738
|
+
const x = Object.assign(newRow(), r);
|
|
739
|
+
x[as] = r[o.date] === null ? null : fiscalYear(r[o.date], o.date);
|
|
740
|
+
return x;
|
|
741
|
+
}) };
|
|
742
|
+
break;
|
|
743
|
+
}
|
|
744
|
+
case "to_fiscal": {
|
|
745
|
+
const o = arg || {};
|
|
746
|
+
const r = toFiscal(cur, String(o.date), String(o.value), o.how ?? "sum", o.freq ?? "M");
|
|
747
|
+
cur = r.table;
|
|
748
|
+
notes.push(...r.notes);
|
|
749
|
+
break;
|
|
750
|
+
}
|
|
751
|
+
case "state_codes":
|
|
752
|
+
cur = addStateCodes(cur, typeof arg === "string" ? arg : String(arg?.col));
|
|
753
|
+
break;
|
|
754
|
+
case "deflate": {
|
|
755
|
+
const o = arg || {};
|
|
756
|
+
if (!o.name || typeof o.name !== "string") {
|
|
757
|
+
throw new ComputeError('deflate: name the price index and its source in "name" (e.g. "CPI-U, BLS CUUR0000SA0"), so the ' +
|
|
758
|
+
"answer can say which dollars these are");
|
|
759
|
+
}
|
|
760
|
+
const value = String(o.value), period = String(o.period);
|
|
761
|
+
need(cur, [value, period], "deflate");
|
|
762
|
+
const idx = table(o.index, "deflate index");
|
|
763
|
+
const ip = String(o.index_period ?? period), iv = String(o.index_value ?? "");
|
|
764
|
+
need(idx, [ip, iv], "deflate index");
|
|
765
|
+
const m = new Map();
|
|
766
|
+
for (const r of idx.rows) {
|
|
767
|
+
const k = cellText(r[ip]);
|
|
768
|
+
if (m.has(k))
|
|
769
|
+
throw new ComputeError(`deflate: the index repeats period ${pyRepr(k)}: one row per period`);
|
|
770
|
+
const v = r[iv];
|
|
771
|
+
if (typeof v !== "number")
|
|
772
|
+
throw new ComputeError(`deflate: index ${iv} = ${pyRepr(v)} at ${pyRepr(k)} is not a number`);
|
|
773
|
+
m.set(k, v);
|
|
774
|
+
}
|
|
775
|
+
const base = o.base === undefined ? null : o.base;
|
|
776
|
+
const out = deflate(cur.rows.map((r) => ({ period: r[period], value: r[value] })), m, base);
|
|
777
|
+
const as = String(o.as ?? `${value}_real`);
|
|
778
|
+
cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r, j) => {
|
|
779
|
+
const x = Object.assign(newRow(), r);
|
|
780
|
+
x[as] = out[j];
|
|
781
|
+
return x;
|
|
782
|
+
}) };
|
|
783
|
+
notes.push(`deflate: ${as} = ${value} in ${base === null ? "the index's own base (used as given)" : `${cellText(base)} dollars`}, ` +
|
|
784
|
+
`deflated by ${o.name}`);
|
|
785
|
+
break;
|
|
786
|
+
}
|
|
787
|
+
case "replicate_se": {
|
|
788
|
+
const r = replicateSe(cur, arg, by);
|
|
789
|
+
cur = r.table;
|
|
790
|
+
notes.push(...r.notes);
|
|
791
|
+
break;
|
|
792
|
+
}
|
|
793
|
+
case "align_check": {
|
|
794
|
+
const o = arg || {};
|
|
795
|
+
const names = strList(o.tables, "align_check tables");
|
|
796
|
+
const frames = Object.fromEntries(names.map((n) => [n, table(n, "align_check")]));
|
|
797
|
+
notes.push(...alignCheck(frames, strList(o.on, "align_check on"), o.units && typeof o.units === "object" ? o.units : {}));
|
|
798
|
+
break;
|
|
799
|
+
}
|
|
800
|
+
case "as":
|
|
801
|
+
if (typeof arg !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) || arg === "this")
|
|
802
|
+
throw new ComputeError("as: a table name (letters, digits, _)");
|
|
803
|
+
env.set(arg, cur);
|
|
804
|
+
break;
|
|
805
|
+
case "from":
|
|
806
|
+
cur = table(arg, "from");
|
|
807
|
+
break;
|
|
589
808
|
}
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
809
|
+
});
|
|
810
|
+
// these keep a subset of the same rows: a list derive owned stays owned (the old list is no longer used)
|
|
811
|
+
if (OWNED.has(before) && ["filter", "select", "orderby", "slice", "dedupe"].includes(verb))
|
|
812
|
+
OWNED.add(cur.rows);
|
|
813
|
+
// a table kept under a name may be read again: its rows are shared from now on
|
|
814
|
+
if (verb === "as")
|
|
815
|
+
OWNED.delete(cur.rows);
|
|
816
|
+
}
|
|
817
|
+
catch (e) {
|
|
818
|
+
// the dry run reports only what the data cannot change: a column or table that does not exist
|
|
819
|
+
if (!dry || (e instanceof ComputeError && STRUCTURAL.test(e.message)))
|
|
820
|
+
throw e;
|
|
821
|
+
break;
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
return { table: cur, notes };
|
|
602
825
|
}
|
|
603
826
|
/** The packs an exhibit's inputs come from: data/<id>/ is <id>; an exhibit carries its lineage; published/ none. */
|
|
604
827
|
export function inferPacks(workspace, inputs) {
|
|
@@ -633,10 +856,39 @@ export function compute(workspace, a) {
|
|
|
633
856
|
lin[save] = entry;
|
|
634
857
|
writeLineage(workspace, lin);
|
|
635
858
|
const n = Math.max(0, Math.min(Number(a.preview ?? 20), 50));
|
|
859
|
+
const pv = previewRows(r.table, n);
|
|
636
860
|
return {
|
|
637
861
|
file: abs, rel: save, rows: r.table.rows.length, columns: r.table.cols,
|
|
638
|
-
preview:
|
|
862
|
+
preview: pv.rows, ...(pv.note ? { preview_note: pv.note } : {}),
|
|
639
863
|
notes: r.notes, packs,
|
|
640
864
|
};
|
|
641
865
|
}
|
|
866
|
+
/** Preview limits: the file has every value exactly; the preview is for reading, so it is narrow and short. */
|
|
867
|
+
export const PREVIEW_MAX_COLS = 20;
|
|
868
|
+
export const PREVIEW_MAX_CHARS = 4000;
|
|
869
|
+
/** A number as the preview shows it: 6 significant digits for a fraction (integers and text as written). */
|
|
870
|
+
function previewCell(v) {
|
|
871
|
+
if (typeof v === "number" && Number.isFinite(v) && !Number.isInteger(v))
|
|
872
|
+
return String(Number(v.toPrecision(6)));
|
|
873
|
+
return cellText(v);
|
|
874
|
+
}
|
|
875
|
+
/** The first n rows, at most PREVIEW_MAX_COLS columns and about PREVIEW_MAX_CHARS characters, with a note on any cut. */
|
|
876
|
+
export function previewRows(t, n) {
|
|
877
|
+
const cols = t.cols.slice(0, PREVIEW_MAX_COLS);
|
|
878
|
+
const rows = [];
|
|
879
|
+
let size = 0;
|
|
880
|
+
for (const row of t.rows.slice(0, n)) {
|
|
881
|
+
const o = Object.fromEntries(cols.map((c) => [c, previewCell(row[c])]));
|
|
882
|
+
size += JSON.stringify(o).length;
|
|
883
|
+
if (rows.length && size > PREVIEW_MAX_CHARS)
|
|
884
|
+
break;
|
|
885
|
+
rows.push(o);
|
|
886
|
+
}
|
|
887
|
+
const cut = [];
|
|
888
|
+
if (cols.length < t.cols.length)
|
|
889
|
+
cut.push(`the first ${cols.length} of ${t.cols.length} columns`);
|
|
890
|
+
if (rows.length < Math.min(n, t.rows.length))
|
|
891
|
+
cut.push(`${rows.length} rows (size limit)`);
|
|
892
|
+
return { rows, ...(cut.length ? { note: `preview shows ${cut.join(" and ")}; fractions rounded to 6 digits; the file has every value exactly` } : {}) };
|
|
893
|
+
}
|
|
642
894
|
//# sourceMappingURL=compute.js.map
|