data-analyst-mcp 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/analysis/combine.js +3 -2
  2. package/dist/analysis/combine.js.map +1 -1
  3. package/dist/analysis/compute.d.ts +26 -5
  4. package/dist/analysis/compute.js +402 -159
  5. package/dist/analysis/compute.js.map +1 -1
  6. package/dist/analysis/expr.js +2 -1
  7. package/dist/analysis/expr.js.map +1 -1
  8. package/dist/analysis/handoff.d.ts +1 -1
  9. package/dist/analysis/handoff.js +44 -3
  10. package/dist/analysis/handoff.js.map +1 -1
  11. package/dist/analysis/table.d.ts +7 -0
  12. package/dist/analysis/table.js +30 -1
  13. package/dist/analysis/table.js.map +1 -1
  14. package/dist/analysis/tools.d.ts +10 -7
  15. package/dist/analysis/tools.js +32 -43
  16. package/dist/analysis/tools.js.map +1 -1
  17. package/dist/core/compact.d.ts +15 -0
  18. package/dist/core/compact.js +98 -0
  19. package/dist/core/compact.js.map +1 -0
  20. package/dist/core/package.d.ts +8 -0
  21. package/dist/core/package.js +30 -1
  22. package/dist/core/package.js.map +1 -1
  23. package/dist/core/packs.d.ts +4 -2
  24. package/dist/core/sections.d.ts +21 -0
  25. package/dist/core/sections.js +53 -0
  26. package/dist/core/sections.js.map +1 -0
  27. package/dist/deep/opm.js +10 -2
  28. package/dist/deep/opm.js.map +1 -1
  29. package/dist/deep/treasury.js +7 -3
  30. package/dist/deep/treasury.js.map +1 -1
  31. package/dist/envelope.js +1 -1
  32. package/dist/envelope.js.map +1 -1
  33. package/dist/index.d.ts +57 -4
  34. package/dist/report/build_report.d.ts +18 -0
  35. package/dist/report/build_report.js +52 -7
  36. package/dist/report/build_report.js.map +1 -1
  37. package/dist/report/tools.d.ts +26 -1
  38. package/dist/report/tools.js +74 -17
  39. package/dist/report/tools.js.map +1 -1
  40. package/dist/server.d.ts +1 -1
  41. package/dist/server.js +52 -43
  42. package/dist/server.js.map +1 -1
  43. package/dist/session.d.ts +2 -0
  44. package/dist/session.js +2 -0
  45. package/dist/session.js.map +1 -1
  46. package/dist/tools.d.ts +98 -6
  47. package/dist/tools.js +154 -17
  48. package/dist/tools.js.map +1 -1
  49. package/guide/WORKFLOW.md +7 -4
  50. package/guide/references/compute.md +51 -0
  51. package/guide/references/report_quick.md +60 -0
  52. package/package.json +1 -1
  53. package/packs/census/docs/microdata.md +77 -0
  54. package/packs/census/docs/timeseries.md +71 -0
  55. package/packs/census/source.md +10 -142
  56. package/packs/congress/docs/nominations_crs.md +26 -0
  57. package/packs/congress/docs/votes.md +30 -0
  58. package/packs/congress/source.md +4 -48
  59. package/packs/govinfo/docs/collections.md +88 -0
  60. package/packs/govinfo/source.md +6 -86
  61. package/packs/irs-soi/docs/eo.md +15 -0
  62. package/packs/irs-soi/docs/tables.md +50 -0
  63. package/packs/irs-soi/source.md +6 -57
@@ -15,11 +15,12 @@
15
15
  */
16
16
  import fs from "node:fs";
17
17
  import path from "node:path";
18
+ import { namesHint } from "../core/compact.js";
18
19
  import { addStateCodes, alignCheck, CombineError, deflate, fiscalYear, toFiscal } from "./combine.js";
19
- import { compare, compile, evaluate, ExprError, isTruthy } from "./expr.js";
20
+ import { compare, compile, evaluate, ExprError, inspect, isTruthy, parse as parseExpr } from "./expr.js";
20
21
  import { pyRepr } from "./pyfmt.js";
21
- import { replicateSe } from "./replicate.js";
22
- import { cellText, newRow, ParquetInputError, parseTable, readParquetTable, sha256, tableCsv } from "./table.js";
22
+ import { replicateColumns, replicateSe } from "./replicate.js";
23
+ import { cellText, csvGrid, headerOf, newRow, PARQUET_MAX_CELLS, parquetColumns, ParquetInputError, readParquetTable, sha256, tableCsv, tableFromGrid } from "./table.js";
23
24
  export class ComputeError extends Error {
24
25
  }
25
26
  export const LINEAGE = "exhibits/lineage.json";
@@ -60,8 +61,82 @@ export function normInput(alias, v) {
60
61
  }
61
62
  throw new ComputeError(`input ${alias}: a workspace path, or {path, columns: [names]}, not ${pyRepr(v)}`);
62
63
  }
64
+ /** The Parquet copy fetch writes beside a large data/<id>/<table>.csv, when it is there and not older than the CSV. */
65
+ function parquetCopy(csvAbs) {
66
+ if (!/[\\/]data[\\/][^\\/]+[\\/][^\\/]+\.csv$/i.test(csvAbs))
67
+ return null;
68
+ const pq = csvAbs.replace(/\.csv$/i, ".parquet");
69
+ try {
70
+ return fs.statSync(pq).mtimeMs >= fs.statSync(csvAbs).mtimeMs ? pq : null;
71
+ }
72
+ catch {
73
+ return null;
74
+ }
75
+ }
76
+ /** An input's column names, read before the table (the header line of a CSV, the schema of a Parquet file). */
77
+ function inputHeader(alias, rel, abs, buf, parquet) {
78
+ if (parquet) {
79
+ try {
80
+ return parquetColumns(buf, abs);
81
+ }
82
+ catch (e) {
83
+ throw new ComputeError(`input ${alias}: ${relPath(rel)} could not be read as Parquet: ${e?.message || e}`);
84
+ }
85
+ }
86
+ return headerOf(buf.subarray(0, Math.min(buf.length, 1 << 20)).toString("utf8").replace(/^\uFEFF/, ""));
87
+ }
88
+ /** The most cells of a CSV parsed at once (every column of every row, before any column is dropped). A 37 MB ACS PUMS
89
+ * file of 62,777 rows x 175 columns (11M cells) took 1.5 s and about 360 MB to parse; 60M is about 2 GB. */
90
+ export const CSV_MAX_RAW_CELLS = { value: 60_000_000 };
91
+ /** Parsed CSV grids kept across calls, keyed by path and content hash: compute and the rerun read the same large
92
+ * microdata file again and again. Only files of at least CACHE_MIN_BYTES; at most the budget in total (about ten
93
+ * times the file size each), least recently used dropped first. */
94
+ const GRID_CACHE = new Map();
95
+ const CACHE_MIN_BYTES = 5_000_000;
96
+ function cacheBudget() {
97
+ const mb = Number(process.env.DATA_ANALYST_PARSE_CACHE_MB);
98
+ return (Number.isFinite(mb) && mb >= 0 ? mb : 512) * 1024 * 1024;
99
+ }
100
+ function cachedGrid(abs, sha, buf) {
101
+ const key = `${abs}\u0000${sha}`;
102
+ const hit = GRID_CACHE.get(key);
103
+ if (hit) {
104
+ GRID_CACHE.delete(key); // most recently used last
105
+ GRID_CACHE.set(key, hit);
106
+ return hit.grid;
107
+ }
108
+ const grid = csvGrid(buf.toString("utf8"));
109
+ const bytes = buf.length * 10;
110
+ const budget = cacheBudget();
111
+ if (buf.length >= CACHE_MIN_BYTES && bytes <= budget) {
112
+ let used = [...GRID_CACHE.values()].reduce((a, v) => a + v.bytes, 0);
113
+ for (const [k, v] of GRID_CACHE) {
114
+ if (used + bytes <= budget)
115
+ break;
116
+ GRID_CACHE.delete(k);
117
+ used -= v.bytes;
118
+ }
119
+ GRID_CACHE.set(key, { grid, bytes });
120
+ }
121
+ return grid;
122
+ }
123
+ /** Drop every cached grid (tests). */
124
+ export function clearParseCache() {
125
+ GRID_CACHE.clear();
126
+ }
127
+ /** How many parsed grids are cached (tests). */
128
+ export function parseCacheSize() {
129
+ return GRID_CACHE.size;
130
+ }
131
+ /** Line ends in a buffer (an upper bound on a CSV's rows, for the cell caps). */
132
+ function lineEnds(buf) {
133
+ let n = 0;
134
+ for (let i = buf.indexOf(10); i !== -1; i = buf.indexOf(10, i + 1))
135
+ n++;
136
+ return n;
137
+ }
63
138
  /** Read one input as a typed table (only its named columns, when given). */
64
- function readInput(alias, abs, inp, buf) {
139
+ function readInput(alias, abs, inp, buf, sha, header) {
65
140
  if (relPath(inp.path).endsWith(".parquet")) {
66
141
  try {
67
142
  return readParquetTable(buf, abs, inp.columns);
@@ -72,19 +147,104 @@ function readInput(alias, abs, inp, buf) {
72
147
  throw new ComputeError(`input ${alias}: ${relPath(inp.path)} could not be read as Parquet: ${e?.message || e}`);
73
148
  }
74
149
  }
75
- const t = parseTable(buf.toString("utf8"));
76
150
  const want = inp.columns;
77
- if (!want)
78
- return t;
79
- const miss = want.filter((c) => !t.cols.includes(c));
80
- if (miss.length)
81
- throw new ComputeError(`input ${alias}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; the columns are ${pyRepr(t.cols)}`);
82
- return { cols: want, rows: t.rows.map((r) => {
83
- const o = newRow();
84
- for (const c of want)
85
- o[c] = r[c];
86
- return o;
87
- }) };
151
+ if (!GRID_CACHE.has(`${abs}\u0000${sha}`)) {
152
+ const rows = lineEnds(buf);
153
+ const raw = rows * header.length, typed = rows * (want ? want.length : header.length);
154
+ if (raw > CSV_MAX_RAW_CELLS.value || typed > PARQUET_MAX_CELLS.value) {
155
+ const n = (x) => x.toLocaleString("en-US");
156
+ throw new ComputeError(`input ${alias}: ${relPath(inp.path)} has about ${n(rows)} rows x ${n(header.length)} columns, ` +
157
+ `more than compute reads at once (${n(CSV_MAX_RAW_CELLS.value)} cells parsed, ${n(PARQUET_MAX_CELLS.value)} kept). ` +
158
+ (raw > CSV_MAX_RAW_CELLS.value
159
+ ? "Read the .parquet file beside it when the fetch wrote one ({path, columns}), or fetch fewer rows or variables."
160
+ : `Name only the columns the steps need: {"${alias}": {"path": "${relPath(inp.path)}", "columns": [...]}}.`));
161
+ }
162
+ }
163
+ return tableFromGrid(header, cachedGrid(abs, sha, buf), want);
164
+ }
165
+ /** The input columns a single-input recipe reads, when its steps say so: every column the steps name up to the first
166
+ * step that keeps only named columns (select, rollup, replicate_se with estimates, pivot_wider with id_cols). Null when
167
+ * any step could read columns it does not name (dedupe: true, other tables, ...) or no step narrows the table, so
168
+ * every column reaches the output: then the whole table is read. */
169
+ export function neededColumns(p, header) {
170
+ try {
171
+ const need = new Set();
172
+ const add = (c) => { if (typeof c === "string")
173
+ need.add(c); };
174
+ const ref = (src) => { for (const c of inspect(parseExpr(src)).cols)
175
+ need.add(c); };
176
+ const done = () => {
177
+ const out = header.filter((c) => need.has(c));
178
+ return out.length && out.length < header.length ? out : null;
179
+ };
180
+ for (const step of p.steps || []) {
181
+ const verbs = Object.keys(step || {}).filter((k) => VERBS.includes(k));
182
+ if (verbs.length !== 1)
183
+ return null;
184
+ const verb = verbs[0], arg = step[verb];
185
+ if (step.groupby !== undefined)
186
+ strList(step.groupby, "groupby").forEach(add);
187
+ switch (verb) {
188
+ case "filter":
189
+ ref(arg);
190
+ break;
191
+ case "derive":
192
+ for (const [, e] of exprs(arg, "derive"))
193
+ ref(e);
194
+ break;
195
+ case "rename":
196
+ for (const [a] of exprs(arg, "rename"))
197
+ add(a);
198
+ break;
199
+ case "orderby":
200
+ strList(arg, "orderby").forEach((c) => add(c.replace(/^-/, "")));
201
+ break;
202
+ case "slice": break;
203
+ case "dedupe":
204
+ if (arg === true)
205
+ return null;
206
+ strList(arg, "dedupe").forEach(add);
207
+ break;
208
+ case "fiscal_year":
209
+ add(typeof arg === "string" ? arg : arg?.date);
210
+ break;
211
+ case "state_codes":
212
+ add(typeof arg === "string" ? arg : arg?.col);
213
+ break;
214
+ case "select":
215
+ strList(arg, "select").forEach(add);
216
+ return done();
217
+ case "rollup":
218
+ for (const [, e] of exprs(arg, "rollup"))
219
+ ref(e);
220
+ return done();
221
+ case "replicate_se": {
222
+ if (!arg || typeof arg !== "object" || !arg.estimates || typeof arg.estimates !== "object")
223
+ return null;
224
+ add(arg.weight);
225
+ replicateColumns(arg.replicates).forEach(add);
226
+ for (const e of Object.values(arg.estimates))
227
+ ref(e);
228
+ for (const e of Object.values(arg.also || {}))
229
+ ref(e);
230
+ return done();
231
+ }
232
+ case "pivot_wider":
233
+ if (!arg?.id_cols)
234
+ return null;
235
+ strList(arg.id_cols, "id_cols").forEach(add);
236
+ add(arg.names_from);
237
+ add(arg.values_from);
238
+ return done();
239
+ default:
240
+ return null; // other tables, value-dependent columns
241
+ }
242
+ }
243
+ return null;
244
+ }
245
+ catch {
246
+ return null; // a malformed step: the full read reports it
247
+ }
88
248
  }
89
249
  export function readLineage(workspace) {
90
250
  try {
@@ -126,7 +286,7 @@ const EXTRA = { filter: ["groupby"], derive: ["groupby"], rollup: ["groupby"], r
126
286
  export function need(t, cols, what) {
127
287
  const miss = cols.filter((c) => !t.cols.includes(c));
128
288
  if (miss.length)
129
- throw new ComputeError(`${what}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; the columns are ${pyRepr(t.cols)}`);
289
+ throw new ComputeError(`${what}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; ${namesHint(miss, t.cols, "columns", pyRepr)}`);
130
290
  }
131
291
  export function strList(v, what) {
132
292
  const xs = typeof v === "string" ? [v] : v;
@@ -166,9 +326,13 @@ function exprs(v, what) {
166
326
  throw new ComputeError(`${what}: give at least one {name: expression}`);
167
327
  return out;
168
328
  }
329
+ /** Row lists this pipeline made itself and nothing else holds: derive may add columns to their rows in place instead
330
+ * of copying every row (an input table, a parse-cache entry or an "as" table is never in it). */
331
+ const OWNED = new WeakSet();
169
332
  function derive(t, spec, by) {
170
333
  let cols = [...t.cols];
171
- const rows = t.rows.map((r) => Object.assign(newRow(), r));
334
+ const rows = OWNED.has(t.rows) ? t.rows : t.rows.map((r) => Object.assign(newRow(), r));
335
+ OWNED.add(rows);
172
336
  for (const [name, src] of exprs(spec, "derive")) {
173
337
  const node = compile(src, cols, { what: "derive" });
174
338
  for (const g of groupRows({ cols, rows }, by, false)) {
@@ -421,8 +585,8 @@ export function runPipeline(workspace, p, parsed) {
421
585
  throw new ComputeError('inputs: {alias: "data/<id>/<table>.csv" | "data/<id>/<table>.parquet" | "exhibits/<x>.csv" | ' +
422
586
  "{path, columns}}, at least one");
423
587
  }
424
- const env = new Map();
425
588
  const inputs = {};
589
+ const prepared = [];
426
590
  for (const [alias, raw] of Object.entries(p.inputs)) {
427
591
  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(alias) || alias === "this" || alias.startsWith("__"))
428
592
  throw new ComputeError(`input alias ${pyRepr(alias)}: letters, digits and _ only`);
@@ -434,14 +598,46 @@ export function runPipeline(workspace, p, parsed) {
434
598
  const buf = fs.readFileSync(abs);
435
599
  const sha = sha256(buf);
436
600
  inputs[alias] = { path: relPath(rel), sha256: sha, ...(inp.columns ? { columns: inp.columns } : {}) };
437
- const key = `${abs}\u0000${sha}\u0000${inp.columns ? JSON.stringify(inp.columns) : ""}`;
601
+ const parquet = relPath(rel).endsWith(".parquet");
602
+ prepared.push({ alias, inp, abs, buf, sha, parquet, header: inputHeader(alias, rel, abs, buf, parquet) });
603
+ }
604
+ for (const x of prepared) {
605
+ const miss = (x.inp.columns || []).filter((c) => !x.header.includes(c));
606
+ if (!miss.length)
607
+ continue;
608
+ const what = `no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}`;
609
+ throw new ComputeError(x.parquet // the messages the readers give (the Parquet one names the file)
610
+ ? `input ${x.alias} (${relPath(x.inp.path)}): ${what}; ${namesHint(miss, x.header, "columns", (v) => v.join(", "))}`
611
+ : `input ${x.alias}: ${what}; ${namesHint(miss, x.header, "columns", pyRepr)}`);
612
+ }
613
+ // a dry run of the steps on empty tables with the inputs' headers: a misspelt column fails here, before a large
614
+ // file is parsed (only what the data cannot change is reported; data-dependent steps end the dry run)
615
+ runSteps(p, new Map(prepared.map((x) => [x.alias, { cols: x.inp.columns ?? x.header, rows: [] }])), true);
616
+ // one input and no columns named: read only the columns the steps use, when the steps say which those are
617
+ const auto = prepared.length === 1 && !prepared[0].inp.columns ? neededColumns(p, prepared[0].header) : null;
618
+ const env = new Map();
619
+ for (const x of prepared) {
620
+ const want = x.inp.columns ?? auto ?? undefined;
621
+ const key = `${x.abs}\u0000${x.sha}\u0000${want ? JSON.stringify(want) : ""}`;
438
622
  let t = parsed?.get(key);
439
623
  if (!t) {
440
- t = readInput(alias, abs, inp, buf);
624
+ // only some columns of a fetched CSV that has a Parquet copy: decode those columns from the copy (same table)
625
+ const pq = want && !x.parquet ? parquetCopy(x.abs) : null;
626
+ t = pq ? readInput(x.alias, pq, { path: relPath(x.inp.path).replace(/\.csv$/, ".parquet"), columns: want }, fs.readFileSync(pq), x.sha, x.header)
627
+ : readInput(x.alias, x.abs, { path: x.inp.path, columns: want }, x.buf, x.sha, x.header);
441
628
  parsed?.set(key, t);
442
629
  }
443
- env.set(alias, t);
630
+ env.set(x.alias, t);
444
631
  }
632
+ const r = runSteps(p, env, false);
633
+ return { table: r.table, notes: r.notes, inputs };
634
+ }
635
+ /** Verbs whose result depends on the data's values (new columns from values, lookups, checks): a dry run stops there. */
636
+ const DRY_STOP = new Set(["pivot_wider", "pivot_longer", "to_fiscal", "deflate", "replicate_se", "align_check"]);
637
+ /** Errors a dry run reports: a column or table that does not exist. */
638
+ const STRUCTURAL = /unknown column|no column|is not a table here|is not an input alias/;
639
+ /** Run the steps over the tables in env (`dry`: on empty tables, as a check before the inputs are read). */
640
+ function runSteps(p, env, dry) {
445
641
  const start = p.from ?? Object.keys(p.inputs)[0];
446
642
  if (!env.has(start))
447
643
  throw new ComputeError(`from: ${pyRepr(start)} is not an input alias (${Object.keys(p.inputs).join(", ")})`);
@@ -457,7 +653,8 @@ export function runPipeline(workspace, p, parsed) {
457
653
  };
458
654
  if (!Array.isArray(p.steps))
459
655
  throw new ComputeError("steps: a list of steps");
460
- p.steps.forEach((step, i) => {
656
+ for (let i = 0; i < p.steps.length; i++) {
657
+ const step = p.steps[i];
461
658
  if (!step || typeof step !== "object" || Array.isArray(step))
462
659
  throw new ComputeError(`step ${i + 1}: an object such as {"filter": "year >= 2020"}`);
463
660
  const verbs = Object.keys(step).filter((k) => VERBS.includes(k));
@@ -470,144 +667,161 @@ export function runPipeline(workspace, p, parsed) {
470
667
  if (extra.length)
471
668
  throw new ComputeError(`step ${i + 1} (${verb}): unknown key(s) ${pyRepr(extra)}; allowed with ${verb}: ${pyRepr(EXTRA[verb] || [])}`);
472
669
  const arg = step[verb];
473
- wrap(i, verb, () => {
474
- const by = step.groupby === undefined ? [] : strList(step.groupby, "groupby");
475
- if (by.length)
476
- need(cur, by, "groupby");
477
- switch (verb) {
478
- case "filter":
479
- cur = filter(cur, arg, by);
480
- break;
481
- case "derive":
482
- cur = derive(cur, arg, by);
483
- break;
484
- case "rollup":
485
- cur = rollup(cur, arg, by);
486
- break;
487
- case "select": {
488
- const cols = strList(arg, "select");
489
- need(cur, cols, "select");
490
- cur = { cols: [...new Set(cols)], rows: cur.rows };
491
- break;
492
- }
493
- case "rename": {
494
- const pairs = exprs(arg, "rename").map(([a, b]) => [a, String(b)]);
495
- need(cur, pairs.map(([a]) => a), "rename");
496
- const map = new Map(pairs);
497
- const cols = cur.cols.map((c) => map.get(c) ?? c);
498
- const dup = cols.filter((c, j) => cols.indexOf(c) !== j);
499
- if (dup.length)
500
- throw new ComputeError(`rename: ${pyRepr([...new Set(dup)])} would name two columns`);
501
- cur = { cols, rows: cur.rows.map((r) => {
502
- const o = newRow();
503
- cur.cols.forEach((c, j) => (o[cols[j]] = r[c]));
504
- return o;
505
- }) };
506
- break;
507
- }
508
- case "join": {
509
- const r = join(cur, table(arg, "join"), step);
510
- cur = r.table;
511
- notes.push(...r.notes);
512
- break;
513
- }
514
- case "pivot_wider":
515
- cur = pivotWider(cur, arg);
516
- break;
517
- case "pivot_longer":
518
- cur = pivotLonger(cur, arg);
519
- break;
520
- case "orderby":
521
- cur = orderby(cur, arg);
522
- break;
523
- case "slice":
524
- cur = slice(cur, arg);
525
- break;
526
- case "dedupe":
527
- cur = dedupe(cur, arg);
528
- break;
529
- case "concat":
530
- cur = concat([cur, ...(Array.isArray(arg) ? arg : [arg]).map((a) => table(a, "concat"))]);
531
- break;
532
- case "fiscal_year": {
533
- const o = typeof arg === "string" ? { date: arg } : arg || {};
534
- const as = String(o.as ?? "fiscal_year");
535
- need(cur, [String(o.date)], "fiscal_year");
536
- cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r) => {
537
- const x = Object.assign(newRow(), r);
538
- x[as] = r[o.date] === null ? null : fiscalYear(r[o.date], o.date);
539
- return x;
540
- }) };
541
- break;
542
- }
543
- case "to_fiscal": {
544
- const o = arg || {};
545
- const r = toFiscal(cur, String(o.date), String(o.value), o.how ?? "sum", o.freq ?? "M");
546
- cur = r.table;
547
- notes.push(...r.notes);
548
- break;
549
- }
550
- case "state_codes":
551
- cur = addStateCodes(cur, typeof arg === "string" ? arg : String(arg?.col));
552
- break;
553
- case "deflate": {
554
- const o = arg || {};
555
- if (!o.name || typeof o.name !== "string") {
556
- throw new ComputeError('deflate: name the price index and its source in "name" (e.g. "CPI-U, BLS CUUR0000SA0"), so the ' +
557
- "answer can say which dollars these are");
670
+ if (dry && DRY_STOP.has(verb))
671
+ break;
672
+ const before = cur.rows;
673
+ try {
674
+ wrap(i, verb, () => {
675
+ const by = step.groupby === undefined ? [] : strList(step.groupby, "groupby");
676
+ if (by.length)
677
+ need(cur, by, "groupby");
678
+ switch (verb) {
679
+ case "filter":
680
+ cur = filter(cur, arg, by);
681
+ break;
682
+ case "derive":
683
+ cur = derive(cur, arg, by);
684
+ break;
685
+ case "rollup":
686
+ cur = rollup(cur, arg, by);
687
+ break;
688
+ case "select": {
689
+ const cols = strList(arg, "select");
690
+ need(cur, cols, "select");
691
+ cur = { cols: [...new Set(cols)], rows: cur.rows };
692
+ break;
558
693
  }
559
- const value = String(o.value), period = String(o.period);
560
- need(cur, [value, period], "deflate");
561
- const idx = table(o.index, "deflate index");
562
- const ip = String(o.index_period ?? period), iv = String(o.index_value ?? "");
563
- need(idx, [ip, iv], "deflate index");
564
- const m = new Map();
565
- for (const r of idx.rows) {
566
- const k = cellText(r[ip]);
567
- if (m.has(k))
568
- throw new ComputeError(`deflate: the index repeats period ${pyRepr(k)}: one row per period`);
569
- const v = r[iv];
570
- if (typeof v !== "number")
571
- throw new ComputeError(`deflate: index ${iv} = ${pyRepr(v)} at ${pyRepr(k)} is not a number`);
572
- m.set(k, v);
694
+ case "rename": {
695
+ const pairs = exprs(arg, "rename").map(([a, b]) => [a, String(b)]);
696
+ need(cur, pairs.map(([a]) => a), "rename");
697
+ const map = new Map(pairs);
698
+ const cols = cur.cols.map((c) => map.get(c) ?? c);
699
+ const dup = cols.filter((c, j) => cols.indexOf(c) !== j);
700
+ if (dup.length)
701
+ throw new ComputeError(`rename: ${pyRepr([...new Set(dup)])} would name two columns`);
702
+ cur = { cols, rows: cur.rows.map((r) => {
703
+ const o = newRow();
704
+ cur.cols.forEach((c, j) => (o[cols[j]] = r[c]));
705
+ return o;
706
+ }) };
707
+ break;
573
708
  }
574
- const base = o.base === undefined ? null : o.base;
575
- const out = deflate(cur.rows.map((r) => ({ period: r[period], value: r[value] })), m, base);
576
- const as = String(o.as ?? `${value}_real`);
577
- cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r, j) => {
578
- const x = Object.assign(newRow(), r);
579
- x[as] = out[j];
580
- return x;
581
- }) };
582
- notes.push(`deflate: ${as} = ${value} in ${base === null ? "the index's own base (used as given)" : `${cellText(base)} dollars`}, ` +
583
- `deflated by ${o.name}`);
584
- break;
585
- }
586
- case "replicate_se": {
587
- const r = replicateSe(cur, arg, by);
588
- cur = r.table;
589
- notes.push(...r.notes);
590
- break;
591
- }
592
- case "align_check": {
593
- const o = arg || {};
594
- const names = strList(o.tables, "align_check tables");
595
- const frames = Object.fromEntries(names.map((n) => [n, table(n, "align_check")]));
596
- notes.push(...alignCheck(frames, strList(o.on, "align_check on"), o.units && typeof o.units === "object" ? o.units : {}));
597
- break;
709
+ case "join": {
710
+ const r = join(cur, table(arg, "join"), step);
711
+ cur = r.table;
712
+ notes.push(...r.notes);
713
+ break;
714
+ }
715
+ case "pivot_wider":
716
+ cur = pivotWider(cur, arg);
717
+ break;
718
+ case "pivot_longer":
719
+ cur = pivotLonger(cur, arg);
720
+ break;
721
+ case "orderby":
722
+ cur = orderby(cur, arg);
723
+ break;
724
+ case "slice":
725
+ cur = slice(cur, arg);
726
+ break;
727
+ case "dedupe":
728
+ cur = dedupe(cur, arg);
729
+ break;
730
+ case "concat":
731
+ cur = concat([cur, ...(Array.isArray(arg) ? arg : [arg]).map((a) => table(a, "concat"))]);
732
+ break;
733
+ case "fiscal_year": {
734
+ const o = typeof arg === "string" ? { date: arg } : arg || {};
735
+ const as = String(o.as ?? "fiscal_year");
736
+ need(cur, [String(o.date)], "fiscal_year");
737
+ cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r) => {
738
+ const x = Object.assign(newRow(), r);
739
+ x[as] = r[o.date] === null ? null : fiscalYear(r[o.date], o.date);
740
+ return x;
741
+ }) };
742
+ break;
743
+ }
744
+ case "to_fiscal": {
745
+ const o = arg || {};
746
+ const r = toFiscal(cur, String(o.date), String(o.value), o.how ?? "sum", o.freq ?? "M");
747
+ cur = r.table;
748
+ notes.push(...r.notes);
749
+ break;
750
+ }
751
+ case "state_codes":
752
+ cur = addStateCodes(cur, typeof arg === "string" ? arg : String(arg?.col));
753
+ break;
754
+ case "deflate": {
755
+ const o = arg || {};
756
+ if (!o.name || typeof o.name !== "string") {
757
+ throw new ComputeError('deflate: name the price index and its source in "name" (e.g. "CPI-U, BLS CUUR0000SA0"), so the ' +
758
+ "answer can say which dollars these are");
759
+ }
760
+ const value = String(o.value), period = String(o.period);
761
+ need(cur, [value, period], "deflate");
762
+ const idx = table(o.index, "deflate index");
763
+ const ip = String(o.index_period ?? period), iv = String(o.index_value ?? "");
764
+ need(idx, [ip, iv], "deflate index");
765
+ const m = new Map();
766
+ for (const r of idx.rows) {
767
+ const k = cellText(r[ip]);
768
+ if (m.has(k))
769
+ throw new ComputeError(`deflate: the index repeats period ${pyRepr(k)}: one row per period`);
770
+ const v = r[iv];
771
+ if (typeof v !== "number")
772
+ throw new ComputeError(`deflate: index ${iv} = ${pyRepr(v)} at ${pyRepr(k)} is not a number`);
773
+ m.set(k, v);
774
+ }
775
+ const base = o.base === undefined ? null : o.base;
776
+ const out = deflate(cur.rows.map((r) => ({ period: r[period], value: r[value] })), m, base);
777
+ const as = String(o.as ?? `${value}_real`);
778
+ cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r, j) => {
779
+ const x = Object.assign(newRow(), r);
780
+ x[as] = out[j];
781
+ return x;
782
+ }) };
783
+ notes.push(`deflate: ${as} = ${value} in ${base === null ? "the index's own base (used as given)" : `${cellText(base)} dollars`}, ` +
784
+ `deflated by ${o.name}`);
785
+ break;
786
+ }
787
+ case "replicate_se": {
788
+ const r = replicateSe(cur, arg, by);
789
+ cur = r.table;
790
+ notes.push(...r.notes);
791
+ break;
792
+ }
793
+ case "align_check": {
794
+ const o = arg || {};
795
+ const names = strList(o.tables, "align_check tables");
796
+ const frames = Object.fromEntries(names.map((n) => [n, table(n, "align_check")]));
797
+ notes.push(...alignCheck(frames, strList(o.on, "align_check on"), o.units && typeof o.units === "object" ? o.units : {}));
798
+ break;
799
+ }
800
+ case "as":
801
+ if (typeof arg !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) || arg === "this")
802
+ throw new ComputeError("as: a table name (letters, digits, _)");
803
+ env.set(arg, cur);
804
+ break;
805
+ case "from":
806
+ cur = table(arg, "from");
807
+ break;
598
808
  }
599
- case "as":
600
- if (typeof arg !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) || arg === "this")
601
- throw new ComputeError("as: a table name (letters, digits, _)");
602
- env.set(arg, cur);
603
- break;
604
- case "from":
605
- cur = table(arg, "from");
606
- break;
607
- }
608
- });
609
- });
610
- return { table: cur, notes, inputs };
809
+ });
810
+ // these keep a subset of the same rows: a list derive owned stays owned (the old list is no longer used)
811
+ if (OWNED.has(before) && ["filter", "select", "orderby", "slice", "dedupe"].includes(verb))
812
+ OWNED.add(cur.rows);
813
+ // a table kept under a name may be read again: its rows are shared from now on
814
+ if (verb === "as")
815
+ OWNED.delete(cur.rows);
816
+ }
817
+ catch (e) {
818
+ // the dry run reports only what the data cannot change: a column or table that does not exist
819
+ if (!dry || (e instanceof ComputeError && STRUCTURAL.test(e.message)))
820
+ throw e;
821
+ break;
822
+ }
823
+ }
824
+ return { table: cur, notes };
611
825
  }
612
826
  /** The packs an exhibit's inputs come from: data/<id>/ is <id>; an exhibit carries its lineage; published/ none. */
613
827
  export function inferPacks(workspace, inputs) {
@@ -642,10 +856,39 @@ export function compute(workspace, a) {
642
856
  lin[save] = entry;
643
857
  writeLineage(workspace, lin);
644
858
  const n = Math.max(0, Math.min(Number(a.preview ?? 20), 50));
859
+ const pv = previewRows(r.table, n);
645
860
  return {
646
861
  file: abs, rel: save, rows: r.table.rows.length, columns: r.table.cols,
647
- preview: r.table.rows.slice(0, n).map((row) => Object.fromEntries(r.table.cols.map((c) => [c, cellText(row[c])]))),
862
+ preview: pv.rows, ...(pv.note ? { preview_note: pv.note } : {}),
648
863
  notes: r.notes, packs,
649
864
  };
650
865
  }
866
+ /** Preview limits: the file has every value exactly; the preview is for reading, so it is narrow and short. */
867
+ export const PREVIEW_MAX_COLS = 20;
868
+ export const PREVIEW_MAX_CHARS = 4000;
869
+ /** A number as the preview shows it: 6 significant digits for a fraction (integers and text as written). */
870
+ function previewCell(v) {
871
+ if (typeof v === "number" && Number.isFinite(v) && !Number.isInteger(v))
872
+ return String(Number(v.toPrecision(6)));
873
+ return cellText(v);
874
+ }
875
+ /** The first n rows, at most PREVIEW_MAX_COLS columns and about PREVIEW_MAX_CHARS characters, with a note on any cut. */
876
+ export function previewRows(t, n) {
877
+ const cols = t.cols.slice(0, PREVIEW_MAX_COLS);
878
+ const rows = [];
879
+ let size = 0;
880
+ for (const row of t.rows.slice(0, n)) {
881
+ const o = Object.fromEntries(cols.map((c) => [c, previewCell(row[c])]));
882
+ size += JSON.stringify(o).length;
883
+ if (rows.length && size > PREVIEW_MAX_CHARS)
884
+ break;
885
+ rows.push(o);
886
+ }
887
+ const cut = [];
888
+ if (cols.length < t.cols.length)
889
+ cut.push(`the first ${cols.length} of ${t.cols.length} columns`);
890
+ if (rows.length < Math.min(n, t.rows.length))
891
+ cut.push(`${rows.length} rows (size limit)`);
892
+ return { rows, ...(cut.length ? { note: `preview shows ${cut.join(" and ")}; fractions rounded to 6 digits; the file has every value exactly` } : {}) };
893
+ }
651
894
  //# sourceMappingURL=compute.js.map