data-analyst-mcp 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +9 -2
  2. package/dist/adapters/_census_micro.d.ts +2 -0
  3. package/dist/adapters/_census_micro.js +24 -1
  4. package/dist/adapters/_census_micro.js.map +1 -1
  5. package/dist/analysis/combine.js +3 -2
  6. package/dist/analysis/combine.js.map +1 -1
  7. package/dist/analysis/compute.d.ts +30 -7
  8. package/dist/analysis/compute.js +414 -162
  9. package/dist/analysis/compute.js.map +1 -1
  10. package/dist/analysis/expr.js +6 -2
  11. package/dist/analysis/expr.js.map +1 -1
  12. package/dist/analysis/handoff.d.ts +1 -1
  13. package/dist/analysis/handoff.js +44 -3
  14. package/dist/analysis/handoff.js.map +1 -1
  15. package/dist/analysis/rerun.js +2 -1
  16. package/dist/analysis/rerun.js.map +1 -1
  17. package/dist/analysis/rerun_job.d.ts +30 -0
  18. package/dist/analysis/rerun_job.js +173 -0
  19. package/dist/analysis/rerun_job.js.map +1 -0
  20. package/dist/analysis/rerun_worker.d.ts +1 -0
  21. package/dist/analysis/rerun_worker.js +13 -0
  22. package/dist/analysis/rerun_worker.js.map +1 -0
  23. package/dist/analysis/table.d.ts +7 -0
  24. package/dist/analysis/table.js +30 -1
  25. package/dist/analysis/table.js.map +1 -1
  26. package/dist/analysis/tools.d.ts +10 -7
  27. package/dist/analysis/tools.js +43 -45
  28. package/dist/analysis/tools.js.map +1 -1
  29. package/dist/core/compact.d.ts +15 -0
  30. package/dist/core/compact.js +98 -0
  31. package/dist/core/compact.js.map +1 -0
  32. package/dist/core/package.d.ts +8 -0
  33. package/dist/core/package.js +30 -1
  34. package/dist/core/package.js.map +1 -1
  35. package/dist/core/packs.d.ts +4 -2
  36. package/dist/core/sections.d.ts +21 -0
  37. package/dist/core/sections.js +53 -0
  38. package/dist/core/sections.js.map +1 -0
  39. package/dist/deep/opm.js +10 -2
  40. package/dist/deep/opm.js.map +1 -1
  41. package/dist/deep/treasury.js +7 -3
  42. package/dist/deep/treasury.js.map +1 -1
  43. package/dist/envelope.d.ts +1 -1
  44. package/dist/envelope.js +1 -1
  45. package/dist/envelope.js.map +1 -1
  46. package/dist/index.d.ts +57 -4
  47. package/dist/report/audit_answer.d.ts +1 -1
  48. package/dist/report/audit_answer.js +2 -2
  49. package/dist/report/audit_answer.js.map +1 -1
  50. package/dist/report/audit_report.d.ts +1 -1
  51. package/dist/report/audit_report.js +2 -2
  52. package/dist/report/audit_report.js.map +1 -1
  53. package/dist/report/build_report.d.ts +18 -0
  54. package/dist/report/build_report.js +52 -7
  55. package/dist/report/build_report.js.map +1 -1
  56. package/dist/report/method_audit.d.ts +4 -3
  57. package/dist/report/method_audit.js +8 -5
  58. package/dist/report/method_audit.js.map +1 -1
  59. package/dist/report/tools.d.ts +31 -1
  60. package/dist/report/tools.js +101 -23
  61. package/dist/report/tools.js.map +1 -1
  62. package/dist/server.d.ts +2 -2
  63. package/dist/server.js +54 -41
  64. package/dist/server.js.map +1 -1
  65. package/dist/session.d.ts +8 -2
  66. package/dist/session.js +9 -0
  67. package/dist/session.js.map +1 -1
  68. package/dist/tools.d.ts +98 -6
  69. package/dist/tools.js +154 -17
  70. package/dist/tools.js.map +1 -1
  71. package/dist/workspace_tool.d.ts +47 -0
  72. package/dist/workspace_tool.js +94 -0
  73. package/dist/workspace_tool.js.map +1 -0
  74. package/guide/WORKFLOW.md +7 -4
  75. package/guide/references/compute.md +51 -0
  76. package/guide/references/report_quick.md +60 -0
  77. package/package.json +1 -1
  78. package/packs/census/docs/microdata.md +77 -0
  79. package/packs/census/docs/timeseries.md +71 -0
  80. package/packs/census/source.md +10 -140
  81. package/packs/congress/docs/nominations_crs.md +26 -0
  82. package/packs/congress/docs/votes.md +30 -0
  83. package/packs/congress/source.md +4 -48
  84. package/packs/govinfo/docs/collections.md +88 -0
  85. package/packs/govinfo/source.md +6 -86
  86. package/packs/irs-soi/docs/eo.md +15 -0
  87. package/packs/irs-soi/docs/tables.md +50 -0
  88. package/packs/irs-soi/source.md +6 -57
@@ -15,11 +15,12 @@
15
15
  */
16
16
  import fs from "node:fs";
17
17
  import path from "node:path";
18
+ import { namesHint } from "../core/compact.js";
18
19
  import { addStateCodes, alignCheck, CombineError, deflate, fiscalYear, toFiscal } from "./combine.js";
19
- import { compare, compile, evaluate, ExprError, isTruthy } from "./expr.js";
20
+ import { compare, compile, evaluate, ExprError, inspect, isTruthy, parse as parseExpr } from "./expr.js";
20
21
  import { pyRepr } from "./pyfmt.js";
21
- import { replicateSe } from "./replicate.js";
22
- import { cellText, newRow, ParquetInputError, parseTable, readParquetTable, sha256, tableCsv } from "./table.js";
22
+ import { replicateColumns, replicateSe } from "./replicate.js";
23
+ import { cellText, csvGrid, headerOf, newRow, PARQUET_MAX_CELLS, parquetColumns, ParquetInputError, readParquetTable, sha256, tableCsv, tableFromGrid } from "./table.js";
23
24
  export class ComputeError extends Error {
24
25
  }
25
26
  export const LINEAGE = "exhibits/lineage.json";
@@ -60,8 +61,82 @@ export function normInput(alias, v) {
60
61
  }
61
62
  throw new ComputeError(`input ${alias}: a workspace path, or {path, columns: [names]}, not ${pyRepr(v)}`);
62
63
  }
64
+ /** The Parquet copy fetch writes beside a large data/<id>/<table>.csv, when it is there and not older than the CSV. */
65
+ function parquetCopy(csvAbs) {
66
+ if (!/[\\/]data[\\/][^\\/]+[\\/][^\\/]+\.csv$/i.test(csvAbs))
67
+ return null;
68
+ const pq = csvAbs.replace(/\.csv$/i, ".parquet");
69
+ try {
70
+ return fs.statSync(pq).mtimeMs >= fs.statSync(csvAbs).mtimeMs ? pq : null;
71
+ }
72
+ catch {
73
+ return null;
74
+ }
75
+ }
76
+ /** An input's column names, read before the table (the header line of a CSV, the schema of a Parquet file). */
77
+ function inputHeader(alias, rel, abs, buf, parquet) {
78
+ if (parquet) {
79
+ try {
80
+ return parquetColumns(buf, abs);
81
+ }
82
+ catch (e) {
83
+ throw new ComputeError(`input ${alias}: ${relPath(rel)} could not be read as Parquet: ${e?.message || e}`);
84
+ }
85
+ }
86
+ return headerOf(buf.subarray(0, Math.min(buf.length, 1 << 20)).toString("utf8").replace(/^\uFEFF/, ""));
87
+ }
88
+ /** The most cells of a CSV parsed at once (every column of every row, before any column is dropped). A 37 MB ACS PUMS
89
+ * file of 62,777 rows x 175 columns (11M cells) took 1.5 s and about 360 MB to parse; 60M is about 2 GB. */
90
+ export const CSV_MAX_RAW_CELLS = { value: 60_000_000 };
91
+ /** Parsed CSV grids kept across calls, keyed by path and content hash: compute and the rerun read the same large
92
+ * microdata file again and again. Only files of at least CACHE_MIN_BYTES; at most the budget in total (about ten
93
+ * times the file size each), least recently used dropped first. */
94
+ const GRID_CACHE = new Map();
95
+ const CACHE_MIN_BYTES = 5_000_000;
96
+ function cacheBudget() {
97
+ const mb = Number(process.env.DATA_ANALYST_PARSE_CACHE_MB);
98
+ return (Number.isFinite(mb) && mb >= 0 ? mb : 512) * 1024 * 1024;
99
+ }
100
+ function cachedGrid(abs, sha, buf) {
101
+ const key = `${abs}\u0000${sha}`;
102
+ const hit = GRID_CACHE.get(key);
103
+ if (hit) {
104
+ GRID_CACHE.delete(key); // most recently used last
105
+ GRID_CACHE.set(key, hit);
106
+ return hit.grid;
107
+ }
108
+ const grid = csvGrid(buf.toString("utf8"));
109
+ const bytes = buf.length * 10;
110
+ const budget = cacheBudget();
111
+ if (buf.length >= CACHE_MIN_BYTES && bytes <= budget) {
112
+ let used = [...GRID_CACHE.values()].reduce((a, v) => a + v.bytes, 0);
113
+ for (const [k, v] of GRID_CACHE) {
114
+ if (used + bytes <= budget)
115
+ break;
116
+ GRID_CACHE.delete(k);
117
+ used -= v.bytes;
118
+ }
119
+ GRID_CACHE.set(key, { grid, bytes });
120
+ }
121
+ return grid;
122
+ }
123
+ /** Drop every cached grid (tests). */
124
+ export function clearParseCache() {
125
+ GRID_CACHE.clear();
126
+ }
127
+ /** How many parsed grids are cached (tests). */
128
+ export function parseCacheSize() {
129
+ return GRID_CACHE.size;
130
+ }
131
+ /** Line ends in a buffer (an upper bound on a CSV's rows, for the cell caps). */
132
+ function lineEnds(buf) {
133
+ let n = 0;
134
+ for (let i = buf.indexOf(10); i !== -1; i = buf.indexOf(10, i + 1))
135
+ n++;
136
+ return n;
137
+ }
63
138
  /** Read one input as a typed table (only its named columns, when given). */
64
- function readInput(alias, abs, inp, buf) {
139
+ function readInput(alias, abs, inp, buf, sha, header) {
65
140
  if (relPath(inp.path).endsWith(".parquet")) {
66
141
  try {
67
142
  return readParquetTable(buf, abs, inp.columns);
@@ -72,19 +147,104 @@ function readInput(alias, abs, inp, buf) {
72
147
  throw new ComputeError(`input ${alias}: ${relPath(inp.path)} could not be read as Parquet: ${e?.message || e}`);
73
148
  }
74
149
  }
75
- const t = parseTable(buf.toString("utf8"));
76
150
  const want = inp.columns;
77
- if (!want)
78
- return t;
79
- const miss = want.filter((c) => !t.cols.includes(c));
80
- if (miss.length)
81
- throw new ComputeError(`input ${alias}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; the columns are ${pyRepr(t.cols)}`);
82
- return { cols: want, rows: t.rows.map((r) => {
83
- const o = newRow();
84
- for (const c of want)
85
- o[c] = r[c];
86
- return o;
87
- }) };
151
+ if (!GRID_CACHE.has(`${abs}\u0000${sha}`)) {
152
+ const rows = lineEnds(buf);
153
+ const raw = rows * header.length, typed = rows * (want ? want.length : header.length);
154
+ if (raw > CSV_MAX_RAW_CELLS.value || typed > PARQUET_MAX_CELLS.value) {
155
+ const n = (x) => x.toLocaleString("en-US");
156
+ throw new ComputeError(`input ${alias}: ${relPath(inp.path)} has about ${n(rows)} rows x ${n(header.length)} columns, ` +
157
+ `more than compute reads at once (${n(CSV_MAX_RAW_CELLS.value)} cells parsed, ${n(PARQUET_MAX_CELLS.value)} kept). ` +
158
+ (raw > CSV_MAX_RAW_CELLS.value
159
+ ? "Read the .parquet file beside it when the fetch wrote one ({path, columns}), or fetch fewer rows or variables."
160
+ : `Name only the columns the steps need: {"${alias}": {"path": "${relPath(inp.path)}", "columns": [...]}}.`));
161
+ }
162
+ }
163
+ return tableFromGrid(header, cachedGrid(abs, sha, buf), want);
164
+ }
165
+ /** The input columns a single-input recipe reads, when its steps say so: every column the steps name up to the first
166
+ * step that keeps only named columns (select, rollup, replicate_se with estimates, pivot_wider with id_cols). Null when
167
+ * any step could read columns it does not name (dedupe: true, other tables, ...) or no step narrows the table, so
168
+ * every column reaches the output: then the whole table is read. */
169
+ export function neededColumns(p, header) {
170
+ try {
171
+ const need = new Set();
172
+ const add = (c) => { if (typeof c === "string")
173
+ need.add(c); };
174
+ const ref = (src) => { for (const c of inspect(parseExpr(src)).cols)
175
+ need.add(c); };
176
+ const done = () => {
177
+ const out = header.filter((c) => need.has(c));
178
+ return out.length && out.length < header.length ? out : null;
179
+ };
180
+ for (const step of p.steps || []) {
181
+ const verbs = Object.keys(step || {}).filter((k) => VERBS.includes(k));
182
+ if (verbs.length !== 1)
183
+ return null;
184
+ const verb = verbs[0], arg = step[verb];
185
+ if (step.groupby !== undefined)
186
+ strList(step.groupby, "groupby").forEach(add);
187
+ switch (verb) {
188
+ case "filter":
189
+ ref(arg);
190
+ break;
191
+ case "derive":
192
+ for (const [, e] of exprs(arg, "derive"))
193
+ ref(e);
194
+ break;
195
+ case "rename":
196
+ for (const [a] of exprs(arg, "rename"))
197
+ add(a);
198
+ break;
199
+ case "orderby":
200
+ strList(arg, "orderby").forEach((c) => add(c.replace(/^-/, "")));
201
+ break;
202
+ case "slice": break;
203
+ case "dedupe":
204
+ if (arg === true)
205
+ return null;
206
+ strList(arg, "dedupe").forEach(add);
207
+ break;
208
+ case "fiscal_year":
209
+ add(typeof arg === "string" ? arg : arg?.date);
210
+ break;
211
+ case "state_codes":
212
+ add(typeof arg === "string" ? arg : arg?.col);
213
+ break;
214
+ case "select":
215
+ strList(arg, "select").forEach(add);
216
+ return done();
217
+ case "rollup":
218
+ for (const [, e] of exprs(arg, "rollup"))
219
+ ref(e);
220
+ return done();
221
+ case "replicate_se": {
222
+ if (!arg || typeof arg !== "object" || !arg.estimates || typeof arg.estimates !== "object")
223
+ return null;
224
+ add(arg.weight);
225
+ replicateColumns(arg.replicates).forEach(add);
226
+ for (const e of Object.values(arg.estimates))
227
+ ref(e);
228
+ for (const e of Object.values(arg.also || {}))
229
+ ref(e);
230
+ return done();
231
+ }
232
+ case "pivot_wider":
233
+ if (!arg?.id_cols)
234
+ return null;
235
+ strList(arg.id_cols, "id_cols").forEach(add);
236
+ add(arg.names_from);
237
+ add(arg.values_from);
238
+ return done();
239
+ default:
240
+ return null; // other tables, value-dependent columns
241
+ }
242
+ }
243
+ return null;
244
+ }
245
+ catch {
246
+ return null; // a malformed step: the full read reports it
247
+ }
88
248
  }
89
249
  export function readLineage(workspace) {
90
250
  try {
@@ -126,7 +286,7 @@ const EXTRA = { filter: ["groupby"], derive: ["groupby"], rollup: ["groupby"], r
126
286
  export function need(t, cols, what) {
127
287
  const miss = cols.filter((c) => !t.cols.includes(c));
128
288
  if (miss.length)
129
- throw new ComputeError(`${what}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; the columns are ${pyRepr(t.cols)}`);
289
+ throw new ComputeError(`${what}: no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}; ${namesHint(miss, t.cols, "columns", pyRepr)}`);
130
290
  }
131
291
  export function strList(v, what) {
132
292
  const xs = typeof v === "string" ? [v] : v;
@@ -166,14 +326,18 @@ function exprs(v, what) {
166
326
  throw new ComputeError(`${what}: give at least one {name: expression}`);
167
327
  return out;
168
328
  }
329
+ /** Row lists this pipeline made itself and nothing else holds: derive may add columns to their rows in place instead
330
+ * of copying every row (an input table, a parse-cache entry or an "as" table is never in it). */
331
+ const OWNED = new WeakSet();
169
332
  function derive(t, spec, by) {
170
333
  let cols = [...t.cols];
171
- const rows = t.rows.map((r) => Object.assign(newRow(), r));
334
+ const rows = OWNED.has(t.rows) ? t.rows : t.rows.map((r) => Object.assign(newRow(), r));
335
+ OWNED.add(rows);
172
336
  for (const [name, src] of exprs(spec, "derive")) {
173
337
  const node = compile(src, cols, { what: "derive" });
174
338
  for (const g of groupRows({ cols, rows }, by, false)) {
175
- const ctx = { i: 0, group: g, cache: new Map() };
176
- const vals = g.map((r, i) => evaluate(node, { ...ctx, row: r, i }));
339
+ const ctx = { i: 0, group: g, cache: new Map() }; // moved row by row: evaluate keeps no reference to it
340
+ const vals = g.map((r, i) => ((ctx.row = r), (ctx.i = i), evaluate(node, ctx)));
177
341
  g.forEach((r, i) => (r[name] = vals[i]));
178
342
  }
179
343
  if (!cols.includes(name))
@@ -413,14 +577,16 @@ function wrap(i, verb, fn) {
413
577
  throw e;
414
578
  }
415
579
  }
416
- /** Read the inputs and run the steps (no file is written). */
417
- export function runPipeline(workspace, p) {
580
+ /** Read the inputs and run the steps (no file is written). `parsed` keeps each input read once across calls (the
581
+ * rerun replays many recipes over one large microdata file); it is keyed by path, content hash and columns, and is
582
+ * safe to share because no step changes its input table in place (every verb builds new rows). */
583
+ export function runPipeline(workspace, p, parsed) {
418
584
  if (!p.inputs || typeof p.inputs !== "object" || !Object.keys(p.inputs).length) {
419
585
  throw new ComputeError('inputs: {alias: "data/<id>/<table>.csv" | "data/<id>/<table>.parquet" | "exhibits/<x>.csv" | ' +
420
586
  "{path, columns}}, at least one");
421
587
  }
422
- const env = new Map();
423
588
  const inputs = {};
589
+ const prepared = [];
424
590
  for (const [alias, raw] of Object.entries(p.inputs)) {
425
591
  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(alias) || alias === "this" || alias.startsWith("__"))
426
592
  throw new ComputeError(`input alias ${pyRepr(alias)}: letters, digits and _ only`);
@@ -430,9 +596,48 @@ export function runPipeline(workspace, p) {
430
596
  if (!fs.existsSync(abs))
431
597
  throw new ComputeError(`input ${alias}: no file ${relPath(rel)} in the workspace (fetch it first, or check the path)`);
432
598
  const buf = fs.readFileSync(abs);
433
- inputs[alias] = { path: relPath(rel), sha256: sha256(buf), ...(inp.columns ? { columns: inp.columns } : {}) };
434
- env.set(alias, readInput(alias, abs, inp, buf));
599
+ const sha = sha256(buf);
600
+ inputs[alias] = { path: relPath(rel), sha256: sha, ...(inp.columns ? { columns: inp.columns } : {}) };
601
+ const parquet = relPath(rel).endsWith(".parquet");
602
+ prepared.push({ alias, inp, abs, buf, sha, parquet, header: inputHeader(alias, rel, abs, buf, parquet) });
603
+ }
604
+ for (const x of prepared) {
605
+ const miss = (x.inp.columns || []).filter((c) => !x.header.includes(c));
606
+ if (!miss.length)
607
+ continue;
608
+ const what = `no column${miss.length > 1 ? "s" : ""} ${miss.join(", ")}`;
609
+ throw new ComputeError(x.parquet // the messages the readers give (the Parquet one names the file)
610
+ ? `input ${x.alias} (${relPath(x.inp.path)}): ${what}; ${namesHint(miss, x.header, "columns", (v) => v.join(", "))}`
611
+ : `input ${x.alias}: ${what}; ${namesHint(miss, x.header, "columns", pyRepr)}`);
612
+ }
613
+ // a dry run of the steps on empty tables with the inputs' headers: a misspelt column fails here, before a large
614
+ // file is parsed (only what the data cannot change is reported; data-dependent steps end the dry run)
615
+ runSteps(p, new Map(prepared.map((x) => [x.alias, { cols: x.inp.columns ?? x.header, rows: [] }])), true);
616
+ // one input and no columns named: read only the columns the steps use, when the steps say which those are
617
+ const auto = prepared.length === 1 && !prepared[0].inp.columns ? neededColumns(p, prepared[0].header) : null;
618
+ const env = new Map();
619
+ for (const x of prepared) {
620
+ const want = x.inp.columns ?? auto ?? undefined;
621
+ const key = `${x.abs}\u0000${x.sha}\u0000${want ? JSON.stringify(want) : ""}`;
622
+ let t = parsed?.get(key);
623
+ if (!t) {
624
+ // only some columns of a fetched CSV that has a Parquet copy: decode those columns from the copy (same table)
625
+ const pq = want && !x.parquet ? parquetCopy(x.abs) : null;
626
+ t = pq ? readInput(x.alias, pq, { path: relPath(x.inp.path).replace(/\.csv$/, ".parquet"), columns: want }, fs.readFileSync(pq), x.sha, x.header)
627
+ : readInput(x.alias, x.abs, { path: x.inp.path, columns: want }, x.buf, x.sha, x.header);
628
+ parsed?.set(key, t);
629
+ }
630
+ env.set(x.alias, t);
435
631
  }
632
+ const r = runSteps(p, env, false);
633
+ return { table: r.table, notes: r.notes, inputs };
634
+ }
635
+ /** Verbs whose result depends on the data's values (new columns from values, lookups, checks): a dry run stops there. */
636
+ const DRY_STOP = new Set(["pivot_wider", "pivot_longer", "to_fiscal", "deflate", "replicate_se", "align_check"]);
637
+ /** Errors a dry run reports: a column or table that does not exist. */
638
+ const STRUCTURAL = /unknown column|no column|is not a table here|is not an input alias/;
639
+ /** Run the steps over the tables in env (`dry`: on empty tables, as a check before the inputs are read). */
640
+ function runSteps(p, env, dry) {
436
641
  const start = p.from ?? Object.keys(p.inputs)[0];
437
642
  if (!env.has(start))
438
643
  throw new ComputeError(`from: ${pyRepr(start)} is not an input alias (${Object.keys(p.inputs).join(", ")})`);
@@ -448,7 +653,8 @@ export function runPipeline(workspace, p) {
448
653
  };
449
654
  if (!Array.isArray(p.steps))
450
655
  throw new ComputeError("steps: a list of steps");
451
- p.steps.forEach((step, i) => {
656
+ for (let i = 0; i < p.steps.length; i++) {
657
+ const step = p.steps[i];
452
658
  if (!step || typeof step !== "object" || Array.isArray(step))
453
659
  throw new ComputeError(`step ${i + 1}: an object such as {"filter": "year >= 2020"}`);
454
660
  const verbs = Object.keys(step).filter((k) => VERBS.includes(k));
@@ -461,144 +667,161 @@ export function runPipeline(workspace, p) {
461
667
  if (extra.length)
462
668
  throw new ComputeError(`step ${i + 1} (${verb}): unknown key(s) ${pyRepr(extra)}; allowed with ${verb}: ${pyRepr(EXTRA[verb] || [])}`);
463
669
  const arg = step[verb];
464
- wrap(i, verb, () => {
465
- const by = step.groupby === undefined ? [] : strList(step.groupby, "groupby");
466
- if (by.length)
467
- need(cur, by, "groupby");
468
- switch (verb) {
469
- case "filter":
470
- cur = filter(cur, arg, by);
471
- break;
472
- case "derive":
473
- cur = derive(cur, arg, by);
474
- break;
475
- case "rollup":
476
- cur = rollup(cur, arg, by);
477
- break;
478
- case "select": {
479
- const cols = strList(arg, "select");
480
- need(cur, cols, "select");
481
- cur = { cols: [...new Set(cols)], rows: cur.rows };
482
- break;
483
- }
484
- case "rename": {
485
- const pairs = exprs(arg, "rename").map(([a, b]) => [a, String(b)]);
486
- need(cur, pairs.map(([a]) => a), "rename");
487
- const map = new Map(pairs);
488
- const cols = cur.cols.map((c) => map.get(c) ?? c);
489
- const dup = cols.filter((c, j) => cols.indexOf(c) !== j);
490
- if (dup.length)
491
- throw new ComputeError(`rename: ${pyRepr([...new Set(dup)])} would name two columns`);
492
- cur = { cols, rows: cur.rows.map((r) => {
493
- const o = newRow();
494
- cur.cols.forEach((c, j) => (o[cols[j]] = r[c]));
495
- return o;
496
- }) };
497
- break;
498
- }
499
- case "join": {
500
- const r = join(cur, table(arg, "join"), step);
501
- cur = r.table;
502
- notes.push(...r.notes);
503
- break;
504
- }
505
- case "pivot_wider":
506
- cur = pivotWider(cur, arg);
507
- break;
508
- case "pivot_longer":
509
- cur = pivotLonger(cur, arg);
510
- break;
511
- case "orderby":
512
- cur = orderby(cur, arg);
513
- break;
514
- case "slice":
515
- cur = slice(cur, arg);
516
- break;
517
- case "dedupe":
518
- cur = dedupe(cur, arg);
519
- break;
520
- case "concat":
521
- cur = concat([cur, ...(Array.isArray(arg) ? arg : [arg]).map((a) => table(a, "concat"))]);
522
- break;
523
- case "fiscal_year": {
524
- const o = typeof arg === "string" ? { date: arg } : arg || {};
525
- const as = String(o.as ?? "fiscal_year");
526
- need(cur, [String(o.date)], "fiscal_year");
527
- cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r) => {
528
- const x = Object.assign(newRow(), r);
529
- x[as] = r[o.date] === null ? null : fiscalYear(r[o.date], o.date);
530
- return x;
531
- }) };
532
- break;
533
- }
534
- case "to_fiscal": {
535
- const o = arg || {};
536
- const r = toFiscal(cur, String(o.date), String(o.value), o.how ?? "sum", o.freq ?? "M");
537
- cur = r.table;
538
- notes.push(...r.notes);
539
- break;
540
- }
541
- case "state_codes":
542
- cur = addStateCodes(cur, typeof arg === "string" ? arg : String(arg?.col));
543
- break;
544
- case "deflate": {
545
- const o = arg || {};
546
- if (!o.name || typeof o.name !== "string") {
547
- throw new ComputeError('deflate: name the price index and its source in "name" (e.g. "CPI-U, BLS CUUR0000SA0"), so the ' +
548
- "answer can say which dollars these are");
670
+ if (dry && DRY_STOP.has(verb))
671
+ break;
672
+ const before = cur.rows;
673
+ try {
674
+ wrap(i, verb, () => {
675
+ const by = step.groupby === undefined ? [] : strList(step.groupby, "groupby");
676
+ if (by.length)
677
+ need(cur, by, "groupby");
678
+ switch (verb) {
679
+ case "filter":
680
+ cur = filter(cur, arg, by);
681
+ break;
682
+ case "derive":
683
+ cur = derive(cur, arg, by);
684
+ break;
685
+ case "rollup":
686
+ cur = rollup(cur, arg, by);
687
+ break;
688
+ case "select": {
689
+ const cols = strList(arg, "select");
690
+ need(cur, cols, "select");
691
+ cur = { cols: [...new Set(cols)], rows: cur.rows };
692
+ break;
549
693
  }
550
- const value = String(o.value), period = String(o.period);
551
- need(cur, [value, period], "deflate");
552
- const idx = table(o.index, "deflate index");
553
- const ip = String(o.index_period ?? period), iv = String(o.index_value ?? "");
554
- need(idx, [ip, iv], "deflate index");
555
- const m = new Map();
556
- for (const r of idx.rows) {
557
- const k = cellText(r[ip]);
558
- if (m.has(k))
559
- throw new ComputeError(`deflate: the index repeats period ${pyRepr(k)}: one row per period`);
560
- const v = r[iv];
561
- if (typeof v !== "number")
562
- throw new ComputeError(`deflate: index ${iv} = ${pyRepr(v)} at ${pyRepr(k)} is not a number`);
563
- m.set(k, v);
694
+ case "rename": {
695
+ const pairs = exprs(arg, "rename").map(([a, b]) => [a, String(b)]);
696
+ need(cur, pairs.map(([a]) => a), "rename");
697
+ const map = new Map(pairs);
698
+ const cols = cur.cols.map((c) => map.get(c) ?? c);
699
+ const dup = cols.filter((c, j) => cols.indexOf(c) !== j);
700
+ if (dup.length)
701
+ throw new ComputeError(`rename: ${pyRepr([...new Set(dup)])} would name two columns`);
702
+ cur = { cols, rows: cur.rows.map((r) => {
703
+ const o = newRow();
704
+ cur.cols.forEach((c, j) => (o[cols[j]] = r[c]));
705
+ return o;
706
+ }) };
707
+ break;
564
708
  }
565
- const base = o.base === undefined ? null : o.base;
566
- const out = deflate(cur.rows.map((r) => ({ period: r[period], value: r[value] })), m, base);
567
- const as = String(o.as ?? `${value}_real`);
568
- cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r, j) => {
569
- const x = Object.assign(newRow(), r);
570
- x[as] = out[j];
571
- return x;
572
- }) };
573
- notes.push(`deflate: ${as} = ${value} in ${base === null ? "the index's own base (used as given)" : `${cellText(base)} dollars`}, ` +
574
- `deflated by ${o.name}`);
575
- break;
576
- }
577
- case "replicate_se": {
578
- const r = replicateSe(cur, arg, by);
579
- cur = r.table;
580
- notes.push(...r.notes);
581
- break;
582
- }
583
- case "align_check": {
584
- const o = arg || {};
585
- const names = strList(o.tables, "align_check tables");
586
- const frames = Object.fromEntries(names.map((n) => [n, table(n, "align_check")]));
587
- notes.push(...alignCheck(frames, strList(o.on, "align_check on"), o.units && typeof o.units === "object" ? o.units : {}));
588
- break;
709
+ case "join": {
710
+ const r = join(cur, table(arg, "join"), step);
711
+ cur = r.table;
712
+ notes.push(...r.notes);
713
+ break;
714
+ }
715
+ case "pivot_wider":
716
+ cur = pivotWider(cur, arg);
717
+ break;
718
+ case "pivot_longer":
719
+ cur = pivotLonger(cur, arg);
720
+ break;
721
+ case "orderby":
722
+ cur = orderby(cur, arg);
723
+ break;
724
+ case "slice":
725
+ cur = slice(cur, arg);
726
+ break;
727
+ case "dedupe":
728
+ cur = dedupe(cur, arg);
729
+ break;
730
+ case "concat":
731
+ cur = concat([cur, ...(Array.isArray(arg) ? arg : [arg]).map((a) => table(a, "concat"))]);
732
+ break;
733
+ case "fiscal_year": {
734
+ const o = typeof arg === "string" ? { date: arg } : arg || {};
735
+ const as = String(o.as ?? "fiscal_year");
736
+ need(cur, [String(o.date)], "fiscal_year");
737
+ cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r) => {
738
+ const x = Object.assign(newRow(), r);
739
+ x[as] = r[o.date] === null ? null : fiscalYear(r[o.date], o.date);
740
+ return x;
741
+ }) };
742
+ break;
743
+ }
744
+ case "to_fiscal": {
745
+ const o = arg || {};
746
+ const r = toFiscal(cur, String(o.date), String(o.value), o.how ?? "sum", o.freq ?? "M");
747
+ cur = r.table;
748
+ notes.push(...r.notes);
749
+ break;
750
+ }
751
+ case "state_codes":
752
+ cur = addStateCodes(cur, typeof arg === "string" ? arg : String(arg?.col));
753
+ break;
754
+ case "deflate": {
755
+ const o = arg || {};
756
+ if (!o.name || typeof o.name !== "string") {
757
+ throw new ComputeError('deflate: name the price index and its source in "name" (e.g. "CPI-U, BLS CUUR0000SA0"), so the ' +
758
+ "answer can say which dollars these are");
759
+ }
760
+ const value = String(o.value), period = String(o.period);
761
+ need(cur, [value, period], "deflate");
762
+ const idx = table(o.index, "deflate index");
763
+ const ip = String(o.index_period ?? period), iv = String(o.index_value ?? "");
764
+ need(idx, [ip, iv], "deflate index");
765
+ const m = new Map();
766
+ for (const r of idx.rows) {
767
+ const k = cellText(r[ip]);
768
+ if (m.has(k))
769
+ throw new ComputeError(`deflate: the index repeats period ${pyRepr(k)}: one row per period`);
770
+ const v = r[iv];
771
+ if (typeof v !== "number")
772
+ throw new ComputeError(`deflate: index ${iv} = ${pyRepr(v)} at ${pyRepr(k)} is not a number`);
773
+ m.set(k, v);
774
+ }
775
+ const base = o.base === undefined ? null : o.base;
776
+ const out = deflate(cur.rows.map((r) => ({ period: r[period], value: r[value] })), m, base);
777
+ const as = String(o.as ?? `${value}_real`);
778
+ cur = { cols: cur.cols.includes(as) ? cur.cols : [...cur.cols, as], rows: cur.rows.map((r, j) => {
779
+ const x = Object.assign(newRow(), r);
780
+ x[as] = out[j];
781
+ return x;
782
+ }) };
783
+ notes.push(`deflate: ${as} = ${value} in ${base === null ? "the index's own base (used as given)" : `${cellText(base)} dollars`}, ` +
784
+ `deflated by ${o.name}`);
785
+ break;
786
+ }
787
+ case "replicate_se": {
788
+ const r = replicateSe(cur, arg, by);
789
+ cur = r.table;
790
+ notes.push(...r.notes);
791
+ break;
792
+ }
793
+ case "align_check": {
794
+ const o = arg || {};
795
+ const names = strList(o.tables, "align_check tables");
796
+ const frames = Object.fromEntries(names.map((n) => [n, table(n, "align_check")]));
797
+ notes.push(...alignCheck(frames, strList(o.on, "align_check on"), o.units && typeof o.units === "object" ? o.units : {}));
798
+ break;
799
+ }
800
+ case "as":
801
+ if (typeof arg !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) || arg === "this")
802
+ throw new ComputeError("as: a table name (letters, digits, _)");
803
+ env.set(arg, cur);
804
+ break;
805
+ case "from":
806
+ cur = table(arg, "from");
807
+ break;
589
808
  }
590
- case "as":
591
- if (typeof arg !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) || arg === "this")
592
- throw new ComputeError("as: a table name (letters, digits, _)");
593
- env.set(arg, cur);
594
- break;
595
- case "from":
596
- cur = table(arg, "from");
597
- break;
598
- }
599
- });
600
- });
601
- return { table: cur, notes, inputs };
809
+ });
810
+ // these keep a subset of the same rows: a list derive owned stays owned (the old list is no longer used)
811
+ if (OWNED.has(before) && ["filter", "select", "orderby", "slice", "dedupe"].includes(verb))
812
+ OWNED.add(cur.rows);
813
+ // a table kept under a name may be read again: its rows are shared from now on
814
+ if (verb === "as")
815
+ OWNED.delete(cur.rows);
816
+ }
817
+ catch (e) {
818
+ // the dry run reports only what the data cannot change: a column or table that does not exist
819
+ if (!dry || (e instanceof ComputeError && STRUCTURAL.test(e.message)))
820
+ throw e;
821
+ break;
822
+ }
823
+ }
824
+ return { table: cur, notes };
602
825
  }
603
826
  /** The packs an exhibit's inputs come from: data/<id>/ is <id>; an exhibit carries its lineage; published/ none. */
604
827
  export function inferPacks(workspace, inputs) {
@@ -633,10 +856,39 @@ export function compute(workspace, a) {
633
856
  lin[save] = entry;
634
857
  writeLineage(workspace, lin);
635
858
  const n = Math.max(0, Math.min(Number(a.preview ?? 20), 50));
859
+ const pv = previewRows(r.table, n);
636
860
  return {
637
861
  file: abs, rel: save, rows: r.table.rows.length, columns: r.table.cols,
638
- preview: r.table.rows.slice(0, n).map((row) => Object.fromEntries(r.table.cols.map((c) => [c, cellText(row[c])]))),
862
+ preview: pv.rows, ...(pv.note ? { preview_note: pv.note } : {}),
639
863
  notes: r.notes, packs,
640
864
  };
641
865
  }
866
+ /** Preview limits: the file has every value exactly; the preview is for reading, so it is narrow and short. */
867
+ export const PREVIEW_MAX_COLS = 20;
868
+ export const PREVIEW_MAX_CHARS = 4000;
869
+ /** A number as the preview shows it: 6 significant digits for a fraction (integers and text as written). */
870
+ function previewCell(v) {
871
+ if (typeof v === "number" && Number.isFinite(v) && !Number.isInteger(v))
872
+ return String(Number(v.toPrecision(6)));
873
+ return cellText(v);
874
+ }
875
+ /** The first n rows, at most PREVIEW_MAX_COLS columns and about PREVIEW_MAX_CHARS characters, with a note on any cut. */
876
+ export function previewRows(t, n) {
877
+ const cols = t.cols.slice(0, PREVIEW_MAX_COLS);
878
+ const rows = [];
879
+ let size = 0;
880
+ for (const row of t.rows.slice(0, n)) {
881
+ const o = Object.fromEntries(cols.map((c) => [c, previewCell(row[c])]));
882
+ size += JSON.stringify(o).length;
883
+ if (rows.length && size > PREVIEW_MAX_CHARS)
884
+ break;
885
+ rows.push(o);
886
+ }
887
+ const cut = [];
888
+ if (cols.length < t.cols.length)
889
+ cut.push(`the first ${cols.length} of ${t.cols.length} columns`);
890
+ if (rows.length < Math.min(n, t.rows.length))
891
+ cut.push(`${rows.length} rows (size limit)`);
892
+ return { rows, ...(cut.length ? { note: `preview shows ${cut.join(" and ")}; fractions rounded to 6 digits; the file has every value exactly` } : {}) };
893
+ }
642
894
  //# sourceMappingURL=compute.js.map