gloomberb 0.20.16 → 0.20.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/package.json +1 -1
  2. package/src/api-client/data.ts +13 -3
  3. package/src/api-client/types/econ.ts +3 -0
  4. package/src/cli/commands/fred-series.ts +3 -1
  5. package/src/cli/commands/overview.ts +65 -13
  6. package/src/cli/csv-export.ts +73 -0
  7. package/src/cli/desktop-pane-shot.ts +3 -2
  8. package/src/cli/help.ts +4 -2
  9. package/src/cli/pane-functions/headless.ts +7 -2
  10. package/src/cli/pane-functions/index.ts +2 -2
  11. package/src/cli/pane-functions/screenshot-bridge.ts +4 -2
  12. package/src/cli/report-tables.ts +17 -3
  13. package/src/cli/result.ts +3 -6
  14. package/src/components/chart/composite/legend.tsx +37 -7
  15. package/src/components/ui/data-table/display-columns.ts +4 -1
  16. package/src/components/ui/data-table/hooks.ts +2 -3
  17. package/src/components/ui/data-table/index.tsx +4 -1
  18. package/src/components/ui/data-table/types.ts +11 -4
  19. package/src/components/ui/data-table/visible-range.ts +11 -1
  20. package/src/plugins/builtin/awards/pane.tsx +6 -2
  21. package/src/plugins/builtin/chart-composer/pane.tsx +4 -1
  22. package/src/plugins/builtin/credit-conditions/series.ts +2 -2
  23. package/src/plugins/builtin/earnings/board-model.ts +12 -0
  24. package/src/plugins/builtin/earnings/board-table.ts +7 -4
  25. package/src/plugins/builtin/earnings/board.tsx +2 -2
  26. package/src/plugins/builtin/earnings/highlight.ts +1 -1
  27. package/src/plugins/builtin/econ/calendar-model.ts +2 -0
  28. package/src/plugins/builtin/econ/fred-catalog.ts +1 -256
  29. package/src/plugins/builtin/econ-statistics/cache.ts +1 -1
  30. package/src/plugins/builtin/econ-statistics/client.ts +11 -10
  31. package/src/plugins/builtin/econ-statistics/detail.tsx +1 -0
  32. package/src/plugins/builtin/econ-statistics/headless.ts +42 -6
  33. package/src/plugins/builtin/econ-statistics/index.tsx +1 -0
  34. package/src/plugins/builtin/econ-statistics/pane.tsx +7 -2
  35. package/src/plugins/builtin/econ-statistics/stats.ts +2 -1
  36. package/src/plugins/builtin/econ-statistics/view.ts +9 -3
  37. package/src/plugins/builtin/shared/fred-catalog.ts +256 -0
  38. package/src/plugins/builtin/shared/series-cache.ts +3 -0
  39. package/src/plugins/builtin/shared/series-detail-chart.tsx +3 -0
  40. package/src/plugins/builtin/short-interest/watch-pane.tsx +7 -7
  41. package/src/plugins/builtin/ticker-detail/adr-pair/model.ts +16 -0
  42. package/src/plugins/builtin/ticker-detail/adr-pair/tab.tsx +8 -4
  43. package/src/plugins/builtin/time-sales/headless.ts +4 -1
  44. package/src/plugins/builtin/time-sales/pane.tsx +2 -0
  45. package/src/plugins/builtin/time-sales/window.ts +12 -0
  46. package/src/plugins/builtin/treasury-auctions/client.ts +4 -0
  47. package/src/plugins/builtin/treasury-auctions/treasury-direct.ts +2 -0
  48. package/src/plugins/builtin/treasury-daily/client.ts +23 -11
  49. package/src/renderers/electrobun/view/cli-pane-shot-entry.tsx +2 -0
  50. package/src/sources/gloomberb-cloud/fred-publisher.ts +25 -0
  51. package/src/sources/gloomberb-cloud/fred-series.ts +1 -1
  52. package/src/time-series/base-series.ts +3 -0
  53. package/src/time-series/ice-credit.ts +1 -0
  54. package/src/time-series/types.ts +1 -0
  55. package/src/types/headless.ts +4 -2
  56. package/src/utils/throttled-fetch.ts +15 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "gloomberb",
3
- "version": "0.20.16",
3
+ "version": "0.20.17",
4
4
  "packageManager": "bun@1.3.14",
5
5
  "description": "Market research and portfolio tracker for the terminal",
6
6
  "module": "src/public/index.ts",
@@ -56,7 +56,7 @@ import type { GpuBoardPayload, GpuEventsPayload, GpuHistoryPayload, GpuHistoryQu
56
56
  import type { CpiBoardPayload } from "./cpi";
57
57
  import type { TapeRange, TapeSnapshot } from "./tape";
58
58
  import { LIVE_SCAN_PATH, type LiveScanPayload, type LiveScanWireDefinition } from "./live-scan";
59
- import { abortable, signalWithTimeout } from "../utils/async-deadline";
59
+ import { abortable, signalWithTimeout, withDeadline } from "../utils/async-deadline";
60
60
  import { ApiRequestError, isRetryableApiError, markAskedTwice } from "./errors";
61
61
  import type { ExchangeRateSnapshot } from "../types/exchange-rate";
62
62
  import type { RatePathPayload } from "./rates";
@@ -211,6 +211,12 @@ function batchAnswers<K extends string, T>(response: CloudMarketResponse<CloudBa
211
211
 
212
212
  /** How long a listing read waits before asking again after the service said its source failed. */
213
213
  const LISTING_RETRY_DELAY_MS = 400;
214
+ /**
215
+ * A company never looked at is searched while it is asked about, for a few
216
+ * seconds at most (the search goes on and the list says pending): a list
217
+ * that has not come in this long is a failure the pane says, not a minute's wait.
218
+ */
219
+ const EARNINGS_CALLS_LIST_TIMEOUT_MS = 20_000;
214
220
 
215
221
  /**
216
222
  * How long the service's answer that it has no quote for a listing stands.
@@ -1506,8 +1512,12 @@ export class CloudDataApi {
1506
1512
  async getCloudEarningsCalls(
1507
1513
  params: CloudEarningsCallsParams = {},
1508
1514
  ): Promise<CloudEarningsCallListPayload> {
1509
- return this.request<CloudEarningsCallListPayload>(
1510
- cloudEarningsCallsPath(params),
1515
+ const abandon = new AbortController();
1516
+ return withDeadline(
1517
+ this.request<CloudEarningsCallListPayload>(cloudEarningsCallsPath(params), { signal: abandon.signal }),
1518
+ EARNINGS_CALLS_LIST_TIMEOUT_MS,
1519
+ `No answer within ${EARNINGS_CALLS_LIST_TIMEOUT_MS / 1000} s.`,
1520
+ () => abandon.abort(),
1511
1521
  );
1512
1522
  }
1513
1523
 
@@ -147,6 +147,9 @@ export interface CloudFredObservationPayload {
147
147
  * 1 Sep and were released 20 Oct. Absent on daily series and older points.
148
148
  */
149
149
  releasedOn?: string;
150
+ firstReleased?: string;
151
+ revisedOn?: string;
152
+ firstReleaseValue?: number;
150
153
  }
151
154
 
152
155
  export interface CloudFredSeriesInfoPayload {
@@ -119,9 +119,11 @@ export function basisPointsCell(value: unknown): string {
119
119
  * reading the source has put out.
120
120
  */
121
121
  const LAGGING_SERIES: Readonly<Record<string, string>> = {
122
- UMCSENT: "FRED publishes the University of Michigan survey with a lag; latest preliminary reading: fn ECO",
122
+ UMCSENT: "University of Michigan sentiment is delayed one month on FRED at the publisher's request; newer preliminary readings are in fn ECO.",
123
123
  };
124
124
 
125
+ export const FRED_VINTAGE_NOTE = "Release dates are the first numeric ALFRED vintage available; archive coverage can begin after an older figure's original publication. Missing release provenance stays blank. The as_of field dates the current value's revision when available, not its first release.";
126
+
125
127
  export function laggingSeriesNote(seriesId: string): string | null {
126
128
  return LAGGING_SERIES[seriesId.toUpperCase()] ?? null;
127
129
  }
@@ -4,7 +4,8 @@ import { ApiRequestError } from "../../api-client/errors";
4
4
  import type { CliCommandDef } from "../../types/plugin";
5
5
  import { withCliServices, withConfigData, withMarketData } from "../context";
6
6
  import { CLI_ERROR_CODE, failWithFields } from "../errors";
7
- import type { CliResultColumn } from "../result";
7
+ import type { CliResult, CliResultColumn, CliResultRenderOptions } from "../result";
8
+ import { fredCitation, fredPublisher } from "../../sources/gloomberb-cloud/fred-publisher";
8
9
  import { MISSING_VALUE, formatCompact } from "../../utils/format";
9
10
  import { DATE_OPTION_EXAMPLES, DATE_OPTION_FORMATS, dateOptionError, readCalendarDate } from "../../utils/calendar-date";
10
11
  import {
@@ -102,6 +103,7 @@ import {
102
103
  fredValueHeader,
103
104
  isSpreadSeries,
104
105
  laggingSeriesNote,
106
+ FRED_VINTAGE_NOTE,
105
107
  parseFredChange,
106
108
  } from "./fred-series";
107
109
  import { fredProvenance, fredPublicationTime, notOnFredSource } from "../../sources/gloomberb-cloud/fred-attribution";
@@ -745,12 +747,12 @@ async function runEcon(args: string[], ctx: Parameters<CliCommandDef["execute"]>
745
747
  }),
746
748
  // The exports add what a script needs to read a row; the text table keeps the columns it had.
747
749
  columns: [...econExportHeadColumns(viewerZone, publisherTimed), ...econFigureColumns(false), ...econExportColumns],
748
- textColumns: [...econTextHeadColumns(viewerZone, publisherTimed), ...econFigureColumns(true), ECON_PUBLISHER_COLUMN],
750
+ textColumns: [...econTextHeadColumns(viewerZone, publisherTimed), { key: "referencePeriod", header: "Period" }, ...econFigureColumns(true), ECON_PUBLISHER_COLUMN],
749
751
  });
750
752
  });
751
753
  }
752
754
 
753
- const FRED_USAGE = "fred <series-id> [--start <date>] [--sort desc|asc] [--change 1D|1W]";
755
+ const FRED_USAGE = "fred <series-id> [series-id ...] [--start <date>] [--sort desc|asc] [--change 1D|1W]";
754
756
  const FRED_LIST_USAGE = "fred --list [filter]";
755
757
 
756
758
  /** A year or a month stands for the first day of it. */
@@ -844,12 +846,55 @@ async function runFred(rawArgs: string[], ctx: Parameters<CliCommandDef["execute
844
846
  if (changeGiven && !horizon) {
845
847
  ctx.fail(change ? `Unknown change "${change}".` : "--change needs 1D or 1W.", "Use 1D (from the previous observation) or 1W (from a week before).");
846
848
  }
849
+ if (args.length > 1 || args[0]?.includes(",")) {
850
+ const seriesIds = [...new Set(args.flatMap((value) => value.split(",")).map((value) => value.trim().toUpperCase()).filter(Boolean))];
851
+ if (!seriesIds.length || seriesIds.some((seriesId) => !/^[A-Z0-9_]+$/.test(seriesId))) ctx.fail("Use space- or comma-separated series IDs.", `Usage: gloom ${FRED_USAGE}`);
852
+ if (seriesIds.length > 20) ctx.fail("fred accepts at most 20 series in one aligned report.");
853
+ return withSavedFredSeries(ctx, () => printFredAligned(ctx, seriesIds, { start, startDate, sortOrder, horizon }));
854
+ }
847
855
  const asked = requireOneArg(args, FRED_USAGE, "series", ctx);
848
856
  // A series by its name (`3 month t-bill` is DTB3) when one alone fits it.
849
857
  const seriesId = (/\s/.test(asked.trim()) ? fredSeriesIdForName(asked) : null) ?? asked.toUpperCase();
850
858
  await withSavedFredSeries(ctx, () => printFredSeries(ctx, { seriesId, start, startDate, sortOrder, horizon }));
851
859
  }
852
860
 
861
+ async function printFredAligned(ctx: Parameters<CliCommandDef["execute"]>[1], seriesIds: string[],
862
+ options: { start?: string; startDate: string; sortOrder: "asc" | "desc"; horizon: ReturnType<typeof parseFredChange> }) {
863
+ const reports: Array<{ seriesId: string; result: CliResult; render: CliResultRenderOptions }> = [];
864
+ for (const seriesId of seriesIds) {
865
+ await printFredSeries({ ...ctx, printResult: (result, render = {}) => { reports.push({ seriesId, result, render: render as unknown as CliResultRenderOptions }); } }, { ...options, start: options.start, seriesId });
866
+ }
867
+ const byDate = new Map<string, Record<string, unknown>>();
868
+ const metadata: Record<string, unknown> = {};
869
+ const notes = ["Aligned on observation dates, not release dates. Missing observations remain blank; values are not forward-filled."];
870
+ const suffixes = ["unit", "interval", "source", "released", "first_released", "revised_on", "first_release_value", ...options.horizon ? ["change", "changeFrom"] : [], "as_of"];
871
+ for (const { seriesId, result, render } of reports) {
872
+ metadata[seriesId] = result.metadata;
873
+ notes.push(...render.exportNotes ?? []);
874
+ for (const entry of result.data as Record<string, unknown>[]) {
875
+ const date = String(entry.date);
876
+ const row = byDate.get(date) ?? { date };
877
+ const identity = typeof render.exportIdentity === "function" ? render.exportIdentity(entry) : render.exportIdentity;
878
+ row[seriesId] = entry.value;
879
+ row[`${seriesId}_unit`] = identity?.currencyOrUnit;
880
+ row[`${seriesId}_interval`] = identity?.interval;
881
+ row[`${seriesId}_source`] = result.freshness?.source;
882
+ for (const suffix of suffixes.slice(3)) row[`${seriesId}_${suffix}`] = entry[suffix];
883
+ row[`${seriesId}_as_of`] = identity?.asOf;
884
+ byDate.set(date, row);
885
+ }
886
+ }
887
+ const rows = [...byDate.values()].sort((left, right) => options.sortOrder === "asc"
888
+ ? String(left.date).localeCompare(String(right.date)) : String(right.date).localeCompare(String(left.date)));
889
+ ctx.printResult({ data: rows, metadata: { series: metadata, alignment: "observation date" },
890
+ warnings: reports.flatMap(({ result }) => result.warnings ?? []),
891
+ freshness: rowsFreshness(rows, { source: [...new Set(reports.map(({ result }) => result.freshness?.source).filter(Boolean))].join("; "),
892
+ status: "not-a-feed", basis: "published statistics", observedKey: "date", oldest: null }),
893
+ }, { dateKey: "date", columns: ["date", ...seriesIds, ...seriesIds.flatMap((seriesId) => suffixes.map((suffix) => `${seriesId}_${suffix}`))].map((key) => ({ key, header: key })),
894
+ exportIdentity: { symbolOrSeries: seriesIds.join(" "), currencyOrUnit: "see per-series unit columns", interval: "see per-series interval columns" },
895
+ exportNotes: [...new Set(notes)], footnote: [...new Set(notes)].join("\n") });
896
+ }
897
+
853
898
  async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { seriesId, start, startDate, sortOrder, horizon }: {
854
899
  seriesId: string;
855
900
  start: string | undefined;
@@ -870,11 +915,11 @@ async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { s
870
915
  if (mismatch) ctx.fail(mismatch);
871
916
  // Each row's change in the unit its value exports in, and the date it is from; a holiday's stays blank.
872
917
  const changes = horizon ? fredChanges(seriesId, data.observations, horizon) : null;
873
- const releasedOf = (observation: { publisher?: string; publishedAt?: string; releasedOn?: string }) =>
874
- observation.releasedOn ?? (observation.publisher?.trim() ? observation.publishedAt : undefined);
918
+ const releasedOf = (observation: { publisher?: string; publishedAt?: string; releasedOn?: string; firstReleased?: string }) =>
919
+ observation.firstReleased ?? observation.releasedOn ?? (observation.publisher?.trim() ? observation.publishedAt : undefined);
875
920
  // A value FRED does not list yet carries its publisher and when it published, in every export.
876
921
  // A row from the copy an earlier read saved, the refresh having failed, carries its mark: the source line says so.
877
- const exportRow = (observation: { date: string; value: number | null; publisher?: string; publishedAt?: string; releasedOn?: string }) => ({
922
+ const exportRow = (observation: CloudFredSeriesPayload["observations"][number]) => ({
878
923
  ...savedCopyMark(data),
879
924
  date: observation.date,
880
925
  value: fredExportValue(seriesId, observation.value),
@@ -882,6 +927,9 @@ async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { s
882
927
  ...(observation.publisher?.trim() ? { publisher: observation.publisher.trim(), ...(observation.publishedAt ? { publishedAt: observation.publishedAt } : {}) } : {}),
883
928
  // When the figure came out, where the data says: only a series' newest observation does. `date` is the period it covers.
884
929
  ...releasedOf(observation) ? { released: releasedOf(observation) } : {},
930
+ ...(observation.firstReleased ? { first_released: observation.firstReleased } : {}),
931
+ ...(observation.revisedOn ? { revised_on: observation.revisedOn } : {}),
932
+ ...(observation.firstReleaseValue != null ? { first_release_value: fredExportValue(seriesId, observation.firstReleaseValue) } : {}),
885
933
  });
886
934
  // A --start before the series begins comes back padded with blank days: the rows start at its first observation.
887
935
  const fromStart = fredObservationsFromStart(data.observations, data.info?.observationStart)
@@ -892,6 +940,7 @@ async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { s
892
940
  ...fredHistoryNotes({ startDate, startGiven: start != null, seriesStart: data.info?.observationStart }),
893
941
  laggingSeriesNote(seriesId),
894
942
  computedFredSeriesNote(seriesId),
943
+ ...(data.observations.some((observation) => observation.firstReleased) ? [FRED_VINTAGE_NOTE] : []),
895
944
  // A publisher's licence asks for its attribution wherever its series is shown (MAS's SORA).
896
945
  seriesNotOnFred(seriesId)?.attribution ?? null,
897
946
  ].filter((note): note is string => note != null);
@@ -913,7 +962,7 @@ async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { s
913
962
  // A newest value FRED does not list yet names its publisher, Treasury for a yield; FRED's own stay "FRED".
914
963
  const ahead = fredProvenance(data.observations.filter((observation) => observation.date === newestDate), seriesId);
915
964
  // FRED names the series' publisher when Gloom Cloud passes it on, and when it last updated the series.
916
- const publisher = ahead?.publisher ?? data.info?.source?.trim();
965
+ const publisher = ahead?.publisher ?? fredPublisher(seriesId, data.info?.source);
917
966
  const publishedAt = ahead ? ahead.publishedAt : fredPublicationTime(data.info, newestDate);
918
967
  // When the newest figure was released: the report is as of then, `Sep 2026, released Tue 20 Oct`.
919
968
  const releasedOn = ahead ? undefined : data.observations.find((observation) => observation.date === newestDate)?.releasedOn;
@@ -926,7 +975,7 @@ async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { s
926
975
  ...(latest && !closedDay ? { latestObservation: exportRow(latest) } : {}),
927
976
  },
928
977
  freshness: rowsFreshness(rows, {
929
- source: ahead?.source ?? notOnFredSource(seriesId) ?? "FRED", status: "not-a-feed", basis: "published statistics", observedKey: "date", oldest: null,
978
+ source: ahead?.source ?? notOnFredSource(seriesId) ?? fredCitation(seriesId, data.info?.source), status: "not-a-feed", basis: "published statistics", observedKey: "date", oldest: null,
930
979
  // A monthly or quarterly observation is dated the first of its period, not a day it was seen.
931
980
  ...(periodicity ? { periodicity } : {}),
932
981
  ...(releasedOn ? { releasedOn } : {}),
@@ -937,16 +986,18 @@ async function printFredSeries(ctx: Parameters<CliCommandDef["execute"]>[1], { s
937
986
  dateKey: "date",
938
987
  empty: fredEmptyMessage(startDate, latest == null ? null : exportRow(latest)),
939
988
  ...(notes.length ? { footnote: notes.join("\n"), exportNotes: notes } : {}),
940
- // `as_of` is the release. A figure with none stated is as of its own day when the series is known to observe
989
+ // `as_of` is the revision or release. Without either, it is its own day when the series is known to observe
941
990
  // days (a daily or weekly rate), and of nothing otherwise, never the first day of the period it covers.
942
991
  exportDated: true,
992
+ exportAppendedKeys: ["first_released", "revised_on", "first_release_value"],
943
993
  exportIdentity: (row) => {
944
994
  const released = typeof row.released === "string" ? row.released : null;
995
+ const revised = typeof row.revised_on === "string" ? row.revised_on : null;
945
996
  const date = typeof row.date === "string" ? row.date : null;
946
- return { symbolOrSeries: seriesId, currencyOrUnit: unit, interval, asOf: released ?? (observesDays ? date : null), period: date, released };
997
+ return { symbolOrSeries: seriesId, currencyOrUnit: unit, interval, asOf: revised ?? released ?? (observesDays ? date : null), period: date, released };
947
998
  },
948
999
  // The release closes the row with the other dating columns, so it is not one of the observation's own.
949
- columns: [...new Set(rows.flatMap((row) => Object.keys(row)))].filter((key) => key !== "released").map((key) => ({ key, header: key })),
1000
+ columns: [...new Set([...rows.flatMap((row) => Object.keys(row)), "first_released", "revised_on", "first_release_value"])].filter((key) => key !== "released").map((key) => ({ key, header: key })),
950
1001
  textColumns: [
951
1002
  { key: "date", header: "Date" },
952
1003
  isSpreadSeries(seriesId)
@@ -1140,7 +1191,7 @@ export const overviewCliCommands: CliCommandDef[] = [
1140
1191
  options: [
1141
1192
  { flags: "--country <regions or countries>", description: "US, G7, EU, all (default), or countries by their two-letter codes, comma separated, such as MX or MX,BR,ZA" },
1142
1193
  { flags: "--impact <level>", description: "high, medium, low, or all (default all)" },
1143
- { flags: "--search <words>", description: "Events whose name or publisher has these words, case, spaces and hyphens aside, such as nonfarm, fed funds or USDA" },
1194
+ { flags: "--search <words>", description: "Match all words in an event name or publisher; OR joins alternatives, such as \"housing OR mortgage OR NAHB\". Case, spaces and hyphens are ignored" },
1144
1195
  { flags: "--from <date>", description: `First day listed, as ${DATE_OPTION_EXAMPLES} (default today, UTC); an earlier day lists past events from the stored calendar and says where it starts` },
1145
1196
  { flags: "--tz <zone>", description: "Add a column with each release's time on that clock, an IANA zone such as America/New_York; with the weekday when it falls on another day than the UTC date" },
1146
1197
  ],
@@ -1168,9 +1219,10 @@ export const overviewCliCommands: CliCommandDef[] = [
1168
1219
  title: "Series",
1169
1220
  lines: [
1170
1221
  "Gloom Cloud serves a curated set of FRED series and needs no sign-in for them. gloom fred (or fred --list) prints them by topic; any other ID fails with that pointer.",
1222
+ "Several IDs, separated by spaces or commas, align on observation dates without forward filling. Each series keeps its own unit, source and release fields.",
1171
1223
  ],
1172
1224
  }],
1173
- examples: ["fred CPIAUCSL", "fred DGS10 --tail 5", "fred DGS10 --change 1D --tail 5", "fred UNRATE --start 2020-01-01 --csv", "fred", "fred --list credit", "fred --list fx"],
1225
+ examples: ["fred CPIAUCSL", "fred DGS10 --tail 5", "fred DGS10 --change 1D --tail 5", "fred UNRATE --start 2020-01-01 --csv", "fred PERMIT HOUST --start 2026-01-01 --csv", "fred", "fred --list credit", "fred --list fx"],
1174
1226
  },
1175
1227
  execute: runFred,
1176
1228
  },
@@ -0,0 +1,73 @@
1
+ import { csvDecimal, serializeCsv } from "../utils/csv";
2
+ import { splitCsvFooter } from "./report-tables";
3
+
4
+ function csvRows(text: string, separator: string): string[][] {
5
+ const rows: string[][] = [];
6
+ let row: string[] = [], cell = "", quoted = false;
7
+ for (let index = 0; index < text.length; index++) {
8
+ const character = text[index];
9
+ if (character === '"') {
10
+ if (quoted && text[index + 1] === '"') { cell += '"'; index++; }
11
+ else quoted = !quoted;
12
+ } else if (!quoted && character === separator) {
13
+ row.push(cell); cell = "";
14
+ } else if (!quoted && (character === "\n" || character === "\r")) {
15
+ if (character === "\r" && text[index + 1] === "\n") index++;
16
+ rows.push([...row, cell]); row = []; cell = "";
17
+ } else cell += character;
18
+ }
19
+ if (row.length || cell) rows.push([...row, cell]);
20
+ return rows;
21
+ }
22
+
23
+ export function portableCsv(text: string, warnings: readonly string[] = [], extraNotes: readonly string[] = []): string {
24
+ const decimal = csvDecimal();
25
+ const separator = decimal === "comma" ? ";" : ",";
26
+ const { csv, footer } = splitCsvFooter(text);
27
+ const tables: Array<{ section: string; columns: string[]; rows: string[][] }> = [];
28
+ let section = "", current: (typeof tables)[number] | null = null;
29
+ for (const row of csvRows(csv, separator)) {
30
+ if (row.length === 1 && row[0] === "") { current = null; continue; }
31
+ if (!current && row[0]?.startsWith("# section: ")) {
32
+ section = row.join(separator).slice("# section: ".length); current = null; continue;
33
+ }
34
+ if (!current) {
35
+ current = { section, columns: row.map((name) => name.replace(/\r\n?|\n/g, "\\n")), rows: [] }; tables.push(current);
36
+ } else current.rows.push(row);
37
+ }
38
+ const usedNames = new Set(tables.flatMap((table) => table.columns));
39
+ const variants = new Map<string, string[]>();
40
+ for (const table of tables) {
41
+ const occurrences = new Map<string, number>();
42
+ table.columns = table.columns.map((name) => {
43
+ const occurrence = occurrences.get(name) ?? 0;
44
+ occurrences.set(name, occurrence + 1);
45
+ const names = variants.get(name) ?? [name];
46
+ if (names[occurrence] === undefined) {
47
+ let suffix = occurrence + 1;
48
+ while (usedNames.has(`${name}_${suffix}`)) suffix++;
49
+ names[occurrence] = `${name}_${suffix}`;
50
+ usedNames.add(names[occurrence]!);
51
+ }
52
+ variants.set(name, names);
53
+ return names[occurrence]!;
54
+ });
55
+ }
56
+ const columns = [...new Set(tables.flatMap((table) => table.columns))];
57
+ const tagged = tables.length > 1 || tables.some((table) => table.section);
58
+ let sectionKey = "section", notesKey = "export_notes";
59
+ while (usedNames.has(sectionKey)) sectionKey = `_${sectionKey}`;
60
+ while (usedNames.has(notesKey)) notesKey = `_${notesKey}`;
61
+ const notes = [...new Set([...footer, ...extraNotes.flatMap((note) => note.split(/\r\n?|\n/).filter(Boolean).map((line) => `note: ${line}`)), ...warnings.map((warning) => `warning: ${warning}`)].map((note) => note.replace(/\r\n?|\n/g, " | ")))].join(" | ");
62
+ const header = [...columns, ...tagged ? [sectionKey] : [], notesKey];
63
+ const rows = tables.flatMap((table) => {
64
+ const indices = columns.map((column) => table.columns.indexOf(column));
65
+ return table.rows.map((row) => [...indices.map((index) => index < 0 ? "" : row[index] ?? ""), ...tagged ? [table.section] : [], notes]);
66
+ });
67
+ if (!rows.length && notes) rows.push([...header.slice(0, -1).map(() => ""), notes]);
68
+ const records = [header, ...rows];
69
+ const normalized = decimal === "comma"
70
+ ? records.map((row) => row.map((cell) => /^[+-]?(?:\d+(?:,\d+)?|,\d+)(?:e[+-]?\d+)?%?$/i.test(cell) ? cell.replace(",", ".") : cell))
71
+ : records;
72
+ return serializeCsv(normalized);
73
+ }
@@ -122,7 +122,8 @@ export interface DesktopPaneShotApiProxy {
122
122
  */
123
123
  export interface DesktopPaneShotBridge {
124
124
  marketData(operation: string, args: unknown[]): Promise<unknown>;
125
- httpFetch(request: HttpProxyRequestEnvelope): Promise<HttpProxyResponseEnvelope>;
125
+ /** `signal` ends the read when the page gives up on it (its own deadline), so it holds nothing past that. */
126
+ httpFetch(request: HttpProxyRequestEnvelope, signal?: AbortSignal): Promise<HttpProxyResponseEnvelope>;
126
127
  }
127
128
 
128
129
  interface DesktopPaneShotRenderedCell {
@@ -494,7 +495,7 @@ async function runShotHttpBridge(
494
495
  if (!bridge) return Response.json({ ok: false, error: "No HTTP bridge" }, { status: 404 });
495
496
  try {
496
497
  const payload = await request.json() as HttpProxyRequestEnvelope;
497
- const data = await bridge.httpFetch(payload);
498
+ const data = await bridge.httpFetch(payload, request.signal);
498
499
  return Response.json({ ok: true, data });
499
500
  } catch (error) {
500
501
  return Response.json({ ok: false, error: errorMessage(error) });
package/src/cli/help.ts CHANGED
@@ -45,11 +45,11 @@ const START_HERE: Array<[string, string]> = [
45
45
 
46
46
  const GLOBAL_OPTIONS: Array<[string, string]> = [
47
47
  ["--json", "Print the full result as JSON"],
48
- ["--csv", "Print rows as CSV under the columns the text shows; each row ends with what it is, its source, as_of and retrieved_at (when it was read), then any columns a report adds (a formula's leg contracts, a board's Group), and fn reports and fundamentals put each table under # section:. The closing source and note lines go to stderr, not into the rows or an --output file"],
48
+ ["--csv", "Print one rectangular CSV with a single header, readable by pandas.read_csv with no options. Existing columns stay in order; duplicate headers gain unique numeric suffixes, and extra tables add columns and a section column. Header line breaks are escaped as \\n; data cells preserve quoted newlines. The final export_notes column stays single-line and repeats source details, caveats, warnings and truncation notes joined with |, including in files and pipes"],
49
49
  ["--ndjson", "Print one JSON object per table row"],
50
50
  ["--output <path>, --out <path>", "Write a --csv, --json or --ndjson export to this file in UTF-8, and print where it went (a redirect is UTF-16 in Windows PowerShell 5.1)"],
51
51
  ["--decimal comma|dot", "Write a --csv export's numbers with a decimal comma (17,2), cells split by ;, for a spreadsheet in a decimal-comma locale; or a dot. gloom config set csvDecimal comma makes the comma the default. Text and JSON keep the dot"],
52
- ["--limit <n>", "Show at most n rows, the first n as printed"],
52
+ ["--limit <n>", "For row/table reports, keep the first n rows in printed order, not the newest n (use --tail). Applies to text, JSON, CSV and NDJSON, not shot or scalar results. Pane reports may set a per-table limit or a source cap; see help fn <function>. It does not request all available source rows"],
53
53
  ["--tail <n>", "Show the newest n rows of a dated series (history, fred, econ) in their printed order, or the last n of another list"],
54
54
  ["--refresh", "Fetch fresh data instead of reading the cache"],
55
55
  ["--dry-run", "Preview config, cache, notes, alerts, plugin on/off, and remote changes without saving"],
@@ -173,6 +173,8 @@ export function renderCommandHelp(command: CliCommandDef): string {
173
173
  { width, termStyle: cliStyles.command },
174
174
  ));
175
175
  }
176
+ lines.push("", renderSection("Output And Row Limits"));
177
+ lines.push(...renderDefinitions(GLOBAL_OPTIONS.filter(([flag]) => ["--csv", "--limit <n>", "--tail <n>"].includes(flag)), { width, termStyle: cliStyles.command }));
176
178
 
177
179
  for (const section of help.sections ?? []) {
178
180
  lines.push("", renderSection(section.title));
@@ -891,6 +891,9 @@ export function headlessReportTables(
891
891
  // A section of one name or one date says so for every line it exports.
892
892
  const named = {
893
893
  ...(section.symbol?.trim() ? { symbolOrSeries: section.symbol.trim() } : {}),
894
+ ...(section.unit?.trim() ? { currencyOrUnit: section.unit.trim() } : {}),
895
+ ...(section.interval?.trim() ? { interval: section.interval.trim() } : {}),
896
+ ...("source" in section && section.source?.trim() ? { source: section.source.trim() } : {}),
894
897
  ...(section.asOf === null ? { asOf: null } : section.asOf?.trim() ? { asOf: section.asOf.trim() } : {}),
895
898
  ...(section.period?.trim() ? { period: section.period.trim() } : {}),
896
899
  ...(section.released?.trim() ? { released: section.released.trim() } : {}),
@@ -913,14 +916,16 @@ export function headlessReportTables(
913
916
  const rowIdentities = section.entries.some((entry) => own(entry)) ? section.entries.map(own) : undefined;
914
917
  return { ...entries, ...(Object.keys(named).length > 0 ? { identity: named } : {}), ...(rowIdentities ? { rowIdentities } : {}) };
915
918
  }
916
- const table = exportRowsTable(section.title, displayedColumns(columnsFor(section.rows, section.columns, definition.columns)), section.rows, { status: true });
919
+ const table = exportRowsTable(section.title, displayedColumns(columnsFor(section.rows, section.columns, definition.columns)), section.rows,
920
+ { status: true, appendedColumns: section.appendedColumns });
917
921
  const identity = {
918
922
  ...named,
919
923
  ...(section.unit?.trim() ? { currencyOrUnit: section.unit.trim() } : {}),
920
924
  ...(section.interval?.trim() ? { interval: section.interval.trim() } : {}),
921
925
  };
922
926
  const rowIdentities = rowIdentitiesOf(section.rows, section);
923
- return { ...table, ...(Object.keys(identity).length > 0 ? { identity } : {}), ...(rowIdentities ? { rowIdentities } : {}) };
927
+ return { ...table, ...(section.appendedColumns ? { appendedColumns: section.appendedColumns } : {}),
928
+ ...(Object.keys(identity).length > 0 ? { identity } : {}), ...(rowIdentities ? { rowIdentities } : {}) };
924
929
  });
925
930
  break;
926
931
  case "series": {
@@ -50,6 +50,7 @@ import { DATA_SERVICE_UNREACHABLE_MESSAGE, isNetworkFailure } from "../../utils/
50
50
  import { isTickerLinkPeer } from "../../layout/ticker-navigation";
51
51
  import { knownGapMessage, commandPointer, commandPointerLeads, topicPointer } from "../known-gaps";
52
52
  import { selectReportTables, splitCsvFooter, withRefusedRows, type CliReportTables } from "../report-tables";
53
+ import { portableCsv } from "../csv-export";
53
54
  import { cliOutputWritten, printCliResult, serializeCliResult, type CliErrorObject } from "../result";
54
55
  import { checkOutputPath, resolveOutputPath, writeOutputFile } from "../output-file";
55
56
  import type { CatalogCommandHit } from "../unknown-command";
@@ -471,10 +472,9 @@ export async function runPaneFunction(args: string[], ctx: CliCommandContext) {
471
472
  // A report with no data writes no file: a header alone would read as a screen that matched nothing. It fails
472
473
  // as plain --json fails, with its code and whether to retry.
473
474
  if (failure) failWithError({ ...failure, details: `Nothing was written to ${outputPath}.` });
474
- // A CSV file holds its rows alone; its closing source and note lines go to stderr, as printed CSV's do.
475
475
  const written = renderForFile(() => serializeCliResult(result, ctx.cliOptions, render));
476
476
  const { csv, footer } = ctx.cliOptions.format === "csv" ? splitCsvFooter(written) : { csv: written, footer: [] };
477
- await writeOutputFile(outputPath, `${csv}\n`);
477
+ await writeOutputFile(outputPath, `${ctx.cliOptions.format === "csv" ? portableCsv(written, warnings) : csv}\n`);
478
478
  if (!ctx.cliOptions.quiet) for (const line of footer) console.error(line);
479
479
  // What was written and where; warnings go to stderr as they would beside printed rows, and JSON keeps its envelope.
480
480
  const saved = { output: outputPath, format: ctx.cliOptions.format, rows: report.data.rowCount, replaced: replacing };
@@ -8,6 +8,7 @@ import {
8
8
  type HttpProxyRequestEnvelope,
9
9
  type HttpProxyResponseEnvelope,
10
10
  } from "../../utils/http-proxy-response";
11
+ import { signalWithTimeout } from "../../utils/async-deadline";
11
12
  import { fetchCheckingRedirects, readBytesWithin } from "../../utils/http-proxy-upstream";
12
13
  import { isPublicAddress } from "../../utils/public-address";
13
14
  import type { DesktopPaneShotApiProxy, DesktopPaneShotBridge } from "../desktop-pane-shot";
@@ -146,7 +147,7 @@ export function createDesktopShotBridge(
146
147
  args,
147
148
  );
148
149
  },
149
- httpFetch: (request) => runDesktopShotHttpFetch(request, httpOptions),
150
+ httpFetch: (request, signal) => runDesktopShotHttpFetch(request, httpOptions, signal),
150
151
  };
151
152
  }
152
153
 
@@ -196,6 +197,7 @@ async function checkShotHttpTarget(
196
197
  async function runDesktopShotHttpFetch(
197
198
  request: HttpProxyRequestEnvelope,
198
199
  options: ShotHttpOptions,
200
+ signal?: AbortSignal,
199
201
  ): Promise<HttpProxyResponseEnvelope> {
200
202
  const resolve = options.resolveHost ?? resolveHost;
201
203
  const addresses = new Map<string, string>();
@@ -218,7 +220,7 @@ async function runDesktopShotHttpFetch(
218
220
  return fetchShotHttpAddress(url, init ?? {}, address);
219
221
  }) as typeof fetch,
220
222
  check: checkTarget,
221
- signal: AbortSignal.timeout(SHOT_BRIDGE_HTTP_TIMEOUT_MS),
223
+ signal: signalWithTimeout(signal, SHOT_BRIDGE_HTTP_TIMEOUT_MS),
222
224
  });
223
225
  if ("error" in result) throw new Error(result.error);
224
226
  const { response } = result;
@@ -539,7 +539,7 @@ export function exportRowsTable<Row extends Record<string, unknown>>(
539
539
  title: string,
540
540
  columns: readonly CliReportColumn<Row>[],
541
541
  rows: readonly Row[],
542
- { status = false, rowUnitKey }: { status?: boolean; rowUnitKey?: string } = {},
542
+ { status = false, rowUnitKey, appendedColumns = 0 }: { status?: boolean; rowUnitKey?: string; appendedColumns?: number } = {},
543
543
  ): CliReportTable {
544
544
  const reasons = rows.map((): Array<{ header: string; text: string }> => []);
545
545
  const units = columns.map(columnUnit);
@@ -587,7 +587,14 @@ export function exportRowsTable<Row extends Record<string, unknown>>(
587
587
  header: columns.some((column) => column.header.trim().toLowerCase() === "status") ? "value_status" : "status",
588
588
  values: reasons.map(statusText),
589
589
  } : undefined;
590
- return assembleTable(title, resolved, rows.length, statusColumn);
590
+ const table = assembleTable(title, resolved, rows.length, statusColumn);
591
+ if (appendedColumns && statusColumn) {
592
+ const before = table.columns.length - 1 - appendedColumns;
593
+ const reorder = <Value,>(values: Value[]) => [...values.slice(0, before), values.at(-1)!, ...values.slice(before, -1)];
594
+ table.columns = reorder(table.columns);
595
+ table.rows = table.rows.map(reorder);
596
+ }
597
+ return appendedColumns ? { ...table, appendedColumns } : table;
591
598
  }
592
599
 
593
600
  /** Label/value lines as `Metric,Value`; a number's unit moves into its label. */
@@ -787,7 +794,14 @@ export function splitCsvFooter(text: string): { csv: string; footer: string[] }
787
794
  const last = blocks.at(-1) ?? "";
788
795
  const lines = last.split("\n");
789
796
  if (!last || !lines.every(isComment) || lines.some((line) => line.startsWith("# section: "))) return { csv: text, footer: [] };
790
- return { csv: blocks.slice(0, -1).join("\n\n"), footer: lines.map((line) => line.slice(2)) };
797
+ const csv = blocks.slice(0, -1).join("\n\n");
798
+ let quoted = false;
799
+ for (let index = 0; index < csv.length; index++) {
800
+ if (csv[index] !== '"') continue;
801
+ if (quoted && csv[index + 1] === '"') index++;
802
+ else quoted = !quoted;
803
+ }
804
+ return quoted ? { csv: text, footer: [] } : { csv, footer: lines.map((line) => line.slice(2)) };
791
805
  }
792
806
 
793
807
  /** CSV with its closing `#` lines after a blank line. */
package/src/cli/result.ts CHANGED
@@ -15,6 +15,7 @@ import {
15
15
  type CliTableColumn,
16
16
  } from "../utils/cli-output";
17
17
  import { serializeCsv } from "../utils/csv";
18
+ import { portableCsv } from "./csv-export";
18
19
  import { logFailure } from "../utils/log-failure";
19
20
  import { formatUtcTime, isEpochMilliseconds, isZonedIsoDateTime } from "../utils/utc-time";
20
21
  import { formatFreshnessLine, withSavedCopy, type ReportFreshness } from "./pane-functions/freshness";
@@ -732,17 +733,15 @@ export function printCliResult<T, Row extends Record<string, unknown> = Record<s
732
733
  const serialized = hinted(toFile
733
734
  ? renderForFile(() => serializeCliResult(cells, options, renderOptions))
734
735
  : serializeCliResult(cells, options, renderOptions));
735
- // A CSV's closing source and note lines go to stderr with the warnings: in the file they are rows of junk to a spreadsheet.
736
736
  const { csv, footer } = options.format === "csv" ? splitCsvFooter(serialized) : { csv: serialized, footer: [] };
737
- const output = csv;
737
+ const output = options.format === "csv" ? portableCsv(serialized, result.warnings,
738
+ [...renderOptions.exportNotes ?? [], ...renderOptions.footnote ? [renderOptions.footnote] : []]) : csv;
738
739
  if (toFile) fileExport.push(output);
739
740
  // Bun's console writer can truncate a large pipe write after stdout has been
740
741
  // initialized. The stream queues the remaining bytes until the reader drains.
741
742
  else if (output && !process.stdout.write(`${output}\n`)) {
742
743
  stdoutWritten = new Promise((resolve) => process.stdout.once("drain", () => resolve()));
743
744
  }
744
- // JSON carries warnings in its envelope. Text, CSV and NDJSON are only the
745
- // rows, so a warning goes to stderr, out of a file the rows are piped into.
746
745
  if (options.format !== "json" && !options.quiet) {
747
746
  // Each wraps to a narrow --width, as a note does.
748
747
  for (const warning of result.warnings ?? []) {
@@ -750,8 +749,6 @@ export function printCliResult<T, Row extends Record<string, unknown> = Record<s
750
749
  }
751
750
  }
752
751
  if (!options.quiet) for (const line of footer) console.error(line);
753
- // CSV says what its rows do not in closing `note:` lines, on stderr; NDJSON has no place for one among its rows,
754
- // so the same notes go to stderr, where the warnings are.
755
752
  if (options.format === "ndjson" && !options.quiet) {
756
753
  for (const note of exportNdjsonNotes(result, options, renderOptions)) console.error(`note: ${note}`);
757
754
  }