@vimoxshah/tokenflow 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/CONTRIBUTING.md +84 -0
  2. package/LICENSE +21 -0
  3. package/README.md +250 -0
  4. package/Refresh & Open Dashboard.command +22 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/tokenflow.js +1342 -0
  7. package/docs/architecture.md +193 -0
  8. package/docs/cli.md +390 -0
  9. package/docs/configuration.md +281 -0
  10. package/docs/creating-provider.md +262 -0
  11. package/docs/data-model.md +213 -0
  12. package/docs/getting-started.md +266 -0
  13. package/docs/live-mode.md +199 -0
  14. package/docs/media/architecture-hero.svg +86 -0
  15. package/docs/media/cost-editorial-dark.png +0 -0
  16. package/docs/media/health-terminal-light.png +0 -0
  17. package/docs/media/menubar-dark.png +0 -0
  18. package/docs/media/menubar-light.png +0 -0
  19. package/docs/media/models-terminal-dark.png +0 -0
  20. package/docs/media/overview-aurora-dark.png +0 -0
  21. package/docs/media/time-aurora-light.png +0 -0
  22. package/docs/providers.md +309 -0
  23. package/docs/skill.md +64 -0
  24. package/docs/troubleshooting.md +207 -0
  25. package/examples/config.example.yaml +92 -0
  26. package/examples/demo-data/README.md +38 -0
  27. package/examples/demo-data/sample-usage.csv +11 -0
  28. package/package.json +74 -0
  29. package/scripts/build-dmg.sh +33 -0
  30. package/scripts/build-menubar-app.sh +67 -0
  31. package/scripts/lint.js +111 -0
  32. package/scripts/validate-install.js +140 -0
  33. package/skills/tokenflow/SKILL.md +392 -0
  34. package/skills/tokenflow/examples/config.yaml +92 -0
  35. package/skills/tokenflow/examples/generic-mapping.json +26 -0
  36. package/skills/tokenflow/examples/session-transcript.md +191 -0
  37. package/skills/tokenflow/providers/adapter-template.js +135 -0
  38. package/skills/tokenflow/providers/detection-matrix.md +142 -0
  39. package/skills/tokenflow/schemas/config.schema.json +107 -0
  40. package/skills/tokenflow/schemas/normalized-record.json +63 -0
  41. package/src/analytics/aggregate.js +247 -0
  42. package/src/analytics/anomalies.js +222 -0
  43. package/src/analytics/capacity.js +278 -0
  44. package/src/analytics/comparison.js +96 -0
  45. package/src/analytics/dimensions.js +230 -0
  46. package/src/analytics/efficiency.js +138 -0
  47. package/src/analytics/forecast.js +202 -0
  48. package/src/analytics/index.js +327 -0
  49. package/src/analytics/insights.js +283 -0
  50. package/src/analytics/milestones.js +91 -0
  51. package/src/analytics/peak.js +106 -0
  52. package/src/analytics/productivity.js +166 -0
  53. package/src/analytics/token-usage.js +267 -0
  54. package/src/commands/diagnostics.js +88 -0
  55. package/src/commands/digest.js +155 -0
  56. package/src/commands/models-compare.js +96 -0
  57. package/src/core/budget.js +142 -0
  58. package/src/core/bundle.js +191 -0
  59. package/src/core/config.js +202 -0
  60. package/src/core/delivery.js +109 -0
  61. package/src/core/geo.js +99 -0
  62. package/src/core/ingest.js +457 -0
  63. package/src/core/interface-map.js +55 -0
  64. package/src/core/jsonl.js +124 -0
  65. package/src/core/live-status.js +417 -0
  66. package/src/core/model-map.js +157 -0
  67. package/src/core/notify.js +83 -0
  68. package/src/core/pricing.js +288 -0
  69. package/src/core/prompt-analytics.js +127 -0
  70. package/src/core/registry.js +107 -0
  71. package/src/core/restore.js +261 -0
  72. package/src/core/schedule.js +120 -0
  73. package/src/core/schema.js +316 -0
  74. package/src/core/sqlite.js +96 -0
  75. package/src/core/store.js +493 -0
  76. package/src/core/sync.js +151 -0
  77. package/src/core/units.js +147 -0
  78. package/src/core/validate.js +123 -0
  79. package/src/core/watch.js +287 -0
  80. package/src/core/yaml.js +209 -0
  81. package/src/export/bundler.js +107 -0
  82. package/src/export/csv.js +100 -0
  83. package/src/export/html-snapshot.js +101 -0
  84. package/src/export/menubar.js +158 -0
  85. package/src/index.js +18 -0
  86. package/src/providers/anthropic/index.js +294 -0
  87. package/src/providers/cline/index.js +120 -0
  88. package/src/providers/cursor/index.js +143 -0
  89. package/src/providers/generic/index.js +268 -0
  90. package/src/providers/git/index.js +188 -0
  91. package/src/providers/headroom/index.js +114 -0
  92. package/src/providers/hermes/index.js +299 -0
  93. package/src/providers/mock/index.js +117 -0
  94. package/src/providers/openai/index.js +370 -0
  95. package/src/providers/opencode/index.js +245 -0
  96. package/src/sdk.js +46 -0
  97. package/src/server/server.js +264 -0
  98. package/src/ui/app.js +2473 -0
  99. package/src/ui/charts.js +925 -0
  100. package/src/ui/index.html +42 -0
  101. package/src/ui/styles.css +644 -0
@@ -0,0 +1,193 @@
1
+ # Architecture
2
+
3
+ ```
4
+ Data sources ~/.claude* ~/.codex ~/.cline ~/.cursor ~/.headroom git *.csv
5
+
6
+
7
+ Ingestion layer src/providers/*/index.js ← adapters PARSE, nothing else
8
+ │ src/core/registry.js ← discovery + contract validation
9
+
10
+ Normalization layer src/core/ingest.js (enrich) ← classify, date, price, id, dedup
11
+ │ src/core/schema.js ← the unified record + null contract
12
+
13
+ Store src/core/store.js
14
+ │ data/records/YYYY-MM.jsonl request-level facts
15
+ │ data/cube.json additive pre-aggregate
16
+ │ data/sessions.json one row per session
17
+ │ data/activity.json daily work rollup
18
+ │ data/state.json per-file offsets (incremental engine)
19
+
20
+ Analytics engine src/analytics/*.js ← pure, no Node imports
21
+ │ computeView(bundle, filters) the single entry point
22
+
23
+ Live intelligence src/analytics/{forecast,anomalies,capacity}.js
24
+ │ (pure too — same modules in CLI, server and browser)
25
+
26
+ Live snapshot src/core/live-status.js data/status.json
27
+ │ written by `tokenflow watch` (src/core/watch.js),
28
+ ▼ read by menu bar / --bar commands / dashboard pill
29
+ Filtering / aggregation in-process, in the browser ← zero API calls per filter change
30
+
31
+
32
+ Dashboard UI src/ui/{index.html,app.js,charts.js,styles.css}
33
+
34
+
35
+ Export src/export/{csv.js,html-snapshot.js,bundler.js,menubar.js}
36
+ ```
37
+
38
+ ## Layer boundaries, and why they hold
39
+
40
+ **Adapters parse; the engine enriches.** An adapter emits a partial record and nothing else. The
41
+ engine does model→vendor classification, interface classification, timezone resolution, cost
42
+ estimation, id assignment, dedup bookkeeping, and every rollup. That is why an adapter is ~80
43
+ lines and why adding one cannot change how anything is counted.
44
+
45
+ **Analytics are pure and Node-free.** `src/analytics/*` imports only from `src/core/schema.js` and
46
+ `src/core/pricing.js` — no `fs`, no `path`. That is what lets the **exact same modules** run in the
47
+ CLI, in the API, and in the browser. There is no "backend calculation" and "frontend calculation"
48
+ to drift apart: `tokenflow status` and the dashboard cannot disagree, because they call the same
49
+ function.
50
+
51
+ **No logic in UI components.** `src/ui/app.js` renders `computeView()` output. It contains no
52
+ arithmetic beyond formatting. Every number on screen has a named function behind it —
53
+ `calculateDailyUsage`, `calculateProviderUsage`, `calculatePeakUsage`,
54
+ `calculateAverageUsage`, `calculateCacheRatio`, `calculateUsageTrend`, `calculateHourlyUsage`,
55
+ `calculateInterfaceUsage`, `calculatePeriodComparison`, `calculateCorrelations`,
56
+ `generateInsights` — and each is unit-tested against its formula.
57
+
58
+ ## The live layer keeps the same discipline
59
+
60
+ `forecast.js`, `anomalies.js` and `capacity.js` import nothing from Node —
61
+ they are analytics like any other, so `tokenflow forecast`, the Live tab and
62
+ the menu bar literally cannot disagree. Wall-clock facts enter as arguments
63
+ (`today`, `nowMs`, timezone offset), which is what makes reset countdowns
64
+ testable and identical across surfaces. The watcher (`src/core/watch.js`) is
65
+ the only writer of `data/status.json`; it is single-instanced per data home,
66
+ backs off exponentially on failure, and never lets one bad provider stop the
67
+ loop. Staleness is a property of *reading*, not of writing: every consumer
68
+ recomputes it against the current clock.
69
+
70
+ ## Why a cube
71
+ A dashboard that filters must either round-trip to a server on every interaction or hold the data
72
+ locally. Holding a million request-level records in a browser tab is not viable; holding a
73
+ pre-aggregated fact table is.
74
+
75
+ ```
76
+ dims: date, hour, dow, provider, model, model_family, client,
77
+ interface, gateway, project, repository, measurement
78
+ measures: in, out, cr, cw, cf, rs, req, cost, costMeasured, costReq,
79
+ naIn, naOut, naCr, naCw
80
+ ```
81
+
82
+ Properties that matter:
83
+
84
+ - **Additive.** An incremental refresh adds into cells; no rebuild needed. This is what makes a
85
+ 2-second no-op refresh possible on a multi-gigabyte corpus.
86
+ - **Small.** Real cardinality on a heavy 8-month corpus of 69,520 records: **2,470 rows**. The whole
87
+ bundle, including 4,359 sessions, is a few hundred KB.
88
+ - **Complete for every question the UI asks.** Every filter dimension is a column, so any
89
+ combination of filters is one linear scan — microseconds, in the browser, offline.
90
+ - **Honest.** The four `na*` counters travel with the measures, so "not reported" survives
91
+ aggregation instead of being flattened into a zero.
92
+
93
+ Request-level facts stay on disk for the Data Explorer and full CSV export, streamed and filtered
94
+ server-side with a bounded top-K buffer so a 100-row page never materialises a million objects.
95
+
96
+ ## Why incremental refresh is exact
97
+
98
+ Session transcripts are append-only, so `state.json` remembers each file's
99
+ `{size, mtimeMs, offset, gen}`:
100
+
101
+ | Observation | Action |
102
+ |---|---|
103
+ | same size **and** mtime | skip entirely — zero reads |
104
+ | grew | resume reading at `offset`; ingested bytes are never re-read |
105
+ | shrank / rewritten | bump `gen`; the old generation's records become stale |
106
+
107
+ Dedup is therefore **structural**, not probabilistic — there is no hash index to size, tune or
108
+ outgrow, and "duplicate records: 0" is a property of the design rather than a measurement.
109
+
110
+ Two refinements make it airtight:
111
+
112
+ 1. `readLines` returns the offset of the last **complete** line, so a partially-written trailing
113
+ line is never consumed.
114
+ 2. An adapter whose logical unit can straddle EOF (a streaming message, an in-flight turn)
115
+ persists what it already emitted in `ref.state`, and emits only the **delta** when the unit
116
+ continues. Without this, every settled file silently lost its last record forever — which is
117
+ exactly the bug this mechanism was built to fix.
118
+
119
+ A scoped `--full` (one adapter) marks that adapter's generations stale, re-ingests, compacts the
120
+ shards, and rebuilds the aggregates from the surviving facts — so one adapter can be re-read from
121
+ scratch without disturbing any other adapter's numbers. A test asserts three consecutive scoped
122
+ re-ingests leave the totals unchanged.
123
+
124
+ ## Time budget and resumability
125
+
126
+ `refresh({ deadlineMs })` checks the clock on every file boundary, persists state, and returns
127
+ `done: false`. The CLI, the HTTP endpoint and CI all use the same path. This is what lets a 5 GB
128
+ first ingest complete inside 45-second shells, or a browser request, without ever leaving the
129
+ store in a half-written condition.
130
+
131
+ ## Plugin points
132
+
133
+ Four extension seams, none of which require touching the core:
134
+
135
+ | Extend | How | Discovered from |
136
+ |---|---|---|
137
+ | **Sources** | `createProvider({...})` | `src/providers/*/index.js`, `$TOKENFLOW_HOME/providers/*.js` |
138
+ | **Metrics** | `registerAnalytics({id, compute})` | result appears at `view.plugins.<id>` |
139
+ | **Model → vendor** | `modelMappings:` in config | prepended to the built-in ruleset |
140
+ | **Prices** | `pricing.json` / `tokenflow pricing --set` | overrides the built-in table |
141
+
142
+ A user adapter shadows a built-in with the same id, which makes local experimentation cheap and
143
+ reversible.
144
+
145
+ ## Zero dependencies, on purpose
146
+
147
+ Not minimalism for its own sake — three concrete properties:
148
+
149
+ - **It installs and runs where a package manager can't.** Sandboxes, air-gapped machines, and
150
+ short-lived shells with no network. A tool for auditing your own data should not need to phone
151
+ a registry first.
152
+ - **There is no supply chain.** A dashboard that reads your session logs is exactly the wrong
153
+ place for 400 transitive packages.
154
+ - **There is no build step in the dev path.** The browser imports the same ES modules the CLI
155
+ does, straight from `src/`. Edit, reload, done.
156
+
157
+ The cost is a hand-written SVG chart library (`src/ui/charts.js`, ~800 lines) and a ~100-line
158
+ ES-module bundler used *only* for the offline HTML snapshot. Both are honest trades. `node:sqlite`
159
+ covers SQLite sources; `node:test` covers testing.
160
+
161
+ The snapshot bundler keeps each module in its own function scope and wires exports through a tiny
162
+ require registry, rather than renaming identifiers — which is where naive concatenating bundlers
163
+ break. `buildSnapshot` parses the result with `vm.Script` before writing, because a snapshot that
164
+ doesn't parse is worse than a failed export.
165
+
166
+ ## Charts
167
+
168
+ `src/ui/charts.js` implements the mark specs directly: thin marks, hairline solid gridlines, a 2px
169
+ surface gap between touching fills, a 2px surface ring on overlapping markers, selective direct
170
+ labels, hit targets larger than the marks, and a crosshair that snaps to the nearest X.
171
+
172
+ Colour is assigned **by entity, in fixed order, never by rank**, so filtering a series out never
173
+ repaints the survivors. Past eight keys everything folds into one muted "Other" rather than
174
+ generating a ninth hue. Scatter and bubble forms cap colour groups at three, because they need
175
+ all-pairs colour separation rather than adjacent-pair. Both palettes (dark primary, light
176
+ selected) were checked against the lightness band, chroma floor, adjacent-pair CVD separation,
177
+ normal-vision floor and surface contrast before being written down.
178
+
179
+ Every chart ships a **table twin** toggled per card, so no value is reachable only by hovering.
180
+
181
+ ## Privacy by construction
182
+
183
+ There is no HTTP client anywhere in this codebase — not for telemetry, not for pricing, not for
184
+ updates. The server binds to loopback and refuses non-GET requests from a foreign origin without a
185
+ token. Adapters read token counts and metadata and discard content; the schema has nowhere to put
186
+ a prompt. SQLite sources are read from a temp snapshot so a live editor is never disturbed.
187
+
188
+ ## Team-readiness without team infrastructure
189
+
190
+ `user`, `machine`, `session_id`, `project` and `repository` are first-class, and the cube is
191
+ additive — so cubes from several machines can be merged by concatenating rows and re-summing
192
+ duplicated dimension tuples. V1 ships no identity, auth or sync, deliberately. Nothing here makes
193
+ them impossible to add.
package/docs/cli.md ADDED
@@ -0,0 +1,390 @@
1
+ # CLI reference
2
+
3
+ ```
4
+ tokenflow <command> [flags]
5
+ ```
6
+
7
+ Every command is safe to run repeatedly, writes only inside `$TOKENFLOW_HOME` (default
8
+ `~/.tokenflow`), and makes no network requests.
9
+
10
+ Global flags: `--json`, `--quiet`, `--debug`, `--help`, `--version`.
11
+
12
+ ---
13
+
14
+ ## Getting started
15
+
16
+ ### `setup`
17
+ Runs every adapter's `detect()`, writes `config.yaml` with what was found, and sets the timezone
18
+ and identity. Reads no usage data. Safe to re-run — it refreshes the detected provider list
19
+ without discarding your source paths or pricing.
20
+
21
+ ### `up` (alias: `open`)
22
+ The "I just want to look at it" command, and what `npm start` and the double-clickable
23
+ `Refresh & Open Dashboard.command` both run.
24
+
25
+ ```
26
+ refresh (budgeted, looped until the engine reports done)
27
+ → rebuild tokenflow-dashboard.html beside it
28
+ → serve + open the live dashboard
29
+ ```
30
+
31
+ | Flag | Effect |
32
+ |---|---|
33
+ | `--budget <seconds>` | per-pass ingest budget (default 60) |
34
+ | `--passes <n>` | maximum ingest passes before giving up (default 20) |
35
+ | `--no-refresh` | skip the ingest, just rebuild + serve |
36
+ | `--no-snapshot` | skip rebuilding the offline HTML file |
37
+ | `--snapshot <file>` | write the offline file somewhere else |
38
+ | `--port`, `--host`, `--no-open` | passed through to `dashboard` |
39
+
40
+ **Keeping it fresh unattended.** `tokenflow up --no-serve` does the refresh and rebuilds the
41
+ offline file without starting a server, which is the thing to schedule. macOS (`launchd`), saved
42
+ as `~/Library/LaunchAgents/com.local.tokenflow.plist` and loaded with
43
+ `launchctl load ~/Library/LaunchAgents/com.local.tokenflow.plist`:
44
+
45
+ ```xml
46
+ <plist version="1.0"><dict>
47
+ <key>Label</key><string>com.local.tokenflow</string>
48
+ <key>ProgramArguments</key>
49
+ <array>
50
+ <string>/usr/local/bin/node</string>
51
+ <string>/path/to/tokenflow/bin/tokenflow.js</string>
52
+ <string>up</string><string>--no-serve</string>
53
+ </array>
54
+ <key>WorkingDirectory</key><string>/path/to/tokenflow</string>
55
+ <key>StartCalendarInterval</key><dict><key>Hour</key><integer>9</integer><key>Minute</key><integer>0</integer></dict>
56
+ <key>RunAtLoad</key><false/>
57
+ </dict></plist>
58
+ ```
59
+
60
+ Linux (systemd user timer) or plain cron works the same way:
61
+
62
+ ```cron
63
+ 0 9 * * * cd /path/to/tokenflow && node bin/tokenflow.js up --no-serve >/dev/null 2>&1
64
+ ```
65
+
66
+ Windows, with Task Scheduler:
67
+
68
+ ```bat
69
+ schtasks /create /tn "Tokenflow daily" /sc daily /st 09:00 ^
70
+ /tr "cmd /c cd /d C:\path\to\tokenflow && node bin\tokenflow.js up --no-serve"
71
+ ```
72
+
73
+ ### `refresh`
74
+ Ingests new usage. Incremental by default: unchanged files are skipped without being read.
75
+
76
+ | Flag | Effect |
77
+ |---|---|
78
+ | `--provider <id[,id]>` | only these adapters |
79
+ | `--full` | re-ingest from scratch. With `--provider`, scoped: that adapter's old records are superseded, compacted away, and the aggregates rebuilt — other adapters are untouched. Without `--provider`, everything is reset. Refuses to run if a source that contributed stored records is not reachable right now (see below). |
80
+ | `--force` | with `--full`, proceed even though an unreachable source's records will be dropped and cannot be rebuilt |
81
+ | `--budget <seconds>` | stop cleanly on a file boundary and report `done: false`; run again to continue |
82
+ | `--strict` | validate every record as it is written (slower; use when developing an adapter) |
83
+ | `--quiet` | no progress line |
84
+ | `--json` | machine-readable report |
85
+
86
+ ```bash
87
+ tokenflow refresh
88
+ tokenflow refresh --provider anthropic --full
89
+ tokenflow refresh --budget 30 # then run it again
90
+ ```
91
+
92
+ **Why `--full` can refuse.** A full re-ingest drops the stored records for the sources in
93
+ scope and rebuilds them from the source logs. That is only safe if those logs are reachable.
94
+ Run the same command on another machine, inside a sandbox, or after the log directory has
95
+ moved, and the adapter simply detects as unavailable — an unguarded reset would delete a
96
+ corpus it cannot rebuild. So detection runs *before* anything destructive, and the command
97
+ exits non-zero with the record count at risk and the reason each source is unreachable.
98
+ Fix the path and retry, run an incremental `refresh`, restore from an export, or pass
99
+ `--force` if you really do mean discard.
100
+
101
+ ### `dashboard` (aliases: `serve`, `ui`)
102
+ Starts the local server and opens a browser.
103
+
104
+ | Flag | Default |
105
+ |---|---|
106
+ | `--port <n>` | `7799` |
107
+ | `--host <addr>` | `127.0.0.1` — loopback only |
108
+ | `--no-open` | don't launch a browser |
109
+
110
+ ### `demo`
111
+ Generates deterministic synthetic data and opens the dashboard. `--days <n>`, `--seed <n>`,
112
+ `--no-dashboard`. Everything it produces is labelled as demo data in the UI.
113
+
114
+ ---
115
+
116
+ ## Everyday
117
+
118
+ ### `status`
119
+ Totals, coverage, composition, peak, cost, data health and the top insights — the same numbers
120
+ the dashboard shows, from the same analytics code. `--json` for scripting.
121
+
122
+ | Flag | Effect |
123
+ |---|---|
124
+ | `--bar` | print only the one-line menu-bar summary (see `menubar`) |
125
+ | `--mode <m>` | with `--bar`: `auto` (default) / `tokens` / `cost` / `limit` |
126
+ | `--prefix <s>` | with `--bar`: leading label (default `TF`) |
127
+
128
+ ---
129
+
130
+ ### `digest`
131
+ A shareable summary of a usage window, rendered from the same cube as everything else: headline
132
+ tokens/cost/requests, by source, by provider, top models, pace vs your 14-day average, nearest
133
+ capacity limit and any high-severity alerts. Markdown by default; paste it into Slack, an issue,
134
+ or a stand-up note.
135
+
136
+ ```bash
137
+ tokenflow digest # last 7 days, markdown to stdout
138
+ tokenflow digest --format text # plain text
139
+ tokenflow digest --from 2026-08-01 --to 2026-08-07
140
+ tokenflow digest --out week.md # write instead of print
141
+ ```
142
+
143
+ ---
144
+
145
+ ### `providers`
146
+ What is detected, connected, or disabled, with the reason for each. `--json`.
147
+
148
+ ### `provider add|remove|list <id>`
149
+ Enable or disable an adapter in `config.yaml`.
150
+
151
+ ### `export`
152
+ | Flag | Effect |
153
+ |---|---|
154
+ | `--csv [file]` | request-level CSV; default name `tokenflow-usage-YYYY-MM-DD.csv` |
155
+ | `--all` | every record, ignoring filters |
156
+ | `--from` / `--to` / `--provider` / `--model` / `--client` / `--interface` / `--project` | filter the export |
157
+ | `--html [file]` | one self-contained offline dashboard file |
158
+ | `--maxRecords <n>` | records embedded in the HTML snapshot (default 20,000) |
159
+ | `--out <dir>` | output directory when no filename is given |
160
+
161
+ ```bash
162
+ tokenflow export --csv --from 2026-03-14 --provider anthropic
163
+ tokenflow export --csv --all
164
+ tokenflow export --html ~/Desktop/tokenflow.html
165
+ ```
166
+
167
+ Missing values are written as empty cells, never as `0`.
168
+
169
+ ---
170
+
171
+ ## Live
172
+
173
+ These four read the watcher's snapshot (`data/status.json`) when it is fresh
174
+ and compute on the spot otherwise — so they are fast to run repeatedly and
175
+ always agree with the dashboard.
176
+
177
+ ### `usage`
178
+ Today / yesterday / week-to-date / month-to-date: tokens by bucket, requests,
179
+ sessions, estimated cost, plus today's top providers and models. `--json`
180
+ returns the full slices.
181
+
182
+ ### `cost`
183
+ Estimated vs measured spend for the same windows, today's spend by provider,
184
+ and the month-end projection with its confidence. `--json`.
185
+
186
+ ### `capacity`
187
+ Every configured limit (see [live-mode.md](live-mode.md#capacity--budgets) and
188
+ [configuration.md](configuration.md)): bar, % of cap, burn rate, projected
189
+ exhaustion ETA, reset countdown, sorted most-urgent first. With no limits
190
+ configured it prints the exact YAML to paste instead of pretending to know
191
+ vendor quotas. `--json` includes validation errors for bad definitions.
192
+
193
+ ### `forecast`
194
+ Tomorrow (with likely range), next 7 days, month-end projection, confidence,
195
+ sample size — followed by active anomaly alerts with their arithmetic.
196
+ `--json` returns `{ forecast, anomalies }`.
197
+
198
+ ### `watch`
199
+ The background refresher. One cycle = incremental refresh → rebuild
200
+ `data/status.json` → (opt-in) notify transitions.
201
+
202
+ ```bash
203
+ tokenflow watch # every watch.intervalSeconds (default 120)
204
+ tokenflow watch --interval 30 # this run only
205
+ tokenflow watch --once # single cycle, exit (cron-friendly)
206
+ tokenflow watch --notify # OS notifications for threshold crossings
207
+ tokenflow watch --status # running? pid? how fresh is the data?
208
+ tokenflow watch --stop # stop a running watcher
209
+ ```
210
+
211
+ Single-instance per data home; failures back off exponentially (15-minute
212
+ ceiling) and land in `lastError`. See [live-mode.md](live-mode.md).
213
+
214
+ ### `menubar`
215
+ Menu bar surfaces. On macOS the primary is **TokenFlow's own native app**:
216
+
217
+ ```bash
218
+ tokenflow menubar --app # build TokenFlow.app (swiftc), install
219
+ # to ~/Applications and launch it
220
+ tokenflow menubar --app --login-item # also add it to Login Items
221
+ ```
222
+
223
+ The app shows an adaptive status item (limit % with state color > today's
224
+ cost > today's tokens). Its dropdown carries today/week/month usage, a
225
+ per-provider breakdown, capacity meters with reset countdowns and ETAs, the
226
+ forecast with confidence, high-severity anomaly alerts, freshness/watcher
227
+ badges, and Refresh-now / Open-Dashboard / Start-Stop-watcher actions.
228
+
229
+ Cross-platform text protocol for other bars:
230
+
231
+ ```bash
232
+ tokenflow menubar --render # print xbar/SwiftBar-format text
233
+ tokenflow menubar --swiftbar # install into ~/Library/Plugins
234
+ tokenflow menubar --xbar # install into xbar's plugin dir
235
+ tokenflow menubar --out <dir> # any compatible bar's plugin dir
236
+ tokenflow menubar --mode <m> # auto | tokens | cost | limit
237
+ ```
238
+
239
+ Requires `swiftc` for `--app` (`xcode-select --install` provides it).
240
+
241
+ ---
242
+
243
+ ## Configure
244
+
245
+ ### `pricing`
246
+ With no flags: every model in the store, its token volume, estimated cost, effective $/1M, and
247
+ which source its rate came from.
248
+
249
+ ```bash
250
+ tokenflow pricing
251
+ tokenflow pricing --sources # provenance + tier multipliers
252
+ tokenflow pricing --set "claude-opus-5=5,25,0.5,6.25" # input,output[,cacheRead[,cacheWrite]]
253
+ tokenflow pricing --unset claude-opus-5
254
+ ```
255
+
256
+ Rates are USD per 1,000,000 tokens. Run `tokenflow refresh --full` afterwards to re-cost history.
257
+ Anything left unpriced shows as "no price", never as `$0`.
258
+
259
+ `--sources` prints, for every group of built-in rates, the URL it was taken from, the date it was
260
+ fetched, and whether the source is official or third-party — plus the service-tier multipliers
261
+ that get applied per request, and the one premium tier that is deliberately *not* applied
262
+ (long-context).
263
+
264
+ ### `import <file>`
265
+ Generic import for CSV / TSV / JSON / JSONL / SQLite. With no `--field` flags it prints the
266
+ columns it found, infers a mapping, and previews five normalized rows.
267
+
268
+ | Flag | Effect |
269
+ |---|---|
270
+ | `--name <n>` | mapping name (default: the filename) |
271
+ | `--format csv\|tsv\|json\|jsonl\|sqlite` | override detection |
272
+ | `--table <t>` / `--query <sql>` | SQLite source |
273
+ | `--field <schemaField>=<column>` | repeatable |
274
+ | `--default <field>=<value>` | repeatable, e.g. `--default client=openrouter` |
275
+ | `--timestamp-format iso\|epoch_ms\|epoch_s` | |
276
+ | `--dry-run` | preview only, save nothing |
277
+
278
+ ```bash
279
+ tokenflow import ~/Downloads/usage.csv --dry-run
280
+ tokenflow import ~/Downloads/usage.csv --field timestamp=created_at --field input_tokens=prompt_tokens
281
+ ```
282
+
283
+ ### `restore <file.csv>`
284
+ Rebuilds the store from a full export (`tokenflow export --csv --all`) and re-prices every
285
+ *estimated* cost with the current price table. Measured costs — a gateway's own billing
286
+ numbers — are preserved verbatim and never re-estimated.
287
+
288
+ | Flag | Effect |
289
+ |---|---|
290
+ | `--yes` | required when the store already holds records; a restore replaces the dataset |
291
+ | `--no-reprice` | keep the costs exactly as exported |
292
+
293
+ ```bash
294
+ tokenflow export --csv --all # on the machine that has the logs
295
+ tokenflow restore tokenflow-usage-2026-08-20.csv --yes
296
+ ```
297
+
298
+ Three reasons to reach for it:
299
+
300
+ - **portability** — move a dataset between machines, or into a team roll-up, without
301
+ shipping the vendors' raw session logs. The CSV holds counts and identifiers; prompts,
302
+ conversations and source code never leave the machine that produced them.
303
+ - **recovery** — rebuild after the source logs have been rotated, pruned, or moved.
304
+ - **re-pricing** — apply a price-table update to all of history without re-reading
305
+ gigabytes of logs.
306
+
307
+ What restore cannot recover: per-record `metadata` (working directory, streaming audit
308
+ trail, price provenance) is not part of the CSV contract. Restored records carry
309
+ `metadata.restored_from` instead.
310
+
311
+ A restored slice is provisional. The next `refresh` that actually reaches a source's logs
312
+ marks the restored records for that source stale, compacts them away and rebuilds the
313
+ aggregates — so a restore and a real re-read can never be double counted.
314
+
315
+ ### `config show|path|export|import`
316
+ ```bash
317
+ tokenflow config show # effective config as YAML
318
+ tokenflow config path # the data home
319
+ tokenflow config export backup.json # config + pricing + mappings (no usage data)
320
+ tokenflow config import backup.json # restore on another machine
321
+ ```
322
+
323
+ ---
324
+
325
+ ## Maintain
326
+
327
+ ### `doctor`
328
+ Runtime, `node:sqlite` availability, timezone, every path, store size, cube rows, sessions,
329
+ pending compaction, and every adapter's detection status. Start here when something is off.
330
+
331
+ ### `validate`
332
+ Re-validates every stored record against the schema and reports the failure modes by frequency.
333
+ Exits non-zero if anything is invalid.
334
+
335
+ ### `compact`
336
+ Rewrites the record shards without superseded generations and rebuilds the aggregates. Normally
337
+ automatic after a scoped `--full` or after a restored slice is superseded; run it manually if
338
+ `doctor` reports pending stale generations.
339
+
340
+ | Flag | Effect |
341
+ |---|---|
342
+ | `--recount` | rebuild and re-derive the record counts even when there is nothing to compact |
343
+
344
+ Record counts in `state.json` accumulate as records are ingested. After a restore, a scoped
345
+ re-ingest, or a compaction they are re-derived from the records that actually survived — so the
346
+ stored count is the store's real size, not a lifetime total. `--recount` forces that derivation
347
+ on demand.
348
+
349
+ ### `reset --yes`
350
+ Deletes ingested data. Config, pricing and mappings are kept.
351
+
352
+ ---
353
+
354
+ ## npm script wrappers
355
+
356
+ ```bash
357
+ npm run setup npm run refresh npm run status
358
+ npm run dashboard npm run demo
359
+ npm test npm run lint npm run typecheck npm run validate
360
+ ```
361
+
362
+ ## Environment
363
+
364
+ | Variable | Effect |
365
+ |---|---|
366
+ | `TOKENFLOW_HOME` | data + config directory (default `~/.tokenflow`) |
367
+ | `TOKENFLOW_DEMO=1` | enable the mock provider |
368
+ | `CLAUDE_CONFIG_DIR` | extra Claude Code home(s) for the anthropic adapter |
369
+ | `CODEX_HOME` | extra Codex home for the openai adapter |
370
+ | `NO_COLOR` | plain output |
371
+
372
+ ## Exit codes
373
+
374
+ `0` success · `1` a command failed, or `validate` found invalid records.
375
+
376
+ ## Scripting
377
+
378
+ `--json` on `refresh`, `status` and `providers` emits machine-readable output:
379
+
380
+ ```bash
381
+ tokenflow refresh --json | jq '.newRecords, .done'
382
+ tokenflow status --json | jq '.health.grade, .meta.coverage'
383
+ tokenflow providers --json | jq '.[] | select(.available) | .id'
384
+ ```
385
+
386
+ A nightly refresh needs nothing more than:
387
+
388
+ ```cron
389
+ 17 3 * * * cd ~/tokenflow && node bin/tokenflow.js refresh --quiet
390
+ ```