@vimoxshah/tokenflow 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/CONTRIBUTING.md +84 -0
  2. package/LICENSE +21 -0
  3. package/README.md +250 -0
  4. package/Refresh & Open Dashboard.command +22 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/tokenflow.js +1342 -0
  7. package/docs/architecture.md +193 -0
  8. package/docs/cli.md +390 -0
  9. package/docs/configuration.md +281 -0
  10. package/docs/creating-provider.md +262 -0
  11. package/docs/data-model.md +213 -0
  12. package/docs/getting-started.md +266 -0
  13. package/docs/live-mode.md +199 -0
  14. package/docs/media/architecture-hero.svg +86 -0
  15. package/docs/media/cost-editorial-dark.png +0 -0
  16. package/docs/media/health-terminal-light.png +0 -0
  17. package/docs/media/menubar-dark.png +0 -0
  18. package/docs/media/menubar-light.png +0 -0
  19. package/docs/media/models-terminal-dark.png +0 -0
  20. package/docs/media/overview-aurora-dark.png +0 -0
  21. package/docs/media/time-aurora-light.png +0 -0
  22. package/docs/providers.md +309 -0
  23. package/docs/skill.md +64 -0
  24. package/docs/troubleshooting.md +207 -0
  25. package/examples/config.example.yaml +92 -0
  26. package/examples/demo-data/README.md +38 -0
  27. package/examples/demo-data/sample-usage.csv +11 -0
  28. package/package.json +74 -0
  29. package/scripts/build-dmg.sh +33 -0
  30. package/scripts/build-menubar-app.sh +67 -0
  31. package/scripts/lint.js +111 -0
  32. package/scripts/validate-install.js +140 -0
  33. package/skills/tokenflow/SKILL.md +392 -0
  34. package/skills/tokenflow/examples/config.yaml +92 -0
  35. package/skills/tokenflow/examples/generic-mapping.json +26 -0
  36. package/skills/tokenflow/examples/session-transcript.md +191 -0
  37. package/skills/tokenflow/providers/adapter-template.js +135 -0
  38. package/skills/tokenflow/providers/detection-matrix.md +142 -0
  39. package/skills/tokenflow/schemas/config.schema.json +107 -0
  40. package/skills/tokenflow/schemas/normalized-record.json +63 -0
  41. package/src/analytics/aggregate.js +247 -0
  42. package/src/analytics/anomalies.js +222 -0
  43. package/src/analytics/capacity.js +278 -0
  44. package/src/analytics/comparison.js +96 -0
  45. package/src/analytics/dimensions.js +230 -0
  46. package/src/analytics/efficiency.js +138 -0
  47. package/src/analytics/forecast.js +202 -0
  48. package/src/analytics/index.js +327 -0
  49. package/src/analytics/insights.js +283 -0
  50. package/src/analytics/milestones.js +91 -0
  51. package/src/analytics/peak.js +106 -0
  52. package/src/analytics/productivity.js +166 -0
  53. package/src/analytics/token-usage.js +267 -0
  54. package/src/commands/diagnostics.js +88 -0
  55. package/src/commands/digest.js +155 -0
  56. package/src/commands/models-compare.js +96 -0
  57. package/src/core/budget.js +142 -0
  58. package/src/core/bundle.js +191 -0
  59. package/src/core/config.js +202 -0
  60. package/src/core/delivery.js +109 -0
  61. package/src/core/geo.js +99 -0
  62. package/src/core/ingest.js +457 -0
  63. package/src/core/interface-map.js +55 -0
  64. package/src/core/jsonl.js +124 -0
  65. package/src/core/live-status.js +417 -0
  66. package/src/core/model-map.js +157 -0
  67. package/src/core/notify.js +83 -0
  68. package/src/core/pricing.js +288 -0
  69. package/src/core/prompt-analytics.js +127 -0
  70. package/src/core/registry.js +107 -0
  71. package/src/core/restore.js +261 -0
  72. package/src/core/schedule.js +120 -0
  73. package/src/core/schema.js +316 -0
  74. package/src/core/sqlite.js +96 -0
  75. package/src/core/store.js +493 -0
  76. package/src/core/sync.js +151 -0
  77. package/src/core/units.js +147 -0
  78. package/src/core/validate.js +123 -0
  79. package/src/core/watch.js +287 -0
  80. package/src/core/yaml.js +209 -0
  81. package/src/export/bundler.js +107 -0
  82. package/src/export/csv.js +100 -0
  83. package/src/export/html-snapshot.js +101 -0
  84. package/src/export/menubar.js +158 -0
  85. package/src/index.js +18 -0
  86. package/src/providers/anthropic/index.js +294 -0
  87. package/src/providers/cline/index.js +120 -0
  88. package/src/providers/cursor/index.js +143 -0
  89. package/src/providers/generic/index.js +268 -0
  90. package/src/providers/git/index.js +188 -0
  91. package/src/providers/headroom/index.js +114 -0
  92. package/src/providers/hermes/index.js +299 -0
  93. package/src/providers/mock/index.js +117 -0
  94. package/src/providers/openai/index.js +370 -0
  95. package/src/providers/opencode/index.js +245 -0
  96. package/src/sdk.js +46 -0
  97. package/src/server/server.js +264 -0
  98. package/src/ui/app.js +2473 -0
  99. package/src/ui/charts.js +925 -0
  100. package/src/ui/index.html +42 -0
  101. package/src/ui/styles.css +644 -0
@@ -0,0 +1,213 @@
1
+ # Data model
2
+
3
+ Every adapter emits the same record shape. Nothing downstream — analytics, UI, export — knows
4
+ anything about a specific vendor. This document is the contract.
5
+
6
+ ## The normalized record
7
+
8
+ ```ts
9
+ interface NormalizedUsageRecord {
10
+ id: string; // stable dedup key
11
+ timestamp: string; // ISO-8601, UTC
12
+ date: string; // YYYY-MM-DD in the capture timezone
13
+ hour: number; // 0-23 in the capture timezone
14
+ dow: number; // 0=Mon .. 6=Sun in the capture timezone
15
+ tz_offset: number; // capture tz offset, minutes
16
+
17
+ provider: string; // canonical vendor slug: anthropic | openai | deepseek | zai | ...
18
+ provider_label: string;
19
+ gateway: string | null; // routing layer (proxy/router), if any — NOT the vendor
20
+ model: string; // raw model identifier from the source
21
+ model_family: string; // human tier, e.g. "Claude Opus 5", "GPT 5.6 (sol)"
22
+
23
+ client: string; // tool that made the call: claude-code | codex | cline | cursor
24
+ application: string; // human label for the client
25
+ interface: Interface; // CLI | IDE | Desktop App | Web | API | SDK | Extension | Unknown
26
+
27
+ input_tokens: number | null; // fresh prompt tokens, EXCLUDING cache
28
+ cache_read_tokens: number | null; // prompt tokens served from cache
29
+ cache_write_tokens: number | null; // prompt tokens written into cache
30
+ output_tokens: number | null; // generated tokens
31
+ cache_refresh_tokens: number | null; // SUBSET of cache_write (long-TTL / refreshed)
32
+ reasoning_tokens: number | null; // SUBSET of output (thinking / reasoning)
33
+ total_tokens: number | null; // derived, never trusted from the source
34
+ total_is_partial: boolean; // some billable field was not available
35
+
36
+ session_id: string | null;
37
+ conversation_id: string | null;
38
+ request_id: string | null;
39
+
40
+ project: string | null;
41
+ repository: string | null;
42
+ git_branch: string | null;
43
+ category: string | null; // main | subagent | commit | ai-edit:* | ...
44
+
45
+ estimated_cost: number | null;
46
+ cost_basis: 'measured' | 'estimated' | null;
47
+
48
+ source: string; // adapter id
49
+ measurement: Measurement; // primary | overlay | activity
50
+ user: string | null; // multi-user ready
51
+ machine: string | null;
52
+ duration_ms: number | null;
53
+ metadata: object; // source-specific, never interpreted by analytics
54
+ }
55
+ ```
56
+
57
+ ## Missing-value contract
58
+
59
+ This is the load-bearing rule of the whole project.
60
+
61
+ | Value | Meaning |
62
+ |---|---|
63
+ | `null` | the source does not report this field — **not available** |
64
+ | `undefined` | normalised to `null` on construction |
65
+ | `0` | the source reported zero — a real, **measured** zero |
66
+
67
+ Analytics **never** coerce `null` to `0`. Sums skip nulls and carry a parallel not-available
68
+ counter per field (`naIn`, `naOut`, `naCr`, `naCw`), which is why the UI can say
69
+ "cache tokens unreported by 22% of records in this slice" instead of drawing a confident zero.
70
+ The linter enforces this: `|| 0` on a line mentioning a token field is a build error.
71
+
72
+ Practical consequence: Cline logs sessions but no token counts, so **every** Cline token field is
73
+ `null`, the record is `measurement: activity`, and its sessions count towards activity metrics
74
+ without dragging every token average towards zero.
75
+
76
+ ## Token accounting
77
+
78
+ ### The four billable buckets are mutually exclusive
79
+
80
+ ```
81
+ total_tokens = input_tokens + cache_read_tokens + cache_write_tokens + output_tokens
82
+ ```
83
+
84
+ ### The two breakdown fields are subsets and are never added again
85
+
86
+ ```
87
+ cache_refresh_tokens ⊆ cache_write_tokens (long-TTL / refreshed cache writes)
88
+ reasoning_tokens ⊆ output_tokens (thinking / reasoning)
89
+ ```
90
+
91
+ `validateUsage` rejects a record where a subset exceeds its parent, and the test suite asserts
92
+ both invariants.
93
+
94
+ ### Vendors disagree about `input_tokens`
95
+
96
+ | Vendor | What the source calls `input_tokens` | Adapter must |
97
+ |---|---|---|
98
+ | Anthropic | fresh prompt tokens, **excluding** cache read and cache creation | pass through |
99
+ | OpenAI / Codex | prompt tokens **including** `cached_input_tokens` | subtract: `fresh = input − cached` |
100
+
101
+ Getting this wrong double-counts every cached prompt token — once as fresh input and once as
102
+ cache read. Each adapter's test asserts its own convention.
103
+
104
+ ### Ratios and what they actually divide by
105
+
106
+ | Metric | Formula | Read it as |
107
+ |---|---|---|
108
+ | Output / input | `out / in` | generated per **fresh** prompt token. With a cache-heavy agent, `in` is tiny, so this number is large and not very meaningful on its own. |
109
+ | Output / prompt sent | `out / (in + cr + cw)` | generated per prompt token **actually sent**. This is the honest "prompt-heavy vs output-heavy" measure. |
110
+ | Cache / total | `(cr + cw) / total` | share of all token activity that was cache traffic |
111
+ | Cache hit rate | `cr / (in + cr)` | share of prompt tokens served from cache rather than re-sent |
112
+ | Fresh per cached prompt | `in / cr` | below 1 means the cache is carrying the context |
113
+
114
+ ## Cost, and what an estimate is allowed to claim
115
+
116
+ `estimated_cost` + `cost_basis` are the only cost fields, and they mean two different things:
117
+
118
+ | `cost_basis` | Meaning |
119
+ |---|---|
120
+ | `measured` | a source actually billed this request (a gateway's own `cost_usd`) |
121
+ | `estimated` | computed here, from a published rate table |
122
+ | `null` | the model has no configured rate — **and no number is shown** |
123
+
124
+ Three things make the estimate defensible rather than decorative:
125
+
126
+ 1. **Every rate names a source.** Each entry in the built-in table carries a `src` key into
127
+ `PRICING_SOURCES`, which records the URL, the date it was fetched, and whether the source is
128
+ official or third-party. `tokenflow pricing --sources` prints it; the Cost page shows it.
129
+ 2. **Service tier is a multiplier, not a label.** OpenAI's Fast mode (renamed from "priority" on
130
+ 2026-07-30) bills at **4x** standard; Anthropic's Batch API at 0.5x. `service_tier` is a
131
+ first-class field and a cube dimension, and the multiplier is applied per request. Ignoring it
132
+ under-reports a Fast-mode-heavy workload by up to 4x.
133
+ 3. **The cache write tiers are priced separately.** `cache_refresh_tokens` (the long-TTL subset) is
134
+ billed at the 1-hour rate and the remainder at the 5-minute rate — for Claude Opus 5 that is
135
+ $10 vs $6.25 per MTok, so on a cache-heavy workload the split is worth real money.
136
+
137
+ **Known gap, stated rather than hidden:** long-context premium tiers (Anthropic above 200K,
138
+ OpenAI's long-context rows) are *not* applied, because doing so needs a per-request prompt size
139
+ plus a per-model threshold and premium that are not uniformly published. A long-context-heavy
140
+ workload is therefore **under**-estimated, and the Cost page says so.
141
+
142
+ ## Measurement kinds
143
+
144
+ | `measurement` | Meaning | Counted in token totals? |
145
+ |---|---|---|
146
+ | `primary` | authoritative per-request usage from the model API | yes |
147
+ | `overlay` | a second view of traffic a client adapter already recorded (gateway/proxy logs) | **no** by default — it would double count. Used for measured cost. |
148
+ | `activity` | AI activity with no token accounting at all (IDE edits, sessions without a usage block, commits) | never — activity and correlation only |
149
+
150
+ Both non-primary kinds are toggleable in the filter bar, and the Data Health page explains each
151
+ one in place.
152
+
153
+ ## Provider vs gateway
154
+
155
+ A local proxy or router is **not** a model vendor. When Codex reports
156
+ `model_provider: "headroom"`, the record gets `gateway: "headroom"` and `provider: "openai"` —
157
+ the vendor is derived from the model string, which is the only real evidence of who made the
158
+ model. This keeps "who served this request" and "who built this model" as separate, filterable
159
+ dimensions, and lets you answer "how much of my OpenAI traffic goes through the proxy?".
160
+
161
+ A provider hint from a source **never** overrides evidence in the model name.
162
+
163
+ ## Interface classification
164
+
165
+ `interface` is derived **only** from an explicit surface field in the source record — `entrypoint`,
166
+ `originator`, `source`, an IDE marker, a client name. It is never inferred from the model or the
167
+ provider: "it's a Claude model so it must be Claude Desktop" is exactly the mistake this rule
168
+ exists to prevent. With no signal, the value is `Unknown` and the UI says what share that is.
169
+
170
+ `interfaceClass()` groups the eight values into the four buckets the CLI-vs-GUI comparison uses:
171
+ `CLI / headless` (CLI, SDK), `IDE` (IDE, Extension), `Desktop / Web`, `API`.
172
+
173
+ ## Time and timezone
174
+
175
+ `date`, `hour` and `dow` are resolved **once at ingest**, in the capture timezone
176
+ (`config.timezone`, defaulting to the machine's zone), and stored. The UI never re-derives them,
177
+ so "my peak hour" means your local peak hour and a late-evening UTC event is filed on the right
178
+ local day. `dow` is Monday-first (0=Mon), matching how the charts are labelled.
179
+
180
+ ## Dedup and identity
181
+
182
+ `id` is a stable 64-bit hash of `(source, session, request||timestamp, model, sequence)`.
183
+
184
+ Dedup is **structural**, not probabilistic: ingest resumes at a per-file byte offset, so a byte is
185
+ never read twice and a record cannot be ingested twice. That is why the Data Health page can state
186
+ `Duplicate records: 0` as a fact about the design rather than an estimate. Streaming duplicates
187
+ *within* a source file are a different problem, solved inside each adapter — see
188
+ [providers.md](providers.md).
189
+
190
+ ## Storage layout
191
+
192
+ ```
193
+ data/records/YYYY-MM.jsonl request-level facts, short-key codec, nulls omitted
194
+ data/cube.json pre-aggregated fact table (dims + measures), what the browser loads
195
+ data/sessions.json one row per session
196
+ data/activity.json daily work-activity rollup (commits, churn, AI edits)
197
+ data/state.json per-file {size, mtime, offset, gen} — the incremental engine
198
+ ```
199
+
200
+ The cube's dimensions are `date, hour, dow, provider, model, model_family, client, interface,
201
+ gateway, project, repository, measurement`; its measures are the six token fields plus
202
+ `requests`, `cost`, `costMeasured`, `costReq` and the four not-available counters. It is
203
+ **additive**, which is what lets an incremental refresh update it without a rebuild.
204
+
205
+ Set `store.keepRaw: false` to skip the request-level shards entirely — the cube, sessions and the
206
+ whole dashboard still work; only the Data Explorer and full CSV export need the raw facts.
207
+
208
+ ## Multi-user readiness
209
+
210
+ `user`, `machine` and `session_id` are first-class fields, and `project` / `repository` are
211
+ dimensions. A future team deployment can aggregate the same records across machines without a
212
+ schema change. V1 deliberately ships no server-side identity, auth or sync — but nothing here
213
+ makes those impossible to add later.
@@ -0,0 +1,266 @@
1
+ # Getting started
2
+
3
+ ## Requirements
4
+
5
+ - **Node 22.5 or newer.** 22.5 is where `node:sqlite` and the built-in test runner landed; both
6
+ are used. Check with `node -v`.
7
+ - Nothing else. This project has **zero runtime dependencies** — `npm install` does nothing and
8
+ is not required.
9
+
10
+ ## Install
11
+
12
+ **macOS, from a DMG:** download `TokenFlow-<version>.dmg` from the
13
+ [latest release](https://github.com/vimoxshah/tokenflow/releases/latest), open it, drag
14
+ **TokenFlow.app** to Applications, launch from Launchpad. The app builds and launches everything
15
+ it needs — refresh, watcher, dashboard. It is unsigned and not notarized, so macOS shows a
16
+ security warning on first launch:
17
+
18
+ 1. If launch is blocked with *"Apple cannot check it for malicious software"*:
19
+ open **System Settings → Privacy & Security**, scroll to **Security**, click
20
+ **Open Anyway** next to the blocked-app message, then confirm **Open**.
21
+ 2. Alternatively right-click `TokenFlow.app` → **Open** → **Open**.
22
+
23
+ This one-time approval is needed because the project has no paid Apple Developer signature;
24
+ the app runs entirely locally either way.
25
+
26
+ The DMG is produced by [`scripts/build-dmg.sh`](../scripts/build-dmg.sh) and attached to releases
27
+ automatically by [`.github/workflows/release.yml`](../.github/workflows/release.yml). Releases
28
+ also carry `tokenflow-dashboard-demo.html` — a fully offline demo dashboard you can open in any
29
+ browser without installing anything.
30
+
31
+ **From source (all platforms):**
32
+
33
+ ```bash
34
+ git clone <this repo> tokenflow
35
+ cd tokenflow
36
+ node bin/tokenflow.js --help
37
+ ```
38
+
39
+ To get a global `tokenflow` command:
40
+
41
+ ```bash
42
+ npm link # or: npm i -g .
43
+ tokenflow --help
44
+ ```
45
+
46
+ Everything below works either way; `npm run <script>` wrappers exist for `setup`, `refresh`,
47
+ `status`, `dashboard`, and `demo`. The bare `tokenflow <command>` examples throughout this doc
48
+ assume the global command from `npm link`; otherwise substitute
49
+ `node bin/tokenflow.js <command>` or the matching `npm run` script.
50
+
51
+ ## 1. Look around first (optional)
52
+
53
+ ```bash
54
+ npm run demo
55
+ ```
56
+
57
+ Generates deterministic synthetic data and opens the dashboard. Every demo record is flagged, and
58
+ the dashboard shows a red **DEMO DATA** banner for as long as any of it is in scope, so it can
59
+ never be mistaken for your own usage. To clear it later:
60
+
61
+ ```bash
62
+ tokenflow provider remove mock
63
+ tokenflow refresh --full
64
+ ```
65
+
66
+ ## 2. Detect your tools
67
+
68
+ ```bash
69
+ tokenflow setup
70
+ ```
71
+
72
+ ```
73
+ Tokenflow — setup
74
+ config home: /Users/you/.tokenflow
75
+
76
+ ✓ Anthropic (Claude Code / Agent SDK) ~/.claude, ~/.claude-work
77
+ ✓ OpenAI (Codex CLI / IDE / Desktop) /Users/you/.codex
78
+ ✓ Cline CLI 22 session directories · no token fields reported
79
+ ✓ Cursor (AI code activity) activity source — AI-authored edits + commits
80
+ ○ Headroom gateway (overlay) No ~/.headroom/savings_events.jsonl found
81
+ ○ Git activity (correlation) no repositories found — set sources.git.scanRoots
82
+
83
+ ✓ wrote /Users/you/.tokenflow/config.yaml
84
+ ```
85
+
86
+ `setup` only writes config. It reads no usage data yet.
87
+
88
+ **Nothing detected?** That is a normal answer, not a failure. Either the tools store their logs
89
+ somewhere non-standard (point at it: `sources.<id>.paths`), or you want
90
+ [`tokenflow import`](cli.md#import) for an export file, or `tokenflow demo` to explore.
91
+
92
+ ## 3. Ingest
93
+
94
+ ```bash
95
+ tokenflow refresh
96
+ ```
97
+
98
+ ```
99
+ ✓ anthropic 44,820 new records · 5,110 files read, 0 unchanged of 5,110 found · 1.4 GB
100
+ ✓ openai 4,460 new records · 625 files read, 0 unchanged of 625 found · 3.8 GB
101
+ ✓ cline 22 new records · 22 files read, 0 unchanged of 22 found · 92.2 KB
102
+ ✓ cursor 12,480 new records
103
+ ○ git no repositories found — set sources.git.repos or sources.git.scanRoots
104
+
105
+ 61,782 new records · 5.2 GB read · 0 files skipped as unchanged · 29s
106
+ ```
107
+
108
+ Run it again and it costs almost nothing — unchanged files are skipped without being read:
109
+
110
+ ```
111
+ 0 new records · 0 B read · 5,757 files skipped as unchanged · 2s
112
+ ```
113
+
114
+ If a first ingest is too big for the time you have, cap it and resume:
115
+
116
+ ```bash
117
+ tokenflow refresh --budget 30 # stop cleanly after ~30s
118
+ tokenflow refresh --budget 30 # continue exactly where it stopped
119
+ ```
120
+
121
+ ## 4. Open the dashboard
122
+
123
+ ```bash
124
+ tokenflow dashboard
125
+ ```
126
+
127
+ The server binds to `127.0.0.1` only. The browser downloads one pre-aggregated bundle and then
128
+ does every filter and aggregation locally, so filtering is instant and offline.
129
+
130
+ Click **↻ Refresh data** any time — it re-ingests, streams progress, and keeps your filters.
131
+
132
+ ### The one-command version
133
+
134
+ ```bash
135
+ npm start # = tokenflow up
136
+ ```
137
+
138
+ On macOS you can double-click **`Refresh & Open Dashboard.command`** in the project folder
139
+ instead (it is a bash script, so on Windows stay with `npm start`). Either way it refreshes, rebuilds the offline `tokenflow-dashboard.html` beside it, and
140
+ opens the live dashboard. That is the thing to use when you come back after a few days.
141
+
142
+ The offline file, opened on its own, states how old its data is and looks for a live dashboard on
143
+ loopback; if it finds one it offers to hand over and refresh there. To keep the file current
144
+ without thinking about it, schedule `tokenflow up --no-serve` (see
145
+ [cli.md § up](cli.md#up-alias-open) for a ready-made launchd plist and cron line).
146
+
147
+ ## 5. Pick a look (optional)
148
+
149
+ Three skins, each with a dark and a light mode. Switch from the header (**◑ Aurora**), or set the
150
+ default:
151
+
152
+ ```yaml
153
+ # ~/.tokenflow/config.yaml
154
+ ui:
155
+ skin: aurora # aurora | terminal | editorial
156
+ mode: dark # dark | light
157
+ port: 7799
158
+ ```
159
+
160
+ Series colours belong to the mode, not the skin, and were validated for colour-blind separation
161
+ against every skin's chart surface — so changing the look never changes what a colour means.
162
+
163
+ Every field is documented in **[configuration.md](configuration.md)** — that is the page to open
164
+ when you want to change something and are not sure what it affects.
165
+
166
+ ## 6. Set the default window (optional)
167
+
168
+ If your store contains older records you don't normally want in scope:
169
+
170
+ ```yaml
171
+ # ~/.tokenflow/config.yaml
172
+ ui:
173
+ defaultRange: all # all | 7d | 30d | 90d | mtd
174
+ defaultFrom: "2026-03-14"
175
+ ```
176
+
177
+ `defaultFrom` is a floor for the default view, not a filter on the data — everything stays in the
178
+ store and in `--all` exports.
179
+
180
+ ## 7. Turn on cost (optional)
181
+
182
+ Cost is blank until a model has a price. See what is missing:
183
+
184
+ ```bash
185
+ tokenflow pricing
186
+ ```
187
+
188
+ ```
189
+ MODEL TOKENS STATUS
190
+ claude-opus-5 6.35B no price
191
+ gpt-5.6-luna 1.71B no price
192
+ claude-3-5-sonnet-20241022 88.1M priced $264.30 est.
193
+ ```
194
+
195
+ Add rates in USD per million tokens — `input,output[,cacheRead[,cacheWrite]]`:
196
+
197
+ ```bash
198
+ tokenflow pricing --set "claude-opus-5=15,75,1.5,18.75"
199
+ tokenflow refresh --full # re-cost the history
200
+ ```
201
+
202
+ Or use the **Pricing** dialog in the dashboard, which lists your models by volume and saves +
203
+ re-costs in one step. Anything you leave blank stays unpriced — the dashboard shows "no price"
204
+ rather than a plausible-looking `$0`.
205
+
206
+ ## 8. Correlate with your actual work (optional)
207
+
208
+ ```yaml
209
+ sources:
210
+ git:
211
+ scanRoots: ["~/code", "~/work"]
212
+ autoFromUsage: true # also use the working directories seen in usage records
213
+ ```
214
+
215
+ ```bash
216
+ tokenflow provider add git && tokenflow refresh
217
+ ```
218
+
219
+ The Productivity page will then correlate daily AI usage with commits, files changed and line
220
+ churn — over the overlapping days only, with `n` reported, and labelled as a correlation. With
221
+ fewer than 10 overlapping days it tells you that instead of showing a number.
222
+
223
+ ## 9. Export
224
+
225
+ ```bash
226
+ tokenflow export --csv # the current filter
227
+ tokenflow export --csv --all # everything
228
+ tokenflow export --html # one self-contained offline dashboard file
229
+ ```
230
+
231
+ ## Where everything lives
232
+
233
+ ```
234
+ ~/.tokenflow/
235
+ config.yaml providers, source paths, preferences
236
+ pricing.json your rate overrides
237
+ mappings/ saved generic-import field mappings
238
+ providers/ your own adapters (*.js), loaded automatically
239
+ data/
240
+ records/ YYYY-MM.jsonl — request-level facts
241
+ cube.json the pre-aggregated table the dashboard loads
242
+ sessions.json one row per session
243
+ activity.json daily work-activity rollup
244
+ state.json per-file ingest offsets (this is what makes refresh incremental)
245
+ cache/
246
+ ```
247
+
248
+ Move it with `TOKENFLOW_HOME=/some/path tokenflow ...`, back it up with
249
+ `tokenflow config export`, and restore with `tokenflow config import`.
250
+
251
+ ## Verify the install
252
+
253
+ ```bash
254
+ npm run validate
255
+ ```
256
+
257
+ Checks the runtime, the config, every adapter's `detect()`, and — importantly — that the
258
+ aggregates still agree with the stored records. If they have drifted, it tells you to run
259
+ `tokenflow compact`.
260
+
261
+ ## Next
262
+
263
+ - Numbers look wrong? → [troubleshooting.md](troubleshooting.md)
264
+ - Want another source? → [providers.md](providers.md), [creating-provider.md](creating-provider.md)
265
+ - Want an agent to set this up on a new machine? → [skill.md](skill.md)
266
+ - Changing a setting? → [configuration.md](configuration.md)
@@ -0,0 +1,199 @@
1
+ # Live mode — the watcher, the native menu bar app, capacity and alerts
2
+
3
+ Everything in this document runs **locally**. The watcher reads your logs, the
4
+ menu bar app reads one JSON file, notifications come from your operating
5
+ system. No network client exists anywhere in the codebase.
6
+
7
+ ```
8
+ tokenflow watch ──every N s──▶ incremental refresh
9
+
10
+
11
+ data/status.json ◀── read by ──▶ TokenFlow.app (native macOS menu bar)
12
+ │ tokenflow usage / cost / capacity / forecast
13
+ │ tokenflow status --bar
14
+ ▼ dashboard header pill + Live tab
15
+ OS notifications (opt-in): limit crossings, same-day high-severity anomalies
16
+ ```
17
+
18
+ ## The watcher
19
+
20
+ ```bash
21
+ tokenflow watch # start; Ctrl+C stops it
22
+ tokenflow watch --interval 30
23
+ tokenflow watch --once # one cycle and exit — what cron wants
24
+ tokenflow watch --status # running? how fresh is the data?
25
+ tokenflow watch --stop # stop a running watcher
26
+ tokenflow watch --notify # enable OS notifications for this run
27
+ ```
28
+
29
+ Properties that matter:
30
+
31
+ - **Single instance.** A pidfile guards the store. A crashed run leaves a dead
32
+ pid behind; the next start detects it and takes over.
33
+ - **Failure isolation with backoff.** A failing refresh doubles the interval,
34
+ up to a 15-minute ceiling, and records `lastError` in the status file
35
+ instead of crashing or spamming.
36
+ - **Sleep/wake is free.** Each tick reschedules from wall-clock reality, so a
37
+ laptop that slept for three hours wakes to exactly one refresh, never a
38
+ catch-up burst.
39
+ - **Honest freshness.** Every surface recomputes "stale?" at *read* time from
40
+ `lastRefresh`, against `watch.staleAfterSeconds` (default 600). Old data
41
+ says so; nothing silently pretends to be real-time.
42
+
43
+ Defaults live in `config.yaml`:
44
+
45
+ ```yaml
46
+ watch:
47
+ intervalSeconds: 120
48
+ notifications: false # or pass --notify per run
49
+ staleAfterSeconds: 600
50
+ ```
51
+
52
+ `tokenflow up` remains the pull-based path (`refresh → snapshot → open`); the
53
+ watcher is the push-based one. They share the same incremental engine and can
54
+ interleave safely — a manual refresh during a watched cycle simply coalesces.
55
+
56
+ ## Capacity & budgets
57
+
58
+ TokenFlow **never invents vendor quota data** — there is no network client, so
59
+ no screen claims to know what Anthropic or OpenAI think your balance is.
60
+ Instead you declare caps, and the engine evaluates them against measured
61
+ consumption:
62
+
63
+ ```yaml
64
+ limits:
65
+ - id: anthropic-monthly
66
+ provider: anthropic # optional filters: provider | model | project
67
+ scope: month # day | week | month (your local calendar)
68
+ metric: tokens # tokens | input | output | requests | cost
69
+ cap: 120000000 # tokens — or dollars when metric is cost
70
+ warnAt: 0.8 # optional: when "approaching" starts
71
+ ```
72
+
73
+ For each limit the Live tab and `tokenflow capacity` derive:
74
+
75
+ | Field | Meaning |
76
+ |---|---|
77
+ | used / remaining | consumption inside the current window |
78
+ | % / bar | fraction of cap consumed |
79
+ | burn/hour | today's pace so far (local wall clock) |
80
+ | burn/day | trailing 7 calendar days, trimmed to dataset coverage |
81
+ | ETA | projected exhaustion at the faster of the two paces |
82
+ | resets in | deterministic countdown to local midnight / Monday / 1st |
83
+
84
+ Limits evaluate against the whole primary dataset regardless of dashboard
85
+ filters — quota windows are facts about your accounts, not filter states.
86
+ Invalid definitions are reported (`tokenflow capacity --json` → `invalid`),
87
+ never silently dropped. Edit them in the dashboard's **Live → Manage limits**
88
+ or by hand in `config.yaml`.
89
+
90
+ ## Forecasting
91
+
92
+ `tokenflow forecast` fits a conservative linear trend over the last 14 days
93
+ (minimum 5), reports tomorrow's likely range, next-7-days and month-end totals
94
+ with cost, and states its confidence (high / medium / low) plus sample size.
95
+ Month-to-date is always shown separately from the projection. Thin or volatile
96
+ history degrades confidence honestly instead of producing confident nonsense.
97
+
98
+ ## Anomalies
99
+
100
+ Detection uses robust statistics (median / MAD, Iglewicz–Hoaglin modified
101
+ z-score ≥ 3.5):
102
+
103
+ - token, cost and request-volume spikes (and weekday collapses)
104
+ - possible ingestion gaps — a zero weekday between two active days
105
+ - models/providers appearing for the first time this week
106
+
107
+ Every alert carries its own arithmetic — observed value, trailing median,
108
+ ratio, z-score — so you can check it rather than trust it. Notifications fire
109
+ only for **same-day high-severity** anomalies; history replaying as alerts on
110
+ a first run would be noise, so it doesn't happen.
111
+
112
+ ## Menu bar — TokenFlow.app (native, macOS)
113
+
114
+ TokenFlow ships its own menu bar application: a ~370 KB native binary compiled
115
+ on your machine from [`menubar/TokenFlow/main.swift`](../menubar/TokenFlow/main.swift)
116
+ with `swiftc` (Xcode Command Line Tools). No Electron, no third-party bar, no
117
+ network. It reads `data/status.json` every 5 seconds and rebuilds its dropdown
118
+ every time it opens.
119
+
120
+ ```bash
121
+ tokenflow menubar --app # build → ~/Applications/TokenFlow.app → launch
122
+ tokenflow menubar --app --login-item
123
+ # …and start it on every login
124
+ ```
125
+
126
+ **Status item (adaptive, most-urgent signal wins):**
127
+
128
+ | Situation | Menu bar shows |
129
+ |---|---|
130
+ | a limit exceeded | `✗ 105%` in red |
131
+ | a limit approaching | `▲ 82%` in orange |
132
+ | healthy limit | `● 42%` in green |
133
+ | no limits, priced usage | `$4.83` |
134
+ | unpriced day | today's tokens |
135
+
136
+ **The dropdown contains everything — no browser hop required:**
137
+
138
+ - header: live/watcher badge + data freshness ("live · data updated just now")
139
+ - Today / Week / Month rows — tokens, requests, estimated cost
140
+ - Today by provider — inline share bars (▰▱), tokens and cost per provider
141
+ - Today by source — the app that wrote the log (claude-code, opencode,
142
+ hermes, git…). Provider attribution names the model's vendor, so Hermes
143
+ traffic appears under each model's vendor there; this section shows it as
144
+ "hermes"
145
+ - Top models today — token and cost leaders per model
146
+ - Capacity — a real meter per declared limit with %, exhaustion ETA and reset
147
+ countdown; "first projected hit" callout when one will cross before reset
148
+ - Forecast — tomorrow / 7-day projections, month-end spend, confidence
149
+ - Alerts — high-severity anomalies with their arithmetic
150
+ - Actions — `Refresh now` (⌘R, runs a watch cycle), `Open Dashboard`,
151
+ `Start/Stop watcher`, `Quit`
152
+ - Appearance button (◐) — cycles system → light → dark; persisted across
153
+ launches in `defaults` under `appearanceOverride`
154
+
155
+ Dark/light follows the appearance override (◐ button, persisted) or the system
156
+ appearance when set to follow; all figures use monospaced digits.
157
+
158
+ ## Menu bar — other platforms (SwiftBar/xbar text protocol)
159
+
160
+ For Linux bars or if you prefer SwiftBar on macOS:
161
+
162
+ ```bash
163
+ tokenflow menubar --swiftbar # install for SwiftBar (~/Library/Plugins)
164
+ tokenflow menubar --xbar # xbar's plugin directory instead
165
+ tokenflow menubar --out <dir> # any compatible bar
166
+ tokenflow menubar --render # print the text protocol (debug)
167
+ tokenflow menubar --mode cost # auto | tokens | cost | limit
168
+ ```
169
+
170
+ The generated script shells to `tokenflow menubar --render` every couple of
171
+ minutes (filename convention). The title adapts automatically — the most
172
+ urgent signal wins:
173
+
174
+ | Situation | Title shows |
175
+ |---|---|
176
+ | a limit approaching | `TF ⚠ anthropic-monthly 82% · 4h 12m` |
177
+ | a limit exceeded | `TF ✗ anthropic-monthly 105% · resets 3d` |
178
+ | no limits, priced usage | `TF $4.83` |
179
+ | unpriced day, week has data | `TF 7d 5.04B` |
180
+
181
+ Any bar speaking the same text protocol (Argos, Waybar custom modules) works
182
+ via `--out`.
183
+
184
+ The SwiftBar dropdown carries the same breakdowns as the native popover:
185
+ today/week/month totals, **by source** (claude-code, hermes, opencode… —
186
+ provider rows name each model's vendor, so this is where Hermes appears under
187
+ its own name), **top models**, limits, forecast and alerts. Appearance is the
188
+ one thing a text-protocol plugin cannot control: SwiftBar renders plain text,
189
+ so its light/dark look follows SwiftBar's own settings — use the native app's
190
+ ◐ button for an in-app toggle.
191
+
192
+ ## Data flow guarantees
193
+
194
+ - `data/status.json` is written atomically (tmp + rename). A reader never sees
195
+ a half-file; a corrupt file degrades to "recompute fresh".
196
+ - Estimated and measured cost stay separate everywhere, including the bar line
197
+ and the dropdown.
198
+ - A one-shot cycle (`--once`) never leaves the file claiming a watcher is
199
+ running; liveness is verified against the pid, not assumed.