@vimoxshah/tokenflow 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +84 -0
- package/LICENSE +21 -0
- package/README.md +250 -0
- package/Refresh & Open Dashboard.command +22 -0
- package/SECURITY.md +42 -0
- package/bin/tokenflow.js +1342 -0
- package/docs/architecture.md +193 -0
- package/docs/cli.md +390 -0
- package/docs/configuration.md +281 -0
- package/docs/creating-provider.md +262 -0
- package/docs/data-model.md +213 -0
- package/docs/getting-started.md +266 -0
- package/docs/live-mode.md +199 -0
- package/docs/media/architecture-hero.svg +86 -0
- package/docs/media/cost-editorial-dark.png +0 -0
- package/docs/media/health-terminal-light.png +0 -0
- package/docs/media/menubar-dark.png +0 -0
- package/docs/media/menubar-light.png +0 -0
- package/docs/media/models-terminal-dark.png +0 -0
- package/docs/media/overview-aurora-dark.png +0 -0
- package/docs/media/time-aurora-light.png +0 -0
- package/docs/providers.md +309 -0
- package/docs/skill.md +64 -0
- package/docs/troubleshooting.md +207 -0
- package/examples/config.example.yaml +92 -0
- package/examples/demo-data/README.md +38 -0
- package/examples/demo-data/sample-usage.csv +11 -0
- package/package.json +74 -0
- package/scripts/build-dmg.sh +33 -0
- package/scripts/build-menubar-app.sh +67 -0
- package/scripts/lint.js +111 -0
- package/scripts/validate-install.js +140 -0
- package/skills/tokenflow/SKILL.md +392 -0
- package/skills/tokenflow/examples/config.yaml +92 -0
- package/skills/tokenflow/examples/generic-mapping.json +26 -0
- package/skills/tokenflow/examples/session-transcript.md +191 -0
- package/skills/tokenflow/providers/adapter-template.js +135 -0
- package/skills/tokenflow/providers/detection-matrix.md +142 -0
- package/skills/tokenflow/schemas/config.schema.json +107 -0
- package/skills/tokenflow/schemas/normalized-record.json +63 -0
- package/src/analytics/aggregate.js +247 -0
- package/src/analytics/anomalies.js +222 -0
- package/src/analytics/capacity.js +278 -0
- package/src/analytics/comparison.js +96 -0
- package/src/analytics/dimensions.js +230 -0
- package/src/analytics/efficiency.js +138 -0
- package/src/analytics/forecast.js +202 -0
- package/src/analytics/index.js +327 -0
- package/src/analytics/insights.js +283 -0
- package/src/analytics/milestones.js +91 -0
- package/src/analytics/peak.js +106 -0
- package/src/analytics/productivity.js +166 -0
- package/src/analytics/token-usage.js +267 -0
- package/src/commands/diagnostics.js +88 -0
- package/src/commands/digest.js +155 -0
- package/src/commands/models-compare.js +96 -0
- package/src/core/budget.js +142 -0
- package/src/core/bundle.js +191 -0
- package/src/core/config.js +202 -0
- package/src/core/delivery.js +109 -0
- package/src/core/geo.js +99 -0
- package/src/core/ingest.js +457 -0
- package/src/core/interface-map.js +55 -0
- package/src/core/jsonl.js +124 -0
- package/src/core/live-status.js +417 -0
- package/src/core/model-map.js +157 -0
- package/src/core/notify.js +83 -0
- package/src/core/pricing.js +288 -0
- package/src/core/prompt-analytics.js +127 -0
- package/src/core/registry.js +107 -0
- package/src/core/restore.js +261 -0
- package/src/core/schedule.js +120 -0
- package/src/core/schema.js +316 -0
- package/src/core/sqlite.js +96 -0
- package/src/core/store.js +493 -0
- package/src/core/sync.js +151 -0
- package/src/core/units.js +147 -0
- package/src/core/validate.js +123 -0
- package/src/core/watch.js +287 -0
- package/src/core/yaml.js +209 -0
- package/src/export/bundler.js +107 -0
- package/src/export/csv.js +100 -0
- package/src/export/html-snapshot.js +101 -0
- package/src/export/menubar.js +158 -0
- package/src/index.js +18 -0
- package/src/providers/anthropic/index.js +294 -0
- package/src/providers/cline/index.js +120 -0
- package/src/providers/cursor/index.js +143 -0
- package/src/providers/generic/index.js +268 -0
- package/src/providers/git/index.js +188 -0
- package/src/providers/headroom/index.js +114 -0
- package/src/providers/hermes/index.js +299 -0
- package/src/providers/mock/index.js +117 -0
- package/src/providers/openai/index.js +370 -0
- package/src/providers/opencode/index.js +245 -0
- package/src/sdk.js +46 -0
- package/src/server/server.js +264 -0
- package/src/ui/app.js +2473 -0
- package/src/ui/charts.js +925 -0
- package/src/ui/index.html +42 -0
- package/src/ui/styles.css +644 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# Data model
|
|
2
|
+
|
|
3
|
+
Every adapter emits the same record shape. Nothing downstream — analytics, UI, export — knows
|
|
4
|
+
anything about a specific vendor. This document is the contract.
|
|
5
|
+
|
|
6
|
+
## The normalized record
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
interface NormalizedUsageRecord {
|
|
10
|
+
id: string; // stable dedup key
|
|
11
|
+
timestamp: string; // ISO-8601, UTC
|
|
12
|
+
date: string; // YYYY-MM-DD in the capture timezone
|
|
13
|
+
hour: number; // 0-23 in the capture timezone
|
|
14
|
+
dow: number; // 0=Mon .. 6=Sun in the capture timezone
|
|
15
|
+
tz_offset: number; // capture tz offset, minutes
|
|
16
|
+
|
|
17
|
+
provider: string; // canonical vendor slug: anthropic | openai | deepseek | zai | ...
|
|
18
|
+
provider_label: string;
|
|
19
|
+
gateway: string | null; // routing layer (proxy/router), if any — NOT the vendor
|
|
20
|
+
model: string; // raw model identifier from the source
|
|
21
|
+
model_family: string; // human tier, e.g. "Claude Opus 5", "GPT 5.6 (sol)"
|
|
22
|
+
|
|
23
|
+
client: string; // tool that made the call: claude-code | codex | cline | cursor
|
|
24
|
+
application: string; // human label for the client
|
|
25
|
+
interface: Interface; // CLI | IDE | Desktop App | Web | API | SDK | Extension | Unknown
|
|
26
|
+
|
|
27
|
+
input_tokens: number | null; // fresh prompt tokens, EXCLUDING cache
|
|
28
|
+
cache_read_tokens: number | null; // prompt tokens served from cache
|
|
29
|
+
cache_write_tokens: number | null; // prompt tokens written into cache
|
|
30
|
+
output_tokens: number | null; // generated tokens
|
|
31
|
+
cache_refresh_tokens: number | null; // SUBSET of cache_write (long-TTL / refreshed)
|
|
32
|
+
reasoning_tokens: number | null; // SUBSET of output (thinking / reasoning)
|
|
33
|
+
total_tokens: number | null; // derived, never trusted from the source
|
|
34
|
+
total_is_partial: boolean; // some billable field was not available
|
|
35
|
+
|
|
36
|
+
session_id: string | null;
|
|
37
|
+
conversation_id: string | null;
|
|
38
|
+
request_id: string | null;
|
|
39
|
+
|
|
40
|
+
project: string | null;
|
|
41
|
+
repository: string | null;
|
|
42
|
+
git_branch: string | null;
|
|
43
|
+
category: string | null; // main | subagent | commit | ai-edit:* | ...
|
|
44
|
+
|
|
45
|
+
estimated_cost: number | null;
|
|
46
|
+
cost_basis: 'measured' | 'estimated' | null;
|
|
47
|
+
|
|
48
|
+
source: string; // adapter id
|
|
49
|
+
measurement: Measurement; // primary | overlay | activity
|
|
50
|
+
user: string | null; // multi-user ready
|
|
51
|
+
machine: string | null;
|
|
52
|
+
duration_ms: number | null;
|
|
53
|
+
metadata: object; // source-specific, never interpreted by analytics
|
|
54
|
+
}
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Missing-value contract
|
|
58
|
+
|
|
59
|
+
This is the load-bearing rule of the whole project.
|
|
60
|
+
|
|
61
|
+
| Value | Meaning |
|
|
62
|
+
|---|---|
|
|
63
|
+
| `null` | the source does not report this field — **not available** |
|
|
64
|
+
| `undefined` | normalised to `null` on construction |
|
|
65
|
+
| `0` | the source reported zero — a real, **measured** zero |
|
|
66
|
+
|
|
67
|
+
Analytics **never** coerce `null` to `0`. Sums skip nulls and carry a parallel not-available
|
|
68
|
+
counter per field (`naIn`, `naOut`, `naCr`, `naCw`), which is why the UI can say
|
|
69
|
+
"cache tokens unreported by 22% of records in this slice" instead of drawing a confident zero.
|
|
70
|
+
The linter enforces this: `|| 0` on a line mentioning a token field is a build error.
|
|
71
|
+
|
|
72
|
+
Practical consequence: Cline logs sessions but no token counts, so **every** Cline token field is
|
|
73
|
+
`null`, the record is `measurement: activity`, and its sessions count towards activity metrics
|
|
74
|
+
without dragging every token average towards zero.
|
|
75
|
+
|
|
76
|
+
## Token accounting
|
|
77
|
+
|
|
78
|
+
### The four billable buckets are mutually exclusive
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
total_tokens = input_tokens + cache_read_tokens + cache_write_tokens + output_tokens
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### The two breakdown fields are subsets and are never added again
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
cache_refresh_tokens ⊆ cache_write_tokens (long-TTL / refreshed cache writes)
|
|
88
|
+
reasoning_tokens ⊆ output_tokens (thinking / reasoning)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`validateUsage` rejects a record where a subset exceeds its parent, and the test suite asserts
|
|
92
|
+
both invariants.
|
|
93
|
+
|
|
94
|
+
### Vendors disagree about `input_tokens`
|
|
95
|
+
|
|
96
|
+
| Vendor | What the source calls `input_tokens` | Adapter must |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| Anthropic | fresh prompt tokens, **excluding** cache read and cache creation | pass through |
|
|
99
|
+
| OpenAI / Codex | prompt tokens **including** `cached_input_tokens` | subtract: `fresh = input − cached` |
|
|
100
|
+
|
|
101
|
+
Getting this wrong double-counts every cached prompt token — once as fresh input and once as
|
|
102
|
+
cache read. Each adapter's test asserts its own convention.
|
|
103
|
+
|
|
104
|
+
### Ratios and what they actually divide by
|
|
105
|
+
|
|
106
|
+
| Metric | Formula | Read it as |
|
|
107
|
+
|---|---|---|
|
|
108
|
+
| Output / input | `out / in` | generated per **fresh** prompt token. With a cache-heavy agent, `in` is tiny, so this number is large and not very meaningful on its own. |
|
|
109
|
+
| Output / prompt sent | `out / (in + cr + cw)` | generated per prompt token **actually sent**. This is the honest "prompt-heavy vs output-heavy" measure. |
|
|
110
|
+
| Cache / total | `(cr + cw) / total` | share of all token activity that was cache traffic |
|
|
111
|
+
| Cache hit rate | `cr / (in + cr)` | share of prompt tokens served from cache rather than re-sent |
|
|
112
|
+
| Fresh per cached prompt | `in / cr` | below 1 means the cache is carrying the context |
|
|
113
|
+
|
|
114
|
+
## Cost, and what an estimate is allowed to claim
|
|
115
|
+
|
|
116
|
+
`estimated_cost` + `cost_basis` are the only cost fields, and they mean two different things:
|
|
117
|
+
|
|
118
|
+
| `cost_basis` | Meaning |
|
|
119
|
+
|---|---|
|
|
120
|
+
| `measured` | a source actually billed this request (a gateway's own `cost_usd`) |
|
|
121
|
+
| `estimated` | computed here, from a published rate table |
|
|
122
|
+
| `null` | the model has no configured rate — **and no number is shown** |
|
|
123
|
+
|
|
124
|
+
Three things make the estimate defensible rather than decorative:
|
|
125
|
+
|
|
126
|
+
1. **Every rate names a source.** Each entry in the built-in table carries a `src` key into
|
|
127
|
+
`PRICING_SOURCES`, which records the URL, the date it was fetched, and whether the source is
|
|
128
|
+
official or third-party. `tokenflow pricing --sources` prints it; the Cost page shows it.
|
|
129
|
+
2. **Service tier is a multiplier, not a label.** OpenAI's Fast mode (renamed from "priority" on
|
|
130
|
+
2026-07-30) bills at **4x** standard; Anthropic's Batch API at 0.5x. `service_tier` is a
|
|
131
|
+
first-class field and a cube dimension, and the multiplier is applied per request. Ignoring it
|
|
132
|
+
under-reports a Fast-mode-heavy workload by up to 4x.
|
|
133
|
+
3. **The cache write tiers are priced separately.** `cache_refresh_tokens` (the long-TTL subset) is
|
|
134
|
+
billed at the 1-hour rate and the remainder at the 5-minute rate — for Claude Opus 5 that is
|
|
135
|
+
$10 vs $6.25 per MTok, so on a cache-heavy workload the split is worth real money.
|
|
136
|
+
|
|
137
|
+
**Known gap, stated rather than hidden:** long-context premium tiers (Anthropic above 200K,
|
|
138
|
+
OpenAI's long-context rows) are *not* applied, because doing so needs a per-request prompt size
|
|
139
|
+
plus a per-model threshold and premium that are not uniformly published. A long-context-heavy
|
|
140
|
+
workload is therefore **under**-estimated, and the Cost page says so.
|
|
141
|
+
|
|
142
|
+
## Measurement kinds
|
|
143
|
+
|
|
144
|
+
| `measurement` | Meaning | Counted in token totals? |
|
|
145
|
+
|---|---|---|
|
|
146
|
+
| `primary` | authoritative per-request usage from the model API | yes |
|
|
147
|
+
| `overlay` | a second view of traffic a client adapter already recorded (gateway/proxy logs) | **no** by default — it would double count. Used for measured cost. |
|
|
148
|
+
| `activity` | AI activity with no token accounting at all (IDE edits, sessions without a usage block, commits) | never — activity and correlation only |
|
|
149
|
+
|
|
150
|
+
Both non-primary kinds are toggleable in the filter bar, and the Data Health page explains each
|
|
151
|
+
one in place.
|
|
152
|
+
|
|
153
|
+
## Provider vs gateway
|
|
154
|
+
|
|
155
|
+
A local proxy or router is **not** a model vendor. When Codex reports
|
|
156
|
+
`model_provider: "headroom"`, the record gets `gateway: "headroom"` and `provider: "openai"` —
|
|
157
|
+
the vendor is derived from the model string, which is the only real evidence of who made the
|
|
158
|
+
model. This keeps "who served this request" and "who built this model" as separate, filterable
|
|
159
|
+
dimensions, and lets you answer "how much of my OpenAI traffic goes through the proxy?".
|
|
160
|
+
|
|
161
|
+
A provider hint from a source **never** overrides evidence in the model name.
|
|
162
|
+
|
|
163
|
+
## Interface classification
|
|
164
|
+
|
|
165
|
+
`interface` is derived **only** from an explicit surface field in the source record — `entrypoint`,
|
|
166
|
+
`originator`, `source`, an IDE marker, a client name. It is never inferred from the model or the
|
|
167
|
+
provider: "it's a Claude model so it must be Claude Desktop" is exactly the mistake this rule
|
|
168
|
+
exists to prevent. With no signal, the value is `Unknown` and the UI says what share that is.
|
|
169
|
+
|
|
170
|
+
`interfaceClass()` groups the eight values into the four buckets the CLI-vs-GUI comparison uses:
|
|
171
|
+
`CLI / headless` (CLI, SDK), `IDE` (IDE, Extension), `Desktop / Web`, `API`.
|
|
172
|
+
|
|
173
|
+
## Time and timezone
|
|
174
|
+
|
|
175
|
+
`date`, `hour` and `dow` are resolved **once at ingest**, in the capture timezone
|
|
176
|
+
(`config.timezone`, defaulting to the machine's zone), and stored. The UI never re-derives them,
|
|
177
|
+
so "my peak hour" means your local peak hour and a late-evening UTC event is filed on the right
|
|
178
|
+
local day. `dow` is Monday-first (0=Mon), matching how the charts are labelled.
|
|
179
|
+
|
|
180
|
+
## Dedup and identity
|
|
181
|
+
|
|
182
|
+
`id` is a stable 64-bit hash of `(source, session, request||timestamp, model, sequence)`.
|
|
183
|
+
|
|
184
|
+
Dedup is **structural**, not probabilistic: ingest resumes at a per-file byte offset, so a byte is
|
|
185
|
+
never read twice and a record cannot be ingested twice. That is why the Data Health page can state
|
|
186
|
+
`Duplicate records: 0` as a fact about the design rather than an estimate. Streaming duplicates
|
|
187
|
+
*within* a source file are a different problem, solved inside each adapter — see
|
|
188
|
+
[providers.md](providers.md).
|
|
189
|
+
|
|
190
|
+
## Storage layout
|
|
191
|
+
|
|
192
|
+
```
|
|
193
|
+
data/records/YYYY-MM.jsonl request-level facts, short-key codec, nulls omitted
|
|
194
|
+
data/cube.json pre-aggregated fact table (dims + measures), what the browser loads
|
|
195
|
+
data/sessions.json one row per session
|
|
196
|
+
data/activity.json daily work-activity rollup (commits, churn, AI edits)
|
|
197
|
+
data/state.json per-file {size, mtime, offset, gen} — the incremental engine
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
The cube's dimensions are `date, hour, dow, provider, model, model_family, client, interface,
|
|
201
|
+
gateway, project, repository, measurement`; its measures are the six token fields plus
|
|
202
|
+
`requests`, `cost`, `costMeasured`, `costReq` and the four not-available counters. It is
|
|
203
|
+
**additive**, which is what lets an incremental refresh update it without a rebuild.
|
|
204
|
+
|
|
205
|
+
Set `store.keepRaw: false` to skip the request-level shards entirely — the cube, sessions and the
|
|
206
|
+
whole dashboard still work; only the Data Explorer and full CSV export need the raw facts.
|
|
207
|
+
|
|
208
|
+
## Multi-user readiness
|
|
209
|
+
|
|
210
|
+
`user`, `machine` and `session_id` are first-class fields, and `project` / `repository` are
|
|
211
|
+
dimensions. A future team deployment can aggregate the same records across machines without a
|
|
212
|
+
schema change. V1 deliberately ships no server-side identity, auth or sync — but nothing here
|
|
213
|
+
makes those impossible to add later.
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# Getting started
|
|
2
|
+
|
|
3
|
+
## Requirements
|
|
4
|
+
|
|
5
|
+
- **Node 22.5 or newer.** 22.5 is where `node:sqlite` and the built-in test runner landed; both
|
|
6
|
+
are used. Check with `node -v`.
|
|
7
|
+
- Nothing else. This project has **zero runtime dependencies** — `npm install` does nothing and
|
|
8
|
+
is not required.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
**macOS, from a DMG:** download `TokenFlow-<version>.dmg` from the
|
|
13
|
+
[latest release](https://github.com/vimoxshah/tokenflow/releases/latest), open it, drag
|
|
14
|
+
**TokenFlow.app** to Applications, launch from Launchpad. The app builds and launches everything
|
|
15
|
+
it needs — refresh, watcher, dashboard. It is unsigned and not notarized, so macOS shows a
|
|
16
|
+
security warning on first launch:
|
|
17
|
+
|
|
18
|
+
1. If launch is blocked with *"Apple cannot check it for malicious software"*:
|
|
19
|
+
open **System Settings → Privacy & Security**, scroll to **Security**, click
|
|
20
|
+
**Open Anyway** next to the blocked-app message, then confirm **Open**.
|
|
21
|
+
2. Alternatively right-click `TokenFlow.app` → **Open** → **Open**.
|
|
22
|
+
|
|
23
|
+
This one-time approval is needed because the project has no paid Apple Developer signature;
|
|
24
|
+
the app runs entirely locally either way.
|
|
25
|
+
|
|
26
|
+
The DMG is produced by [`scripts/build-dmg.sh`](../scripts/build-dmg.sh) and attached to releases
|
|
27
|
+
automatically by [`.github/workflows/release.yml`](../.github/workflows/release.yml). Releases
|
|
28
|
+
also carry `tokenflow-dashboard-demo.html` — a fully offline demo dashboard you can open in any
|
|
29
|
+
browser without installing anything.
|
|
30
|
+
|
|
31
|
+
**From source (all platforms):**
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
git clone <this repo> tokenflow
|
|
35
|
+
cd tokenflow
|
|
36
|
+
node bin/tokenflow.js --help
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
To get a global `tokenflow` command:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
npm link # or: npm i -g .
|
|
43
|
+
tokenflow --help
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Everything below works either way; `npm run <script>` wrappers exist for `setup`, `refresh`,
|
|
47
|
+
`status`, `dashboard`, and `demo`. The bare `tokenflow <command>` examples throughout this doc
|
|
48
|
+
assume the global command from `npm link`; otherwise substitute
|
|
49
|
+
`node bin/tokenflow.js <command>` or the matching `npm run` script.
|
|
50
|
+
|
|
51
|
+
## 1. Look around first (optional)
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
npm run demo
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Generates deterministic synthetic data and opens the dashboard. Every demo record is flagged, and
|
|
58
|
+
the dashboard shows a red **DEMO DATA** banner for as long as any of it is in scope, so it can
|
|
59
|
+
never be mistaken for your own usage. To clear it later:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
tokenflow provider remove mock
|
|
63
|
+
tokenflow refresh --full
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## 2. Detect your tools
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
tokenflow setup
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
Tokenflow — setup
|
|
74
|
+
config home: /Users/you/.tokenflow
|
|
75
|
+
|
|
76
|
+
✓ Anthropic (Claude Code / Agent SDK) ~/.claude, ~/.claude-work
|
|
77
|
+
✓ OpenAI (Codex CLI / IDE / Desktop) /Users/you/.codex
|
|
78
|
+
✓ Cline CLI 22 session directories · no token fields reported
|
|
79
|
+
✓ Cursor (AI code activity) activity source — AI-authored edits + commits
|
|
80
|
+
○ Headroom gateway (overlay) No ~/.headroom/savings_events.jsonl found
|
|
81
|
+
○ Git activity (correlation) no repositories found — set sources.git.scanRoots
|
|
82
|
+
|
|
83
|
+
✓ wrote /Users/you/.tokenflow/config.yaml
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`setup` only writes config. It reads no usage data yet.
|
|
87
|
+
|
|
88
|
+
**Nothing detected?** That is a normal answer, not a failure. Either the tools store their logs
|
|
89
|
+
somewhere non-standard (point at it: `sources.<id>.paths`), or you want
|
|
90
|
+
[`tokenflow import`](cli.md#import) for an export file, or `tokenflow demo` to explore.
|
|
91
|
+
|
|
92
|
+
## 3. Ingest
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
tokenflow refresh
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
✓ anthropic 44,820 new records · 5,110 files read, 0 unchanged of 5,110 found · 1.4 GB
|
|
100
|
+
✓ openai 4,460 new records · 625 files read, 0 unchanged of 625 found · 3.8 GB
|
|
101
|
+
✓ cline 22 new records · 22 files read, 0 unchanged of 22 found · 92.2 KB
|
|
102
|
+
✓ cursor 12,480 new records
|
|
103
|
+
○ git no repositories found — set sources.git.repos or sources.git.scanRoots
|
|
104
|
+
|
|
105
|
+
61,782 new records · 5.2 GB read · 0 files skipped as unchanged · 29s
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Run it again and it costs almost nothing — unchanged files are skipped without being read:
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
0 new records · 0 B read · 5,757 files skipped as unchanged · 2s
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
If a first ingest is too big for the time you have, cap it and resume:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
tokenflow refresh --budget 30 # stop cleanly after ~30s
|
|
118
|
+
tokenflow refresh --budget 30 # continue exactly where it stopped
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## 4. Open the dashboard
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
tokenflow dashboard
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The server binds to `127.0.0.1` only. The browser downloads one pre-aggregated bundle and then
|
|
128
|
+
does every filter and aggregation locally, so filtering is instant and offline.
|
|
129
|
+
|
|
130
|
+
Click **↻ Refresh data** any time — it re-ingests, streams progress, and keeps your filters.
|
|
131
|
+
|
|
132
|
+
### The one-command version
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
npm start # = tokenflow up
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
On macOS you can double-click **`Refresh & Open Dashboard.command`** in the project folder
|
|
139
|
+
instead (it is a bash script, so on Windows stay with `npm start`). Either way it refreshes, rebuilds the offline `tokenflow-dashboard.html` beside it, and
|
|
140
|
+
opens the live dashboard. That is the thing to use when you come back after a few days.
|
|
141
|
+
|
|
142
|
+
The offline file, opened on its own, states how old its data is and looks for a live dashboard on
|
|
143
|
+
loopback; if it finds one it offers to hand over and refresh there. To keep the file current
|
|
144
|
+
without thinking about it, schedule `tokenflow up --no-serve` (see
|
|
145
|
+
[cli.md § up](cli.md#up-alias-open) for a ready-made launchd plist and cron line).
|
|
146
|
+
|
|
147
|
+
## 5. Pick a look (optional)
|
|
148
|
+
|
|
149
|
+
Three skins, each with a dark and a light mode. Switch from the header (**◑ Aurora**), or set the
|
|
150
|
+
default:
|
|
151
|
+
|
|
152
|
+
```yaml
|
|
153
|
+
# ~/.tokenflow/config.yaml
|
|
154
|
+
ui:
|
|
155
|
+
skin: aurora # aurora | terminal | editorial
|
|
156
|
+
mode: dark # dark | light
|
|
157
|
+
port: 7799
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Series colours belong to the mode, not the skin, and were validated for colour-blind separation
|
|
161
|
+
against every skin's chart surface — so changing the look never changes what a colour means.
|
|
162
|
+
|
|
163
|
+
Every field is documented in **[configuration.md](configuration.md)** — that is the page to open
|
|
164
|
+
when you want to change something and are not sure what it affects.
|
|
165
|
+
|
|
166
|
+
## 6. Set the default window (optional)
|
|
167
|
+
|
|
168
|
+
If your store contains older records you don't normally want in scope:
|
|
169
|
+
|
|
170
|
+
```yaml
|
|
171
|
+
# ~/.tokenflow/config.yaml
|
|
172
|
+
ui:
|
|
173
|
+
defaultRange: all # all | 7d | 30d | 90d | mtd
|
|
174
|
+
defaultFrom: "2026-03-14"
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
`defaultFrom` is a floor for the default view, not a filter on the data — everything stays in the
|
|
178
|
+
store and in `--all` exports.
|
|
179
|
+
|
|
180
|
+
## 7. Turn on cost (optional)
|
|
181
|
+
|
|
182
|
+
Cost is blank until a model has a price. See what is missing:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
tokenflow pricing
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
```
|
|
189
|
+
MODEL TOKENS STATUS
|
|
190
|
+
claude-opus-5 6.35B no price
|
|
191
|
+
gpt-5.6-luna 1.71B no price
|
|
192
|
+
claude-3-5-sonnet-20241022 88.1M priced $264.30 est.
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Add rates in USD per million tokens — `input,output[,cacheRead[,cacheWrite]]`:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
tokenflow pricing --set "claude-opus-5=15,75,1.5,18.75"
|
|
199
|
+
tokenflow refresh --full # re-cost the history
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Or use the **Pricing** dialog in the dashboard, which lists your models by volume and saves +
|
|
203
|
+
re-costs in one step. Anything you leave blank stays unpriced — the dashboard shows "no price"
|
|
204
|
+
rather than a plausible-looking `$0`.
|
|
205
|
+
|
|
206
|
+
## 8. Correlate with your actual work (optional)
|
|
207
|
+
|
|
208
|
+
```yaml
|
|
209
|
+
sources:
|
|
210
|
+
git:
|
|
211
|
+
scanRoots: ["~/code", "~/work"]
|
|
212
|
+
autoFromUsage: true # also use the working directories seen in usage records
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
tokenflow provider add git && tokenflow refresh
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
The Productivity page will then correlate daily AI usage with commits, files changed and line
|
|
220
|
+
churn — over the overlapping days only, with `n` reported, and labelled as a correlation. With
|
|
221
|
+
fewer than 10 overlapping days it tells you that instead of showing a number.
|
|
222
|
+
|
|
223
|
+
## 9. Export
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
tokenflow export --csv # the current filter
|
|
227
|
+
tokenflow export --csv --all # everything
|
|
228
|
+
tokenflow export --html # one self-contained offline dashboard file
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
## Where everything lives
|
|
232
|
+
|
|
233
|
+
```
|
|
234
|
+
~/.tokenflow/
|
|
235
|
+
config.yaml providers, source paths, preferences
|
|
236
|
+
pricing.json your rate overrides
|
|
237
|
+
mappings/ saved generic-import field mappings
|
|
238
|
+
providers/ your own adapters (*.js), loaded automatically
|
|
239
|
+
data/
|
|
240
|
+
records/ YYYY-MM.jsonl — request-level facts
|
|
241
|
+
cube.json the pre-aggregated table the dashboard loads
|
|
242
|
+
sessions.json one row per session
|
|
243
|
+
activity.json daily work-activity rollup
|
|
244
|
+
state.json per-file ingest offsets (this is what makes refresh incremental)
|
|
245
|
+
cache/
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Move it with `TOKENFLOW_HOME=/some/path tokenflow ...`, back it up with
|
|
249
|
+
`tokenflow config export`, and restore with `tokenflow config import`.
|
|
250
|
+
|
|
251
|
+
## Verify the install
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
npm run validate
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Checks the runtime, the config, every adapter's `detect()`, and — importantly — that the
|
|
258
|
+
aggregates still agree with the stored records. If they have drifted, it tells you to run
|
|
259
|
+
`tokenflow compact`.
|
|
260
|
+
|
|
261
|
+
## Next
|
|
262
|
+
|
|
263
|
+
- Numbers look wrong? → [troubleshooting.md](troubleshooting.md)
|
|
264
|
+
- Want another source? → [providers.md](providers.md), [creating-provider.md](creating-provider.md)
|
|
265
|
+
- Want an agent to set this up on a new machine? → [skill.md](skill.md)
|
|
266
|
+
- Changing a setting? → [configuration.md](configuration.md)
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Live mode — the watcher, the native menu bar app, capacity and alerts
|
|
2
|
+
|
|
3
|
+
Everything in this document runs **locally**. The watcher reads your logs, the
|
|
4
|
+
menu bar app reads one JSON file, notifications come from your operating
|
|
5
|
+
system. No network client exists anywhere in the codebase.
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
tokenflow watch ──every N s──▶ incremental refresh
|
|
9
|
+
│
|
|
10
|
+
▼
|
|
11
|
+
data/status.json ◀── read by ──▶ TokenFlow.app (native macOS menu bar)
|
|
12
|
+
│ tokenflow usage / cost / capacity / forecast
|
|
13
|
+
│ tokenflow status --bar
|
|
14
|
+
▼ dashboard header pill + Live tab
|
|
15
|
+
OS notifications (opt-in): limit crossings, same-day high-severity anomalies
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## The watcher
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
tokenflow watch # start; Ctrl+C stops it
|
|
22
|
+
tokenflow watch --interval 30
|
|
23
|
+
tokenflow watch --once # one cycle and exit — what cron wants
|
|
24
|
+
tokenflow watch --status # running? how fresh is the data?
|
|
25
|
+
tokenflow watch --stop # stop a running watcher
|
|
26
|
+
tokenflow watch --notify # enable OS notifications for this run
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Properties that matter:
|
|
30
|
+
|
|
31
|
+
- **Single instance.** A pidfile guards the store. A crashed run leaves a dead
|
|
32
|
+
pid behind; the next start detects it and takes over.
|
|
33
|
+
- **Failure isolation with backoff.** A failing refresh doubles the interval,
|
|
34
|
+
up to a 15-minute ceiling, and records `lastError` in the status file
|
|
35
|
+
instead of crashing or spamming.
|
|
36
|
+
- **Sleep/wake is free.** Each tick reschedules from wall-clock reality, so a
|
|
37
|
+
laptop that slept for three hours wakes to exactly one refresh, never a
|
|
38
|
+
catch-up burst.
|
|
39
|
+
- **Honest freshness.** Every surface recomputes "stale?" at *read* time from
|
|
40
|
+
`lastRefresh`, against `watch.staleAfterSeconds` (default 600). Old data
|
|
41
|
+
says so; nothing silently pretends to be real-time.
|
|
42
|
+
|
|
43
|
+
Defaults live in `config.yaml`:
|
|
44
|
+
|
|
45
|
+
```yaml
|
|
46
|
+
watch:
|
|
47
|
+
intervalSeconds: 120
|
|
48
|
+
notifications: false # or pass --notify per run
|
|
49
|
+
staleAfterSeconds: 600
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`tokenflow up` remains the pull-based path (`refresh → snapshot → open`); the
|
|
53
|
+
watcher is the push-based one. They share the same incremental engine and can
|
|
54
|
+
interleave safely — a manual refresh during a watched cycle simply coalesces.
|
|
55
|
+
|
|
56
|
+
## Capacity & budgets
|
|
57
|
+
|
|
58
|
+
TokenFlow **never invents vendor quota data** — there is no network client, so
|
|
59
|
+
no screen claims to know what Anthropic or OpenAI think your balance is.
|
|
60
|
+
Instead you declare caps, and the engine evaluates them against measured
|
|
61
|
+
consumption:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
limits:
|
|
65
|
+
- id: anthropic-monthly
|
|
66
|
+
provider: anthropic # optional filters: provider | model | project
|
|
67
|
+
scope: month # day | week | month (your local calendar)
|
|
68
|
+
metric: tokens # tokens | input | output | requests | cost
|
|
69
|
+
cap: 120000000 # tokens — or dollars when metric is cost
|
|
70
|
+
warnAt: 0.8 # optional: when "approaching" starts
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
For each limit the Live tab and `tokenflow capacity` derive:
|
|
74
|
+
|
|
75
|
+
| Field | Meaning |
|
|
76
|
+
|---|---|
|
|
77
|
+
| used / remaining | consumption inside the current window |
|
|
78
|
+
| % / bar | fraction of cap consumed |
|
|
79
|
+
| burn/hour | today's pace so far (local wall clock) |
|
|
80
|
+
| burn/day | trailing 7 calendar days, trimmed to dataset coverage |
|
|
81
|
+
| ETA | projected exhaustion at the faster of the two paces |
|
|
82
|
+
| resets in | deterministic countdown to local midnight / Monday / 1st |
|
|
83
|
+
|
|
84
|
+
Limits evaluate against the whole primary dataset regardless of dashboard
|
|
85
|
+
filters — quota windows are facts about your accounts, not filter states.
|
|
86
|
+
Invalid definitions are reported (`tokenflow capacity --json` → `invalid`),
|
|
87
|
+
never silently dropped. Edit them in the dashboard's **Live → Manage limits**
|
|
88
|
+
or by hand in `config.yaml`.
|
|
89
|
+
|
|
90
|
+
## Forecasting
|
|
91
|
+
|
|
92
|
+
`tokenflow forecast` fits a conservative linear trend over the last 14 days
|
|
93
|
+
(minimum 5), reports tomorrow's likely range, next-7-days and month-end totals
|
|
94
|
+
with cost, and states its confidence (high / medium / low) plus sample size.
|
|
95
|
+
Month-to-date is always shown separately from the projection. Thin or volatile
|
|
96
|
+
history degrades confidence honestly instead of producing confident nonsense.
|
|
97
|
+
|
|
98
|
+
## Anomalies
|
|
99
|
+
|
|
100
|
+
Detection uses robust statistics (median / MAD, Iglewicz–Hoaglin modified
|
|
101
|
+
z-score ≥ 3.5):
|
|
102
|
+
|
|
103
|
+
- token, cost and request-volume spikes (and weekday collapses)
|
|
104
|
+
- possible ingestion gaps — a zero weekday between two active days
|
|
105
|
+
- models/providers appearing for the first time this week
|
|
106
|
+
|
|
107
|
+
Every alert carries its own arithmetic — observed value, trailing median,
|
|
108
|
+
ratio, z-score — so you can check it rather than trust it. Notifications fire
|
|
109
|
+
only for **same-day high-severity** anomalies; history replaying as alerts on
|
|
110
|
+
a first run would be noise, so it doesn't happen.
|
|
111
|
+
|
|
112
|
+
## Menu bar — TokenFlow.app (native, macOS)
|
|
113
|
+
|
|
114
|
+
TokenFlow ships its own menu bar application: a ~370 KB native binary compiled
|
|
115
|
+
on your machine from [`menubar/TokenFlow/main.swift`](../menubar/TokenFlow/main.swift)
|
|
116
|
+
with `swiftc` (Xcode Command Line Tools). No Electron, no third-party bar, no
|
|
117
|
+
network. It reads `data/status.json` every 5 seconds and rebuilds its dropdown
|
|
118
|
+
every time it opens.
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
tokenflow menubar --app # build → ~/Applications/TokenFlow.app → launch
|
|
122
|
+
tokenflow menubar --app --login-item
|
|
123
|
+
# …and start it on every login
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
**Status item (adaptive, most-urgent signal wins):**
|
|
127
|
+
|
|
128
|
+
| Situation | Menu bar shows |
|
|
129
|
+
|---|---|
|
|
130
|
+
| a limit exceeded | `✗ 105%` in red |
|
|
131
|
+
| a limit approaching | `▲ 82%` in orange |
|
|
132
|
+
| healthy limit | `● 42%` in green |
|
|
133
|
+
| no limits, priced usage | `$4.83` |
|
|
134
|
+
| unpriced day | today's tokens |
|
|
135
|
+
|
|
136
|
+
**The dropdown contains everything — no browser hop required:**
|
|
137
|
+
|
|
138
|
+
- header: live/watcher badge + data freshness ("live · data updated just now")
|
|
139
|
+
- Today / Week / Month rows — tokens, requests, estimated cost
|
|
140
|
+
- Today by provider — inline share bars (▰▱), tokens and cost per provider
|
|
141
|
+
- Today by source — the app that wrote the log (claude-code, opencode,
|
|
142
|
+
hermes, git…). Provider attribution names the model's vendor, so Hermes
|
|
143
|
+
traffic appears under each model's vendor there; this section shows it as
|
|
144
|
+
"hermes"
|
|
145
|
+
- Top models today — token and cost leaders per model
|
|
146
|
+
- Capacity — a real meter per declared limit with %, exhaustion ETA and reset
|
|
147
|
+
countdown; "first projected hit" callout when one will cross before reset
|
|
148
|
+
- Forecast — tomorrow / 7-day projections, month-end spend, confidence
|
|
149
|
+
- Alerts — high-severity anomalies with their arithmetic
|
|
150
|
+
- Actions — `Refresh now` (⌘R, runs a watch cycle), `Open Dashboard`,
|
|
151
|
+
`Start/Stop watcher`, `Quit`
|
|
152
|
+
- Appearance button (◐) — cycles system → light → dark; persisted across
|
|
153
|
+
launches in `defaults` under `appearanceOverride`
|
|
154
|
+
|
|
155
|
+
Dark/light follows the appearance override (◐ button, persisted) or the system
|
|
156
|
+
appearance when set to follow; all figures use monospaced digits.
|
|
157
|
+
|
|
158
|
+
## Menu bar — other platforms (SwiftBar/xbar text protocol)
|
|
159
|
+
|
|
160
|
+
For Linux bars or if you prefer SwiftBar on macOS:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
tokenflow menubar --swiftbar # install for SwiftBar (~/Library/Plugins)
|
|
164
|
+
tokenflow menubar --xbar # xbar's plugin directory instead
|
|
165
|
+
tokenflow menubar --out <dir> # any compatible bar
|
|
166
|
+
tokenflow menubar --render # print the text protocol (debug)
|
|
167
|
+
tokenflow menubar --mode cost # auto | tokens | cost | limit
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The generated script shells to `tokenflow menubar --render` every couple of
|
|
171
|
+
minutes (filename convention). The title adapts automatically — the most
|
|
172
|
+
urgent signal wins:
|
|
173
|
+
|
|
174
|
+
| Situation | Title shows |
|
|
175
|
+
|---|---|
|
|
176
|
+
| a limit approaching | `TF ⚠ anthropic-monthly 82% · 4h 12m` |
|
|
177
|
+
| a limit exceeded | `TF ✗ anthropic-monthly 105% · resets 3d` |
|
|
178
|
+
| no limits, priced usage | `TF $4.83` |
|
|
179
|
+
| unpriced day, week has data | `TF 7d 5.04B` |
|
|
180
|
+
|
|
181
|
+
Any bar speaking the same text protocol (Argos, Waybar custom modules) works
|
|
182
|
+
via `--out`.
|
|
183
|
+
|
|
184
|
+
The SwiftBar dropdown carries the same breakdowns as the native popover:
|
|
185
|
+
today/week/month totals, **by source** (claude-code, hermes, opencode… —
|
|
186
|
+
provider rows name each model's vendor, so this is where Hermes appears under
|
|
187
|
+
its own name), **top models**, limits, forecast and alerts. Appearance is the
|
|
188
|
+
one thing a text-protocol plugin cannot control: SwiftBar renders plain text,
|
|
189
|
+
so its light/dark look follows SwiftBar's own settings — use the native app's
|
|
190
|
+
◐ button for an in-app toggle.
|
|
191
|
+
|
|
192
|
+
## Data flow guarantees
|
|
193
|
+
|
|
194
|
+
- `data/status.json` is written atomically (tmp + rename). A reader never sees
|
|
195
|
+
a half-file; a corrupt file degrades to "recompute fresh".
|
|
196
|
+
- Estimated and measured cost stay separate everywhere, including the bar line
|
|
197
|
+
and the dropdown.
|
|
198
|
+
- A one-shot cycle (`--once`) never leaves the file claiming a watcher is
|
|
199
|
+
running; liveness is verified against the pid, not assumed.
|