@vimoxshah/tokenflow 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +84 -0
- package/LICENSE +21 -0
- package/README.md +250 -0
- package/Refresh & Open Dashboard.command +22 -0
- package/SECURITY.md +42 -0
- package/bin/tokenflow.js +1342 -0
- package/docs/architecture.md +193 -0
- package/docs/cli.md +390 -0
- package/docs/configuration.md +281 -0
- package/docs/creating-provider.md +262 -0
- package/docs/data-model.md +213 -0
- package/docs/getting-started.md +266 -0
- package/docs/live-mode.md +199 -0
- package/docs/media/architecture-hero.svg +86 -0
- package/docs/media/cost-editorial-dark.png +0 -0
- package/docs/media/health-terminal-light.png +0 -0
- package/docs/media/menubar-dark.png +0 -0
- package/docs/media/menubar-light.png +0 -0
- package/docs/media/models-terminal-dark.png +0 -0
- package/docs/media/overview-aurora-dark.png +0 -0
- package/docs/media/time-aurora-light.png +0 -0
- package/docs/providers.md +309 -0
- package/docs/skill.md +64 -0
- package/docs/troubleshooting.md +207 -0
- package/examples/config.example.yaml +92 -0
- package/examples/demo-data/README.md +38 -0
- package/examples/demo-data/sample-usage.csv +11 -0
- package/package.json +74 -0
- package/scripts/build-dmg.sh +33 -0
- package/scripts/build-menubar-app.sh +67 -0
- package/scripts/lint.js +111 -0
- package/scripts/validate-install.js +140 -0
- package/skills/tokenflow/SKILL.md +392 -0
- package/skills/tokenflow/examples/config.yaml +92 -0
- package/skills/tokenflow/examples/generic-mapping.json +26 -0
- package/skills/tokenflow/examples/session-transcript.md +191 -0
- package/skills/tokenflow/providers/adapter-template.js +135 -0
- package/skills/tokenflow/providers/detection-matrix.md +142 -0
- package/skills/tokenflow/schemas/config.schema.json +107 -0
- package/skills/tokenflow/schemas/normalized-record.json +63 -0
- package/src/analytics/aggregate.js +247 -0
- package/src/analytics/anomalies.js +222 -0
- package/src/analytics/capacity.js +278 -0
- package/src/analytics/comparison.js +96 -0
- package/src/analytics/dimensions.js +230 -0
- package/src/analytics/efficiency.js +138 -0
- package/src/analytics/forecast.js +202 -0
- package/src/analytics/index.js +327 -0
- package/src/analytics/insights.js +283 -0
- package/src/analytics/milestones.js +91 -0
- package/src/analytics/peak.js +106 -0
- package/src/analytics/productivity.js +166 -0
- package/src/analytics/token-usage.js +267 -0
- package/src/commands/diagnostics.js +88 -0
- package/src/commands/digest.js +155 -0
- package/src/commands/models-compare.js +96 -0
- package/src/core/budget.js +142 -0
- package/src/core/bundle.js +191 -0
- package/src/core/config.js +202 -0
- package/src/core/delivery.js +109 -0
- package/src/core/geo.js +99 -0
- package/src/core/ingest.js +457 -0
- package/src/core/interface-map.js +55 -0
- package/src/core/jsonl.js +124 -0
- package/src/core/live-status.js +417 -0
- package/src/core/model-map.js +157 -0
- package/src/core/notify.js +83 -0
- package/src/core/pricing.js +288 -0
- package/src/core/prompt-analytics.js +127 -0
- package/src/core/registry.js +107 -0
- package/src/core/restore.js +261 -0
- package/src/core/schedule.js +120 -0
- package/src/core/schema.js +316 -0
- package/src/core/sqlite.js +96 -0
- package/src/core/store.js +493 -0
- package/src/core/sync.js +151 -0
- package/src/core/units.js +147 -0
- package/src/core/validate.js +123 -0
- package/src/core/watch.js +287 -0
- package/src/core/yaml.js +209 -0
- package/src/export/bundler.js +107 -0
- package/src/export/csv.js +100 -0
- package/src/export/html-snapshot.js +101 -0
- package/src/export/menubar.js +158 -0
- package/src/index.js +18 -0
- package/src/providers/anthropic/index.js +294 -0
- package/src/providers/cline/index.js +120 -0
- package/src/providers/cursor/index.js +143 -0
- package/src/providers/generic/index.js +268 -0
- package/src/providers/git/index.js +188 -0
- package/src/providers/headroom/index.js +114 -0
- package/src/providers/hermes/index.js +299 -0
- package/src/providers/mock/index.js +117 -0
- package/src/providers/openai/index.js +370 -0
- package/src/providers/opencode/index.js +245 -0
- package/src/sdk.js +46 -0
- package/src/server/server.js +264 -0
- package/src/ui/app.js +2473 -0
- package/src/ui/charts.js +925 -0
- package/src/ui/index.html +42 -0
- package/src/ui/styles.css +644 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
# Troubleshooting
|
|
2
|
+
|
|
3
|
+
Start here:
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
tokenflow doctor # runtime, paths, store, every adapter's detection status
|
|
7
|
+
npm run validate # incl. whether the aggregates still agree with the stored records
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## "No usage sources found"
|
|
13
|
+
|
|
14
|
+
That is a normal answer, not a failure. Check where your tool actually keeps its logs, then point
|
|
15
|
+
at it:
|
|
16
|
+
|
|
17
|
+
```yaml
|
|
18
|
+
# ~/.tokenflow/config.yaml
|
|
19
|
+
sources:
|
|
20
|
+
anthropic:
|
|
21
|
+
paths: ["/custom/claude/home", "~/.claude-work"]
|
|
22
|
+
openai:
|
|
23
|
+
paths: ["/custom/codex/home"]
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
An adapter is only "available" if the specific directory or file it names exists. `tokenflow
|
|
27
|
+
providers` prints the exact reason for each one.
|
|
28
|
+
|
|
29
|
+
No supported tool at all? Either `tokenflow import <file>` an export, or `tokenflow demo` to explore.
|
|
30
|
+
|
|
31
|
+
## "node:sqlite is unavailable"
|
|
32
|
+
|
|
33
|
+
You are on Node < 22.5. Upgrade Node; the SQLite-backed adapters (`cursor`, and `generic` with
|
|
34
|
+
`--format sqlite`) will start working. Everything else is unaffected.
|
|
35
|
+
|
|
36
|
+
## Refresh is slow
|
|
37
|
+
|
|
38
|
+
The first ingest reads everything once; after that it should be seconds. If it stays slow:
|
|
39
|
+
|
|
40
|
+
- `tokenflow doctor` → is `files tracked` growing every run? A source whose files are rewritten
|
|
41
|
+
rather than appended can't be resumed, and will be re-read each time.
|
|
42
|
+
- Network drives and FUSE mounts are the usual culprit. The store handles mounts that refuse
|
|
43
|
+
`rename`/`unlink`, but read throughput is what it is.
|
|
44
|
+
- Cap the work and resume: `tokenflow refresh --budget 30`, repeatedly.
|
|
45
|
+
|
|
46
|
+
## Refresh hit the time budget
|
|
47
|
+
|
|
48
|
+
Expected, by design. Progress was saved. Run it again — it continues at the byte offset where it
|
|
49
|
+
stopped:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
until node bin/tokenflow.js refresh --budget 30 --quiet | grep -q 'budget reached'; do :; done
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## The numbers look too high
|
|
56
|
+
|
|
57
|
+
This is the failure mode worth being suspicious about, and there are three real causes.
|
|
58
|
+
|
|
59
|
+
**Streaming logs re-report usage.** Both shipped adapters handle it, but a custom adapter probably
|
|
60
|
+
doesn't. Symptom: one day, or one session, dwarfs everything else by 10–100x. Check
|
|
61
|
+
`metadata.token_count_events` / `metadata.usage_segments` in the Data Explorer — a record built
|
|
62
|
+
from hundreds of events is a reconstruction, and if your adapter summed them it inflated them.
|
|
63
|
+
See [creating-provider.md](creating-provider.md#5-watch-for-re-reported-usage).
|
|
64
|
+
|
|
65
|
+
**Overlay double counting.** If "Include gateway overlay" is on, proxy records are being added to
|
|
66
|
+
client records for the same traffic. Turn it off (it is off by default) — overlay exists for the
|
|
67
|
+
measured *cost*, not for tokens.
|
|
68
|
+
|
|
69
|
+
**Cache read counted as input.** If your adapter passed a vendor's inclusive `input_tokens`
|
|
70
|
+
straight through, every cached prompt token is counted twice. `input + cache_read` should be the
|
|
71
|
+
prompt side, with no overlap.
|
|
72
|
+
|
|
73
|
+
Sanity check any suspicious day against the raw source:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
tokenflow export --csv --from 2026-08-15 --to 2026-08-15
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
…then compare with whatever independent record you have (a gateway log, a billing page). Two
|
|
80
|
+
sources agreeing is worth more than one source being plausible.
|
|
81
|
+
|
|
82
|
+
## The numbers look too low
|
|
83
|
+
|
|
84
|
+
- **Active days** counts days with *measured tokens*. Days where only a no-token source (Cline,
|
|
85
|
+
Cursor, git) was active are reported separately as "activity-only" — that is why the count can
|
|
86
|
+
be lower than you expect.
|
|
87
|
+
- Check the **date filter**. `ui.defaultFrom` sets a floor for the default view; the data is still
|
|
88
|
+
there. Click "Since …" or set an explicit range.
|
|
89
|
+
- Check **Data health → Sources**: each adapter's real coverage window is listed. A source that
|
|
90
|
+
only started logging in July does not cover March.
|
|
91
|
+
- Many tools prune their own logs (Claude Code defaults to ~30 days). Nothing can recover data the
|
|
92
|
+
source already deleted.
|
|
93
|
+
|
|
94
|
+
## Cost shows nothing / "no price"
|
|
95
|
+
|
|
96
|
+
By design: no rate in the table, no number. See what is missing and fix it:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
tokenflow pricing
|
|
100
|
+
tokenflow pricing --set "your-model=15,75,1.5,18.75"
|
|
101
|
+
tokenflow refresh --full
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Newly released models are the usual case — the built-in table only carries rates that were
|
|
105
|
+
published when it was built (its version is shown in the dashboard footer).
|
|
106
|
+
|
|
107
|
+
## Cost coverage is under 100%
|
|
108
|
+
|
|
109
|
+
Expected when some models are priced and others aren't. The Cost page states the covered share and
|
|
110
|
+
lists the unpriced models by volume. The **estimated** total covers only priced requests; the
|
|
111
|
+
**measured** figure, if present, covers only gateway-routed traffic. They are shown separately
|
|
112
|
+
because they measure different things.
|
|
113
|
+
|
|
114
|
+
## "Data health: Good" instead of "Excellent"
|
|
115
|
+
|
|
116
|
+
The grade is driven by the share of token fields the *sources* didn't report. A dataset containing
|
|
117
|
+
Cline or Cursor records will never be "Excellent" — those sources report no tokens at all. Check
|
|
118
|
+
**Data health → Field availability** for the per-field breakdown.
|
|
119
|
+
|
|
120
|
+
## Interface shows "Unknown"
|
|
121
|
+
|
|
122
|
+
The source record had no surface field. Interface is deliberately never inferred from the model.
|
|
123
|
+
If you know what a client is, say so:
|
|
124
|
+
|
|
125
|
+
```yaml
|
|
126
|
+
interfaceOverrides:
|
|
127
|
+
my-custom-client: "API"
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## The dashboard is empty but `status` shows data
|
|
131
|
+
|
|
132
|
+
Almost always the date filter. Click **Since …**, or clear filters via the breadcrumb. The
|
|
133
|
+
breadcrumb above the tabs always lists every active filter.
|
|
134
|
+
|
|
135
|
+
## "a refresh is already running"
|
|
136
|
+
|
|
137
|
+
One refresh at a time, so two writers can't interleave into the store. Wait for it, or restart the
|
|
138
|
+
server.
|
|
139
|
+
|
|
140
|
+
## Aggregates disagree with the records
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
npm run validate # reports drift between the cube and the stored facts
|
|
144
|
+
tokenflow compact # rewrites shards without superseded records and rebuilds
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
This can happen if a process was killed mid-refresh. `compact` is always safe to run.
|
|
148
|
+
|
|
149
|
+
## Port already in use
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
tokenflow dashboard --port 8080
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
## `refresh --full` says it is "refusing a full re-ingest"
|
|
156
|
+
|
|
157
|
+
The command was about to drop stored records belonging to a source whose logs it cannot read
|
|
158
|
+
right now, so it stopped and told you how many and why. This is the guard working, not a
|
|
159
|
+
failure. Usual causes:
|
|
160
|
+
|
|
161
|
+
- you are running from a different machine, container or sandbox than the one that ingested
|
|
162
|
+
- a source path in `config.yaml` moved (`~/.claude`, `~/.codex`, a Cursor DB)
|
|
163
|
+
- the log directory was pruned or archived
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
tokenflow doctor # which paths resolve, which adapters detect
|
|
167
|
+
tokenflow refresh # incremental is always safe
|
|
168
|
+
tokenflow restore export.csv --yes # rebuild from a full export instead
|
|
169
|
+
tokenflow refresh --full --force # only if you really mean "discard those records"
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## I moved to a new machine / the logs are gone
|
|
173
|
+
|
|
174
|
+
Export on the machine that has the logs, restore on the one that does not. The CSV carries
|
|
175
|
+
counts and identifiers only — prompts, conversations and code stay put.
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
# old machine
|
|
179
|
+
tokenflow export --csv --all
|
|
180
|
+
# new machine
|
|
181
|
+
tokenflow restore tokenflow-usage-2026-08-20.csv --yes
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Estimated costs are recomputed with the current price table on restore (`--no-reprice` to
|
|
185
|
+
keep them as exported); measured costs are preserved. Per-record `metadata` is not part of a
|
|
186
|
+
CSV export and is not restored. If the real logs later become reachable again, just
|
|
187
|
+
`refresh` — the restored slice is superseded automatically rather than double counted.
|
|
188
|
+
|
|
189
|
+
## I want to start over
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
tokenflow reset --yes # deletes ingested data, keeps config, pricing and mappings
|
|
193
|
+
tokenflow refresh
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Or delete the whole thing: `rm -rf ~/.tokenflow`. Nothing lives anywhere else.
|
|
197
|
+
|
|
198
|
+
## Something is genuinely broken
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
node bin/tokenflow.js doctor
|
|
202
|
+
node bin/tokenflow.js refresh --strict --debug --provider <suspect>
|
|
203
|
+
npm test
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
`--strict` validates every record as it is written and prints the first failures with the adapter
|
|
207
|
+
that produced them, which is usually enough to localise the problem to one adapter and one field.
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# tokenflow — a fully worked configuration
|
|
2
|
+
# Lives at ~/.tokenflow/config.yaml (or $TOKENFLOW_HOME/config.yaml).
|
|
3
|
+
# config.json is also accepted if you prefer JSON.
|
|
4
|
+
|
|
5
|
+
version: 1
|
|
6
|
+
|
|
7
|
+
# Dates and hours are resolved in this zone AT INGEST and then stored, so "my
|
|
8
|
+
# peak hour" means your local peak hour. null = this machine's zone.
|
|
9
|
+
timezone: Asia/Kolkata
|
|
10
|
+
|
|
11
|
+
# Stamped onto every record. Present so a future team deployment can aggregate
|
|
12
|
+
# across machines without a schema change; unused in single-user mode.
|
|
13
|
+
identity:
|
|
14
|
+
user: vshah
|
|
15
|
+
machine: my-laptop
|
|
16
|
+
team: null
|
|
17
|
+
|
|
18
|
+
# Enabled adapters. An empty list means "everything detected".
|
|
19
|
+
providers:
|
|
20
|
+
- anthropic
|
|
21
|
+
- openai
|
|
22
|
+
- cline
|
|
23
|
+
- cursor
|
|
24
|
+
- headroom
|
|
25
|
+
- git
|
|
26
|
+
|
|
27
|
+
sources:
|
|
28
|
+
# Several Claude Code homes is normal — one per account. Listing them here
|
|
29
|
+
# skips auto-discovery entirely.
|
|
30
|
+
anthropic:
|
|
31
|
+
paths:
|
|
32
|
+
- "~/.claude"
|
|
33
|
+
- "~/.claude-work"
|
|
34
|
+
|
|
35
|
+
# $CODEX_HOME is respected; this overrides it.
|
|
36
|
+
openai:
|
|
37
|
+
paths:
|
|
38
|
+
- "~/.codex"
|
|
39
|
+
|
|
40
|
+
# Reports sessions but NO token counts, so every token field is null and the
|
|
41
|
+
# records are measurement:activity.
|
|
42
|
+
cline:
|
|
43
|
+
path: "~/.cline/data/sessions"
|
|
44
|
+
|
|
45
|
+
# SQLite; needs Node >= 22.5. Read from a temp snapshot so a live editor is
|
|
46
|
+
# never disturbed.
|
|
47
|
+
cursor:
|
|
48
|
+
db: "~/.cursor/ai-tracking/ai-code-tracking.db"
|
|
49
|
+
|
|
50
|
+
# A local gateway. measurement:overlay — excluded from token totals by
|
|
51
|
+
# default (it would double count the client adapter) but it carries the only
|
|
52
|
+
# MEASURED cost available anywhere.
|
|
53
|
+
headroom:
|
|
54
|
+
path: "~/.headroom"
|
|
55
|
+
|
|
56
|
+
# The independent work signal for the Productivity correlations.
|
|
57
|
+
git:
|
|
58
|
+
scanRoots:
|
|
59
|
+
- "~/code"
|
|
60
|
+
- "~/work"
|
|
61
|
+
autoFromUsage: true # also use cwds seen in ingested usage records
|
|
62
|
+
author: "you@example.com" # optional: only your own commits
|
|
63
|
+
since: "2026-01-01" # optional
|
|
64
|
+
|
|
65
|
+
store:
|
|
66
|
+
keepRaw: true # false still builds the cube; only the Data
|
|
67
|
+
rawRetentionDays: null # Explorer and full CSV export need the shards
|
|
68
|
+
|
|
69
|
+
analytics:
|
|
70
|
+
includeOverlaySources: false # leave false; see the headroom note above
|
|
71
|
+
minSessionGapMinutes: 30 # only used for sources with no session id
|
|
72
|
+
|
|
73
|
+
# Prepended to the built-in model -> vendor ruleset, so an internal alias maps
|
|
74
|
+
# to the right vendor instead of falling through to "unknown".
|
|
75
|
+
modelMappings:
|
|
76
|
+
- match: "^acme-llm-"
|
|
77
|
+
provider: acme
|
|
78
|
+
label: "Acme AI"
|
|
79
|
+
- match: "^internal-glm"
|
|
80
|
+
provider: zai
|
|
81
|
+
label: "Z.ai / GLM"
|
|
82
|
+
|
|
83
|
+
# Interface is never inferred from the model. If you know a client's surface and
|
|
84
|
+
# the source doesn't record one, say so here.
|
|
85
|
+
interfaceOverrides:
|
|
86
|
+
my-batch-runner: "API"
|
|
87
|
+
team-slackbot: "Web"
|
|
88
|
+
|
|
89
|
+
ui:
|
|
90
|
+
theme: dark
|
|
91
|
+
defaultRange: all
|
|
92
|
+
defaultFrom: "2026-03-14" # floor for the DEFAULT VIEW only
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Sample data
|
|
2
|
+
|
|
3
|
+
Two ways to get a populated dashboard without connecting anything real.
|
|
4
|
+
|
|
5
|
+
## 1. The mock provider (recommended)
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm run demo
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Deterministic synthetic usage across four providers, three clients and four interfaces, with
|
|
12
|
+
weekday seasonality, a mild upward trend, genuinely idle days and a bimodal daily rhythm — so the
|
|
13
|
+
time-pattern and trend views have something real-shaped to show. Same seed, same output.
|
|
14
|
+
|
|
15
|
+
Every record carries `metadata.demo = true` and `machine: "demo-machine"`, and the dashboard shows
|
|
16
|
+
a persistent **DEMO DATA** banner while any of it is in scope. Clear it with:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
tokenflow provider remove mock && tokenflow refresh --full
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## 2. The generic importer (exercises a different code path)
|
|
23
|
+
|
|
24
|
+
`sample-usage.csv` is a tiny hand-written export in the shape a third-party console tends to
|
|
25
|
+
produce. Importing it exercises the mapping layer rather than the generator:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
tokenflow import examples/demo-data/sample-usage.csv --dry-run # see the inferred mapping
|
|
29
|
+
tokenflow import examples/demo-data/sample-usage.csv
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Things it deliberately contains:
|
|
33
|
+
|
|
34
|
+
- an **empty** `cached_tokens` cell — must import as `null` (not available), never `0`
|
|
35
|
+
- a model with no configured price (`glm-4.6`) — must import with `estimated_cost: null`
|
|
36
|
+
- a quoted field containing a comma
|
|
37
|
+
- three vendors, so provider classification has something to do
|
|
38
|
+
- a `client` column, so interface classification has a real surface signal
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
created_at,model,client,surface,prompt_tokens,completion_tokens,cached_tokens,cost_usd,generation_id,project
|
|
2
|
+
2026-08-05T09:12:00Z,claude-3-5-sonnet-20241022,my-runner,api,12400,860,9800,0.0412,gen-0001,billing-service
|
|
3
|
+
2026-08-05T09:31:00Z,claude-3-5-sonnet-20241022,my-runner,api,13100,1240,11200,0.0455,gen-0002,billing-service
|
|
4
|
+
2026-08-05T11:02:00Z,gpt-4o,vscode-ext,vscode,4200,510,0,0.0156,gen-0003,web-app
|
|
5
|
+
2026-08-05T14:45:00Z,deepseek-chat,cli-tool,cli,8800,1900,6400,0.0045,gen-0004,data-pipeline
|
|
6
|
+
2026-08-06T10:05:00Z,gpt-4o,vscode-ext,vscode,5100,720,1200,0.0184,gen-0005,web-app
|
|
7
|
+
2026-08-06T10:40:00Z,glm-4.6,cli-tool,cli,9600,1400,,0.0031,gen-0006,"infra, terraform"
|
|
8
|
+
2026-08-06T16:20:00Z,claude-3-5-sonnet-20241022,my-runner,api,21000,2100,18400,0.0620,gen-0007,billing-service
|
|
9
|
+
2026-08-07T09:00:00Z,deepseek-chat,cli-tool,cli,3300,410,2100,0.0014,gen-0008,data-pipeline
|
|
10
|
+
2026-08-07T13:15:00Z,glm-4.6,cli-tool,cli,15200,2600,,0.0058,gen-0009,web-app
|
|
11
|
+
2026-08-07T20:55:00Z,claude-3-5-sonnet-20241022,my-runner,api,30500,3400,27000,0.0951,gen-0010,billing-service
|
package/package.json
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@vimoxshah/tokenflow",
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"description": "Local-first, provider-agnostic analytics for the AI tokens you actually spend. Zero dependencies, nothing leaves your machine.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"engines": {
|
|
8
|
+
"node": ">=22.5.0"
|
|
9
|
+
},
|
|
10
|
+
"bin": {
|
|
11
|
+
"tokenflow": "bin/tokenflow.js"
|
|
12
|
+
},
|
|
13
|
+
"main": "src/index.js",
|
|
14
|
+
"exports": {
|
|
15
|
+
".": "./src/index.js",
|
|
16
|
+
"./sdk": "./src/sdk.js"
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"bin",
|
|
20
|
+
"src",
|
|
21
|
+
"docs",
|
|
22
|
+
"skills",
|
|
23
|
+
"examples",
|
|
24
|
+
"scripts",
|
|
25
|
+
"README.md",
|
|
26
|
+
"LICENSE",
|
|
27
|
+
"SECURITY.md",
|
|
28
|
+
"CONTRIBUTING.md",
|
|
29
|
+
"Refresh & Open Dashboard.command"
|
|
30
|
+
],
|
|
31
|
+
"scripts": {
|
|
32
|
+
"setup": "node bin/tokenflow.js setup",
|
|
33
|
+
"dashboard": "node bin/tokenflow.js dashboard",
|
|
34
|
+
"refresh": "node bin/tokenflow.js refresh",
|
|
35
|
+
"status": "node bin/tokenflow.js status",
|
|
36
|
+
"demo": "node bin/tokenflow.js demo",
|
|
37
|
+
"test": "node --test \"test/**/*.test.js\"",
|
|
38
|
+
"typecheck": "tsc --noEmit -p jsconfig.json",
|
|
39
|
+
"typecheck:deps": "npm i --no-save --no-package-lock typescript@5 @types/node@22",
|
|
40
|
+
"lint": "node scripts/lint.js",
|
|
41
|
+
"validate": "node scripts/validate-install.js",
|
|
42
|
+
"start": "node bin/tokenflow.js up",
|
|
43
|
+
"up": "node bin/tokenflow.js up"
|
|
44
|
+
},
|
|
45
|
+
"keywords": [
|
|
46
|
+
"ai",
|
|
47
|
+
"tokens",
|
|
48
|
+
"usage",
|
|
49
|
+
"analytics",
|
|
50
|
+
"dashboard",
|
|
51
|
+
"claude",
|
|
52
|
+
"claude-code",
|
|
53
|
+
"openai",
|
|
54
|
+
"codex",
|
|
55
|
+
"cursor",
|
|
56
|
+
"cline",
|
|
57
|
+
"deepseek",
|
|
58
|
+
"observability",
|
|
59
|
+
"local-first",
|
|
60
|
+
"cost",
|
|
61
|
+
"llm"
|
|
62
|
+
],
|
|
63
|
+
"dependencies": {},
|
|
64
|
+
"devDependencies": {},
|
|
65
|
+
"repository": {
|
|
66
|
+
"type": "git",
|
|
67
|
+
"url": "git+https://github.com/vimoxshah/tokenflow.git"
|
|
68
|
+
},
|
|
69
|
+
"bugs": {
|
|
70
|
+
"url": "https://github.com/vimoxshah/tokenflow/issues"
|
|
71
|
+
},
|
|
72
|
+
"homepage": "https://github.com/vimoxshah/tokenflow#readme",
|
|
73
|
+
"author": "Vimox Shah"
|
|
74
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Package TokenFlow.app into a distributable DMG.
|
|
3
|
+
#
|
|
4
|
+
# scripts/build-dmg.sh [version] # → dist/TokenFlow-<version>.dmg
|
|
5
|
+
#
|
|
6
|
+
# Builds the app first (scripts/build-menubar-app.sh), then wraps it in a
|
|
7
|
+
# drag-to-Applications DMG. Requires macOS with hdiutil (built in) and
|
|
8
|
+
# Xcode Command Line Tools for swiftc. The DMG is unsigned — Gatekeeper
|
|
9
|
+
# right-click → Open on first launch, see docs/getting-started.md.
|
|
10
|
+
set -euo pipefail
|
|
11
|
+
|
|
12
|
+
REPO="$(cd "$(dirname "$0")/.." && pwd)"
|
|
13
|
+
VERSION="${1:-$(node -p "require('$REPO/package.json').version")}"
|
|
14
|
+
DIST="$REPO/dist"
|
|
15
|
+
APP="$DIST/TokenFlow.app"
|
|
16
|
+
DMG="$DIST/TokenFlow-$VERSION.dmg"
|
|
17
|
+
|
|
18
|
+
"$REPO/scripts/build-menubar-app.sh" "$DIST" "$VERSION" >/dev/null
|
|
19
|
+
echo "built TokenFlow.app"
|
|
20
|
+
|
|
21
|
+
STAGING="$(mktemp -d)"
|
|
22
|
+
trap 'rm -rf "$STAGING"' EXIT
|
|
23
|
+
|
|
24
|
+
cp -R "$APP" "$STAGING/"
|
|
25
|
+
ln -s /Applications "$STAGING/Applications"
|
|
26
|
+
|
|
27
|
+
rm -f "$DMG"
|
|
28
|
+
hdiutil create -volname "TokenFlow $VERSION" \
|
|
29
|
+
-srcfolder "$STAGING" \
|
|
30
|
+
-ov -format UDZO \
|
|
31
|
+
"$DMG" | tail -1
|
|
32
|
+
|
|
33
|
+
echo "wrote $DMG ($(du -h "$DMG" | cut -f1))"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Build TokenFlow.app — TokenFlow's native macOS menu bar application.
|
|
3
|
+
#
|
|
4
|
+
# scripts/build-menubar-app.sh [output-dir]
|
|
5
|
+
#
|
|
6
|
+
# Compiles menubar/TokenFlow/main.swift with swiftc (Xcode Command Line Tools)
|
|
7
|
+
# into a minimal .app bundle, embedding the absolute paths of this clone's
|
|
8
|
+
# node binary and CLI so the app can drive refresh/watch actions.
|
|
9
|
+
set -euo pipefail
|
|
10
|
+
|
|
11
|
+
REPO="$(cd "$(dirname "$0")/.." && pwd)"
|
|
12
|
+
SRC="$REPO/menubar/TokenFlow/main.swift"
|
|
13
|
+
OUT_DIR="${1:-$REPO/dist}"
|
|
14
|
+
APP="$OUT_DIR/TokenFlow.app"
|
|
15
|
+
# App version: first argument, else package.json version. Embedded into the
|
|
16
|
+
# bundle's Info.plist so release CI can verify tag ↔ bundle consistency.
|
|
17
|
+
VERSION="${2:-$(node -p "require('$REPO/package.json').version")}"
|
|
18
|
+
|
|
19
|
+
command -v swiftc >/dev/null 2>&1 || {
|
|
20
|
+
echo "error: swiftc not found — install Xcode Command Line Tools:" >&2
|
|
21
|
+
echo " xcode-select --install" >&2
|
|
22
|
+
exit 1
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
NODE_BIN="$(command -v node)"
|
|
26
|
+
CLI_JS="$REPO/bin/tokenflow.js"
|
|
27
|
+
[ -f "$CLI_JS" ] || { echo "error: $CLI_JS missing" >&2; exit 1; }
|
|
28
|
+
|
|
29
|
+
TMP="$(mktemp -d)"
|
|
30
|
+
trap 'rm -rf "$TMP"' EXIT
|
|
31
|
+
|
|
32
|
+
mkdir -p "$APP/Contents/MacOS"
|
|
33
|
+
mkdir -p "$APP/Contents/Resources"
|
|
34
|
+
cp "$REPO/menubar/TokenFlow/AppIcon.icns" "$APP/Contents/Resources/AppIcon.icns"
|
|
35
|
+
|
|
36
|
+
cat > "$APP/Contents/Info.plist" <<PLIST
|
|
37
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
38
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
39
|
+
<plist version="1.0">
|
|
40
|
+
<dict>
|
|
41
|
+
<key>CFBundleName</key> <string>TokenFlow</string>
|
|
42
|
+
<key>CFBundleDisplayName</key> <string>TokenFlow</string>
|
|
43
|
+
<key>CFBundleIdentifier</key> <string>app.tokenflow.bar</string>
|
|
44
|
+
<key>CFBundleVersion</key> <string>$VERSION</string>
|
|
45
|
+
<key>CFBundleShortVersionString</key> <string>$VERSION</string>
|
|
46
|
+
<key>CFBundlePackageType</key> <string>APPL</string>
|
|
47
|
+
<key>CFBundleExecutable</key> <string>TokenFlow</string>
|
|
48
|
+
<key>CFBundleIconFile</key> <string>AppIcon</string>
|
|
49
|
+
<key>LSMinimumSystemVersion</key> <string>13.0</string>
|
|
50
|
+
<key>LSUIElement</key> <true/>
|
|
51
|
+
<key>NSHighResolutionCapable</key> <true/>
|
|
52
|
+
<key>NSHumanReadableCopyright</key> <string>MIT — local-first, nothing leaves your machine.</string>
|
|
53
|
+
<key>TokenFlowNodePath</key> <string>$NODE_BIN</string>
|
|
54
|
+
<key>TokenFlowCLIPath</key> <string>$CLI_JS</string>
|
|
55
|
+
</dict>
|
|
56
|
+
</plist>
|
|
57
|
+
PLIST
|
|
58
|
+
|
|
59
|
+
echo "compiling with $(swiftc --version | head -1)"
|
|
60
|
+
swiftc -O -swift-version 5 \
|
|
61
|
+
-o "$APP/Contents/MacOS/TokenFlow" \
|
|
62
|
+
"$SRC" 2>&1 | head -40
|
|
63
|
+
|
|
64
|
+
codesign --force --sign - "$APP" >/dev/null 2>&1 || true
|
|
65
|
+
|
|
66
|
+
SIZE=$(du -h "$APP" | cut -f1 | tr -d ' ')
|
|
67
|
+
echo "built: $APP ($SIZE)"
|
package/scripts/lint.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* A dependency-free linter for this project's own invariants.
|
|
4
|
+
*
|
|
5
|
+
* It is deliberately small and opinionated: it checks the things that would
|
|
6
|
+
* actually break the product (a syntax error, a `|| 0` on a token field, an
|
|
7
|
+
* `innerHTML` built from data, an unguarded localStorage call) rather than
|
|
8
|
+
* re-litigating formatting.
|
|
9
|
+
*/
|
|
10
|
+
import fs from 'node:fs';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import url from 'node:url';
|
|
13
|
+
import { execFileSync } from 'node:child_process';
|
|
14
|
+
|
|
15
|
+
const ROOT = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), '..');
|
|
16
|
+
const problems = [];
|
|
17
|
+
const warnings = [];
|
|
18
|
+
|
|
19
|
+
function walk(dir, out = []) {
|
|
20
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
21
|
+
if (e.name === 'node_modules' || e.name.startsWith('.')) continue;
|
|
22
|
+
const p = path.join(dir, e.name);
|
|
23
|
+
if (e.isDirectory()) walk(p, out);
|
|
24
|
+
else if (e.name.endsWith('.js')) out.push(p);
|
|
25
|
+
}
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const files = [
|
|
30
|
+
...walk(path.join(ROOT, 'src')),
|
|
31
|
+
...walk(path.join(ROOT, 'bin')),
|
|
32
|
+
...walk(path.join(ROOT, 'scripts')),
|
|
33
|
+
...walk(path.join(ROOT, 'test')),
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
const TOKEN_FIELDS = /(input_tokens|output_tokens|cache_read_tokens|cache_write_tokens|cache_refresh_tokens|reasoning_tokens)/;
|
|
37
|
+
|
|
38
|
+
const SELF = path.relative(ROOT, url.fileURLToPath(import.meta.url));
|
|
39
|
+
|
|
40
|
+
for (const f of files) {
|
|
41
|
+
const rel = path.relative(ROOT, f);
|
|
42
|
+
const src = fs.readFileSync(f, 'utf8');
|
|
43
|
+
// The linter's own rule patterns would otherwise trip its own rules.
|
|
44
|
+
const isSelf = rel === SELF || rel.endsWith('scripts/lint.js');
|
|
45
|
+
|
|
46
|
+
// 1. it must parse
|
|
47
|
+
try {
|
|
48
|
+
execFileSync(process.execPath, ['--check', f], { stdio: 'pipe' });
|
|
49
|
+
} catch (err) {
|
|
50
|
+
problems.push(`${rel}: syntax error\n${String(err.stderr || err.message).split('\n').slice(0, 4).join('\n')}`);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const lines = src.split('\n');
|
|
55
|
+
lines.forEach((line, i) => {
|
|
56
|
+
const at = `${rel}:${i + 1}`;
|
|
57
|
+
if (isSelf && i > 30) return;
|
|
58
|
+
if (/\t/.test(line)) problems.push(`${at}: tab character`);
|
|
59
|
+
if (/[ \t]+$/.test(line)) problems.push(`${at}: trailing whitespace`);
|
|
60
|
+
|
|
61
|
+
// 2. the null contract: a token field must never be coerced to 0
|
|
62
|
+
if (TOKEN_FIELDS.test(line) && /\|\|\s*0\b/.test(line) && !/^\s*\*/.test(line)) {
|
|
63
|
+
problems.push(`${at}: "|| 0" on a token field — a missing value must stay null`);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// 3. untrusted labels must never be concatenated into HTML
|
|
67
|
+
if (/\.innerHTML\s*(\+?=)/.test(line) && /[`+]/.test(line.split('innerHTML')[1] || '')) {
|
|
68
|
+
if (!/html:/.test(line)) problems.push(`${at}: innerHTML built from an expression — use textContent`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// 4. browser storage must be guarded, because a snapshot may run from file://
|
|
72
|
+
if (/localStorage|sessionStorage/.test(line) && !/^\s*(\/\/|\*)/.test(line)) {
|
|
73
|
+
const window = lines.slice(Math.max(0, i - 6), i + 6).join('\n');
|
|
74
|
+
if (!/try\s*\{/.test(window)) problems.push(`${at}: unguarded ${/local/.test(line) ? 'localStorage' : 'sessionStorage'} — wrap in try/catch`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// 5. no stray debugging
|
|
78
|
+
if (/\bdebugger\b/.test(line)) problems.push(`${at}: stray debugger statement`);
|
|
79
|
+
if (/console\.(log|debug)\(/.test(line) && rel.startsWith('src/') && !rel.startsWith('src/ui/')) {
|
|
80
|
+
warnings.push(`${at}: console.${/debug/.test(line) ? 'debug' : 'log'} in library code`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// 6. an empty catch should say why it is empty
|
|
84
|
+
if (/catch\s*(\([^)]*\))?\s*\{\s*\}\s*$/.test(line)) {
|
|
85
|
+
problems.push(`${at}: silent empty catch — add a comment explaining what is being ignored`);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
if (!src.endsWith('\n')) problems.push(`${rel}: missing trailing newline`);
|
|
90
|
+
if (src.includes('\r\n')) problems.push(`${rel}: CRLF line endings`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// 7. every provider directory must export a valid provider
|
|
94
|
+
const provDir = path.join(ROOT, 'src', 'providers');
|
|
95
|
+
for (const d of fs.readdirSync(provDir)) {
|
|
96
|
+
const idx = path.join(provDir, d, 'index.js');
|
|
97
|
+
if (!fs.existsSync(idx)) problems.push(`src/providers/${d}: missing index.js`);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const ok = problems.length === 0;
|
|
101
|
+
if (warnings.length) {
|
|
102
|
+
console.log(`\n${warnings.length} warning(s):`);
|
|
103
|
+
for (const w of warnings.slice(0, 30)) console.log(` ! ${w}`);
|
|
104
|
+
}
|
|
105
|
+
if (!ok) {
|
|
106
|
+
console.error(`\n${problems.length} problem(s):`);
|
|
107
|
+
for (const p of problems.slice(0, 60)) console.error(` ✗ ${p}`);
|
|
108
|
+
if (problems.length > 60) console.error(` … and ${problems.length - 60} more`);
|
|
109
|
+
process.exit(1);
|
|
110
|
+
}
|
|
111
|
+
console.log(`\n✓ lint clean — ${files.length} files checked`);
|