@vimoxshah/tokenflow 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +228 -0
- package/Dockerfile.team +20 -0
- package/README.md +30 -11
- package/bin/tokenflow.js +147 -12
- package/design/tokens.yaml +330 -0
- package/docs/architecture.md +5 -4
- package/docs/cli.md +204 -0
- package/docs/configuration.md +117 -2
- package/docs/design-system.md +187 -0
- package/docs/exports-and-budgets.md +85 -0
- package/docs/guard-codex.md +132 -0
- package/docs/ledger.md +144 -0
- package/docs/live-mode.md +40 -0
- package/docs/media/overview-aurora-dark.png +0 -0
- package/docs/media/receipts-aurora-dark.png +0 -0
- package/docs/providers-otel.md +179 -0
- package/docs/providers.md +54 -1
- package/docs/receipt-schema.md +74 -0
- package/docs/roadmap.md +182 -0
- package/docs/team-server.md +170 -0
- package/docs/ui-views.md +322 -0
- package/package.json +7 -2
- package/schemas/receipt.v0.json +160 -0
- package/scripts/build-dmg.sh +11 -2
- package/scripts/build-menubar-app.sh +58 -7
- package/scripts/design-build.js +475 -0
- package/src/analytics/anatomy.js +467 -0
- package/src/analytics/branch-compare.js +159 -0
- package/src/analytics/cache-health.js +141 -0
- package/src/analytics/live-view.js +266 -0
- package/src/analytics/receipt-schema.js +214 -0
- package/src/analytics/receipt.js +709 -0
- package/src/analytics/rhythm.js +184 -0
- package/src/analytics/whatif.js +263 -0
- package/src/commands/budget-scopes.js +133 -0
- package/src/commands/doctor-checks.js +400 -0
- package/src/commands/guard.js +531 -0
- package/src/commands/hooks.js +238 -0
- package/src/commands/pricing-diff.js +316 -0
- package/src/commands/receipt.js +226 -0
- package/src/commands/team-serve.js +407 -0
- package/src/commands/week.js +86 -0
- package/src/core/annotations.js +97 -0
- package/src/core/budget.js +33 -0
- package/src/core/bundle.js +45 -2
- package/src/core/ingest.js +33 -0
- package/src/core/live-status.js +227 -2
- package/src/core/policy.js +103 -0
- package/src/core/receipt-note.js +123 -0
- package/src/core/repo.js +64 -0
- package/src/core/sync.js +163 -26
- package/src/core/team.js +0 -0
- package/src/export/html-snapshot.js +28 -1
- package/src/export/menubar.js +21 -0
- package/src/export/receipt-card.js +210 -0
- package/src/export/week-card.js +185 -0
- package/src/providers/mock/index.js +383 -52
- package/src/providers/openai/index.js +31 -1
- package/src/providers/otel/index.js +656 -0
- package/src/server/routes/annotations.js +42 -0
- package/src/server/routes/cache-health.js +95 -0
- package/src/server/routes/index.js +54 -0
- package/src/server/routes/session.js +157 -0
- package/src/server/server.js +47 -1
- package/src/ui/app.js +541 -308
- package/src/ui/charts.js +95 -0
- package/src/ui/first-run.js +144 -0
- package/src/ui/index.html +4 -1
- package/src/ui/palette.js +335 -0
- package/src/ui/styles/anatomy.css +117 -0
- package/src/ui/styles/annotations.css +40 -0
- package/src/ui/styles/branches.css +99 -0
- package/src/ui/styles/cache.css +6 -0
- package/src/ui/styles/first-run.css +31 -0
- package/src/ui/styles/live.css +100 -0
- package/src/ui/styles/palette.css +85 -0
- package/src/ui/styles/rhythm.css +8 -0
- package/src/ui/styles/whatif.css +55 -0
- package/src/ui/styles.css +303 -196
- package/src/ui/views/anatomy.js +567 -0
- package/src/ui/views/annotations.js +121 -0
- package/src/ui/views/branches.js +304 -0
- package/src/ui/views/cache.js +232 -0
- package/src/ui/views/index.js +85 -0
- package/src/ui/views/live.js +683 -0
- package/src/ui/views/rhythm.js +206 -0
- package/src/ui/views/whatif.js +196 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `tokenflow receipt` — what a branch or pull request cost.
|
|
3
|
+
*
|
|
4
|
+
* Joins the two things only this machine has side by side: the session logs
|
|
5
|
+
* (which turn ran on which branch) and the repository (which branch became
|
|
6
|
+
* which pull request). The analytics live in src/analytics/receipt.js and are
|
|
7
|
+
* pure; this file does the Node work — scanning the store, seeing through git
|
|
8
|
+
* worktrees, and optionally asking `gh` for the merged pull requests.
|
|
9
|
+
*
|
|
10
|
+
* tokenflow receipt every repo, top branches by spend
|
|
11
|
+
* tokenflow receipt --repo ~/code/api --gh one repo, joined to its merged PRs
|
|
12
|
+
* tokenflow receipt --repo api --branch feat/x one branch
|
|
13
|
+
* tokenflow receipt --repo ~/code/api --gh --pr 478 --md a PR-comment receipt
|
|
14
|
+
* tokenflow receipt --sessions where the money goes across sessions
|
|
15
|
+
* tokenflow receipt --prs prs.json PR list from `gh pr list --json ...`
|
|
16
|
+
*
|
|
17
|
+
* Repository identity: a worktree under `.worktrees/<x>` is the SAME repository
|
|
18
|
+
* as its main checkout. The adapters record `project` as the basename of the
|
|
19
|
+
* working directory, which splits one repo's spend across every worktree, so
|
|
20
|
+
* this command walks up from each recorded cwd to `.git`, follows a worktree's
|
|
21
|
+
* `gitdir:` pointer back to the main checkout, and names the repo by that
|
|
22
|
+
* directory. No subprocess, no network.
|
|
23
|
+
*/
|
|
24
|
+
import fs from 'node:fs';
|
|
25
|
+
import path from 'node:path';
|
|
26
|
+
import os from 'node:os';
|
|
27
|
+
import { execFileSync } from 'node:child_process';
|
|
28
|
+
import { paths, loadConfig } from '../core/config.js';
|
|
29
|
+
import { Store, decodeRecord, readJson } from '../core/store.js';
|
|
30
|
+
import { buildPriceBook } from '../core/pricing.js';
|
|
31
|
+
import { MEASUREMENT } from '../core/schema.js';
|
|
32
|
+
import { repoRootOf, makeRepoResolver } from '../core/repo.js';
|
|
33
|
+
import {
|
|
34
|
+
buildReceipts, sessionStats,
|
|
35
|
+
renderReceiptMarkdown, renderReceiptsTable, renderSessionStats,
|
|
36
|
+
} from '../analytics/receipt.js';
|
|
37
|
+
import { csvLine } from '../export/csv.js';
|
|
38
|
+
import { renderReceiptCardSvg, renderSvgToPng } from '../export/receipt-card.js';
|
|
39
|
+
|
|
40
|
+
// Repository identity lives in core so the bundle can use it too; re-exported
|
|
41
|
+
// here because this command is where callers first met it.
|
|
42
|
+
export { repoRootOf, makeRepoResolver } from '../core/repo.js';
|
|
43
|
+
|
|
44
|
+
function monthsBetween(from, to) {
|
|
45
|
+
if (!from && !to) return null;
|
|
46
|
+
const a = (from || '2000-01-01').slice(0, 7);
|
|
47
|
+
const b = (to || '2999-12-31').slice(0, 7);
|
|
48
|
+
const out = [];
|
|
49
|
+
let [y, m] = a.split('-').map(Number);
|
|
50
|
+
const [by, bm] = b.split('-').map(Number);
|
|
51
|
+
while (y < by || (y === by && m <= bm)) {
|
|
52
|
+
out.push(`${y}-${String(m).padStart(2, '0')}`);
|
|
53
|
+
m += 1;
|
|
54
|
+
if (m > 12) { m = 1; y += 1; }
|
|
55
|
+
}
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Stream the store's primary records, decoded, within an optional date window.
|
|
61
|
+
* @param {{from?:string|null, to?:string|null, store?:Store}} [opt]
|
|
62
|
+
* @returns {object[]}
|
|
63
|
+
*/
|
|
64
|
+
export function loadPrimaryRecords(opt = {}) {
|
|
65
|
+
const store = opt.store || new Store();
|
|
66
|
+
const months = monthsBetween(opt.from, opt.to);
|
|
67
|
+
const out = [];
|
|
68
|
+
store.scanRecords((o) => {
|
|
69
|
+
if (o.ms !== MEASUREMENT.PRIMARY) return;
|
|
70
|
+
if (opt.from && o.d < opt.from) return;
|
|
71
|
+
if (opt.to && o.d > opt.to) return;
|
|
72
|
+
out.push(decodeRecord(o));
|
|
73
|
+
}, months ? { months } : {});
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Merged pull requests for a repository checkout, via the GitHub CLI.
|
|
79
|
+
* Returns null (with a hint on the error) when `gh` is unavailable or fails.
|
|
80
|
+
* @param {string} repoPath
|
|
81
|
+
* @param {{limit?:number}} [opt]
|
|
82
|
+
*/
|
|
83
|
+
export function fetchMergedPrs(repoPath, opt = {}) {
|
|
84
|
+
const limit = String(opt.limit || 200);
|
|
85
|
+
try {
|
|
86
|
+
const out = execFileSync('gh', [
|
|
87
|
+
'pr', 'list', '--state', 'merged', '--limit', limit,
|
|
88
|
+
'--json', 'number,headRefName,additions,deletions,title,createdAt,mergedAt',
|
|
89
|
+
], { cwd: repoPath, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
|
90
|
+
return JSON.parse(out);
|
|
91
|
+
} catch (err) {
|
|
92
|
+
const e = /** @type {Error & {hint?:string}} */ (new Error(`could not list pull requests with gh: ${String(err.stderr || err.message).trim().split('\n')[0]}`));
|
|
93
|
+
e.hint = 'Install the GitHub CLI and run `gh auth login`, or pass --prs <file.json> exported with `gh pr list --state merged --json number,headRefName,additions,deletions,title,createdAt,mergedAt`.';
|
|
94
|
+
throw e;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function expandHome(p) {
|
|
99
|
+
return p && p.startsWith('~') ? path.join(os.homedir(), p.slice(1)) : p;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const RECEIPT_CSV_COLUMNS = [
|
|
103
|
+
'repo', 'branch', 'costUsd', 'contextShare', 'sessions', 'turns', 'subagentTurns',
|
|
104
|
+
'first', 'last', 'longLived', 'prNumber', 'mergedAt', 'changedLines', 'costPer100Lines',
|
|
105
|
+
];
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* One row per branch, across every repository in `result`.
|
|
109
|
+
* @param {ReturnType<typeof buildReceipts>} result
|
|
110
|
+
*/
|
|
111
|
+
export function renderReceiptsCsv(result) {
|
|
112
|
+
let out = csvLine(RECEIPT_CSV_COLUMNS);
|
|
113
|
+
for (const R of result.repos) {
|
|
114
|
+
for (const b of R.branches) {
|
|
115
|
+
out += csvLine([
|
|
116
|
+
R.repo, b.key, b.cost, b.contextShare, b.sessions, b.turns, b.subagentTurns,
|
|
117
|
+
b.first, b.last, b.longLived, b.pr ? b.pr.number : null, b.pr ? b.pr.mergedAt : null,
|
|
118
|
+
b.changedLines, b.costPer100Lines,
|
|
119
|
+
]);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return out;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* @param {object} flags parsed CLI flags
|
|
127
|
+
* @returns {{text:string, json:object}}
|
|
128
|
+
*/
|
|
129
|
+
export function run(flags = {}) {
|
|
130
|
+
const book = buildPriceBook(readJson(paths().pricing, {}));
|
|
131
|
+
const from = typeof flags.from === 'string' ? flags.from : null;
|
|
132
|
+
const to = typeof flags.to === 'string' ? flags.to : null;
|
|
133
|
+
const records = loadPrimaryRecords({ from, to });
|
|
134
|
+
|
|
135
|
+
if (flags.sessions) {
|
|
136
|
+
const caps = typeof flags.cap === 'string'
|
|
137
|
+
? flags.cap.split(',').map(Number).filter((n) => n > 0)
|
|
138
|
+
: undefined;
|
|
139
|
+
const s = sessionStats(records, { book, caps });
|
|
140
|
+
return { text: renderSessionStats(s), json: s };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// --repo: a path (has a separator or exists) or a bare repository name.
|
|
144
|
+
let repoName = null;
|
|
145
|
+
let repoPath = null;
|
|
146
|
+
if (typeof flags.repo === 'string') {
|
|
147
|
+
const p = expandHome(flags.repo);
|
|
148
|
+
if (fs.existsSync(p) && fs.statSync(p).isDirectory()) {
|
|
149
|
+
repoPath = fs.realpathSync(p);
|
|
150
|
+
repoName = path.basename(repoRootOf(repoPath) || repoPath);
|
|
151
|
+
} else {
|
|
152
|
+
repoName = flags.repo;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
let prs = [];
|
|
157
|
+
if (typeof flags.prs === 'string') {
|
|
158
|
+
prs = JSON.parse(fs.readFileSync(expandHome(flags.prs), 'utf8'));
|
|
159
|
+
} else if (flags.gh) {
|
|
160
|
+
if (!repoPath) {
|
|
161
|
+
const e = /** @type {Error & {hint?:string}} */ (new Error('--gh needs --repo <path to a checkout> so gh knows which repository to ask'));
|
|
162
|
+
e.hint = 'e.g. tokenflow receipt --repo ~/code/api --gh';
|
|
163
|
+
throw e;
|
|
164
|
+
}
|
|
165
|
+
prs = fetchMergedPrs(repoPath, { limit: Number(flags.limit) || 200 });
|
|
166
|
+
}
|
|
167
|
+
if (repoName) for (const p of prs) p.repo = repoName;
|
|
168
|
+
|
|
169
|
+
const automated = typeof flags.automated === 'string' ? new RegExp(flags.automated) : null;
|
|
170
|
+
const result = buildReceipts(records, {
|
|
171
|
+
book,
|
|
172
|
+
prs,
|
|
173
|
+
repoOf: makeRepoResolver(),
|
|
174
|
+
minTurns: Number(flags['min-turns']) || 1,
|
|
175
|
+
automated,
|
|
176
|
+
});
|
|
177
|
+
if (repoName) result.repos = result.repos.filter((R) => R.repo === repoName);
|
|
178
|
+
|
|
179
|
+
if (flags.csv) {
|
|
180
|
+
return { text: renderReceiptsCsv(result), json: result };
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Single receipt: --branch <name> or --pr <number>
|
|
184
|
+
const wantBranch = typeof flags.branch === 'string' ? flags.branch : null;
|
|
185
|
+
const wantPr = flags.pr !== undefined && flags.pr !== true ? Number(flags.pr) : null;
|
|
186
|
+
if (wantBranch || wantPr !== null) {
|
|
187
|
+
for (const R of result.repos) {
|
|
188
|
+
const b = R.branches.find((x) => (wantBranch && x.key === wantBranch) || (wantPr !== null && x.pr && x.pr.number === wantPr));
|
|
189
|
+
if (b) {
|
|
190
|
+
const md = renderReceiptMarkdown(b, { repo: R.repo, pricingVersion: book.version });
|
|
191
|
+
let text = md;
|
|
192
|
+
if (typeof flags.svg === 'string' || typeof flags.png === 'string') {
|
|
193
|
+
const cfg = loadConfig();
|
|
194
|
+
const skin = cfg.ui?.skin || 'aurora';
|
|
195
|
+
const mode = cfg.ui?.mode || cfg.ui?.theme || 'dark';
|
|
196
|
+
const svgPath = typeof flags.svg === 'string'
|
|
197
|
+
? expandHome(flags.svg)
|
|
198
|
+
: `${expandHome(flags.png)}.svg`;
|
|
199
|
+
fs.writeFileSync(svgPath, renderReceiptCardSvg(b, { repo: R.repo, skin, mode }));
|
|
200
|
+
text += `\n\nWrote SVG receipt card to ${svgPath}`;
|
|
201
|
+
if (typeof flags.png === 'string') {
|
|
202
|
+
const pngPath = expandHome(flags.png);
|
|
203
|
+
// Matches renderReceiptCardSvg's own default aspect ratio (640x460).
|
|
204
|
+
const res = renderSvgToPng(svgPath, pngPath, { width: 640, height: 460 });
|
|
205
|
+
text += res.ok ? `\nWrote PNG receipt card to ${pngPath}` : `\n${res.message}`;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return { text, json: { repo: R.repo, receipt: b } };
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
const what = wantBranch ? `branch ${wantBranch}` : `PR #${wantPr}`;
|
|
212
|
+
const e = /** @type {Error & {hint?:string}} */ (new Error(`no local sessions found for ${what}`));
|
|
213
|
+
e.hint = wantPr !== null && !prs.length
|
|
214
|
+
? 'Matching a PR number needs the PR list: add --gh (with --repo <path>) or --prs <file.json>.'
|
|
215
|
+
: 'Sessions on a detached HEAD or with no branch recorded cannot be attributed. `tokenflow receipt --repo <name>` lists what was.';
|
|
216
|
+
throw e;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (flags.md) {
|
|
220
|
+
// Every branch as a PR-comment block, most expensive first.
|
|
221
|
+
const blocks = [];
|
|
222
|
+
for (const R of result.repos) for (const b of R.branches.slice(0, Number(flags.top) || 10)) blocks.push(renderReceiptMarkdown(b, { repo: R.repo, pricingVersion: book.version }));
|
|
223
|
+
return { text: blocks.join('\n\n'), json: result };
|
|
224
|
+
}
|
|
225
|
+
return { text: renderReceiptsTable(result, { top: Number(flags.top) || 20 }), json: result };
|
|
226
|
+
}
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `tokenflow team serve` — a self-hosted team server.
|
|
3
|
+
*
|
|
4
|
+
* One process on a machine the team owns (LAN or Docker), fed by the SAME
|
|
5
|
+
* per-machine files the folder sync writes (`src/core/sync.js`): each
|
|
6
|
+
* machine POSTs its own `<machineId>.jsonl` (daily rollups) and
|
|
7
|
+
* `<machineId>.receipts.json` here instead of writing them into a shared
|
|
8
|
+
* folder. Everything downstream — `aggregate()`/`renderText()` — is the
|
|
9
|
+
* exact code the folder-sync `tokenflow team` view already uses, so this is
|
|
10
|
+
* a second transport for the same contract, not a new data model.
|
|
11
|
+
*
|
|
12
|
+
* Auth model: a single shared bearer token, checked two ways.
|
|
13
|
+
* - `POST /api/rollup` always requires it (Bearer header) once configured.
|
|
14
|
+
* - `GET /api/team` and `GET /` require it too, once configured — the
|
|
15
|
+
* aggregate carries repo/branch/machine names and costs, which a LAN
|
|
16
|
+
* peer should not see without the secret. Either `Authorization: Bearer
|
|
17
|
+
* <token>` or a `tf_token` cookie satisfies this; visiting `/?token=
|
|
18
|
+
* <token>` once from a browser mints that cookie (HttpOnly, SameSite
|
|
19
|
+
* Strict) and redirects to `/` with the query stripped, so the secret
|
|
20
|
+
* never lingers in the address bar or history.
|
|
21
|
+
* - `GET /health` always answers, but only `{ ok: true }` until
|
|
22
|
+
* authenticated — machine counts and freshness are withheld like
|
|
23
|
+
* everything else the aggregate would carry.
|
|
24
|
+
* With no token configured (the loopback default), every read stays open —
|
|
25
|
+
* there is no per-viewer identity to check on a machine only its owner can
|
|
26
|
+
* reach. A non-loopback bind address always requires a token, so an operator
|
|
27
|
+
* cannot accidentally expose an open server on 0.0.0.0.
|
|
28
|
+
*
|
|
29
|
+
* Nothing here reads prompt or code content: the request bodies are daily
|
|
30
|
+
* token/cost rollups and receipt summaries, the same coarse shape the folder
|
|
31
|
+
* sync already produces. Request bodies are never logged.
|
|
32
|
+
*
|
|
33
|
+
* tokenflow team serve loopback, no auth, port 7790
|
|
34
|
+
* tokenflow team serve --token <shared-secret> required for --host 0.0.0.0
|
|
35
|
+
* TOKENFLOW_TEAM_TOKEN=<shared-secret> tokenflow team serve --host 0.0.0.0
|
|
36
|
+
*/
|
|
37
|
+
import http from 'node:http';
|
|
38
|
+
import fs from 'node:fs';
|
|
39
|
+
import path from 'node:path';
|
|
40
|
+
import { fileURLToPath } from 'node:url';
|
|
41
|
+
import { aggregate, renderText } from '../core/team.js';
|
|
42
|
+
import { paths } from '../core/config.js';
|
|
43
|
+
|
|
44
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
45
|
+
const DEFAULT_PORT = 7790;
|
|
46
|
+
const MAX_BODY_BYTES = 8 * 1024 * 1024; // 8 MB, per the rollup contract
|
|
47
|
+
const MACHINE_ID_RE = /^[A-Za-z0-9_-]{8,64}$/;
|
|
48
|
+
const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1']);
|
|
49
|
+
|
|
50
|
+
// Mirrors scripts/design-build.js's START/END markers (the source of truth
|
|
51
|
+
// for the generated block). Duplicated here as plain strings so this runtime
|
|
52
|
+
// command has no import dependency on the design build tooling.
|
|
53
|
+
const TOKEN_BLOCK_START = '/* @generated design-tokens:start';
|
|
54
|
+
const TOKEN_BLOCK_END = '/* @generated design-tokens:end */';
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Start the team server: accepts rollup uploads from each machine and serves
|
|
58
|
+
* the aggregated team view over HTTP.
|
|
59
|
+
* @param {{dir?:string, host?:string, port?:number, token?:string|null}} [opt]
|
|
60
|
+
* dir defaults to `<paths().root>/team` (created if missing); host defaults
|
|
61
|
+
* to 127.0.0.1; port defaults to 7790; token falls back to the
|
|
62
|
+
* TOKENFLOW_TEAM_TOKEN env var when not passed explicitly.
|
|
63
|
+
* @returns {Promise<{server:import('node:http').Server, url:string, close:()=>Promise<void>}>}
|
|
64
|
+
*/
|
|
65
|
+
export async function startTeamServer(opt = {}) {
|
|
66
|
+
const host = opt.host || '127.0.0.1';
|
|
67
|
+
const port = opt.port ?? DEFAULT_PORT;
|
|
68
|
+
const dir = opt.dir || path.join(paths().root, 'team');
|
|
69
|
+
const token = typeof opt.token === 'string' && opt.token
|
|
70
|
+
? opt.token
|
|
71
|
+
: (process.env.TOKENFLOW_TEAM_TOKEN || null);
|
|
72
|
+
|
|
73
|
+
if (!LOOPBACK_HOSTS.has(host) && !token) {
|
|
74
|
+
throw new Error('refusing to bind a non-loopback host without a token — pass --token or set TOKENFLOW_TEAM_TOKEN');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
78
|
+
|
|
79
|
+
const server = http.createServer((req, res) => {
|
|
80
|
+
handleRequest(req, res, { dir, token }).catch(() => {
|
|
81
|
+
// Never echo the triggering error: it may embed request-derived text
|
|
82
|
+
// (e.g. a JSON.parse SyntaxError snippet of the body).
|
|
83
|
+
try { json(res, { error: 'internal error' }, 500); } catch { /* response already sent or socket gone */ }
|
|
84
|
+
console.error('[tokenflow team-serve] request handler failed');
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
await new Promise((resolve, reject) => {
|
|
89
|
+
server.on('error', reject);
|
|
90
|
+
server.listen(port, host, () => resolve(undefined));
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
const bound = server.address();
|
|
94
|
+
const boundPort = typeof bound === 'object' && bound !== null ? bound.port : port;
|
|
95
|
+
const url = `http://${host}:${boundPort}`;
|
|
96
|
+
return { server, url, close: () => new Promise((resolve) => server.close(() => resolve(undefined))) };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* `tokenflow team serve` CLI entry point. Starts the server and keeps the
|
|
101
|
+
* process alive (Ctrl+C to stop), matching `tokenflow dashboard`'s pattern.
|
|
102
|
+
* @param {object} flags parsed CLI flags: --host, --port, --dir, --token
|
|
103
|
+
*/
|
|
104
|
+
export async function run(flags = {}) {
|
|
105
|
+
const host = typeof flags.host === 'string' ? flags.host : '127.0.0.1';
|
|
106
|
+
const port = flags.port !== undefined ? Number(flags.port) : DEFAULT_PORT;
|
|
107
|
+
const dir = typeof flags.dir === 'string' ? flags.dir : undefined;
|
|
108
|
+
const token = typeof flags.token === 'string' ? flags.token : undefined;
|
|
109
|
+
|
|
110
|
+
const { url } = await startTeamServer({ host, port, dir, token });
|
|
111
|
+
const resolvedDir = dir || path.join(paths().root, 'team');
|
|
112
|
+
const hasToken = !!(token || process.env.TOKENFLOW_TEAM_TOKEN);
|
|
113
|
+
|
|
114
|
+
console.log(`\n TokenFlow team server`);
|
|
115
|
+
console.log(` ${url}`);
|
|
116
|
+
console.log(` folder: ${resolvedDir}`);
|
|
117
|
+
console.log(hasToken
|
|
118
|
+
? ' auth: bearer token (or the tf_token cookie minted by /?token=<token>) required for writes and reads'
|
|
119
|
+
: ' auth: none configured — every route is open. Set --token or TOKENFLOW_TEAM_TOKEN to require one.');
|
|
120
|
+
console.log(' /health always answers, but only { ok: true } until authenticated.');
|
|
121
|
+
console.log(' Ctrl+C to stop\n');
|
|
122
|
+
await new Promise(() => {}); // keep the process alive until interrupted
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ------------------------------------------------------------- routing ---
|
|
126
|
+
|
|
127
|
+
/** @param {{dir:string, token:string|null}} ctx */
|
|
128
|
+
async function handleRequest(req, res, ctx) {
|
|
129
|
+
const { dir, token } = ctx;
|
|
130
|
+
let url;
|
|
131
|
+
try {
|
|
132
|
+
url = new URL(req.url, `http://${req.headers.host || 'localhost'}`);
|
|
133
|
+
} catch {
|
|
134
|
+
return json(res, { error: 'bad request' }, 400);
|
|
135
|
+
}
|
|
136
|
+
const p = url.pathname;
|
|
137
|
+
|
|
138
|
+
if (req.method === 'GET' && p === '/health') {
|
|
139
|
+
if (token && !isAuthorized(req, token)) return json(res, { ok: true });
|
|
140
|
+
return json(res, healthPayload(dir));
|
|
141
|
+
}
|
|
142
|
+
if (req.method === 'GET' && p === '/api/team') {
|
|
143
|
+
if (token && !isAuthorized(req, token)) return json(res, { error: 'unauthorized' }, 401);
|
|
144
|
+
return json(res, aggregate(dir));
|
|
145
|
+
}
|
|
146
|
+
if (req.method === 'GET' && p === '/') return handleIndex(req, res, url, { dir, token });
|
|
147
|
+
if (req.method === 'POST' && p === '/api/rollup') return handleRollup(req, res, { dir, token });
|
|
148
|
+
return json(res, { error: 'not found' }, 404);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* `GET /`: mints the `tf_token` cookie from `?token=` (once, then redirects
|
|
153
|
+
* with the query stripped), otherwise serves the full aggregate view when
|
|
154
|
+
* authorized or a minimal "token required" page when not.
|
|
155
|
+
*/
|
|
156
|
+
function handleIndex(req, res, url, { dir, token }) {
|
|
157
|
+
if (token) {
|
|
158
|
+
const qToken = url.searchParams.get('token');
|
|
159
|
+
if (qToken && qToken === token) {
|
|
160
|
+
res.setHeader('set-cookie', `tf_token=${encodeURIComponent(token)}; HttpOnly; SameSite=Strict; Path=/`);
|
|
161
|
+
res.writeHead(302, { location: '/', 'cache-control': 'no-store' });
|
|
162
|
+
return res.end();
|
|
163
|
+
}
|
|
164
|
+
if (!isAuthorized(req, token)) return html(res, renderUnauthorizedHtml(), 401);
|
|
165
|
+
}
|
|
166
|
+
return html(res, renderHtml(dir));
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** True whenever no token is configured, or the request carries the right Bearer header or `tf_token` cookie. */
|
|
170
|
+
function isAuthorized(req, token) {
|
|
171
|
+
if (!token) return true;
|
|
172
|
+
const auth = req.headers.authorization || '';
|
|
173
|
+
const m = /^Bearer\s+(.+)$/.exec(auth);
|
|
174
|
+
if (m && m[1] === token) return true;
|
|
175
|
+
return parseCookies(req.headers.cookie).tf_token === token;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Minimal `Cookie:` header parser — name/value pairs only, no attributes (those are only ever set by us). */
|
|
179
|
+
function parseCookies(header) {
|
|
180
|
+
const out = {};
|
|
181
|
+
if (!header) return out;
|
|
182
|
+
for (const part of header.split(';')) {
|
|
183
|
+
const eq = part.indexOf('=');
|
|
184
|
+
if (eq === -1) continue;
|
|
185
|
+
const k = part.slice(0, eq).trim();
|
|
186
|
+
if (!k) continue;
|
|
187
|
+
try { out[k] = decodeURIComponent(part.slice(eq + 1).trim()); } catch { out[k] = part.slice(eq + 1).trim(); }
|
|
188
|
+
}
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Validate and atomically write a machine's rollup upload.
|
|
194
|
+
* Never serves files from `dir` directly and never logs the body.
|
|
195
|
+
*/
|
|
196
|
+
async function handleRollup(req, res, { dir, token }) {
|
|
197
|
+
if (token) {
|
|
198
|
+
const auth = req.headers.authorization || '';
|
|
199
|
+
const m = /^Bearer\s+(.+)$/.exec(auth);
|
|
200
|
+
if (!m || m[1] !== token) return json(res, { error: 'unauthorized' }, 401);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
let raw;
|
|
204
|
+
try {
|
|
205
|
+
raw = await readBodyLimited(req, MAX_BODY_BYTES);
|
|
206
|
+
} catch (err) {
|
|
207
|
+
if (err.code === 'PAYLOAD_TOO_LARGE') return json(res, { error: 'payload too large' }, 413);
|
|
208
|
+
return json(res, { error: 'bad request' }, 400);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
let body;
|
|
212
|
+
try { body = JSON.parse(raw || '{}'); } catch { return json(res, { error: 'invalid JSON body' }, 400); }
|
|
213
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) {
|
|
214
|
+
return json(res, { error: 'invalid JSON body' }, 400);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const machineId = body.machineId;
|
|
218
|
+
if (typeof machineId !== 'string' || !MACHINE_ID_RE.test(machineId)) {
|
|
219
|
+
return json(res, { error: 'invalid machineId' }, 400);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const files = body.files;
|
|
223
|
+
if (!files || typeof files !== 'object' || Array.isArray(files)) {
|
|
224
|
+
return json(res, { error: 'files must be an object of name -> text' }, 400);
|
|
225
|
+
}
|
|
226
|
+
const names = Object.keys(files);
|
|
227
|
+
if (!names.length) return json(res, { error: 'no files provided' }, 400);
|
|
228
|
+
|
|
229
|
+
const allowed = new Set([`${machineId}.jsonl`, `${machineId}.receipts.json`]);
|
|
230
|
+
for (const name of names) {
|
|
231
|
+
if (!allowed.has(name)) return json(res, { error: 'unexpected file name' }, 400);
|
|
232
|
+
if (typeof files[name] !== 'string') return json(res, { error: 'file content must be a string' }, 400);
|
|
233
|
+
}
|
|
234
|
+
for (const name of names) {
|
|
235
|
+
const text = files[name];
|
|
236
|
+
if (name.endsWith('.jsonl')) {
|
|
237
|
+
for (const line of text.split('\n')) {
|
|
238
|
+
if (!line.trim()) continue;
|
|
239
|
+
try { JSON.parse(line); } catch { return json(res, { error: 'malformed JSONL line' }, 400); }
|
|
240
|
+
}
|
|
241
|
+
} else {
|
|
242
|
+
try { JSON.parse(text); } catch { return json(res, { error: 'malformed receipts JSON' }, 400); }
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
for (const name of names) writeAtomic(path.join(dir, name), files[name]);
|
|
247
|
+
return json(res, { ok: true, files: names });
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ------------------------------------------------------------------ i/o ---
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Read the request body, capped at maxBytes. Once the cap is passed the
|
|
254
|
+
* chunks collected so far are dropped (no unbounded buffering) but the
|
|
255
|
+
* stream keeps being drained to `end` before rejecting with
|
|
256
|
+
* `code: 'PAYLOAD_TOO_LARGE'` — responding while the client still has
|
|
257
|
+
* unsent body bytes in flight resets the connection instead of delivering
|
|
258
|
+
* the 413.
|
|
259
|
+
*/
|
|
260
|
+
function readBodyLimited(req, maxBytes) {
|
|
261
|
+
return new Promise((resolve, reject) => {
|
|
262
|
+
let total = 0;
|
|
263
|
+
const chunks = [];
|
|
264
|
+
let tooLarge = false;
|
|
265
|
+
req.on('data', (c) => {
|
|
266
|
+
total += c.length;
|
|
267
|
+
if (total > maxBytes) { tooLarge = true; chunks.length = 0; return; }
|
|
268
|
+
chunks.push(c);
|
|
269
|
+
});
|
|
270
|
+
req.on('end', () => {
|
|
271
|
+
if (tooLarge) {
|
|
272
|
+
const err = /** @type {Error & {code?:string}} */ (new Error('payload too large'));
|
|
273
|
+
err.code = 'PAYLOAD_TOO_LARGE';
|
|
274
|
+
reject(err);
|
|
275
|
+
} else {
|
|
276
|
+
resolve(Buffer.concat(chunks).toString('utf8'));
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
req.on('error', reject);
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Write-then-rename so a reader never observes a partial file. */
|
|
284
|
+
function writeAtomic(file, content) {
|
|
285
|
+
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
286
|
+
fs.writeFileSync(tmp, content);
|
|
287
|
+
fs.renameSync(tmp, file);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** What the server has received: file count and freshest mtime, no aggregation. */
|
|
291
|
+
function healthPayload(dir) {
|
|
292
|
+
let machines = 0;
|
|
293
|
+
let updatedAt = null;
|
|
294
|
+
if (fs.existsSync(dir)) {
|
|
295
|
+
for (const f of fs.readdirSync(dir)) {
|
|
296
|
+
if (!f.endsWith('.jsonl')) continue; // one rollup file per machine
|
|
297
|
+
machines++;
|
|
298
|
+
const iso = fs.statSync(path.join(dir, f)).mtime.toISOString();
|
|
299
|
+
if (!updatedAt || iso > updatedAt) updatedAt = iso;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return { ok: true, machines, updatedAt };
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function json(res, obj, code = 200) {
|
|
306
|
+
const s = JSON.stringify(obj);
|
|
307
|
+
res.writeHead(code, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
|
|
308
|
+
res.end(s);
|
|
309
|
+
}
|
|
310
|
+
function html(res, body, code = 200) {
|
|
311
|
+
res.writeHead(code, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' });
|
|
312
|
+
res.end(body);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// ------------------------------------------------------------------ HTML ---
|
|
316
|
+
|
|
317
|
+
function esc(s) {
|
|
318
|
+
return String(s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** Extract the generated design-token block verbatim, read fresh on every request. */
|
|
322
|
+
function extractTokenBlock() {
|
|
323
|
+
let css = '';
|
|
324
|
+
try { css = fs.readFileSync(path.join(ROOT, 'src', 'ui', 'styles.css'), 'utf8'); } catch { return ''; }
|
|
325
|
+
const a = css.indexOf(TOKEN_BLOCK_START);
|
|
326
|
+
const b = css.indexOf(TOKEN_BLOCK_END);
|
|
327
|
+
if (a === -1 || b === -1 || b < a) return '';
|
|
328
|
+
return css.slice(a, b + TOKEN_BLOCK_END.length);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** The unauthenticated view of `/` when a token is configured: no aggregate content. */
|
|
332
|
+
function renderUnauthorizedHtml() {
|
|
333
|
+
const tokenBlock = extractTokenBlock();
|
|
334
|
+
return `<!doctype html>
|
|
335
|
+
<html>
|
|
336
|
+
<head>
|
|
337
|
+
<meta charset="utf-8">
|
|
338
|
+
<title>TokenFlow team server</title>
|
|
339
|
+
<style>
|
|
340
|
+
${tokenBlock}
|
|
341
|
+
body { background: var(--plane); color: var(--text-primary); font-family: var(--sans); padding: var(--sp-24); }
|
|
342
|
+
h1 { font-family: var(--display); }
|
|
343
|
+
code { font-family: var(--mono); }
|
|
344
|
+
</style>
|
|
345
|
+
</head>
|
|
346
|
+
<body>
|
|
347
|
+
<h1>TokenFlow team server</h1>
|
|
348
|
+
<p>A token is required to view this page. Visit <code>/?token=<your-team-token></code> once from a browser you trust —
|
|
349
|
+
it sets a private cookie and the token never appears in the address bar again.</p>
|
|
350
|
+
</body>
|
|
351
|
+
</html>`;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Self-contained HTML view: no external requests, no scripts. Renders the
|
|
356
|
+
* same text the CLI's `tokenflow team` prints, plus a top-repos table when
|
|
357
|
+
* the aggregate exposes `receipts.byRepo` (degrades to the <pre> alone when
|
|
358
|
+
* that shape is absent — the receipts pipeline is landing separately).
|
|
359
|
+
*/
|
|
360
|
+
function renderHtml(dir) {
|
|
361
|
+
const tokenBlock = extractTokenBlock();
|
|
362
|
+
const agg = aggregate(dir);
|
|
363
|
+
const health = healthPayload(dir);
|
|
364
|
+
const text = renderText(agg);
|
|
365
|
+
|
|
366
|
+
let repoSection = '';
|
|
367
|
+
const byRepo = agg && agg.receipts && Array.isArray(agg.receipts.byRepo) ? agg.receipts.byRepo : null;
|
|
368
|
+
if (byRepo && byRepo.length) {
|
|
369
|
+
const rows = byRepo.slice(0, 20).map((r) => {
|
|
370
|
+
const repo = esc(r.repo ?? r.name ?? '(unknown)');
|
|
371
|
+
const tok = r.tokens != null ? esc(r.tokens) : '';
|
|
372
|
+
const costVal = r.estCostUsd ?? r.cost;
|
|
373
|
+
const cost = costVal != null ? esc(`$${Number(costVal).toFixed(2)}`) : '';
|
|
374
|
+
return ` <tr><td>${repo}</td><td>${tok}</td><td>${cost}</td></tr>`;
|
|
375
|
+
}).join('\n');
|
|
376
|
+
repoSection = `
|
|
377
|
+
<h2>Top repositories</h2>
|
|
378
|
+
<table>
|
|
379
|
+
<thead><tr><th>Repo</th><th>Tokens</th><th>Est. cost</th></tr></thead>
|
|
380
|
+
<tbody>
|
|
381
|
+
${rows}
|
|
382
|
+
</tbody>
|
|
383
|
+
</table>`;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
return `<!doctype html>
|
|
387
|
+
<html>
|
|
388
|
+
<head>
|
|
389
|
+
<meta charset="utf-8">
|
|
390
|
+
<title>TokenFlow team server</title>
|
|
391
|
+
<style>
|
|
392
|
+
${tokenBlock}
|
|
393
|
+
body { background: var(--plane); color: var(--text-primary); font-family: var(--sans); padding: var(--sp-24); }
|
|
394
|
+
h1, h2 { font-family: var(--display); }
|
|
395
|
+
pre { background: var(--surface-1); color: var(--text-primary); padding: var(--sp-16); border-radius: var(--radius); overflow-x: auto; white-space: pre-wrap; }
|
|
396
|
+
table { border-collapse: collapse; margin-top: var(--sp-16); }
|
|
397
|
+
th, td { border: 1px solid var(--border); padding: var(--sp-8); text-align: left; }
|
|
398
|
+
.meta { color: var(--text-secondary); font-family: var(--mono); }
|
|
399
|
+
</style>
|
|
400
|
+
</head>
|
|
401
|
+
<body>
|
|
402
|
+
<h1>TokenFlow team server</h1>
|
|
403
|
+
<p class="meta">${health.machines} machine(s) reporting · updated ${health.updatedAt ? esc(health.updatedAt) : 'never'}</p>
|
|
404
|
+
<pre>${esc(text)}</pre>${repoSection}
|
|
405
|
+
</body>
|
|
406
|
+
</html>`;
|
|
407
|
+
}
|