@vimoxshah/tokenflow 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +84 -0
- package/LICENSE +21 -0
- package/README.md +250 -0
- package/Refresh & Open Dashboard.command +22 -0
- package/SECURITY.md +42 -0
- package/bin/tokenflow.js +1342 -0
- package/docs/architecture.md +193 -0
- package/docs/cli.md +390 -0
- package/docs/configuration.md +281 -0
- package/docs/creating-provider.md +262 -0
- package/docs/data-model.md +213 -0
- package/docs/getting-started.md +266 -0
- package/docs/live-mode.md +199 -0
- package/docs/media/architecture-hero.svg +86 -0
- package/docs/media/cost-editorial-dark.png +0 -0
- package/docs/media/health-terminal-light.png +0 -0
- package/docs/media/menubar-dark.png +0 -0
- package/docs/media/menubar-light.png +0 -0
- package/docs/media/models-terminal-dark.png +0 -0
- package/docs/media/overview-aurora-dark.png +0 -0
- package/docs/media/time-aurora-light.png +0 -0
- package/docs/providers.md +309 -0
- package/docs/skill.md +64 -0
- package/docs/troubleshooting.md +207 -0
- package/examples/config.example.yaml +92 -0
- package/examples/demo-data/README.md +38 -0
- package/examples/demo-data/sample-usage.csv +11 -0
- package/package.json +74 -0
- package/scripts/build-dmg.sh +33 -0
- package/scripts/build-menubar-app.sh +67 -0
- package/scripts/lint.js +111 -0
- package/scripts/validate-install.js +140 -0
- package/skills/tokenflow/SKILL.md +392 -0
- package/skills/tokenflow/examples/config.yaml +92 -0
- package/skills/tokenflow/examples/generic-mapping.json +26 -0
- package/skills/tokenflow/examples/session-transcript.md +191 -0
- package/skills/tokenflow/providers/adapter-template.js +135 -0
- package/skills/tokenflow/providers/detection-matrix.md +142 -0
- package/skills/tokenflow/schemas/config.schema.json +107 -0
- package/skills/tokenflow/schemas/normalized-record.json +63 -0
- package/src/analytics/aggregate.js +247 -0
- package/src/analytics/anomalies.js +222 -0
- package/src/analytics/capacity.js +278 -0
- package/src/analytics/comparison.js +96 -0
- package/src/analytics/dimensions.js +230 -0
- package/src/analytics/efficiency.js +138 -0
- package/src/analytics/forecast.js +202 -0
- package/src/analytics/index.js +327 -0
- package/src/analytics/insights.js +283 -0
- package/src/analytics/milestones.js +91 -0
- package/src/analytics/peak.js +106 -0
- package/src/analytics/productivity.js +166 -0
- package/src/analytics/token-usage.js +267 -0
- package/src/commands/diagnostics.js +88 -0
- package/src/commands/digest.js +155 -0
- package/src/commands/models-compare.js +96 -0
- package/src/core/budget.js +142 -0
- package/src/core/bundle.js +191 -0
- package/src/core/config.js +202 -0
- package/src/core/delivery.js +109 -0
- package/src/core/geo.js +99 -0
- package/src/core/ingest.js +457 -0
- package/src/core/interface-map.js +55 -0
- package/src/core/jsonl.js +124 -0
- package/src/core/live-status.js +417 -0
- package/src/core/model-map.js +157 -0
- package/src/core/notify.js +83 -0
- package/src/core/pricing.js +288 -0
- package/src/core/prompt-analytics.js +127 -0
- package/src/core/registry.js +107 -0
- package/src/core/restore.js +261 -0
- package/src/core/schedule.js +120 -0
- package/src/core/schema.js +316 -0
- package/src/core/sqlite.js +96 -0
- package/src/core/store.js +493 -0
- package/src/core/sync.js +151 -0
- package/src/core/units.js +147 -0
- package/src/core/validate.js +123 -0
- package/src/core/watch.js +287 -0
- package/src/core/yaml.js +209 -0
- package/src/export/bundler.js +107 -0
- package/src/export/csv.js +100 -0
- package/src/export/html-snapshot.js +101 -0
- package/src/export/menubar.js +158 -0
- package/src/index.js +18 -0
- package/src/providers/anthropic/index.js +294 -0
- package/src/providers/cline/index.js +120 -0
- package/src/providers/cursor/index.js +143 -0
- package/src/providers/generic/index.js +268 -0
- package/src/providers/git/index.js +188 -0
- package/src/providers/headroom/index.js +114 -0
- package/src/providers/hermes/index.js +299 -0
- package/src/providers/mock/index.js +117 -0
- package/src/providers/openai/index.js +370 -0
- package/src/providers/opencode/index.js +245 -0
- package/src/sdk.js +46 -0
- package/src/server/server.js +264 -0
- package/src/ui/app.js +2473 -0
- package/src/ui/charts.js +925 -0
- package/src/ui/index.html +42 -0
- package/src/ui/styles.css +644 -0
package/src/core/yaml.js
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A deliberately small YAML subset — enough for a config file, with zero
|
|
3
|
+
* dependencies and no surprises.
|
|
4
|
+
*
|
|
5
|
+
* Supported: nested mappings by indentation, block sequences (`- x`),
|
|
6
|
+
* sequences of mappings, scalars (string / number / bool / null), single- and
|
|
7
|
+
* double-quoted strings, inline flow arrays (`[a, b]`), `#` comments,
|
|
8
|
+
* `key:` with an empty value (=> {}), and multi-line values via `|` / `>`.
|
|
9
|
+
*
|
|
10
|
+
* NOT supported (and rejected loudly rather than mis-parsed): anchors,
|
|
11
|
+
* aliases, tags, multiple documents, complex keys, flow mappings.
|
|
12
|
+
* `config.json` is always accepted as an alternative.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export function parseYaml(text) {
|
|
16
|
+
const lines = String(text).split(/\r?\n/);
|
|
17
|
+
const rows = [];
|
|
18
|
+
for (let i = 0; i < lines.length; i++) {
|
|
19
|
+
const raw = lines[i];
|
|
20
|
+
if (/^\s*$/.test(raw)) continue;
|
|
21
|
+
if (/^\s*#/.test(raw)) continue;
|
|
22
|
+
if (/^\s*(---|\.\.\.)\s*$/.test(raw)) continue;
|
|
23
|
+
if (/^\s*[&*!]/.test(raw)) throw new Error(`yaml: anchors/aliases/tags are not supported (line ${i + 1})`);
|
|
24
|
+
const indent = raw.match(/^ */)[0].length;
|
|
25
|
+
rows.push({ indent, text: stripComment(raw.trim()), line: i + 1 });
|
|
26
|
+
}
|
|
27
|
+
const [value] = parseBlock(rows, 0, rows.length > 0 ? rows[0].indent : 0);
|
|
28
|
+
return value ?? {};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function stripComment(s) {
|
|
32
|
+
let out = '';
|
|
33
|
+
let q = null;
|
|
34
|
+
for (let i = 0; i < s.length; i++) {
|
|
35
|
+
const c = s[i];
|
|
36
|
+
if (q) {
|
|
37
|
+
out += c;
|
|
38
|
+
if (c === q && s[i - 1] !== '\\') q = null;
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (c === '"' || c === "'") { q = c; out += c; continue; }
|
|
42
|
+
if (c === '#' && (i === 0 || /\s/.test(s[i - 1]))) break;
|
|
43
|
+
out += c;
|
|
44
|
+
}
|
|
45
|
+
return out.trim();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function parseBlock(rows, i, indent) {
|
|
49
|
+
if (i >= rows.length) return [null, i];
|
|
50
|
+
if (rows[i].text.startsWith('- ') || rows[i].text === '-') return parseSeq(rows, i, indent);
|
|
51
|
+
return parseMap(rows, i, indent);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function parseMap(rows, i, indent) {
|
|
55
|
+
const obj = {};
|
|
56
|
+
while (i < rows.length && rows[i].indent >= indent) {
|
|
57
|
+
if (rows[i].indent > indent) throw new Error(`yaml: unexpected indent (line ${rows[i].line})`);
|
|
58
|
+
const { text, line } = rows[i];
|
|
59
|
+
const m = text.match(/^([^:]+):(?:\s+(.*))?$/);
|
|
60
|
+
if (!m) throw new Error(`yaml: expected "key: value" (line ${line}): ${text}`);
|
|
61
|
+
const key = unquote(m[1].trim());
|
|
62
|
+
const inline = m[2] === undefined ? '' : m[2].trim();
|
|
63
|
+
i++;
|
|
64
|
+
if (inline === '|' || inline === '>' || inline === '|-' || inline === '>-') {
|
|
65
|
+
const childIndent = i < rows.length ? rows[i].indent : indent + 2;
|
|
66
|
+
const parts = [];
|
|
67
|
+
while (i < rows.length && rows[i].indent >= childIndent && childIndent > indent) {
|
|
68
|
+
parts.push(rows[i].text);
|
|
69
|
+
i++;
|
|
70
|
+
}
|
|
71
|
+
obj[key] = inline[0] === '|' ? parts.join('\n') : parts.join(' ');
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (inline !== '') {
|
|
75
|
+
obj[key] = scalar(inline);
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
// Nested block, if the next row is deeper.
|
|
79
|
+
if (i < rows.length && rows[i].indent > indent) {
|
|
80
|
+
const [v, ni] = parseBlock(rows, i, rows[i].indent);
|
|
81
|
+
obj[key] = v;
|
|
82
|
+
i = ni;
|
|
83
|
+
} else if (i < rows.length && rows[i].indent === indent && rows[i].text.startsWith('- ')) {
|
|
84
|
+
// A sequence at the same indent as its key (valid YAML).
|
|
85
|
+
const [v, ni] = parseSeq(rows, i, indent);
|
|
86
|
+
obj[key] = v;
|
|
87
|
+
i = ni;
|
|
88
|
+
} else {
|
|
89
|
+
obj[key] = {};
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return [obj, i];
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function parseSeq(rows, i, indent) {
|
|
96
|
+
const arr = [];
|
|
97
|
+
while (i < rows.length && rows[i].indent === indent && (rows[i].text.startsWith('- ') || rows[i].text === '-')) {
|
|
98
|
+
const body = rows[i].text === '-' ? '' : rows[i].text.slice(2).trim();
|
|
99
|
+
i++;
|
|
100
|
+
if (body === '') {
|
|
101
|
+
if (i < rows.length && rows[i].indent > indent) {
|
|
102
|
+
const [v, ni] = parseBlock(rows, i, rows[i].indent);
|
|
103
|
+
arr.push(v);
|
|
104
|
+
i = ni;
|
|
105
|
+
} else arr.push(null);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (/^[^:\s][^:]*:(\s|$)/.test(body)) {
|
|
109
|
+
// sequence of mappings: "- key: value" plus deeper sibling keys
|
|
110
|
+
const synthetic = [{ indent: indent + 2, text: body, line: rows[i - 1].line }];
|
|
111
|
+
while (i < rows.length && rows[i].indent > indent) {
|
|
112
|
+
synthetic.push({ indent: indent + 2, text: rows[i].text, line: rows[i].line });
|
|
113
|
+
i++;
|
|
114
|
+
}
|
|
115
|
+
const [v] = parseMap(synthetic, 0, indent + 2);
|
|
116
|
+
arr.push(v);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
arr.push(scalar(body));
|
|
120
|
+
}
|
|
121
|
+
return [arr, i];
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function scalar(s) {
|
|
125
|
+
// An anchor, alias or tag in a value position: refuse rather than store the
|
|
126
|
+
// literal text and quietly change the meaning of someone's config.
|
|
127
|
+
if (/^[&*!]/.test(s)) throw new Error(`yaml: anchors/aliases/tags are not supported ("${s}")`);
|
|
128
|
+
if (s.startsWith('[') && s.endsWith(']')) {
|
|
129
|
+
const inner = s.slice(1, -1).trim();
|
|
130
|
+
if (!inner) return [];
|
|
131
|
+
return splitFlow(inner).map((x) => scalar(x.trim()));
|
|
132
|
+
}
|
|
133
|
+
if (s === '{}') return {};
|
|
134
|
+
if (s.startsWith('{')) throw new Error('yaml: flow mappings are not supported (use nested keys)');
|
|
135
|
+
if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) return unquote(s);
|
|
136
|
+
if (s === '~' || s === 'null' || s === 'Null' || s === 'NULL') return null;
|
|
137
|
+
if (s === 'true' || s === 'True' || s === 'yes' || s === 'on') return true;
|
|
138
|
+
if (s === 'false' || s === 'False' || s === 'no' || s === 'off') return false;
|
|
139
|
+
if (/^-?\d+$/.test(s)) return parseInt(s, 10);
|
|
140
|
+
if (/^-?\d*\.\d+(e[-+]?\d+)?$/i.test(s)) return parseFloat(s);
|
|
141
|
+
return s;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function splitFlow(s) {
|
|
145
|
+
const out = [];
|
|
146
|
+
let depth = 0, q = null, cur = '';
|
|
147
|
+
for (const c of s) {
|
|
148
|
+
if (q) { cur += c; if (c === q) q = null; continue; }
|
|
149
|
+
if (c === '"' || c === "'") { q = c; cur += c; continue; }
|
|
150
|
+
if (c === '[') depth++;
|
|
151
|
+
if (c === ']') depth--;
|
|
152
|
+
if (c === ',' && depth === 0) { out.push(cur); cur = ''; continue; }
|
|
153
|
+
cur += c;
|
|
154
|
+
}
|
|
155
|
+
if (cur.trim()) out.push(cur);
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function unquote(s) {
|
|
160
|
+
if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
|
|
161
|
+
const body = s.slice(1, -1);
|
|
162
|
+
return s[0] === '"' ? body.replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\') : body.replace(/''/g, "'");
|
|
163
|
+
}
|
|
164
|
+
return s;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// ---------------------------------------------------------------------------
|
|
168
|
+
|
|
169
|
+
export function stringifyYaml(value, indent = 0) {
|
|
170
|
+
const pad = ' '.repeat(indent);
|
|
171
|
+
if (value === null || value === undefined) return 'null';
|
|
172
|
+
if (Array.isArray(value)) {
|
|
173
|
+
if (value.length === 0) return '[]';
|
|
174
|
+
return value
|
|
175
|
+
.map((v) => {
|
|
176
|
+
if (v !== null && typeof v === 'object') {
|
|
177
|
+
const body = stringifyYaml(v, indent + 2);
|
|
178
|
+
return `${pad}- ${body.trimStart()}`;
|
|
179
|
+
}
|
|
180
|
+
return `${pad}- ${scalarOut(v)}`;
|
|
181
|
+
})
|
|
182
|
+
.join('\n');
|
|
183
|
+
}
|
|
184
|
+
if (typeof value === 'object') {
|
|
185
|
+
const keys = Object.keys(value);
|
|
186
|
+
if (keys.length === 0) return '{}';
|
|
187
|
+
return keys
|
|
188
|
+
.map((k) => {
|
|
189
|
+
const v = value[k];
|
|
190
|
+
if (v !== null && typeof v === 'object' && (Array.isArray(v) ? v.length : Object.keys(v).length)) {
|
|
191
|
+
return `${pad}${k}:\n${stringifyYaml(v, indent + 2)}`;
|
|
192
|
+
}
|
|
193
|
+
if (v !== null && typeof v === 'object') return `${pad}${k}: ${Array.isArray(v) ? '[]' : '{}'}`;
|
|
194
|
+
return `${pad}${k}: ${scalarOut(v)}`;
|
|
195
|
+
})
|
|
196
|
+
.join('\n');
|
|
197
|
+
}
|
|
198
|
+
return scalarOut(value);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function scalarOut(v) {
|
|
202
|
+
if (v === null || v === undefined) return 'null';
|
|
203
|
+
if (typeof v === 'boolean' || typeof v === 'number') return String(v);
|
|
204
|
+
const s = String(v);
|
|
205
|
+
if (s === '' || /^[\s]|[\s]$|[:#\[\]{}&*!|>'"%@`,]|^(true|false|null|yes|no|on|off|~|-?\d)/i.test(s)) {
|
|
206
|
+
return '"' + s.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n') + '"';
|
|
207
|
+
}
|
|
208
|
+
return s;
|
|
209
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A ~100-line ES-module bundler.
|
|
3
|
+
*
|
|
4
|
+
* The dev path has no build step at all — the browser imports the same modules
|
|
5
|
+
* the CLI does. This exists only so `tokenflow export --html` can emit ONE
|
|
6
|
+
* self-contained file that works offline, from a file:// URL, or attached to an
|
|
7
|
+
* email, where `import` from a relative path is not available.
|
|
8
|
+
*
|
|
9
|
+
* Rather than rename identifiers (which is where naive concatenating bundlers
|
|
10
|
+
* break), each module keeps its own function scope and exports are wired
|
|
11
|
+
* through a tiny require registry. The module graph here is acyclic and uses
|
|
12
|
+
* only static imports, which is what makes this safe.
|
|
13
|
+
*/
|
|
14
|
+
import fs from 'node:fs';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
|
|
17
|
+
const IMPORT_RE = /^[ \t]*import\s+(?:([\w$]+)\s*,\s*)?(?:\{([^}]*)\}|\*\s*as\s+([\w$]+)|([\w$]+))?\s*(?:from\s*)?['"]([^'"]+)['"]\s*;?[ \t]*$/gm;
|
|
18
|
+
// `export { a, b }` — possibly spanning many lines, as re-export barrels do.
|
|
19
|
+
const EXPORT_LIST_RE = /^[ \t]*export\s*\{([\s\S]*?)\}[ \t]*;?[ \t]*$/gm;
|
|
20
|
+
const EXPORT_DECL_RE = /^[ \t]*export\s+(async\s+function|function\*?|class|const|let|var)\s+([\w$]+)/gm;
|
|
21
|
+
const EXPORT_DEFAULT_RE = /^[ \t]*export\s+default\s+/m;
|
|
22
|
+
|
|
23
|
+
export function bundle(entry, { root = process.cwd() } = {}) {
|
|
24
|
+
const modules = new Map();
|
|
25
|
+
const order = [];
|
|
26
|
+
|
|
27
|
+
const load = (file) => {
|
|
28
|
+
const abs = path.resolve(file);
|
|
29
|
+
if (modules.has(abs)) return abs;
|
|
30
|
+
const src = fs.readFileSync(abs, 'utf8');
|
|
31
|
+
const mod = { abs, id: rel(abs, root), src, deps: [], exports: new Set(), hasDefault: false };
|
|
32
|
+
modules.set(abs, mod);
|
|
33
|
+
|
|
34
|
+
// ---- collect + rewrite imports
|
|
35
|
+
let body = src.replace(IMPORT_RE, (m, defWithNamed, named, ns, defOnly, spec) => {
|
|
36
|
+
if (!spec.startsWith('.') && !spec.startsWith('/')) {
|
|
37
|
+
throw new Error(`snapshot bundler: bare import "${spec}" in ${mod.id} — only relative imports are supported`);
|
|
38
|
+
}
|
|
39
|
+
const depAbs = resolve(abs, spec);
|
|
40
|
+
mod.deps.push(depAbs);
|
|
41
|
+
const id = rel(depAbs, root);
|
|
42
|
+
const parts = [];
|
|
43
|
+
const defName = defWithNamed || defOnly;
|
|
44
|
+
if (defName && !ns) parts.push(`const ${defName} = __req(${JSON.stringify(id)}).default;`);
|
|
45
|
+
if (ns) parts.push(`const ${ns} = __req(${JSON.stringify(id)});`);
|
|
46
|
+
if (named) {
|
|
47
|
+
const bindings = named.split(',').map((s) => s.trim()).filter(Boolean)
|
|
48
|
+
.map((s) => {
|
|
49
|
+
const mm = s.split(/\s+as\s+/);
|
|
50
|
+
return mm.length === 2 ? `${mm[0].trim()}: ${mm[1].trim()}` : s;
|
|
51
|
+
});
|
|
52
|
+
if (bindings.length) parts.push(`const { ${bindings.join(', ')} } = __req(${JSON.stringify(id)});`);
|
|
53
|
+
}
|
|
54
|
+
return parts.join(' ');
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// ---- collect exports
|
|
58
|
+
for (const m of src.matchAll(EXPORT_DECL_RE)) mod.exports.add(m[2]);
|
|
59
|
+
for (const m of src.matchAll(EXPORT_LIST_RE)) {
|
|
60
|
+
for (const s of m[1].split(',')) {
|
|
61
|
+
const t = s.trim();
|
|
62
|
+
if (!t) continue;
|
|
63
|
+
const parts = t.split(/\s+as\s+/);
|
|
64
|
+
mod.exports.add((parts[1] || parts[0]).trim());
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (EXPORT_DEFAULT_RE.test(src)) mod.hasDefault = true;
|
|
68
|
+
|
|
69
|
+
// ---- strip the `export` keyword / statements
|
|
70
|
+
body = body
|
|
71
|
+
.replace(EXPORT_LIST_RE, '')
|
|
72
|
+
.replace(/^([ \t]*)export\s+default\s+/m, '$1const __default = ')
|
|
73
|
+
.replace(/^([ \t]*)export\s+/gm, '$1');
|
|
74
|
+
mod.body = body;
|
|
75
|
+
|
|
76
|
+
for (const d of mod.deps) load(d);
|
|
77
|
+
order.push(abs);
|
|
78
|
+
return abs;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const entryAbs = load(entry);
|
|
82
|
+
|
|
83
|
+
let out = '';
|
|
84
|
+
out += '(function(){\n"use strict";\nvar __registry = {}, __cache = {};\n';
|
|
85
|
+
out += 'function __req(id){ if(__cache[id]) return __cache[id].e; var m=__registry[id]; if(!m) throw new Error("module not found: "+id); var e={}; __cache[id]={e:e}; m(e); return e; }\n';
|
|
86
|
+
for (const abs of order) {
|
|
87
|
+
const m = modules.get(abs);
|
|
88
|
+
const assigns = [...m.exports].map((n) => ` __exports.${n} = ${n};`).join('\n');
|
|
89
|
+
const def = m.hasDefault ? ' __exports.default = __default;\n' : '';
|
|
90
|
+
out += `__registry[${JSON.stringify(m.id)}] = function(__exports){\n${m.body}\n${assigns}\n${def}};\n`;
|
|
91
|
+
}
|
|
92
|
+
out += `__req(${JSON.stringify(modules.get(entryAbs).id)});\n})();\n`;
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function resolve(fromFile, spec) {
|
|
97
|
+
const p = path.resolve(path.dirname(fromFile), spec);
|
|
98
|
+
if (fs.existsSync(p)) return p;
|
|
99
|
+
for (const ext of ['.js', '.mjs', '/index.js']) {
|
|
100
|
+
if (fs.existsSync(p + ext)) return p + ext;
|
|
101
|
+
}
|
|
102
|
+
throw new Error(`snapshot bundler: cannot resolve "${spec}" from ${fromFile}`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function rel(abs, root) {
|
|
106
|
+
return path.relative(root, abs).split(path.sep).join('/');
|
|
107
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV export.
|
|
3
|
+
*
|
|
4
|
+
* Two modes, both streamed so a full export never buffers the dataset:
|
|
5
|
+
* - "view" the current filtered slice, at request granularity
|
|
6
|
+
* - "all" every normalized record in the store
|
|
7
|
+
*
|
|
8
|
+
* Missing values are written as an empty cell, never as 0 — the whole point of
|
|
9
|
+
* the null contract survives the export, so a spreadsheet can't silently turn
|
|
10
|
+
* "not reported" into "zero".
|
|
11
|
+
*/
|
|
12
|
+
import { Store, decodeRecord } from '../core/store.js';
|
|
13
|
+
|
|
14
|
+
export const RECORD_COLUMNS = [
|
|
15
|
+
['timestamp', 'ts'],
|
|
16
|
+
['date', 'd'],
|
|
17
|
+
['hour', 'h'],
|
|
18
|
+
['provider', 'p'],
|
|
19
|
+
['model', 'm'],
|
|
20
|
+
['model_family', 'mf'],
|
|
21
|
+
['gateway', 'g'],
|
|
22
|
+
['client', 'c'],
|
|
23
|
+
['interface', 'i'],
|
|
24
|
+
['input_tokens', 'in'],
|
|
25
|
+
['output_tokens', 'ou'],
|
|
26
|
+
['cache_read_tokens', 'cr'],
|
|
27
|
+
['cache_write_tokens', 'cw'],
|
|
28
|
+
['cache_refresh_tokens', 'cf'],
|
|
29
|
+
['reasoning_tokens', 'rs'],
|
|
30
|
+
['total_tokens', 'tt'],
|
|
31
|
+
['total_is_partial', 'tp'],
|
|
32
|
+
['session_id', 's'],
|
|
33
|
+
['conversation_id', 'cv'],
|
|
34
|
+
['request_id', 'rq'],
|
|
35
|
+
['project', 'pj'],
|
|
36
|
+
['repository', 'rp'],
|
|
37
|
+
['git_branch', 'br'],
|
|
38
|
+
['category', 'k'],
|
|
39
|
+
['service_tier', 'tr'],
|
|
40
|
+
['estimated_cost', 'co'],
|
|
41
|
+
['cost_basis', 'cb'],
|
|
42
|
+
['measurement', 'ms'],
|
|
43
|
+
['source', 'so'],
|
|
44
|
+
['duration_ms', 'du'],
|
|
45
|
+
['user', 'u'],
|
|
46
|
+
['machine', 'mc'],
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
export function csvCell(v) {
|
|
50
|
+
if (v === null || v === undefined) return '';
|
|
51
|
+
if (typeof v === 'boolean') return v ? 'true' : 'false';
|
|
52
|
+
const s = String(v);
|
|
53
|
+
return /[",\n\r]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function csvLine(values) {
|
|
57
|
+
return values.map(csvCell).join(',') + '\n';
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Stream records as CSV.
|
|
62
|
+
* @param {(chunk:string)=>void} write
|
|
63
|
+
* @param {object} filter same shape as queryRecords
|
|
64
|
+
*/
|
|
65
|
+
export function streamRecordsCsv(write, filter = {}) {
|
|
66
|
+
const store = new Store();
|
|
67
|
+
write(csvLine(RECORD_COLUMNS.map(([name]) => name)));
|
|
68
|
+
let n = 0;
|
|
69
|
+
const from = filter.from || null;
|
|
70
|
+
const to = filter.to || null;
|
|
71
|
+
const sets = {};
|
|
72
|
+
for (const [k, key] of [['provider', 'p'], ['model', 'm'], ['client', 'c'], ['interface', 'i'], ['project', 'pj'], ['source', 'so'], ['measurement', 'ms']]) {
|
|
73
|
+
const v = filter[k];
|
|
74
|
+
if (v) sets[key] = new Set(Array.isArray(v) ? v : String(v).split(',').map((s) => s.trim()));
|
|
75
|
+
}
|
|
76
|
+
let buf = '';
|
|
77
|
+
store.scanRecords((o) => {
|
|
78
|
+
if (from && o.d < from) return;
|
|
79
|
+
if (to && o.d > to) return;
|
|
80
|
+
for (const [key, set] of Object.entries(sets)) if (!set.has(o[key])) return;
|
|
81
|
+
buf += csvLine(RECORD_COLUMNS.map(([, k]) => (k === 'tp' ? !!o[k] : o[k])));
|
|
82
|
+
n++;
|
|
83
|
+
if (buf.length > 1 << 18) { write(buf); buf = ''; }
|
|
84
|
+
});
|
|
85
|
+
if (buf) write(buf);
|
|
86
|
+
return n;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Aggregated CSV of any computed table (used by per-chart "export table"). */
|
|
90
|
+
export function tableToCsv(columns, rows) {
|
|
91
|
+
let out = csvLine(columns.map((c) => c.label ?? c.key));
|
|
92
|
+
for (const r of rows) out += csvLine(columns.map((c) => (typeof c.value === 'function' ? c.value(r) : r[c.key])));
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** tokenflow-usage-2026-08-20.csv */
|
|
97
|
+
export function exportFilename(prefix = 'tokenflow-usage', date = new Date(), ext = 'csv') {
|
|
98
|
+
const d = date.toISOString().slice(0, 10);
|
|
99
|
+
return `${prefix}-${d}.${ext}`;
|
|
100
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-contained HTML snapshot.
|
|
3
|
+
*
|
|
4
|
+
* One file: inlined CSS, an inlined bundle of the same analytics + chart code
|
|
5
|
+
* the live dashboard uses, and the data bundle embedded as JSON. It opens from
|
|
6
|
+
* a file:// URL with no server, no network and no build step, which makes it
|
|
7
|
+
* the thing you can archive, attach, or hand to someone else.
|
|
8
|
+
*
|
|
9
|
+
* The ↻ Refresh button is hidden in a snapshot (there is nothing to re-read),
|
|
10
|
+
* and a bounded sample of request-level records is embedded so the Data
|
|
11
|
+
* Explorer and CSV export still work offline.
|
|
12
|
+
*/
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
import vm from 'node:vm';
|
|
16
|
+
import { bundle } from './bundler.js';
|
|
17
|
+
import { buildBundle, rootDir } from '../core/bundle.js';
|
|
18
|
+
import { Store } from '../core/store.js';
|
|
19
|
+
import { loadConfig } from '../core/config.js';
|
|
20
|
+
|
|
21
|
+
function candidatePorts() {
|
|
22
|
+
let configured = null;
|
|
23
|
+
try { configured = loadConfig().ui?.port || null; } catch { configured = null; }
|
|
24
|
+
return [...new Set([configured, 7799, 7800, 8799].filter(Boolean))];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function buildSnapshot({ maxRecords = 20000, title = 'Tokenflow' } = {}) {
|
|
28
|
+
const ROOT = rootDir();
|
|
29
|
+
const css = fs.readFileSync(path.join(ROOT, 'src', 'ui', 'styles.css'), 'utf8');
|
|
30
|
+
const html = fs.readFileSync(path.join(ROOT, 'src', 'ui', 'index.html'), 'utf8');
|
|
31
|
+
const js = bundle(path.join(ROOT, 'src', 'ui', 'app.js'), { root: ROOT });
|
|
32
|
+
// A snapshot that does not parse is worse than a failed export, so check the
|
|
33
|
+
// bundle compiles before writing it. `new vm.Script` parses without running.
|
|
34
|
+
try {
|
|
35
|
+
new vm.Script(js, { filename: 'tokenflow-snapshot-bundle.js' });
|
|
36
|
+
} catch (err) {
|
|
37
|
+
throw new Error(`snapshot bundle failed to compile: ${err.message}`);
|
|
38
|
+
}
|
|
39
|
+
const data = buildBundle();
|
|
40
|
+
|
|
41
|
+
// Newest records first, capped — an archive should stay a file, not a dump.
|
|
42
|
+
const store = new Store();
|
|
43
|
+
const recs = [];
|
|
44
|
+
store.scanRecords((o) => {
|
|
45
|
+
recs.push(o);
|
|
46
|
+
if (recs.length > maxRecords * 3) {
|
|
47
|
+
recs.sort((a, b) => (a.ts < b.ts ? 1 : -1));
|
|
48
|
+
recs.length = maxRecords;
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
recs.sort((a, b) => (a.ts < b.ts ? 1 : -1));
|
|
52
|
+
const records = recs.slice(0, maxRecords);
|
|
53
|
+
|
|
54
|
+
// NOTE: every replacement below passes a FUNCTION, never a string. A string
|
|
55
|
+
// replacement expands `$&`, `$1` and especially `$'` (the text after the
|
|
56
|
+
// match) — and the code being inlined contains `'$' + n.toFixed(2)`, which
|
|
57
|
+
// would silently splice the rest of the document into the middle of a string
|
|
58
|
+
// literal and produce an unparseable snapshot.
|
|
59
|
+
const dataScript = [
|
|
60
|
+
'<script>',
|
|
61
|
+
`window.__TOKENFLOW_BUNDLE__ = ${safeJson(data)};`,
|
|
62
|
+
`window.__TOKENFLOW_RECORDS__ = ${safeJson(records)};`,
|
|
63
|
+
`window.__TOKENFLOW_SNAPSHOT_AT__ = ${JSON.stringify(new Date().toISOString())};`,
|
|
64
|
+
// Where to look for a live dashboard when this file is opened later: the
|
|
65
|
+
// configured port first, then the usual fallbacks the server tries.
|
|
66
|
+
`window.__TOKENFLOW_PORTS__ = ${JSON.stringify(candidatePorts())};`,
|
|
67
|
+
'</scr' + 'ipt>',
|
|
68
|
+
'<script>',
|
|
69
|
+
js,
|
|
70
|
+
'</scr' + 'ipt>',
|
|
71
|
+
].join('\n');
|
|
72
|
+
|
|
73
|
+
const body = html
|
|
74
|
+
.replace(/<link rel="stylesheet"[^>]*>/, () => `<style>\n${css}\n</style>`)
|
|
75
|
+
.replace(/<script type="module" src="[^"]*"><\/script>/, () => dataScript)
|
|
76
|
+
.replace('<title>Tokenflow</title>', () => `<title>${escapeHtml(title)}</title>`)
|
|
77
|
+
.replace('loading…', () => 'static snapshot');
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
html: body,
|
|
81
|
+
stats: {
|
|
82
|
+
bytes: Buffer.byteLength(body),
|
|
83
|
+
records: records.length,
|
|
84
|
+
recordsTruncated: recs.length > records.length || records.length === maxRecords,
|
|
85
|
+
cubeRows: data.cube.rows.length,
|
|
86
|
+
sessions: data.sessions.length,
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** JSON safe to embed inside a <script> tag. */
|
|
92
|
+
function safeJson(v) {
|
|
93
|
+
return JSON.stringify(v)
|
|
94
|
+
.replace(/</g, '\\u003c')
|
|
95
|
+
.replace(/>/g, '\\u003e')
|
|
96
|
+
.replace(/\u2028/g, '\\u2028')
|
|
97
|
+
.replace(/\u2029/g, '\\u2029');
|
|
98
|
+
}
|
|
99
|
+
function escapeHtml(s) {
|
|
100
|
+
return String(s).replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
|
|
101
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Menu-bar surfaces for external bars (SwiftBar / xbar and compatible).
|
|
3
|
+
*
|
|
4
|
+
* TokenFlow stays zero-dependency and out of your way: rather than shipping an
|
|
5
|
+
* always-running Electron tray, it renders the same live status snapshot into
|
|
6
|
+
* whatever menu bar you already run. One text protocol covers them all —
|
|
7
|
+
* SwiftBar, xbar, Argos, Waybar's custom modules — so this file is pure string
|
|
8
|
+
* shaping over `buildLiveStatus()` output.
|
|
9
|
+
*
|
|
10
|
+
* line 1 becomes the menu-bar title
|
|
11
|
+
* "--" prefixed become dropdown items ("----" alone is a separator)
|
|
12
|
+
* "| key=value…" attach actions (href, bash, refresh…)
|
|
13
|
+
*/
|
|
14
|
+
import { fileURLToPath } from 'node:url';
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import { barLine, countdown, compactTokens, money } from '../core/live-status.js';
|
|
18
|
+
import { processAlive } from '../core/watch.js';
|
|
19
|
+
|
|
20
|
+
/** Absolute path of this clone's CLI — stable across working directories. */
|
|
21
|
+
export function cliPath() {
|
|
22
|
+
return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'bin', 'tokenflow.js');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const GLYPH = { ok: '', warn: '⚠ ', exceeded: '✗' };
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Render the full xbar-format text for a live status snapshot.
|
|
29
|
+
* @param {object} status live status (see core/live-status.js)
|
|
30
|
+
* @param {{mode?:string, dashboardUrl?:string}} opt
|
|
31
|
+
*/
|
|
32
|
+
export function renderXbar(status, opt = {}) {
|
|
33
|
+
const mode = opt.mode || 'auto';
|
|
34
|
+
const url = opt.dashboardUrl || 'http://127.0.0.1:7799';
|
|
35
|
+
const lines = [];
|
|
36
|
+
|
|
37
|
+
const head = barLine(status, mode);
|
|
38
|
+
const empty = head.text.trim().endsWith('—');
|
|
39
|
+
// SwiftBar re-renders on any stdout change; the href makes the whole bar
|
|
40
|
+
// clickable into the dashboard.
|
|
41
|
+
lines.push(head.text + (empty ? '' : ` | href=${url}`));
|
|
42
|
+
if (!status.usage && !status.capacity) {
|
|
43
|
+
lines.push('No data yet — run `tokenflow setup && tokenflow refresh`', `Open docs | href=${url}`);
|
|
44
|
+
return lines.join('\n');
|
|
45
|
+
}
|
|
46
|
+
lines.push('---');
|
|
47
|
+
|
|
48
|
+
const u = status.usage?.today || {};
|
|
49
|
+
const t = u.tokens || {};
|
|
50
|
+
lines.push(
|
|
51
|
+
`Today: ${compactTokens(t.total ?? 0)} tokens${costSuffix(u)} · ${u.requests ?? 0} requests | href=${url}`,
|
|
52
|
+
);
|
|
53
|
+
const w = status.usage?.weekToDate || {};
|
|
54
|
+
const m = status.usage?.monthToDate || {};
|
|
55
|
+
lines.push(`Week: ${compactTokens(w.tokens?.total ?? 0)}${costSuffix(w)}`);
|
|
56
|
+
lines.push(`Month: ${compactTokens(m.tokens?.total ?? 0)}${costSuffix(m)}`);
|
|
57
|
+
if (u.sessions != null) lines.push(`Sessions today: ${u.sessions}`);
|
|
58
|
+
|
|
59
|
+
// ---- by source / by model -------------------------------------------------
|
|
60
|
+
// Source = the tool that wrote the log (claude-code, opencode, hermes…).
|
|
61
|
+
// Provider rows name each model's vendor, so Hermes traffic appears under
|
|
62
|
+
// its vendors there; "by source" is where Hermes shows as itself.
|
|
63
|
+
const sources = status.sourcesToday || [];
|
|
64
|
+
const models = status.modelsToday || [];
|
|
65
|
+
if (sources.length || models.length) {
|
|
66
|
+
lines.push('---');
|
|
67
|
+
for (const p of sources.slice(0, 6)) {
|
|
68
|
+
lines.push(`source ${p.key}: ${compactTokens(p.tokens ?? 0)}${costSuffix(p)} · ${p.requests ?? 0} req | font-size=12`);
|
|
69
|
+
}
|
|
70
|
+
for (const mrow of models.slice(0, 5)) {
|
|
71
|
+
lines.push(`model ${mrow.key}: ${compactTokens(mrow.tokens ?? 0)}${costSuffix(mrow)} | font-size=12`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// ---- limits -------------------------------------------------------------
|
|
76
|
+
const states = status.capacity?.states || [];
|
|
77
|
+
if (states.length) {
|
|
78
|
+
lines.push('---');
|
|
79
|
+
for (const s of states.slice(0, 6)) {
|
|
80
|
+
const pctText = s.pctUsed == null ? '—' : `${Math.round(s.pctUsed * 100)}%`;
|
|
81
|
+
const resetIn = s.resetsInMs != null ? `, resets in ${countdown(s.resetsInMs)}` : '';
|
|
82
|
+
const eta = s.etaHours != null ? ` · ETA ${countdown(s.etaHours * 3600000)}` : '';
|
|
83
|
+
lines.push(`${GLYPH[s.status] || ''}${s.label}: ${pctText}${resetIn}${eta} | font-size=12`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// ---- forecast -----------------------------------------------------------
|
|
88
|
+
const f = status.forecast;
|
|
89
|
+
if (f && f.tomorrow !== null && f.confidence) {
|
|
90
|
+
lines.push('---');
|
|
91
|
+
lines.push(`Tomorrow (projected): ${compactTokens(f.tomorrow)} | font-size=12`);
|
|
92
|
+
if (f.monthEnd !== null) lines.push(`Month-end (projected): ${compactTokens(f.monthEnd)} | font-size=12`);
|
|
93
|
+
if (f.monthEndCost !== null) lines.push(`Month-end spend (projected): ${money(f.monthEndCost)} est. | font-size=12`);
|
|
94
|
+
lines.push(`Confidence: ${f.confidence}${f.n ? ` (${f.n}-day trend)` : ''} | font-size=11`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// ---- anomalies ----------------------------------------------------------
|
|
98
|
+
const alerts = (status.anomalies || []).filter((a) => a.severity === 'high' || a.severity === 'warn');
|
|
99
|
+
if (alerts.length) {
|
|
100
|
+
lines.push('---');
|
|
101
|
+
for (const a of alerts.slice(0, 3)) {
|
|
102
|
+
lines.push(`${a.severity === 'high' ? '‼️' : '⚠️'} ${a.detail.replace(/\|/g, '/')} | font-size=12`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ---- freshness + actions ------------------------------------------------
|
|
107
|
+
lines.push('---');
|
|
108
|
+
const fr = status.freshness || {};
|
|
109
|
+
const age = fr.ageMs != null ? countdown(fr.ageMs) : null;
|
|
110
|
+
lines.push(
|
|
111
|
+
`${fr.stale ? 'Data stale' : 'Updated'}${age ? ` ${age} ago` : ''} | font-size=11`,
|
|
112
|
+
);
|
|
113
|
+
const watcher = status.watcher;
|
|
114
|
+
const alive = !!(watcher?.pid && processAlive(watcher.pid));
|
|
115
|
+
lines.push(
|
|
116
|
+
alive
|
|
117
|
+
? `Watcher: running (pid ${watcher.pid}, every ${watcher.intervalSeconds ?? '?'}s) | font-size=11`
|
|
118
|
+
: 'Watcher: not running — `tokenflow watch` | font-size=11',
|
|
119
|
+
);
|
|
120
|
+
lines.push(`Refresh now | bash=/usr/bin/env param1=node param2=${cliPath()} param3=refresh terminal=true refresh=true`);
|
|
121
|
+
lines.push(`Open Dashboard | href=${url}`);
|
|
122
|
+
return lines.join('\n');
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function costSuffix(slice) {
|
|
126
|
+
const c = slice?.cost ?? slice?.costMeasured;
|
|
127
|
+
return c != null ? ` · ${money(c)}` : '';
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The tiny shell wrapper SwiftBar/xbar execute on their own schedule.
|
|
132
|
+
* Kept minimal on purpose: everything dynamic happens inside Node.
|
|
133
|
+
*/
|
|
134
|
+
export function swiftBarScript({ nodeBin = 'node', cli = 'bin/tokenflow.js', mode = 'auto', intervalMinutes = 2 } = {}) {
|
|
135
|
+
return `#!/bin/bash
|
|
136
|
+
# TokenFlow menu-bar plugin — generated by \`tokenflow menubar --swiftbar\`.
|
|
137
|
+
# Refreshes every ${intervalMinutes} min (filename convention). Requires nothing but Node.
|
|
138
|
+
export TOKENFLOW_MENUBAR_MODE="${mode}"
|
|
139
|
+
exec "${nodeBin}" "${cli}" menubar --render
|
|
140
|
+
`;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Install (or refresh) the plugin into a SwiftBar plugin directory.
|
|
145
|
+
* Returns the written path.
|
|
146
|
+
*/
|
|
147
|
+
export function installSwiftBarPlugin({ dir, mode = 'auto', intervalMinutes = 2 }) {
|
|
148
|
+
const file = path.join(dir, `tokenflow.${intervalMinutes}m.sh`);
|
|
149
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
150
|
+
fs.writeFileSync(file, swiftBarScript({
|
|
151
|
+
nodeBin: process.execPath,
|
|
152
|
+
cli: cliPath(),
|
|
153
|
+
mode,
|
|
154
|
+
intervalMinutes,
|
|
155
|
+
}));
|
|
156
|
+
fs.chmodSync(file, 0o755);
|
|
157
|
+
return file;
|
|
158
|
+
}
|