@vimoxshah/tokenflow 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +84 -0
- package/LICENSE +21 -0
- package/README.md +250 -0
- package/Refresh & Open Dashboard.command +22 -0
- package/SECURITY.md +42 -0
- package/bin/tokenflow.js +1342 -0
- package/docs/architecture.md +193 -0
- package/docs/cli.md +390 -0
- package/docs/configuration.md +281 -0
- package/docs/creating-provider.md +262 -0
- package/docs/data-model.md +213 -0
- package/docs/getting-started.md +266 -0
- package/docs/live-mode.md +199 -0
- package/docs/media/architecture-hero.svg +86 -0
- package/docs/media/cost-editorial-dark.png +0 -0
- package/docs/media/health-terminal-light.png +0 -0
- package/docs/media/menubar-dark.png +0 -0
- package/docs/media/menubar-light.png +0 -0
- package/docs/media/models-terminal-dark.png +0 -0
- package/docs/media/overview-aurora-dark.png +0 -0
- package/docs/media/time-aurora-light.png +0 -0
- package/docs/providers.md +309 -0
- package/docs/skill.md +64 -0
- package/docs/troubleshooting.md +207 -0
- package/examples/config.example.yaml +92 -0
- package/examples/demo-data/README.md +38 -0
- package/examples/demo-data/sample-usage.csv +11 -0
- package/package.json +74 -0
- package/scripts/build-dmg.sh +33 -0
- package/scripts/build-menubar-app.sh +67 -0
- package/scripts/lint.js +111 -0
- package/scripts/validate-install.js +140 -0
- package/skills/tokenflow/SKILL.md +392 -0
- package/skills/tokenflow/examples/config.yaml +92 -0
- package/skills/tokenflow/examples/generic-mapping.json +26 -0
- package/skills/tokenflow/examples/session-transcript.md +191 -0
- package/skills/tokenflow/providers/adapter-template.js +135 -0
- package/skills/tokenflow/providers/detection-matrix.md +142 -0
- package/skills/tokenflow/schemas/config.schema.json +107 -0
- package/skills/tokenflow/schemas/normalized-record.json +63 -0
- package/src/analytics/aggregate.js +247 -0
- package/src/analytics/anomalies.js +222 -0
- package/src/analytics/capacity.js +278 -0
- package/src/analytics/comparison.js +96 -0
- package/src/analytics/dimensions.js +230 -0
- package/src/analytics/efficiency.js +138 -0
- package/src/analytics/forecast.js +202 -0
- package/src/analytics/index.js +327 -0
- package/src/analytics/insights.js +283 -0
- package/src/analytics/milestones.js +91 -0
- package/src/analytics/peak.js +106 -0
- package/src/analytics/productivity.js +166 -0
- package/src/analytics/token-usage.js +267 -0
- package/src/commands/diagnostics.js +88 -0
- package/src/commands/digest.js +155 -0
- package/src/commands/models-compare.js +96 -0
- package/src/core/budget.js +142 -0
- package/src/core/bundle.js +191 -0
- package/src/core/config.js +202 -0
- package/src/core/delivery.js +109 -0
- package/src/core/geo.js +99 -0
- package/src/core/ingest.js +457 -0
- package/src/core/interface-map.js +55 -0
- package/src/core/jsonl.js +124 -0
- package/src/core/live-status.js +417 -0
- package/src/core/model-map.js +157 -0
- package/src/core/notify.js +83 -0
- package/src/core/pricing.js +288 -0
- package/src/core/prompt-analytics.js +127 -0
- package/src/core/registry.js +107 -0
- package/src/core/restore.js +261 -0
- package/src/core/schedule.js +120 -0
- package/src/core/schema.js +316 -0
- package/src/core/sqlite.js +96 -0
- package/src/core/store.js +493 -0
- package/src/core/sync.js +151 -0
- package/src/core/units.js +147 -0
- package/src/core/validate.js +123 -0
- package/src/core/watch.js +287 -0
- package/src/core/yaml.js +209 -0
- package/src/export/bundler.js +107 -0
- package/src/export/csv.js +100 -0
- package/src/export/html-snapshot.js +101 -0
- package/src/export/menubar.js +158 -0
- package/src/index.js +18 -0
- package/src/providers/anthropic/index.js +294 -0
- package/src/providers/cline/index.js +120 -0
- package/src/providers/cursor/index.js +143 -0
- package/src/providers/generic/index.js +268 -0
- package/src/providers/git/index.js +188 -0
- package/src/providers/headroom/index.js +114 -0
- package/src/providers/hermes/index.js +299 -0
- package/src/providers/mock/index.js +117 -0
- package/src/providers/openai/index.js +370 -0
- package/src/providers/opencode/index.js +245 -0
- package/src/sdk.js +46 -0
- package/src/server/server.js +264 -0
- package/src/ui/app.js +2473 -0
- package/src/ui/charts.js +925 -0
- package/src/ui/index.html +42 -0
- package/src/ui/styles.css +644 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Budget forecasting alerts — projected month-end spend vs a configured
|
|
3
|
+
* budget, with persisted alert state to prevent notification spam.
|
|
4
|
+
*
|
|
5
|
+
* Config (config.yaml):
|
|
6
|
+
*
|
|
7
|
+
* budget:
|
|
8
|
+
* monthly: 200 # USD — your monthly cap
|
|
9
|
+
* warnAtPct: 80 # alert when PROJECTED month-end >= 80% of budget
|
|
10
|
+
*
|
|
11
|
+
* States (monotonic per month, tracked in $TOKENFLOW_HOME/data/budget-state.json):
|
|
12
|
+
* safe — projection < warn threshold
|
|
13
|
+
* approaching — projection >= warnAtPct% of budget (fires once per month)
|
|
14
|
+
* over_budget_projected — projection >= 100% of budget (fires once per month)
|
|
15
|
+
* over_budget_actual — measured MTD spend >= budget (fires once per month)
|
|
16
|
+
*
|
|
17
|
+
* Honesty rules:
|
|
18
|
+
* - "projected" is always labelled as a projection; never shown as charged.
|
|
19
|
+
* - If there is insufficient history for a forecast (< 3 days of data this
|
|
20
|
+
* month), the state is "unknown" and no alert fires. Never fabricate $0.
|
|
21
|
+
* - State resets when the calendar month changes, so each month can fire
|
|
22
|
+
* its own approaching/over alerts exactly once.
|
|
23
|
+
*/
|
|
24
|
+
import fs from 'node:fs';
|
|
25
|
+
import path from 'node:path';
|
|
26
|
+
import os from 'node:os';
|
|
27
|
+
|
|
28
|
+
// State file resolves from TOKENFLOW_HOME at CALL time (not import time) so
|
|
29
|
+
// tests can isolate state via env without fighting module-level caching.
|
|
30
|
+
const STATE_FILE = () => {
|
|
31
|
+
const home = process.env.TOKENFLOW_HOME || path.join(os.homedir(), '.tokenflow');
|
|
32
|
+
return path.join(home, 'data', 'budget-state.json');
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Compute the current budget state from a live status snapshot.
|
|
37
|
+
* @param {object} status live status (usage.monthToDate, velocity, forecast)
|
|
38
|
+
* @param {{monthly:number, warnAtPct?:number}} budget config.budget
|
|
39
|
+
* @param {string} today YYYY-MM-DD
|
|
40
|
+
*/
|
|
41
|
+
export function computeBudgetState(status, budget, today) {
|
|
42
|
+
if (!budget?.monthly || !(budget.monthly > 0)) return null;
|
|
43
|
+
const mtd = status.usage?.monthToDate;
|
|
44
|
+
if (!mtd) return null;
|
|
45
|
+
|
|
46
|
+
const spent = mtd.cost ?? null; // estimated; measured kept separate
|
|
47
|
+
if (spent == null) return { state: 'unknown', reason: 'no priced usage yet this month' };
|
|
48
|
+
|
|
49
|
+
const warnPct = (budget.warnAtPct ?? 80) / 100;
|
|
50
|
+
const monthKey = today.slice(0, 7);
|
|
51
|
+
|
|
52
|
+
// Actual overage is checkable immediately and needs no forecast.
|
|
53
|
+
if (spent >= budget.monthly) {
|
|
54
|
+
return {
|
|
55
|
+
state: 'over_budget_actual', monthKey,
|
|
56
|
+
spent, budget: budget.monthly,
|
|
57
|
+
message: `Month-to-date spend ${fmt(spent)} has reached your ${fmt(budget.monthly)} budget.`,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Projected month-end: prefer the engine's linear-trend forecast cost;
|
|
62
|
+
// fall back to pace-based extrapolation only if velocity exists.
|
|
63
|
+
const projected = status.forecast?.monthEndCost ?? projectFromPace(status, today);
|
|
64
|
+
if (projected == null) {
|
|
65
|
+
return {
|
|
66
|
+
state: 'unknown', monthKey,
|
|
67
|
+
spent, budget: budget.monthly,
|
|
68
|
+
reason: 'not enough history this month for a forecast',
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const pct = projected / budget.monthly;
|
|
73
|
+
if (pct >= 1) {
|
|
74
|
+
return {
|
|
75
|
+
state: 'over_budget_projected', monthKey,
|
|
76
|
+
spent, projected, budget: budget.monthly,
|
|
77
|
+
message: `Projected month-end spend ${fmt(projected)} exceeds your ${fmt(budget.monthly)} budget (projection, not an actual charge).`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
if (pct >= warnPct) {
|
|
81
|
+
return {
|
|
82
|
+
state: 'approaching', monthKey,
|
|
83
|
+
spent, projected, budget: budget.monthly,
|
|
84
|
+
message: `Projected month-end spend ${fmt(projected)} is ${Math.round(pct * 100)}% of your ${fmt(budget.monthly)} budget.`,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return { state: 'safe', monthKey, spent, projected, budget: budget.monthly };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function projectFromPace(status, today) {
|
|
91
|
+
const v = status.velocity;
|
|
92
|
+
if (!v?.todayTokensPerHour || !status.usage?.monthToDate?.cost) return null;
|
|
93
|
+
// Pace-based cost projection needs cost-per-token which we do not track
|
|
94
|
+
// here; defer to the engine's forecast rather than inventing a rate.
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function fmt(n) { return `$${(Math.round(n * 100) / 100).toLocaleString('en-US')}`; }
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Decide whether an alert should FIRE now, given persisted state.
|
|
102
|
+
* Dedup: each state fires once per calendar month. Escalation
|
|
103
|
+
* (approaching → over_budget_projected / over_budget_actual) re-fires because
|
|
104
|
+
* it is strictly more urgent than what was already sent.
|
|
105
|
+
* @returns {{fire: boolean, state: object}}
|
|
106
|
+
*/
|
|
107
|
+
export function shouldAlert(newState, opt = {}) {
|
|
108
|
+
if (!newState || newState.state === 'safe' || newState.state === 'unknown') {
|
|
109
|
+
persist({ state: newState?.state || 'safe', monthKey: newState?.monthKey });
|
|
110
|
+
return { fire: false, state: newState };
|
|
111
|
+
}
|
|
112
|
+
const prev = loadState();
|
|
113
|
+
const sameMonth = prev?.monthKey === newState.monthKey;
|
|
114
|
+
const RANK = { safe: 0, unknown: 0, approaching: 1, over_budget_projected: 2, over_budget_actual: 3 };
|
|
115
|
+
|
|
116
|
+
let fire;
|
|
117
|
+
if (!sameMonth) {
|
|
118
|
+
fire = true; // new month → alert again
|
|
119
|
+
} else if ((RANK[newState.state] ?? 0) > (RANK[prev.state] ?? 0)) {
|
|
120
|
+
fire = true; // escalated urgency
|
|
121
|
+
} else {
|
|
122
|
+
fire = false; // already alerted at this level
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (opt.force) fire = true;
|
|
126
|
+
if (fire) persist({ state: newState.state, monthKey: newState.monthKey });
|
|
127
|
+
return { fire, state: newState };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function loadState() {
|
|
131
|
+
try { return JSON.parse(fs.readFileSync(STATE_FILE(), 'utf8')); } catch { return null; }
|
|
132
|
+
}
|
|
133
|
+
function persist(s) {
|
|
134
|
+
try {
|
|
135
|
+
const file = STATE_FILE();
|
|
136
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
137
|
+
fs.writeFileSync(file, JSON.stringify({ ...s, updatedAt: new Date().toISOString() }));
|
|
138
|
+
} catch { /* best-effort */ }
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Reset state (new month detected by callers, or user command). */
|
|
142
|
+
export function resetBudgetState() { try { fs.unlinkSync(STATE_FILE()); } catch { /* absent */ } }
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The data bundle handed to a dashboard (browser or snapshot).
|
|
3
|
+
*
|
|
4
|
+
* It is deliberately small: the pre-aggregated cube, one row per session, the
|
|
5
|
+
* daily work-activity rollup, and metadata. Request-level records stay on disk
|
|
6
|
+
* and are streamed on demand by the Data Explorer, so opening the dashboard
|
|
7
|
+
* never means shipping a gigabyte into a browser tab.
|
|
8
|
+
*/
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import url from 'node:url';
|
|
12
|
+
import { Store, readJson } from './store.js';
|
|
13
|
+
import { paths, loadConfig } from './config.js';
|
|
14
|
+
import { summarizeQuality } from './validate.js';
|
|
15
|
+
import { tzOffsetMinutes } from './schema.js';
|
|
16
|
+
import { PRICING_TABLE_VERSION, PRICING_SOURCES, TIER_MULTIPLIERS } from './pricing.js';
|
|
17
|
+
|
|
18
|
+
export function buildBundle({ config = loadConfig() } = {}) {
|
|
19
|
+
const store = new Store();
|
|
20
|
+
const p = paths();
|
|
21
|
+
const cube = store.cube();
|
|
22
|
+
const sessions = store.sessionList();
|
|
23
|
+
const activity = store.activity();
|
|
24
|
+
const pricing = readJson(p.pricing, {});
|
|
25
|
+
const health = summarizeQuality(cube, store.sessions(), store.state);
|
|
26
|
+
|
|
27
|
+
// Per-adapter date coverage, so "Date range" is never read as if every
|
|
28
|
+
// source covered all of it.
|
|
29
|
+
const perSource = new Map();
|
|
30
|
+
for (const s of sessions) {
|
|
31
|
+
const e = perSource.get(s.so) || { from: null, to: null, sessions: 0, tokens: 0 };
|
|
32
|
+
if (e.from === null || s.d < e.from) e.from = s.d;
|
|
33
|
+
if (e.to === null || s.d > e.to) e.to = s.d;
|
|
34
|
+
e.sessions++;
|
|
35
|
+
e.tokens += s.total || 0;
|
|
36
|
+
perSource.set(s.so, e);
|
|
37
|
+
}
|
|
38
|
+
const providersState = Object.entries(store.state.sources || {}).map(([id, s]) => ({
|
|
39
|
+
id,
|
|
40
|
+
records: s.records || 0,
|
|
41
|
+
files: Object.keys(s.files || {}).length,
|
|
42
|
+
lastRefresh: s.lastRefresh || null,
|
|
43
|
+
coverage: perSource.get(id) ? { from: perSource.get(id).from, to: perSource.get(id).to } : null,
|
|
44
|
+
sessions: perSource.get(id)?.sessions ?? 0,
|
|
45
|
+
tokens: perSource.get(id)?.tokens ?? 0,
|
|
46
|
+
}));
|
|
47
|
+
|
|
48
|
+
const demo = sessions.some((s) => s.so === 'mock') || cube.rows.some((r) => r[6] === 'mock');
|
|
49
|
+
const coverage = health.coverage;
|
|
50
|
+
const tz = cube.tz || config.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
meta: {
|
|
54
|
+
generatedAt: new Date().toISOString(),
|
|
55
|
+
appVersion: readJson(path.join(rootDir(), 'package.json'), {}).version || '0.0.0',
|
|
56
|
+
cubeVersion: cube.version,
|
|
57
|
+
timezone: tz,
|
|
58
|
+
today: localToday(tz),
|
|
59
|
+
// Wall-clock offset of the dataset's timezone right now. Capacity reset
|
|
60
|
+
// countdowns and burn-rate hours are computed against this, so a limit
|
|
61
|
+
// resets when the *user's* calendar rolls over.
|
|
62
|
+
tzOffsetMinutes: tzOffsetMinutes(new Date(), tz),
|
|
63
|
+
pricingTableVersion: PRICING_TABLE_VERSION,
|
|
64
|
+
pricingSources: PRICING_SOURCES,
|
|
65
|
+
tierMultipliers: TIER_MULTIPLIERS,
|
|
66
|
+
lastRefresh: store.state.lastRefresh,
|
|
67
|
+
lastRefreshDurationMs: store.state.lastRefreshDurationMs,
|
|
68
|
+
coverage,
|
|
69
|
+
demo,
|
|
70
|
+
includeOverlayDefault: !!config.analytics?.includeOverlaySources,
|
|
71
|
+
defaultRange: config.ui?.defaultRange || 'all',
|
|
72
|
+
defaultFrom: config.ui?.defaultFrom || null,
|
|
73
|
+
// Theme defaults from config; the browser's own choice wins once set.
|
|
74
|
+
skin: config.ui?.skin || 'aurora',
|
|
75
|
+
mode: config.ui?.mode || config.ui?.theme || 'dark',
|
|
76
|
+
builtAt: cube.builtAt || store.state.lastRefresh || null,
|
|
77
|
+
dataHome: p.root,
|
|
78
|
+
sources: providersState,
|
|
79
|
+
},
|
|
80
|
+
cube,
|
|
81
|
+
sessions,
|
|
82
|
+
activity,
|
|
83
|
+
pricing,
|
|
84
|
+
// User-declared quota/budget caps; analytics/capacity.js evaluates them
|
|
85
|
+
// against the same cube every other surface reads.
|
|
86
|
+
limits: Array.isArray(config.limits) ? config.limits : [],
|
|
87
|
+
health,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Records for the Data Explorer, filtered and paginated server-side. */
|
|
92
|
+
export function queryRecords(q = {}) {
|
|
93
|
+
const store = new Store();
|
|
94
|
+
const limit = Math.min(Number(q.limit) || 100, 5000);
|
|
95
|
+
const offset = Number(q.offset) || 0;
|
|
96
|
+
const search = (q.search || '').toLowerCase();
|
|
97
|
+
const sortKey = q.sort || 'ts';
|
|
98
|
+
const desc = q.dir !== 'asc';
|
|
99
|
+
const wanted = {
|
|
100
|
+
from: q.from || null, to: q.to || null,
|
|
101
|
+
p: split(q.provider), m: split(q.model), c: split(q.client),
|
|
102
|
+
i: split(q.interface), pj: split(q.project), so: split(q.source),
|
|
103
|
+
ms: split(q.measurement),
|
|
104
|
+
};
|
|
105
|
+
const months = monthsBetween(wanted.from, wanted.to);
|
|
106
|
+
const matched = [];
|
|
107
|
+
let scanned = 0;
|
|
108
|
+
let total = 0;
|
|
109
|
+
|
|
110
|
+
// Keep only what a page could need, plus enough to sort: a bounded top-K
|
|
111
|
+
// buffer means a 100-record page never materialises a million objects.
|
|
112
|
+
const cap = offset + limit;
|
|
113
|
+
store.scanRecords((o) => {
|
|
114
|
+
scanned++;
|
|
115
|
+
if (wanted.from && o.d < wanted.from) return;
|
|
116
|
+
if (wanted.to && o.d > wanted.to) return;
|
|
117
|
+
for (const k of ['p', 'm', 'c', 'i', 'pj', 'so', 'ms']) {
|
|
118
|
+
if (wanted[k] && !wanted[k].includes(o[k])) return;
|
|
119
|
+
}
|
|
120
|
+
if (search) {
|
|
121
|
+
const hay = `${o.m} ${o.p} ${o.c} ${o.pj} ${o.s} ${o.i} ${o.rq || ''} ${o.br || ''}`.toLowerCase();
|
|
122
|
+
if (!hay.includes(search)) return;
|
|
123
|
+
}
|
|
124
|
+
total++;
|
|
125
|
+
matched.push(o);
|
|
126
|
+
if (matched.length > cap * 4 + 4000) {
|
|
127
|
+
matched.sort(cmp(sortKey, desc));
|
|
128
|
+
matched.length = cap;
|
|
129
|
+
}
|
|
130
|
+
}, { months });
|
|
131
|
+
|
|
132
|
+
matched.sort(cmp(sortKey, desc));
|
|
133
|
+
return {
|
|
134
|
+
total,
|
|
135
|
+
scanned,
|
|
136
|
+
offset,
|
|
137
|
+
limit,
|
|
138
|
+
rows: matched.slice(offset, offset + limit),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function cmp(key, desc) {
|
|
143
|
+
return (a, b) => {
|
|
144
|
+
const x = a[key];
|
|
145
|
+
const y = b[key];
|
|
146
|
+
if (x === y) return 0;
|
|
147
|
+
if (x === undefined || x === null) return 1;
|
|
148
|
+
if (y === undefined || y === null) return -1;
|
|
149
|
+
const r = x < y ? -1 : 1;
|
|
150
|
+
return desc ? -r : r;
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function split(v) {
|
|
155
|
+
if (!v) return null;
|
|
156
|
+
const a = Array.isArray(v) ? v : String(v).split(',');
|
|
157
|
+
const out = a.map((s) => String(s).trim()).filter(Boolean);
|
|
158
|
+
return out.length ? out : null;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function monthsBetween(from, to) {
|
|
162
|
+
if (!from || !to) return null;
|
|
163
|
+
const out = [];
|
|
164
|
+
let y = Number(from.slice(0, 4));
|
|
165
|
+
let m = Number(from.slice(5, 7));
|
|
166
|
+
const ey = Number(to.slice(0, 4));
|
|
167
|
+
const em = Number(to.slice(5, 7));
|
|
168
|
+
let guard = 0;
|
|
169
|
+
while ((y < ey || (y === ey && m <= em)) && guard++ < 600) {
|
|
170
|
+
out.push(`${y}-${String(m).padStart(2, '0')}`);
|
|
171
|
+
m++;
|
|
172
|
+
if (m > 12) { m = 1; y++; }
|
|
173
|
+
}
|
|
174
|
+
return out;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function localToday(tz) {
|
|
178
|
+
try {
|
|
179
|
+
const f = new Intl.DateTimeFormat('en-CA', { timeZone: tz || undefined, year: 'numeric', month: '2-digit', day: '2-digit' });
|
|
180
|
+
const p = {};
|
|
181
|
+
for (const x of f.formatToParts(new Date())) p[x.type] = x.value;
|
|
182
|
+
return `${p.year}-${p.month}-${p.day}`;
|
|
183
|
+
} catch {
|
|
184
|
+
return new Date().toISOString().slice(0, 10);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function rootDir() {
|
|
189
|
+
// src/core -> project root
|
|
190
|
+
return path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), '..', '..');
|
|
191
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Portable configuration.
|
|
3
|
+
*
|
|
4
|
+
* ~/.tokenflow/
|
|
5
|
+
* config.yaml providers, sources, preferences (config.json also read)
|
|
6
|
+
* pricing.json user-defined pricing overrides
|
|
7
|
+
* mappings/ saved generic-import field mappings
|
|
8
|
+
* data/ normalized records, cube, sessions, ingest state
|
|
9
|
+
* cache/ adapter scratch
|
|
10
|
+
*
|
|
11
|
+
* Override the root with TOKENFLOW_HOME. Everything is local; nothing here is
|
|
12
|
+
* ever transmitted.
|
|
13
|
+
*/
|
|
14
|
+
import fs from 'node:fs';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
import os from 'node:os';
|
|
17
|
+
import { parseYaml, stringifyYaml } from './yaml.js';
|
|
18
|
+
|
|
19
|
+
export const CONFIG_VERSION = 1;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Where everything lives.
|
|
23
|
+
*
|
|
24
|
+
* `TOKENFLOW_HOME` wins, then `~/.tokenflow`. The two legacy names are still
|
|
25
|
+
* honoured so an install that predates the rename keeps its ingested data
|
|
26
|
+
* instead of silently starting from an empty store — which, with an incremental
|
|
27
|
+
* engine keyed on per-file offsets, would look like "all my history vanished".
|
|
28
|
+
* Legacy is only used when the current location does not exist yet.
|
|
29
|
+
*/
|
|
30
|
+
export function homeDir() {
|
|
31
|
+
if (process.env.TOKENFLOW_HOME) return process.env.TOKENFLOW_HOME;
|
|
32
|
+
if (process.env.AI_USAGE_HOME) return process.env.AI_USAGE_HOME;
|
|
33
|
+
const current = path.join(os.homedir(), '.tokenflow');
|
|
34
|
+
if (fs.existsSync(current)) return current;
|
|
35
|
+
const legacy = path.join(os.homedir(), '.ai-usage-dashboard');
|
|
36
|
+
if (fs.existsSync(legacy)) return legacy;
|
|
37
|
+
return current;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export const paths = () => {
|
|
41
|
+
const root = homeDir();
|
|
42
|
+
return {
|
|
43
|
+
root,
|
|
44
|
+
configYaml: path.join(root, 'config.yaml'),
|
|
45
|
+
configJson: path.join(root, 'config.json'),
|
|
46
|
+
pricing: path.join(root, 'pricing.json'),
|
|
47
|
+
mappings: path.join(root, 'mappings'),
|
|
48
|
+
data: path.join(root, 'data'),
|
|
49
|
+
records: path.join(root, 'data', 'records'),
|
|
50
|
+
cube: path.join(root, 'data', 'cube.json'),
|
|
51
|
+
sessions: path.join(root, 'data', 'sessions.json'),
|
|
52
|
+
activity: path.join(root, 'data', 'activity.json'),
|
|
53
|
+
state: path.join(root, 'data', 'state.json'),
|
|
54
|
+
// Written by `tokenflow watch`, read by every live surface (menu bar,
|
|
55
|
+
// CLI --live, dashboard header). Atomic tmp+rename writes only.
|
|
56
|
+
status: path.join(root, 'data', 'status.json'),
|
|
57
|
+
watchPid: path.join(root, 'data', 'watch.pid'),
|
|
58
|
+
cache: path.join(root, 'cache'),
|
|
59
|
+
prefs: path.join(root, 'preferences.json'),
|
|
60
|
+
};
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export const DEFAULT_CONFIG = {
|
|
64
|
+
version: CONFIG_VERSION,
|
|
65
|
+
timezone: null, // null = this machine's zone
|
|
66
|
+
identity: { user: null, machine: null, team: null },
|
|
67
|
+
providers: [], // populated by `tokenflow setup` / `provider add`
|
|
68
|
+
sources: {}, // per-provider source options; `{type: auto}` by default
|
|
69
|
+
store: {
|
|
70
|
+
keepRaw: true, // keep request-level records for the Data Explorer + full export
|
|
71
|
+
rawRetentionDays: null, // null = forever
|
|
72
|
+
},
|
|
73
|
+
analytics: {
|
|
74
|
+
includeOverlaySources: false, // proxy/gateway views excluded from totals by default
|
|
75
|
+
minSessionGapMinutes: 30, // used only for sources without a session id
|
|
76
|
+
},
|
|
77
|
+
modelMappings: [], // user rules, prepended to the built-ins
|
|
78
|
+
interfaceOverrides: {}, // { "<client>": "CLI" }
|
|
79
|
+
// User-declared quota / budget caps, evaluated by analytics/capacity.js.
|
|
80
|
+
// TokenFlow never invents vendor quota data; a limit exists only if you
|
|
81
|
+
// declared it here (or via the dashboard's Live tab).
|
|
82
|
+
limits: [],
|
|
83
|
+
// Monthly spend budget with forecast alerts. OFF until `monthly` is set.
|
|
84
|
+
// budget: { monthly: 200, warnAtPct: 80 }
|
|
85
|
+
// Alerts fire once per state per month (approaching / over-projected /
|
|
86
|
+
// over-actual), persisted in data/budget-state.json to avoid spam.
|
|
87
|
+
budget: {
|
|
88
|
+
monthly: null,
|
|
89
|
+
warnAtPct: 80,
|
|
90
|
+
notify: false, // opt-in OS/delivery notification; state tracking always on
|
|
91
|
+
},
|
|
92
|
+
// Digest delivery channels. Each is OFF until configured. Credentials live
|
|
93
|
+
// only in this local file and are never logged or transmitted elsewhere.
|
|
94
|
+
delivery: {
|
|
95
|
+
telegram: { botToken: null, chatId: null },
|
|
96
|
+
email: { smtpUrl: null, to: null },
|
|
97
|
+
webhook: { url: null },
|
|
98
|
+
},
|
|
99
|
+
// Optional multi-machine aggregation. OFF by default: nothing leaves this
|
|
100
|
+
// machine until you point sync.dir at a folder shared between your machines
|
|
101
|
+
// (iCloud/Dropbox/Syncthing) and set enabled: true. Only daily totals are
|
|
102
|
+
// exchanged — never prompts, code, or credentials. See src/core/sync.js.
|
|
103
|
+
sync: {
|
|
104
|
+
enabled: false,
|
|
105
|
+
dir: null, // e.g. ~/Sync/TokenFlow
|
|
106
|
+
machineName: null, // friendly label; defaults to hostname
|
|
107
|
+
},
|
|
108
|
+
// Prompt-level analytics. STRICTLY opt-in and OFF by default. Even when
|
|
109
|
+
// enabled, only one-way hashes + categories are stored — raw prompt text is
|
|
110
|
+
// captured only if storeRaw is separately set to true.
|
|
111
|
+
promptAnalytics: {
|
|
112
|
+
enabled: false,
|
|
113
|
+
storeRaw: false,
|
|
114
|
+
},
|
|
115
|
+
// The background refresher (`tokenflow watch`). It runs only while you
|
|
116
|
+
// explicitly started it; nothing is installed or auto-started.
|
|
117
|
+
watch: {
|
|
118
|
+
intervalSeconds: 120, // between incremental refresh passes
|
|
119
|
+
notifications: false, // opt-in OS notifications on threshold crossings
|
|
120
|
+
staleAfterSeconds: 600, // when live surfaces should call the data stale
|
|
121
|
+
},
|
|
122
|
+
ui: {
|
|
123
|
+
// Visual theme: skin restyles the chrome, mode flips light/dark. The series
|
|
124
|
+
// colours belong to the mode and are validated per surface, so a skin never
|
|
125
|
+
// changes what a colour means.
|
|
126
|
+
skin: 'aurora', // aurora | terminal | editorial
|
|
127
|
+
mode: 'dark', // dark | light
|
|
128
|
+
theme: 'dark', // legacy single-axis alias, kept for older configs
|
|
129
|
+
// Port the local dashboard binds to (and the port a saved snapshot probes
|
|
130
|
+
// when it looks for a live dashboard to hand over to).
|
|
131
|
+
port: 7799,
|
|
132
|
+
// Which quick range the dashboard opens on: all | 7d | 30d | 90d | mtd
|
|
133
|
+
defaultRange: 'all',
|
|
134
|
+
// Optional hard floor for the default view (YYYY-MM-DD), for when the store
|
|
135
|
+
// holds older records you don't normally want in scope.
|
|
136
|
+
defaultFrom: null,
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
export function loadConfig() {
|
|
141
|
+
const p = paths();
|
|
142
|
+
let cfg = structuredClone(DEFAULT_CONFIG);
|
|
143
|
+
try {
|
|
144
|
+
if (fs.existsSync(p.configYaml)) {
|
|
145
|
+
cfg = merge(cfg, parseYaml(fs.readFileSync(p.configYaml, 'utf8')) || {});
|
|
146
|
+
} else if (fs.existsSync(p.configJson)) {
|
|
147
|
+
cfg = merge(cfg, JSON.parse(fs.readFileSync(p.configJson, 'utf8')));
|
|
148
|
+
}
|
|
149
|
+
} catch (err) {
|
|
150
|
+
const e = /** @type {Error & {hint?:string}} */ (new Error(`Could not read config at ${p.configYaml}: ${err.message}`));
|
|
151
|
+
e.hint = 'Fix the syntax, or delete the file and re-run `tokenflow setup`.';
|
|
152
|
+
throw e;
|
|
153
|
+
}
|
|
154
|
+
return cfg;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function saveConfig(cfg) {
|
|
158
|
+
const p = paths();
|
|
159
|
+
ensureDirs();
|
|
160
|
+
const header = [
|
|
161
|
+
'# tokenflow configuration',
|
|
162
|
+
'# Local-first: nothing in this directory is uploaded anywhere.',
|
|
163
|
+
'# Docs: docs/providers.md Regenerate: tokenflow setup',
|
|
164
|
+
'',
|
|
165
|
+
].join('\n');
|
|
166
|
+
fs.writeFileSync(p.configYaml, header + stringifyYaml(cfg) + '\n');
|
|
167
|
+
return p.configYaml;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export function ensureDirs() {
|
|
171
|
+
const p = paths();
|
|
172
|
+
for (const d of [p.root, p.data, p.records, p.cache, p.mappings]) {
|
|
173
|
+
fs.mkdirSync(d, { recursive: true });
|
|
174
|
+
}
|
|
175
|
+
return p;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export function loadPrefs() {
|
|
179
|
+
const p = paths();
|
|
180
|
+
try {
|
|
181
|
+
return JSON.parse(fs.readFileSync(p.prefs, 'utf8'));
|
|
182
|
+
} catch {
|
|
183
|
+
return {};
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function savePrefs(prefs) {
|
|
188
|
+
const p = paths();
|
|
189
|
+
ensureDirs();
|
|
190
|
+
fs.writeFileSync(p.prefs, JSON.stringify(prefs, null, 2));
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Deep merge where arrays replace and plain objects merge. */
|
|
194
|
+
export function merge(base, over) {
|
|
195
|
+
if (over === null || over === undefined) return base;
|
|
196
|
+
if (Array.isArray(over) || typeof over !== 'object') return over;
|
|
197
|
+
const out = Array.isArray(base) ? [...base] : { ...(base || {}) };
|
|
198
|
+
for (const [k, v] of Object.entries(over)) {
|
|
199
|
+
out[k] = v !== null && typeof v === 'object' && !Array.isArray(v) ? merge(out[k], v) : v;
|
|
200
|
+
}
|
|
201
|
+
return out;
|
|
202
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Digest delivery adapters — strictly opt-in, configured in config.yaml.
|
|
3
|
+
*
|
|
4
|
+
* delivery:
|
|
5
|
+
* telegram:
|
|
6
|
+
* botToken: <from @BotFather> # stored locally, never logged
|
|
7
|
+
* chatId: "<your chat id>"
|
|
8
|
+
* email:
|
|
9
|
+
* smtpUrl: smtp://user:pass@smtp.gmail.com:465
|
|
10
|
+
* to: you@example.com
|
|
11
|
+
* webhook:
|
|
12
|
+
* url: https://example.com/hook
|
|
13
|
+
*
|
|
14
|
+
* Every channel is OFF until its block exists. Credentials live only in
|
|
15
|
+
* $TOKENFLOW_HOME/config.yaml (0600 by umask); they are never logged, never
|
|
16
|
+
* echoed in errors, and never included in packaged files.
|
|
17
|
+
*
|
|
18
|
+
* Zero dependencies: Telegram + webhook use node:https; SMTP is delegated to
|
|
19
|
+
* the local `sendmail`-compatible binary if present (most macOS/Linux boxes
|
|
20
|
+
* have one via postfix) — if not, email reports "transport unavailable"
|
|
21
|
+
* instead of failing silently.
|
|
22
|
+
*/
|
|
23
|
+
import https from 'node:https';
|
|
24
|
+
import { execFile } from 'node:child_process';
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Deliver markdown text via every configured channel.
|
|
28
|
+
* Returns per-channel results; never throws for a channel that isn't configured.
|
|
29
|
+
* @param {object} cfg parsed TokenFlow config
|
|
30
|
+
* @param {string} text markdown digest
|
|
31
|
+
* @param {{subject?: string}} opt
|
|
32
|
+
*/
|
|
33
|
+
export async function deliverAll(cfg, text, opt = {}) {
|
|
34
|
+
const d = cfg?.delivery || {};
|
|
35
|
+
const results = [];
|
|
36
|
+
const subject = opt.subject || 'TokenFlow digest';
|
|
37
|
+
|
|
38
|
+
if (d.telegram?.botToken && d.telegram?.chatId) {
|
|
39
|
+
results.push({ channel: 'telegram', ...(await sendTelegram(d.telegram.botToken, d.telegram.chatId, text)) });
|
|
40
|
+
} else {
|
|
41
|
+
results.push({ channel: 'telegram', skipped: 'not configured' });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (d.email?.smtpUrl && d.email?.to) {
|
|
45
|
+
results.push({ channel: 'email', ...(await sendEmail(d.email.smtpUrl, d.email.to, subject, text)) });
|
|
46
|
+
} else {
|
|
47
|
+
results.push({ channel: 'email', skipped: 'not configured' });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (d.webhook?.url) {
|
|
51
|
+
results.push({ channel: 'webhook', ...(await sendWebhook(d.webhook.url, { subject, text })) });
|
|
52
|
+
} else {
|
|
53
|
+
results.push({ channel: 'webhook', skipped: 'not configured' });
|
|
54
|
+
}
|
|
55
|
+
return results;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function sendTelegram(botToken, chatId, text) {
|
|
59
|
+
try {
|
|
60
|
+
// Markdown digest → Telegram MarkdownV2 needs escaping; send as plain
|
|
61
|
+
// text with disabled preview instead — honest and robust over pretty.
|
|
62
|
+
const body = JSON.stringify({ chat_id: String(chatId), text, disable_web_page_preview: true });
|
|
63
|
+
const status = await postJSON(`api.telegram.org`, `/bot${botToken}/sendMessage`, body);
|
|
64
|
+
return status === 200 ? { ok: true } : { ok: false, error: `HTTP ${status}` };
|
|
65
|
+
} catch (e) {
|
|
66
|
+
return { ok: false, error: e.message };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function postJSON(host, reqPath, body) {
|
|
71
|
+
return new Promise((resolve) => {
|
|
72
|
+
const req = https.request(
|
|
73
|
+
{ host, path: reqPath, method: 'POST', timeout: 15000,
|
|
74
|
+
headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } },
|
|
75
|
+
(res) => { res.resume(); res.on('end', () => resolve(res.statusCode)); });
|
|
76
|
+
req.on('timeout', () => { req.destroy(); resolve(0); });
|
|
77
|
+
req.on('error', () => resolve(0));
|
|
78
|
+
req.write(body);
|
|
79
|
+
req.end();
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function sendWebhook(url, payload) {
|
|
84
|
+
try {
|
|
85
|
+
const u = new URL(url);
|
|
86
|
+
const status = await postJSON(u.host, u.pathname + u.search, JSON.stringify(payload));
|
|
87
|
+
return status >= 200 && status < 300 ? { ok: true } : { ok: false, error: `HTTP ${status}` };
|
|
88
|
+
} catch (e) {
|
|
89
|
+
return { ok: false, error: e.message };
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Email via local sendmail-compatible transport (postfix on macOS).
|
|
95
|
+
* The smtpUrl is used only to decide transport viability today; credentials
|
|
96
|
+
* inside it are NOT parsed or logged. A future version can add direct SMTP
|
|
97
|
+
* AUTH without changing the config shape.
|
|
98
|
+
*/
|
|
99
|
+
async function sendEmail(smtpUrl, to, subject, text) {
|
|
100
|
+
const u = new URL(smtpUrl);
|
|
101
|
+
const host = u.hostname || 'localhost';
|
|
102
|
+
return new Promise((resolve) => {
|
|
103
|
+
// sendmail -t reads recipients from headers; universally available via postfix
|
|
104
|
+
execFile('/usr/sbin/sendmail', ['-t'], { timeout: 20000 }, (err) => {
|
|
105
|
+
if (err) { resolve({ ok: false, error: `local sendmail unavailable (${err.code || err.message})` }); return; }
|
|
106
|
+
resolve({ ok: true });
|
|
107
|
+
}).stdin?.end?.(`To: ${to}\nSubject: ${subject}\nContent-Type: text/plain; charset=utf-8\nX-TokenFlow-Transport: ${host}\n\n${text}\n`);
|
|
108
|
+
});
|
|
109
|
+
}
|