@vimoxshah/tokenflow 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +84 -0
- package/LICENSE +21 -0
- package/README.md +250 -0
- package/Refresh & Open Dashboard.command +22 -0
- package/SECURITY.md +42 -0
- package/bin/tokenflow.js +1342 -0
- package/docs/architecture.md +193 -0
- package/docs/cli.md +390 -0
- package/docs/configuration.md +281 -0
- package/docs/creating-provider.md +262 -0
- package/docs/data-model.md +213 -0
- package/docs/getting-started.md +266 -0
- package/docs/live-mode.md +199 -0
- package/docs/media/architecture-hero.svg +86 -0
- package/docs/media/cost-editorial-dark.png +0 -0
- package/docs/media/health-terminal-light.png +0 -0
- package/docs/media/menubar-dark.png +0 -0
- package/docs/media/menubar-light.png +0 -0
- package/docs/media/models-terminal-dark.png +0 -0
- package/docs/media/overview-aurora-dark.png +0 -0
- package/docs/media/time-aurora-light.png +0 -0
- package/docs/providers.md +309 -0
- package/docs/skill.md +64 -0
- package/docs/troubleshooting.md +207 -0
- package/examples/config.example.yaml +92 -0
- package/examples/demo-data/README.md +38 -0
- package/examples/demo-data/sample-usage.csv +11 -0
- package/package.json +74 -0
- package/scripts/build-dmg.sh +33 -0
- package/scripts/build-menubar-app.sh +67 -0
- package/scripts/lint.js +111 -0
- package/scripts/validate-install.js +140 -0
- package/skills/tokenflow/SKILL.md +392 -0
- package/skills/tokenflow/examples/config.yaml +92 -0
- package/skills/tokenflow/examples/generic-mapping.json +26 -0
- package/skills/tokenflow/examples/session-transcript.md +191 -0
- package/skills/tokenflow/providers/adapter-template.js +135 -0
- package/skills/tokenflow/providers/detection-matrix.md +142 -0
- package/skills/tokenflow/schemas/config.schema.json +107 -0
- package/skills/tokenflow/schemas/normalized-record.json +63 -0
- package/src/analytics/aggregate.js +247 -0
- package/src/analytics/anomalies.js +222 -0
- package/src/analytics/capacity.js +278 -0
- package/src/analytics/comparison.js +96 -0
- package/src/analytics/dimensions.js +230 -0
- package/src/analytics/efficiency.js +138 -0
- package/src/analytics/forecast.js +202 -0
- package/src/analytics/index.js +327 -0
- package/src/analytics/insights.js +283 -0
- package/src/analytics/milestones.js +91 -0
- package/src/analytics/peak.js +106 -0
- package/src/analytics/productivity.js +166 -0
- package/src/analytics/token-usage.js +267 -0
- package/src/commands/diagnostics.js +88 -0
- package/src/commands/digest.js +155 -0
- package/src/commands/models-compare.js +96 -0
- package/src/core/budget.js +142 -0
- package/src/core/bundle.js +191 -0
- package/src/core/config.js +202 -0
- package/src/core/delivery.js +109 -0
- package/src/core/geo.js +99 -0
- package/src/core/ingest.js +457 -0
- package/src/core/interface-map.js +55 -0
- package/src/core/jsonl.js +124 -0
- package/src/core/live-status.js +417 -0
- package/src/core/model-map.js +157 -0
- package/src/core/notify.js +83 -0
- package/src/core/pricing.js +288 -0
- package/src/core/prompt-analytics.js +127 -0
- package/src/core/registry.js +107 -0
- package/src/core/restore.js +261 -0
- package/src/core/schedule.js +120 -0
- package/src/core/schema.js +316 -0
- package/src/core/sqlite.js +96 -0
- package/src/core/store.js +493 -0
- package/src/core/sync.js +151 -0
- package/src/core/units.js +147 -0
- package/src/core/validate.js +123 -0
- package/src/core/watch.js +287 -0
- package/src/core/yaml.js +209 -0
- package/src/export/bundler.js +107 -0
- package/src/export/csv.js +100 -0
- package/src/export/html-snapshot.js +101 -0
- package/src/export/menubar.js +158 -0
- package/src/index.js +18 -0
- package/src/providers/anthropic/index.js +294 -0
- package/src/providers/cline/index.js +120 -0
- package/src/providers/cursor/index.js +143 -0
- package/src/providers/generic/index.js +268 -0
- package/src/providers/git/index.js +188 -0
- package/src/providers/headroom/index.js +114 -0
- package/src/providers/hermes/index.js +299 -0
- package/src/providers/mock/index.js +117 -0
- package/src/providers/openai/index.js +370 -0
- package/src/providers/opencode/index.js +245 -0
- package/src/sdk.js +46 -0
- package/src/server/server.js +264 -0
- package/src/ui/app.js +2473 -0
- package/src/ui/charts.js +925 -0
- package/src/ui/index.html +42 -0
- package/src/ui/styles.css +644 -0
package/src/core/geo.js
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider region map — the geography TokenFlow can honestly claim.
|
|
3
|
+
*
|
|
4
|
+
* Two layers, both explicit about what they are:
|
|
5
|
+
*
|
|
6
|
+
* 1. Vendor regions (offline, always on). A request to Anthropic lands in
|
|
7
|
+
* Anthropic's published datacenter regions; that much is public knowledge
|
|
8
|
+
* and can be attached to measured per-provider usage. What is NOT known is
|
|
9
|
+
* which specific region any individual request hit — providers don't log
|
|
10
|
+
* it locally — so regions are listed per provider as possibilities, never
|
|
11
|
+
* asserted per request, and no fake percentages are generated.
|
|
12
|
+
*
|
|
13
|
+
* 2. "You are here" (opt-in only). One IP geolocation lookup of THIS machine's
|
|
14
|
+
* public IP, gated behind `map.showMyLocation: true` in config.yaml,
|
|
15
|
+
* resolved once and cached in $TOKENFLOW_HOME/data/geo-cache.json with the
|
|
16
|
+
* lookup date. The raw IP is never stored — only city/region/country and
|
|
17
|
+
* coordinates returned by the lookup. Off by default because the project
|
|
18
|
+
* promises "no network client"; enabling it adds exactly one outbound call.
|
|
19
|
+
*/
|
|
20
|
+
import fs from 'node:fs';
|
|
21
|
+
import https from 'node:https';
|
|
22
|
+
import { paths, ensureDirs } from './config.js';
|
|
23
|
+
|
|
24
|
+
/** Publicly documented primary regions per provider (approximate, honest). */
|
|
25
|
+
export const PROVIDER_REGIONS = {
|
|
26
|
+
anthropic: { label: 'Anthropic', regions: ['us-west (Oregon)', 'us-east (Virginia)'] },
|
|
27
|
+
openai: { label: 'OpenAI', regions: ['us-east', 'eu-west (Azure)'] },
|
|
28
|
+
opencode: { label: 'OpenCode gateway', regions: ['multi-vendor routing — region varies by model'] },
|
|
29
|
+
openrouter: { label: 'OpenRouter', regions: ['us-east', 'eu-central'] },
|
|
30
|
+
nous: { label: 'Nous Research', regions: ['us (deepinfra-backed)'] },
|
|
31
|
+
deepseek: { label: 'DeepSeek', regions: ['cn-south'] },
|
|
32
|
+
moonshot: { label: 'Moonshot AI', regions: ['cn-north'] },
|
|
33
|
+
upstage: { label: 'Upstage', regions: ['kr-seoul'] },
|
|
34
|
+
nvidia: { label: 'NVIDIA NIM', regions: ['us-east', 'us-west'] },
|
|
35
|
+
stepfun: { label: 'StepFun', regions: ['cn-east'] },
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const GEO_CACHE = 'geo-cache.json';
|
|
39
|
+
const GEO_TTL_MS = 30 * 24 * 3600 * 1000; // re-resolve monthly at most
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Opt-in "you are here". Returns { lat, lon, city, country, source, date }
|
|
43
|
+
* or null when disabled or unreachable. Never throws; never stores the IP.
|
|
44
|
+
* @param {{config?:object}} opt
|
|
45
|
+
*/
|
|
46
|
+
export async function resolveMyLocation(opt = {}) {
|
|
47
|
+
const config = opt.config;
|
|
48
|
+
if (!config?.map?.showMyLocation) return null;
|
|
49
|
+
|
|
50
|
+
ensureDirs();
|
|
51
|
+
const file = `${paths().data}/${GEO_CACHE}`;
|
|
52
|
+
try {
|
|
53
|
+
const cached = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
54
|
+
if (cached && Date.now() - new Date(cached.date).getTime() < GEO_TTL_MS) {
|
|
55
|
+
return { ...cached, cached: true };
|
|
56
|
+
}
|
|
57
|
+
} catch { /* no cache yet */ }
|
|
58
|
+
|
|
59
|
+
const geo = await fetchGeo();
|
|
60
|
+
if (!geo) return null;
|
|
61
|
+
const record = { ...geo, date: new Date().toISOString() };
|
|
62
|
+
try {
|
|
63
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
64
|
+
fs.writeFileSync(tmp, JSON.stringify(record));
|
|
65
|
+
fs.renameSync(tmp, file);
|
|
66
|
+
} catch { /* cache write is best-effort */ }
|
|
67
|
+
return record;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function fetchGeo() {
|
|
71
|
+
// ipapi.co free tier: one HTTPS GET, returns city/region/country/lat/lon.
|
|
72
|
+
// The public IP travels to the geo service by necessity of the lookup; it is
|
|
73
|
+
// then discarded — only the derived place fields are persisted.
|
|
74
|
+
return new Promise((resolve) => {
|
|
75
|
+
const req = https.get(
|
|
76
|
+
{ host: 'ipapi.co', path: '/json/', timeout: 5000, headers: { 'user-agent': 'tokenflow-local' } },
|
|
77
|
+
(res) => {
|
|
78
|
+
let body = '';
|
|
79
|
+
res.on('data', (c) => { body += c; if (body.length > 65536) req.destroy(); });
|
|
80
|
+
res.on('end', () => {
|
|
81
|
+
try {
|
|
82
|
+
const o = JSON.parse(body);
|
|
83
|
+
if (typeof o.latitude === 'number' && typeof o.longitude === 'number') {
|
|
84
|
+
resolve({
|
|
85
|
+
lat: o.latitude,
|
|
86
|
+
lon: o.longitude,
|
|
87
|
+
city: o.city || null,
|
|
88
|
+
region: o.region || null,
|
|
89
|
+
country: o.country_name || null,
|
|
90
|
+
source: 'ipapi.co',
|
|
91
|
+
});
|
|
92
|
+
} else resolve(null);
|
|
93
|
+
} catch { resolve(null); }
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
req.on('timeout', () => { req.destroy(); resolve(null); });
|
|
97
|
+
req.on('error', () => resolve(null));
|
|
98
|
+
});
|
|
99
|
+
}
|
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ingestion engine.
|
|
3
|
+
*
|
|
4
|
+
* Adapters parse; the engine does everything else — classification, timezone
|
|
5
|
+
* resolution, cost estimation, id assignment, dedup bookkeeping, shard writes,
|
|
6
|
+
* cube and session rollups. That split is what keeps a new adapter to ~80
|
|
7
|
+
* lines and keeps the analytics layer from ever seeing vendor-specific shapes.
|
|
8
|
+
*
|
|
9
|
+
* The engine is time-budgeted and resumable: pass `deadlineMs` and it stops
|
|
10
|
+
* cleanly on a file boundary, persists state, and reports `done: false`.
|
|
11
|
+
* Calling it again picks up exactly where it stopped. That is what lets a
|
|
12
|
+
* multi-gigabyte first ingest run inside short-lived shells or a UI request.
|
|
13
|
+
*/
|
|
14
|
+
import fs from 'node:fs';
|
|
15
|
+
import os from 'node:os';
|
|
16
|
+
import { Store, encodeRecord, decodeRecord, fileId, compactShards } from './store.js';
|
|
17
|
+
import { createRecord, dateParts, hashId, MEASUREMENT, INTERFACE } from './schema.js';
|
|
18
|
+
import { classifyModel, BUILTIN_MODEL_RULES } from './model-map.js';
|
|
19
|
+
import { classifyInterface } from './interface-map.js';
|
|
20
|
+
import { buildPriceBook, estimateCost } from './pricing.js';
|
|
21
|
+
import { validateUsage } from './validate.js';
|
|
22
|
+
import { loadConfig, paths } from './config.js';
|
|
23
|
+
import { readJson, writeJson, truncateFile } from './store.js';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {object} opt
|
|
27
|
+
* @param {string[]} [opt.providers] provider ids; defaults to config.providers or all detected
|
|
28
|
+
* @param {number} [opt.deadlineMs] wall-clock budget for this call
|
|
29
|
+
* @param {boolean} [opt.full] ignore incremental state and re-ingest everything
|
|
30
|
+
* @param {boolean} [opt.strict] run full record validation (slower)
|
|
31
|
+
* @param {(e:object)=>void} [opt.onProgress]
|
|
32
|
+
* @param {object[]} [opt.registry] provider objects
|
|
33
|
+
* @param {object} [opt.config] pre-loaded config; defaults to loadConfig()
|
|
34
|
+
* @param {boolean} [opt.force] allow a full re-ingest even when a source is unreachable
|
|
35
|
+
*/
|
|
36
|
+
export async function refresh(opt = {}) {
|
|
37
|
+
const t0 = Date.now();
|
|
38
|
+
const deadline = opt.deadlineMs ? t0 + opt.deadlineMs : Infinity;
|
|
39
|
+
const config = opt.config || loadConfig();
|
|
40
|
+
const store = new Store();
|
|
41
|
+
const progress = opt.onProgress || (() => {});
|
|
42
|
+
|
|
43
|
+
const userPricing = readJson(paths().pricing, {});
|
|
44
|
+
const priceBook = buildPriceBook(userPricing);
|
|
45
|
+
const rules = [
|
|
46
|
+
...(config.modelMappings || []).map((r) => ({ ...r, label: r.label || r.provider })),
|
|
47
|
+
...BUILTIN_MODEL_RULES,
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
const all = opt.registry || [];
|
|
51
|
+
const wanted = (opt.providers && opt.providers.length ? opt.providers
|
|
52
|
+
: config.providers && config.providers.length ? config.providers
|
|
53
|
+
: all.map((p) => p.id));
|
|
54
|
+
|
|
55
|
+
const ctx = {
|
|
56
|
+
config,
|
|
57
|
+
tz: config.timezone || null,
|
|
58
|
+
home: os.homedir(),
|
|
59
|
+
user: config.identity?.user || os.userInfo().username,
|
|
60
|
+
machine: config.identity?.machine || os.hostname(),
|
|
61
|
+
priceBook,
|
|
62
|
+
rules,
|
|
63
|
+
log: (m) => progress({ type: 'log', message: m }),
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
// ------------------------------------------------------------ preflight ---
|
|
67
|
+
// Detection runs BEFORE anything destructive. A `--full` re-ingest drops the
|
|
68
|
+
// stored records for the sources in scope and rebuilds them from the source
|
|
69
|
+
// logs, which is only safe if those logs are actually reachable: run the same
|
|
70
|
+
// command in a sandbox, on another machine, or after the log directory has
|
|
71
|
+
// been moved, and the sources silently detect as unavailable — an unguarded
|
|
72
|
+
// reset would then delete a corpus it cannot rebuild. So refuse instead, and
|
|
73
|
+
// require an explicit --force from someone who really does mean "discard".
|
|
74
|
+
const detected = new Map();
|
|
75
|
+
for (const id of wanted) {
|
|
76
|
+
const p = all.find((x) => x.id === id);
|
|
77
|
+
if (!p) continue;
|
|
78
|
+
try {
|
|
79
|
+
detected.set(id, await p.detect(ctx));
|
|
80
|
+
} catch (err) {
|
|
81
|
+
detected.set(id, { available: false, detail: err.message, error: true });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const scopedFull = opt.full && opt.providers && opt.providers.length;
|
|
86
|
+
if (opt.full && !opt.force) {
|
|
87
|
+
const restoredBy = store.state.restored?.bySource || {};
|
|
88
|
+
const atRisk = wanted.filter((id) => (
|
|
89
|
+
((store.state.sources[id]?.records || 0) > 0 || (restoredBy[id] || 0) > 0)
|
|
90
|
+
&& !detected.get(id)?.available
|
|
91
|
+
));
|
|
92
|
+
if (atRisk.length) {
|
|
93
|
+
const held = atRisk.reduce((a, id) => a + (store.state.sources[id]?.records || 0) + (restoredBy[id] || 0), 0);
|
|
94
|
+
const why = atRisk.map((id) => `${id}: ${detected.get(id)?.detail || 'not detected'}`);
|
|
95
|
+
const err = /** @type {Error & {code?:string, sources?:string[]}} */ (new Error(
|
|
96
|
+
`refusing a full re-ingest: ${held.toLocaleString()} stored record(s) come from source(s) that are not reachable right now, `
|
|
97
|
+
+ `so they could not be rebuilt after being dropped.\n ${why.join('\n ')}\n`
|
|
98
|
+
+ ' Fix the source path(s) and retry, run an incremental `refresh` instead, '
|
|
99
|
+
+ 'or pass --force to discard those records anyway.',
|
|
100
|
+
));
|
|
101
|
+
err.code = 'FULL_REFRESH_UNSAFE';
|
|
102
|
+
err.sources = atRisk;
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// A restored slice (see `tokenflow restore`) stands in for a source until the
|
|
108
|
+
// real logs are read again. The moment a provider ingests for real, its
|
|
109
|
+
// restored records are marked stale so the two can never be double counted.
|
|
110
|
+
let supersededRestore = false;
|
|
111
|
+
if (store.state.restored?.bySource) {
|
|
112
|
+
for (const id of wanted) {
|
|
113
|
+
if (!detected.get(id)?.available) continue;
|
|
114
|
+
if (!store.state.restored.bySource[id]) continue;
|
|
115
|
+
store.state.stale.push([fileId('restore', id), store.state.restored.gen || 1]);
|
|
116
|
+
delete store.state.restored.bySource[id];
|
|
117
|
+
supersededRestore = true;
|
|
118
|
+
progress({ type: 'log', message: `superseding restored ${id} records with a fresh read of the source logs` });
|
|
119
|
+
}
|
|
120
|
+
if (!Object.keys(store.state.restored.bySource).length) store.state.restored.supersededAt = new Date().toISOString();
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (opt.full && !scopedFull) {
|
|
124
|
+
// Global re-ingest: everything goes.
|
|
125
|
+
store.state.sources = {};
|
|
126
|
+
store.state.stale = [];
|
|
127
|
+
store.state.counters = { records: 0, malformed: 0 };
|
|
128
|
+
store.resetCube();
|
|
129
|
+
store.resetSessions();
|
|
130
|
+
store.resetActivity();
|
|
131
|
+
for (const s of store.listShards()) truncateFile(`${paths().records}/${s}`);
|
|
132
|
+
} else if (scopedFull) {
|
|
133
|
+
// Re-ingest ONE provider without touching the others. Its existing records
|
|
134
|
+
// are marked stale (rather than deleted, which would mean rewriting shared
|
|
135
|
+
// shards mid-ingest) and the aggregates are rebuilt from the shards
|
|
136
|
+
// afterwards, skipping the stale generations.
|
|
137
|
+
for (const id of opt.providers) {
|
|
138
|
+
const st = store.state.sources[id];
|
|
139
|
+
if (!st) continue;
|
|
140
|
+
for (const [key, f] of Object.entries(st.files || {})) {
|
|
141
|
+
store.state.stale.push([fileId(id, key), f.gen || 1]);
|
|
142
|
+
st.files[key] = { size: -1, mtimeMs: -1, offset: 0, gen: (f.gen || 1) + 1, records: 0 };
|
|
143
|
+
}
|
|
144
|
+
// Fetch-based adapters (SQLite, APIs, generators) have no per-file state,
|
|
145
|
+
// so they carry a source-level generation for exactly this purpose.
|
|
146
|
+
store.state.stale.push([fileId(id, id), st.gen || 1]);
|
|
147
|
+
st.gen = (st.gen || 1) + 1;
|
|
148
|
+
st.cursor = null;
|
|
149
|
+
st.records = 0;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Working directories seen in usage records — lets the git adapter
|
|
154
|
+
* auto-discover repositories with zero configuration. */
|
|
155
|
+
const projectPaths = new Set(readJson(`${paths().data}/project-paths.json`, { paths: [] }).paths || []);
|
|
156
|
+
|
|
157
|
+
const report = {
|
|
158
|
+
startedAt: new Date(t0).toISOString(),
|
|
159
|
+
done: true,
|
|
160
|
+
providers: [],
|
|
161
|
+
newRecords: 0,
|
|
162
|
+
filesScanned: 0,
|
|
163
|
+
filesSkipped: 0,
|
|
164
|
+
bytesRead: 0,
|
|
165
|
+
malformed: 0,
|
|
166
|
+
invalid: [],
|
|
167
|
+
durationMs: 0,
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
for (const id of wanted) {
|
|
171
|
+
const p = all.find((x) => x.id === id);
|
|
172
|
+
if (!p) {
|
|
173
|
+
report.providers.push({ id, status: 'unknown-provider' });
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
const pr = { id, name: p.name, status: 'ok', records: 0, files: 0, processed: 0, skipped: 0, bytes: 0, notes: [], detail: null };
|
|
177
|
+
report.providers.push(pr);
|
|
178
|
+
|
|
179
|
+
const det = detected.has(id) ? detected.get(id) : await p.detect(ctx).catch((e) => ({ available: false, detail: e.message, error: true }));
|
|
180
|
+
if (det && det.error) {
|
|
181
|
+
pr.status = 'detect-error';
|
|
182
|
+
pr.notes.push(det.detail);
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (!det || !det.available) {
|
|
186
|
+
pr.status = 'not-detected';
|
|
187
|
+
pr.detail = det?.detail || null;
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
pr.detail = det.detail || null;
|
|
191
|
+
|
|
192
|
+
const sourceState = store.sourceState(p.id);
|
|
193
|
+
let seq = 0;
|
|
194
|
+
|
|
195
|
+
const makeEmit = (fileRef) => (partial) => {
|
|
196
|
+
const rec = enrich(partial, {
|
|
197
|
+
ctx, provider: p, seq: seq++, fileRef,
|
|
198
|
+
});
|
|
199
|
+
if (!rec) return;
|
|
200
|
+
if (opt.strict) {
|
|
201
|
+
const v = validateUsage(rec);
|
|
202
|
+
if (!v.ok && report.invalid.length < 20) report.invalid.push({ id: rec.id, source: p.id, errors: v.errors });
|
|
203
|
+
}
|
|
204
|
+
if (rec.metadata && rec.metadata.cwd && projectPaths.size < 5000) projectPaths.add(rec.metadata.cwd);
|
|
205
|
+
const enc = encodeRecord(rec);
|
|
206
|
+
if (config.store?.keepRaw !== false) store.writer(rec.date).write(enc);
|
|
207
|
+
store.addToCube(rec);
|
|
208
|
+
store.upsertSession(rec);
|
|
209
|
+
store.addToActivity(rec);
|
|
210
|
+
report.newRecords++;
|
|
211
|
+
pr.records++;
|
|
212
|
+
sourceState.records = (sourceState.records || 0) + 1;
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
try {
|
|
216
|
+
if (typeof p.discover === 'function' && typeof p.ingestFile === 'function') {
|
|
217
|
+
const files = (await p.discover(ctx)) || [];
|
|
218
|
+
pr.files = files.length;
|
|
219
|
+
// Oldest first: history fills in monotonically, so a budget-limited
|
|
220
|
+
// first run always leaves a contiguous, explainable dataset.
|
|
221
|
+
files.sort((a, b) => (a.stat?.mtimeMs || 0) - (b.stat?.mtimeMs || 0));
|
|
222
|
+
for (const f of files) {
|
|
223
|
+
if (Date.now() > deadline) { report.done = false; pr.status = 'partial'; break; }
|
|
224
|
+
const stat = f.stat || safeStat(f.path);
|
|
225
|
+
if (!stat) continue;
|
|
226
|
+
const plan = store.planFile(p.id, f.key, stat);
|
|
227
|
+
if (plan.action === 'skip' && !opt.full) { pr.skipped++; report.filesSkipped++; continue; }
|
|
228
|
+
const fileRef = {
|
|
229
|
+
...f, stat, gen: plan.gen,
|
|
230
|
+
start: plan.action === 'append' ? plan.start : 0,
|
|
231
|
+
state: plan.action === 'rewrite' ? {} : (plan.prev?.adapter ?? {}),
|
|
232
|
+
};
|
|
233
|
+
let res;
|
|
234
|
+
try {
|
|
235
|
+
res = (await p.ingestFile(fileRef, ctx, makeEmit(fileRef))) || {};
|
|
236
|
+
} catch (err) {
|
|
237
|
+
pr.notes.push(`${f.key}: ${err.message}`);
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
report.filesScanned++;
|
|
241
|
+
pr.processed++;
|
|
242
|
+
const bytes = (res.offset ?? stat.size) - fileRef.start;
|
|
243
|
+
pr.bytes += Math.max(0, bytes);
|
|
244
|
+
report.bytesRead += Math.max(0, bytes);
|
|
245
|
+
report.malformed += res.malformed || 0;
|
|
246
|
+
store.commitFile(p.id, f.key, stat, plan.gen, res.offset ?? stat.size, res.records ?? 0, plan.prev?.gen);
|
|
247
|
+
store.sourceState(p.id).files[f.key].adapter = fileRef.state;
|
|
248
|
+
if (report.filesScanned % 200 === 0) {
|
|
249
|
+
progress({ type: 'progress', provider: p.id, files: report.filesScanned, records: report.newRecords });
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
} else if (typeof p.fetchUsage === 'function') {
|
|
253
|
+
const fetchRef = { key: p.id, path: null, gen: sourceState.gen || 1, state: sourceState.adapter ?? {} };
|
|
254
|
+
const res = (await p.fetchUsage(ctx, makeEmit(fetchRef), sourceState)) || {};
|
|
255
|
+
sourceState.adapter = fetchRef.state;
|
|
256
|
+
if (res.cursor !== undefined) sourceState.cursor = res.cursor;
|
|
257
|
+
if (res.notes) pr.notes.push(...res.notes);
|
|
258
|
+
report.malformed += res.malformed || 0;
|
|
259
|
+
}
|
|
260
|
+
} catch (err) {
|
|
261
|
+
pr.status = 'error';
|
|
262
|
+
pr.notes.push(err.message);
|
|
263
|
+
}
|
|
264
|
+
sourceState.lastRefresh = new Date().toISOString();
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
store.closeWriters();
|
|
268
|
+
store.state.counters.malformed = (store.state.counters.malformed || 0) + report.malformed;
|
|
269
|
+
store.state.counters.records = (store.state.counters.records || 0) + report.newRecords;
|
|
270
|
+
store.state.lastRefresh = new Date().toISOString();
|
|
271
|
+
report.durationMs = Date.now() - t0;
|
|
272
|
+
store.state.lastRefreshDurationMs = report.durationMs;
|
|
273
|
+
writeJson(`${paths().data}/project-paths.json`, { paths: [...projectPaths] });
|
|
274
|
+
if (scopedFull || supersededRestore) {
|
|
275
|
+
progress({ type: 'log', message: 'compacting superseded records…' });
|
|
276
|
+
store.closeWriters();
|
|
277
|
+
const compacted = compactShards(store);
|
|
278
|
+
progress({ type: 'log', message: 'rebuilding aggregates from stored records…' });
|
|
279
|
+
const stats = rebuildAggregates(store);
|
|
280
|
+
report.rebuilt = { ...stats, ...compacted };
|
|
281
|
+
// Record counts are now derived from what actually survived rather than
|
|
282
|
+
// from a running total. Without this they keep counting superseded and
|
|
283
|
+
// restored records that no longer exist, and `state.json` ends up claiming
|
|
284
|
+
// a corpus larger than the one on disk — which then misinforms the `--full`
|
|
285
|
+
// safety check, which reads exactly these numbers to decide what is at risk.
|
|
286
|
+
store.state.counters.records = stats.records;
|
|
287
|
+
for (const id of Object.keys(store.state.sources)) {
|
|
288
|
+
store.state.sources[id].records = stats.bySource[id] || 0;
|
|
289
|
+
}
|
|
290
|
+
report.recounted = true;
|
|
291
|
+
}
|
|
292
|
+
store.saveCube({ tz: ctx.tz, pricingVersion: priceBook.version });
|
|
293
|
+
store.saveSessions();
|
|
294
|
+
store.saveActivity();
|
|
295
|
+
store.saveState();
|
|
296
|
+
|
|
297
|
+
report.finishedAt = new Date().toISOString();
|
|
298
|
+
progress({ type: 'done', report });
|
|
299
|
+
return report;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Turn an adapter's partial into a fully classified, priced, dated record.
|
|
304
|
+
* Exported so adapter tests can assert normalization in isolation.
|
|
305
|
+
*/
|
|
306
|
+
export function enrich(partial, { ctx, provider, seq = 0, fileRef = null }) {
|
|
307
|
+
if (!partial) return null;
|
|
308
|
+
const ts = partial.timestamp;
|
|
309
|
+
const dp = dateParts(ts, ctx.tz);
|
|
310
|
+
if (!dp) return null;
|
|
311
|
+
|
|
312
|
+
const cls = partial.provider
|
|
313
|
+
? {
|
|
314
|
+
provider: partial.provider,
|
|
315
|
+
provider_label: partial.provider_label || partial.provider,
|
|
316
|
+
model: partial.model || 'unknown',
|
|
317
|
+
model_family: partial.model_family || 'Unknown',
|
|
318
|
+
}
|
|
319
|
+
: classifyModel(partial.model, { rules: ctx.rules, providerHint: partial.providerHint || null });
|
|
320
|
+
|
|
321
|
+
let iface = partial.interface;
|
|
322
|
+
if (!iface) {
|
|
323
|
+
const override = ctx.config.interfaceOverrides?.[partial.client];
|
|
324
|
+
iface = override || classifyInterface(partial.interfaceSignals || []).interface;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const base = {
|
|
328
|
+
timestamp: dp.iso,
|
|
329
|
+
date: dp.date,
|
|
330
|
+
hour: dp.hour,
|
|
331
|
+
dow: dp.dow,
|
|
332
|
+
tz_offset: dp.tz_offset,
|
|
333
|
+
provider: cls.provider,
|
|
334
|
+
provider_label: cls.provider_label,
|
|
335
|
+
model: cls.model,
|
|
336
|
+
model_family: partial.model_family || cls.model_family,
|
|
337
|
+
gateway: partial.gateway ?? null,
|
|
338
|
+
client: partial.client || provider.id,
|
|
339
|
+
application: partial.application || provider.name,
|
|
340
|
+
interface: iface || INTERFACE.UNKNOWN,
|
|
341
|
+
input_tokens: partial.input_tokens,
|
|
342
|
+
output_tokens: partial.output_tokens,
|
|
343
|
+
cache_read_tokens: partial.cache_read_tokens,
|
|
344
|
+
cache_write_tokens: partial.cache_write_tokens,
|
|
345
|
+
cache_refresh_tokens: partial.cache_refresh_tokens,
|
|
346
|
+
reasoning_tokens: partial.reasoning_tokens,
|
|
347
|
+
session_id: partial.session_id ?? null,
|
|
348
|
+
conversation_id: partial.conversation_id ?? null,
|
|
349
|
+
request_id: partial.request_id ?? null,
|
|
350
|
+
project: partial.project ?? null,
|
|
351
|
+
repository: partial.repository ?? null,
|
|
352
|
+
git_branch: partial.git_branch ?? null,
|
|
353
|
+
category: partial.category ?? null,
|
|
354
|
+
service_tier: partial.service_tier ?? partial.metadata?.service_tier ?? null,
|
|
355
|
+
duration_ms: partial.duration_ms ?? null,
|
|
356
|
+
source: provider.id,
|
|
357
|
+
measurement: partial.measurement || provider.measurement || MEASUREMENT.PRIMARY,
|
|
358
|
+
user: partial.user ?? ctx.user,
|
|
359
|
+
machine: partial.machine ?? ctx.machine,
|
|
360
|
+
metadata: partial.metadata || {},
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
// Cost: a measured cost from the source always wins; otherwise estimate,
|
|
364
|
+
// and leave null when the model has no configured price.
|
|
365
|
+
if (partial.measured_cost !== undefined && partial.measured_cost !== null) {
|
|
366
|
+
base.estimated_cost = partial.measured_cost;
|
|
367
|
+
base.cost_basis = 'measured';
|
|
368
|
+
} else {
|
|
369
|
+
// The service tier is a real multiplier, not a footnote: OpenAI's Fast mode
|
|
370
|
+
// bills at 4x standard, Anthropic's Batch API at 0.5x.
|
|
371
|
+
const c = estimateCost(base, base.model, base.provider, ctx.priceBook, { tier: base.service_tier });
|
|
372
|
+
base.estimated_cost = c.cost;
|
|
373
|
+
base.cost_basis = c.basis;
|
|
374
|
+
if (c.partial || (c.tierMult && c.tierMult !== 1) || c.src) {
|
|
375
|
+
base.metadata = {
|
|
376
|
+
...base.metadata,
|
|
377
|
+
...(c.partial ? { cost_partial: true } : {}),
|
|
378
|
+
...(c.tierMult !== 1 ? { cost_tier_multiplier: c.tierMult } : {}),
|
|
379
|
+
...(c.src ? { price_source: c.src } : {}),
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const rec = createRecord(base);
|
|
385
|
+
rec.id = partial.id || hashId(
|
|
386
|
+
provider.id,
|
|
387
|
+
rec.session_id || fileRef?.key || '',
|
|
388
|
+
rec.request_id || rec.timestamp,
|
|
389
|
+
rec.model,
|
|
390
|
+
String(seq),
|
|
391
|
+
);
|
|
392
|
+
if (fileRef) {
|
|
393
|
+
rec._fileId = fileId(provider.id, fileRef.key);
|
|
394
|
+
rec._gen = fileRef.gen ?? 1;
|
|
395
|
+
}
|
|
396
|
+
return rec;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
function safeStat(p) {
|
|
400
|
+
try {
|
|
401
|
+
return fs.statSync(p);
|
|
402
|
+
} catch {
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Walk a directory tree, returning files that match `test`. Symlink-safe. */
|
|
408
|
+
export function walk(root, test, { maxDepth = 12, limit = 200000 } = {}) {
|
|
409
|
+
const out = [];
|
|
410
|
+
const stack = [[root, 0]];
|
|
411
|
+
while (stack.length && out.length < limit) {
|
|
412
|
+
const [dir, depth] = stack.pop();
|
|
413
|
+
if (depth > maxDepth) continue;
|
|
414
|
+
let entries;
|
|
415
|
+
try {
|
|
416
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
417
|
+
} catch {
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
for (const e of entries) {
|
|
421
|
+
const full = `${dir}/${e.name}`;
|
|
422
|
+
if (e.isDirectory()) {
|
|
423
|
+
stack.push([full, depth + 1]);
|
|
424
|
+
} else if (e.isFile() && test(e.name, full)) {
|
|
425
|
+
out.push(full);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return out;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Rebuild the cube, sessions and activity rollups from the stored
|
|
435
|
+
* request-level records, skipping stale generations.
|
|
436
|
+
*
|
|
437
|
+
* This is what makes a per-provider `--full` safe: one adapter can be
|
|
438
|
+
* re-ingested from scratch without disturbing any other adapter's numbers,
|
|
439
|
+
* because the aggregates are always derivable from the facts on disk.
|
|
440
|
+
*/
|
|
441
|
+
export function rebuildAggregates(store) {
|
|
442
|
+
const stale = store.staleSet();
|
|
443
|
+
store.resetCube();
|
|
444
|
+
store.resetSessions();
|
|
445
|
+
store.resetActivity();
|
|
446
|
+
let kept = 0;
|
|
447
|
+
const bySource = {};
|
|
448
|
+
store.scanRecords((o) => {
|
|
449
|
+
const rec = decodeRecord(o);
|
|
450
|
+
store.addToCube(rec);
|
|
451
|
+
store.upsertSession(rec);
|
|
452
|
+
store.addToActivity(rec);
|
|
453
|
+
bySource[rec.source] = (bySource[rec.source] || 0) + 1;
|
|
454
|
+
kept++;
|
|
455
|
+
}, { stale });
|
|
456
|
+
return { records: kept, bySource };
|
|
457
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interface / surface classification.
|
|
3
|
+
*
|
|
4
|
+
* Rule: an interface is only ever derived from an explicit surface signal in
|
|
5
|
+
* the source record (entrypoint, originator, source, IDE marker). It is NEVER
|
|
6
|
+
* inferred from the model or the provider — "it's a Claude model so it must be
|
|
7
|
+
* Claude Desktop" is exactly the mistake this module exists to prevent.
|
|
8
|
+
* No signal => INTERFACE.UNKNOWN.
|
|
9
|
+
*/
|
|
10
|
+
import { INTERFACE } from './schema.js';
|
|
11
|
+
|
|
12
|
+
/** Ordered signal table. First match wins; the value is checked lowercased. */
|
|
13
|
+
/** @type {[RegExp, string][]} */
|
|
14
|
+
const SIGNALS = [
|
|
15
|
+
// --- explicit IDE / editor hosts -------------------------------------
|
|
16
|
+
[/^(vscode|vs-code|visual-?studio(-code)?|code-insiders)$/, INTERFACE.IDE],
|
|
17
|
+
[/(^|[-_])(jetbrains|intellij|pycharm|webstorm|goland|rider|android-?studio)/, INTERFACE.IDE],
|
|
18
|
+
[/^(cursor|windsurf|zed|neovim|nvim|vim|emacs|sublime)$/, INTERFACE.IDE],
|
|
19
|
+
[/(^|[-_])ide($|[-_])/, INTERFACE.IDE],
|
|
20
|
+
[/(^|[-_])(extension|plugin)($|[-_])/, INTERFACE.EXTENSION],
|
|
21
|
+
|
|
22
|
+
// --- desktop apps ----------------------------------------------------
|
|
23
|
+
[/desktop/, INTERFACE.DESKTOP],
|
|
24
|
+
[/(^|[-_])(app|electron|tauri)$/, INTERFACE.DESKTOP],
|
|
25
|
+
|
|
26
|
+
// --- web -------------------------------------------------------------
|
|
27
|
+
[/(^|[-_])(web|browser|chatgpt-web|claude-ai)($|[-_])/, INTERFACE.WEB],
|
|
28
|
+
|
|
29
|
+
// --- SDK / programmatic ---------------------------------------------
|
|
30
|
+
[/sdk/, INTERFACE.SDK],
|
|
31
|
+
[/(^|[-_])(api|rest|http|openai-python|anthropic-python)($|[-_])/, INTERFACE.API],
|
|
32
|
+
|
|
33
|
+
// --- CLI / headless --------------------------------------------------
|
|
34
|
+
[/(^|[-_ ])(cli|tui|exec|terminal|shell|headless|repl)($|[-_ ])/, INTERFACE.CLI],
|
|
35
|
+
[/^(cli|tui|exec)$/, INTERFACE.CLI],
|
|
36
|
+
// Named coding agents that only exist as terminal programs. This is still
|
|
37
|
+
// evidence from a surface field (the originator/client name), not an
|
|
38
|
+
// inference from the model.
|
|
39
|
+
[/(^|[-_ ])(claude[-_ ]?code|aider|cline|opencode|crush|goose|amp|codebuff)($|[-_ ])/, INTERFACE.CLI],
|
|
40
|
+
];
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* @param {(string|null|undefined)[]} signals ordered strongest-first
|
|
44
|
+
* @returns {{interface: string, signal: string|null}}
|
|
45
|
+
*/
|
|
46
|
+
export function classifyInterface(signals) {
|
|
47
|
+
for (const raw of signals) {
|
|
48
|
+
if (!raw) continue;
|
|
49
|
+
const s = String(raw).toLowerCase();
|
|
50
|
+
for (const [re, iface] of SIGNALS) {
|
|
51
|
+
if (re.test(s)) return { interface: iface, signal: String(raw) };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return { interface: INTERFACE.UNKNOWN, signal: signals.find(Boolean) ?? null };
|
|
55
|
+
}
|