@optimuslabs/harness-map-staging 1.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/apply_deferred_vscdb.js +38 -0
- package/dist/bootstrap_constants.js +15 -0
- package/dist/cli/bash_script_generator.js +95 -0
- package/dist/cli.js +144 -0
- package/dist/cli_invocation_match.js +30 -0
- package/dist/compliance_check_runner.js +57 -0
- package/dist/compliance_prompt_gate.js +396 -0
- package/dist/dialog_prefs_cli.js +109 -0
- package/dist/endpoint_client/compress_payload.js +25 -0
- package/dist/endpoint_client/http_transport.js +90 -0
- package/dist/endpoint_client/index.js +3 -0
- package/dist/endpoint_client/registry_api.js +41 -0
- package/dist/endpoint_client/startup_api.js +44 -0
- package/dist/endpoint_client/types.js +4 -0
- package/dist/execute_trusted_restarts.js +60 -0
- package/dist/log_config_files/auth/auth_flow.js +22 -0
- package/dist/log_config_files/auth/auth_key_store.js +14 -0
- package/dist/log_config_files/collection/claude_desktop_extensions_collector.js +248 -0
- package/dist/log_config_files/collection/claude_known_projects_collector.js +42 -0
- package/dist/log_config_files/collection/claude_token_usage_collector.js +203 -0
- package/dist/log_config_files/collection/codex_token_usage_collector.js +342 -0
- package/dist/log_config_files/collection/config_collector.js +245 -0
- package/dist/log_config_files/collection/copilot_token_usage_collector.js +167 -0
- package/dist/log_config_files/collection/cowork_desktop_version_collector.js +60 -0
- package/dist/log_config_files/collection/cowork_session_whitelist.js +89 -0
- package/dist/log_config_files/collection/cursor_project_mcp_collector.js +230 -0
- package/dist/log_config_files/collection/cursor_project_workspace_activity.js +73 -0
- package/dist/log_config_files/collection/cursor_token_usage_collector.js +192 -0
- package/dist/log_config_files/collection/directory_collector.js +126 -0
- package/dist/log_config_files/collection/enrichment_helpers.js +53 -0
- package/dist/log_config_files/collection/ensure_cursor_user_settings_snapshot.js +50 -0
- package/dist/log_config_files/collection/file_type_rules.js +47 -0
- package/dist/log_config_files/collection/grok_cli_version_collector.js +53 -0
- package/dist/log_config_files/collection/hermes_token_usage_collector.js +241 -0
- package/dist/log_config_files/collection/mcp_tool_collector.js +106 -0
- package/dist/log_config_files/collection/metadata_merge.js +43 -0
- package/dist/log_config_files/collection/openclaw_helpers.js +55 -0
- package/dist/log_config_files/collection/openclaw_token_usage_collector.js +223 -0
- package/dist/log_config_files/collection/opencode_token_usage_collector.js +177 -0
- package/dist/log_config_files/collection/pi_token_usage_collector.js +239 -0
- package/dist/log_config_files/collection/plugin_collector.js +223 -0
- package/dist/log_config_files/collection/plugin_version_helpers.js +37 -0
- package/dist/log_config_files/collection/skills_cli_collector.js +162 -0
- package/dist/log_config_files/index.js +19 -0
- package/dist/log_config_files/paths/path_constants_helpers.js +71 -0
- package/dist/log_config_files/paths/pattern_resolver.js +326 -0
- package/dist/log_config_files/readers/composer_shadow_merge.js +67 -0
- package/dist/log_config_files/readers/cursor_shadow_merge_ruleset.js +46 -0
- package/dist/log_config_files/readers/file_readers.js +196 -0
- package/dist/log_config_files/readers/vscdb_config_builder.js +148 -0
- package/dist/log_config_files/readers/vscdb_reactive_storage.js +62 -0
- package/dist/log_config_files/readers/vscdb_reader.js +337 -0
- package/dist/log_config_files/runtime/client_event_reporter.js +347 -0
- package/dist/log_config_files/runtime/compliance_check.js +1013 -0
- package/dist/log_config_files/runtime/compliance_session_log.js +378 -0
- package/dist/log_config_files/runtime/dialog_preferences.js +31 -0
- package/dist/log_config_files/runtime/hardware_uuid.js +36 -0
- package/dist/log_config_files/runtime/hook_logger.js +267 -0
- package/dist/log_config_files/runtime/hook_type_for_request.js +30 -0
- package/dist/log_config_files/runtime/log_metadata.js +122 -0
- package/dist/log_config_files/runtime/main_runner.js +376 -0
- package/dist/log_config_files/runtime/management_storage.js +144 -0
- package/dist/log_config_files/runtime/ops_target_path.js +18 -0
- package/dist/log_config_files/runtime/remediation_apply_tracking.js +152 -0
- package/dist/log_config_files/runtime/remediation_config_path.js +140 -0
- package/dist/log_config_files/runtime/remediation_sync.js +1705 -0
- package/dist/log_config_files/runtime/secret_regex_scan.js +126 -0
- package/dist/log_config_files/runtime/sqlite_binary.js +124 -0
- package/dist/log_config_files/runtime/trusted_restarts.js +113 -0
- package/dist/log_config_files/runtime/workspace_repo.js +170 -0
- package/dist/log_config_files/runtime/worktree_absent.js +50 -0
- package/dist/log_config_files/runtime/worktree_scanner.js +137 -0
- package/dist/log_config_files/sender/batch_sender.js +283 -0
- package/dist/log_config_files/sender/endpoint_config.js +24 -0
- package/dist/log_config_files/sender/signing.js +1 -0
- package/dist/log_sensitive_paths_audit.js +89 -0
- package/dist/log_uuid/auth_key_store.js +37 -0
- package/dist/log_uuid/hardware_uuid.js +35 -0
- package/dist/log_uuid/index.js +11 -0
- package/dist/log_uuid/log_uuid_helper.js +30 -0
- package/dist/log_uuid/startup_sender.js +131 -0
- package/dist/log_uuid/user_profile.js +192 -0
- package/dist/remediation_change_preview.js +182 -0
- package/dist/tofu.js +22 -0
- package/dist/tofu_environment.js +36 -0
- package/dist/types/config_file_types.js +1 -0
- package/package.json +67 -0
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
/**
|
|
5
|
+
* Token-usage / session / version / identity aggregator for OpenAI Codex CLI.
|
|
6
|
+
*
|
|
7
|
+
* Codex writes one JSONL rollout per session under ~/.codex/sessions/<y>/<m>/<d>/rollout-*.jsonl
|
|
8
|
+
* (and ~/.codex/archived_sessions/). A `session_meta` line carries `cli_version` + `model_provider`;
|
|
9
|
+
* `token_count` lines carry a *cumulative* `total_token_usage` ({input,cached_input,output,
|
|
10
|
+
* reasoning_output}_tokens) — so the session total is its LAST token_count, summed across sessions.
|
|
11
|
+
*
|
|
12
|
+
* Identity lives in ~/.codex/auth.json as a JWT (`auth_mode` + an `id_token` whose claims hold
|
|
13
|
+
* email / name / chatgpt_plan_type / account). We decode the id_token's claims locally and emit
|
|
14
|
+
* ONLY those (email, name, plan, account) — never the token, access_token, or API key. Configured
|
|
15
|
+
* model is read from ~/.codex/config.toml. Numbers + identity claims only — no transcript content.
|
|
16
|
+
*
|
|
17
|
+
* Routing config also comes from a scoped, section-aware line scan of config.toml (see
|
|
18
|
+
* readCodexRoutingConfig): top-level `model_provider`/`openai_base_url`/`oss_provider`, and each
|
|
19
|
+
* `[model_providers.<id>]` table's `base_url`/`wire_api` keys only — never `env_key`,
|
|
20
|
+
* `experimental_bearer_token`, `auth.*`, or any other key that can carry credential material or
|
|
21
|
+
* a command string. `[model_providers.amazon-bedrock(.aws)]`'s mere presence is recorded (not its
|
|
22
|
+
* region/profile values) since selecting that built-in provider id is itself the Bedrock signal.
|
|
23
|
+
*
|
|
24
|
+
* Bounded by rollout-file mtime recency so a long session history isn't fully re-parsed each run.
|
|
25
|
+
*/
|
|
26
|
+
const TOKEN_USAGE_RECENCY_DAYS = 30;
|
|
27
|
+
const TOKEN_USAGE_FILE_TYPE = 'codex_token_usage';
|
|
28
|
+
function emptyTotals() {
|
|
29
|
+
return { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0 };
|
|
30
|
+
}
|
|
31
|
+
function num(value) {
|
|
32
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
|
33
|
+
}
|
|
34
|
+
function asObject(value) {
|
|
35
|
+
return value && typeof value === 'object' ? value : {};
|
|
36
|
+
}
|
|
37
|
+
/** Recursively collect rollout-*.jsonl files under a dir whose mtime is within the window. */
|
|
38
|
+
function recentRolloutFiles(dir, cutoffMs) {
|
|
39
|
+
const out = [];
|
|
40
|
+
let entries;
|
|
41
|
+
try {
|
|
42
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
for (const entry of entries) {
|
|
48
|
+
const full = join(dir, entry.name);
|
|
49
|
+
if (entry.isDirectory()) {
|
|
50
|
+
out.push(...recentRolloutFiles(full, cutoffMs));
|
|
51
|
+
}
|
|
52
|
+
else if (entry.isFile() &&
|
|
53
|
+
entry.name.startsWith('rollout-') &&
|
|
54
|
+
entry.name.endsWith('.jsonl')) {
|
|
55
|
+
try {
|
|
56
|
+
if (statSync(full).mtimeMs >= cutoffMs)
|
|
57
|
+
out.push(full);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
/* unreadable — skip */
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
66
|
+
/** Decode a JWT's payload claims (no verification — we only read non-secret identity claims). */
|
|
67
|
+
function decodeJwtClaims(jwt) {
|
|
68
|
+
const parts = jwt.split('.');
|
|
69
|
+
if (parts.length !== 3)
|
|
70
|
+
return {};
|
|
71
|
+
try {
|
|
72
|
+
let b64 = parts[1].replace(/-/g, '+').replace(/_/g, '/');
|
|
73
|
+
b64 += '='.repeat((4 - (b64.length % 4)) % 4);
|
|
74
|
+
return JSON.parse(Buffer.from(b64, 'base64').toString('utf8'));
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return {};
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** Identity from ~/.codex/auth.json: auth_mode + id_token claims (email/name/plan/account). */
|
|
81
|
+
function readCodexIdentity(home) {
|
|
82
|
+
const out = {};
|
|
83
|
+
let auth;
|
|
84
|
+
try {
|
|
85
|
+
auth = JSON.parse(readFileSync(join(home, '.codex', 'auth.json'), 'utf8'));
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
if (typeof auth.auth_mode === 'string' && auth.auth_mode)
|
|
91
|
+
out.auth_mode = auth.auth_mode;
|
|
92
|
+
const idToken = asObject(auth.tokens).id_token;
|
|
93
|
+
if (typeof idToken !== 'string' || !idToken)
|
|
94
|
+
return out;
|
|
95
|
+
const claims = decodeJwtClaims(idToken);
|
|
96
|
+
if (typeof claims.email === 'string')
|
|
97
|
+
out.email = claims.email;
|
|
98
|
+
if (typeof claims.name === 'string')
|
|
99
|
+
out.name = claims.name;
|
|
100
|
+
const openaiAuth = asObject(claims['https://api.openai.com/auth']);
|
|
101
|
+
if (typeof openaiAuth.chatgpt_plan_type === 'string')
|
|
102
|
+
out.plan = openaiAuth.chatgpt_plan_type;
|
|
103
|
+
if (typeof openaiAuth.chatgpt_account_id === 'string') {
|
|
104
|
+
out.account_id = openaiAuth.chatgpt_account_id;
|
|
105
|
+
}
|
|
106
|
+
const orgs = openaiAuth.organizations;
|
|
107
|
+
if (Array.isArray(orgs) && orgs.length > 0) {
|
|
108
|
+
const org = asObject(orgs[0]);
|
|
109
|
+
if (typeof org.id === 'string')
|
|
110
|
+
out.account_id = out.account_id || org.id;
|
|
111
|
+
if (typeof org.title === 'string')
|
|
112
|
+
out.org_name = org.title;
|
|
113
|
+
}
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
/** Configured model from ~/.codex/config.toml (TOML — a simple `model = "..."` line). */
|
|
117
|
+
function readConfiguredModel(home) {
|
|
118
|
+
try {
|
|
119
|
+
const toml = readFileSync(join(home, '.codex', 'config.toml'), 'utf8');
|
|
120
|
+
const m = toml.match(/^\s*model\s*=\s*["']([^"']+)["']/m);
|
|
121
|
+
return m ? m[1] : '';
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
return '';
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function emptyCodexRoutingConfig() {
|
|
128
|
+
return {
|
|
129
|
+
modelProvider: '',
|
|
130
|
+
openaiBaseUrl: '',
|
|
131
|
+
ossProvider: '',
|
|
132
|
+
hasBedrockConfig: false,
|
|
133
|
+
customProviders: {},
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/** Strip quotes from a TOML string value and any trailing inline comment. */
|
|
137
|
+
function stripTomlString(value) {
|
|
138
|
+
const trimmed = value.trim();
|
|
139
|
+
if (trimmed.startsWith('"')) {
|
|
140
|
+
const end = trimmed.indexOf('"', 1);
|
|
141
|
+
return end !== -1 ? trimmed.slice(1, end) : trimmed.slice(1);
|
|
142
|
+
}
|
|
143
|
+
if (trimmed.startsWith("'")) {
|
|
144
|
+
const end = trimmed.indexOf("'", 1);
|
|
145
|
+
return end !== -1 ? trimmed.slice(1, end) : trimmed.slice(1);
|
|
146
|
+
}
|
|
147
|
+
const commentIdx = trimmed.indexOf('#');
|
|
148
|
+
return (commentIdx === -1 ? trimmed : trimmed.slice(0, commentIdx)).trim();
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Scoped, section-aware scan of config.toml for routing-relevant keys ONLY:
|
|
152
|
+
* - top-level `model_provider` / `openai_base_url` / `oss_provider`
|
|
153
|
+
* - each `[model_providers.<id>]` table's `base_url` / `wire_api` keys
|
|
154
|
+
* - `[model_providers.amazon-bedrock]` (or its `.aws` sub-table) presence only — never its
|
|
155
|
+
* region/profile values
|
|
156
|
+
* Deliberately not a full TOML parse: every other key in every other section (MCP servers,
|
|
157
|
+
* hooks, marketplaces, `model_providers.<id>.env_key`/`auth.*`/`experimental_bearer_token`, ...)
|
|
158
|
+
* is never read, since some of those can carry credential material or shell commands.
|
|
159
|
+
*/
|
|
160
|
+
function readCodexRoutingConfig(home) {
|
|
161
|
+
const result = emptyCodexRoutingConfig();
|
|
162
|
+
let text;
|
|
163
|
+
try {
|
|
164
|
+
text = readFileSync(join(home, '.codex', 'config.toml'), 'utf8');
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return result;
|
|
168
|
+
}
|
|
169
|
+
let currentProviderId = '';
|
|
170
|
+
for (const rawLine of text.split('\n')) {
|
|
171
|
+
const line = rawLine.trim();
|
|
172
|
+
if (!line || line.startsWith('#'))
|
|
173
|
+
continue;
|
|
174
|
+
const sectionMatch = line.match(/^\[([^\]]+)\]$/);
|
|
175
|
+
if (sectionMatch) {
|
|
176
|
+
const section = sectionMatch[1].trim();
|
|
177
|
+
currentProviderId = '';
|
|
178
|
+
if (/^model_providers\.amazon-bedrock(\.aws)?$/.test(section)) {
|
|
179
|
+
result.hasBedrockConfig = true;
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const providerMatch = section.match(/^model_providers\.([^.]+)$/);
|
|
183
|
+
if (providerMatch)
|
|
184
|
+
currentProviderId = providerMatch[1].replace(/^["']|["']$/g, '');
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const kv = line.match(/^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*(.*)$/);
|
|
188
|
+
if (!kv)
|
|
189
|
+
continue;
|
|
190
|
+
const [, key, rawValue] = kv;
|
|
191
|
+
if (!currentProviderId) {
|
|
192
|
+
if (key === 'model_provider')
|
|
193
|
+
result.modelProvider = stripTomlString(rawValue);
|
|
194
|
+
else if (key === 'openai_base_url')
|
|
195
|
+
result.openaiBaseUrl = stripTomlString(rawValue);
|
|
196
|
+
else if (key === 'oss_provider')
|
|
197
|
+
result.ossProvider = stripTomlString(rawValue);
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
if (key === 'base_url' || key === 'wire_api') {
|
|
201
|
+
const entry = result.customProviders[currentProviderId] ?? { baseUrl: '', wireApi: '' };
|
|
202
|
+
if (key === 'base_url')
|
|
203
|
+
entry.baseUrl = stripTomlString(rawValue);
|
|
204
|
+
else
|
|
205
|
+
entry.wireApi = stripTomlString(rawValue);
|
|
206
|
+
result.customProviders[currentProviderId] = entry;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return result;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Aggregate Codex token usage / sessions / version across recent rollout transcripts + identity.
|
|
213
|
+
* Returns a single-element array (one per-machine aggregate) or [] when there is no recent data.
|
|
214
|
+
*/
|
|
215
|
+
function collectCodexTokenUsage(home = homedir()) {
|
|
216
|
+
const codexDir = join(home, '.codex');
|
|
217
|
+
const sessionRoots = [join(codexDir, 'sessions'), join(codexDir, 'archived_sessions')];
|
|
218
|
+
if (!sessionRoots.some((d) => existsSync(d)) && !existsSync(join(codexDir, 'auth.json'))) {
|
|
219
|
+
return [];
|
|
220
|
+
}
|
|
221
|
+
const cutoffMs = Date.now() - TOKEN_USAGE_RECENCY_DAYS * 24 * 60 * 60 * 1000;
|
|
222
|
+
const files = sessionRoots.flatMap((d) => (existsSync(d) ? recentRolloutFiles(d, cutoffMs) : []));
|
|
223
|
+
const totals = emptyTotals();
|
|
224
|
+
const models = new Set();
|
|
225
|
+
const providers = new Set();
|
|
226
|
+
const modelTokenSplit = {};
|
|
227
|
+
const sessions = new Set();
|
|
228
|
+
// Per-session cumulative usage, keyed by session id (else file path). total_token_usage is
|
|
229
|
+
// cumulative within a session, so the same session appearing in multiple rollout files
|
|
230
|
+
// (sessions/ + archived_sessions/) must be deduped — keep the largest snapshot rather than
|
|
231
|
+
// summing the files, which would double-count.
|
|
232
|
+
const sessionTotals = new Map();
|
|
233
|
+
let version = '';
|
|
234
|
+
let versionAt = '';
|
|
235
|
+
for (const file of files) {
|
|
236
|
+
let content;
|
|
237
|
+
try {
|
|
238
|
+
content = readFileSync(file, 'utf8');
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
let sessionModel = '';
|
|
244
|
+
let sessionTotal = null; // last (cumulative) token usage
|
|
245
|
+
let sessionId = '';
|
|
246
|
+
for (const line of content.split('\n')) {
|
|
247
|
+
const trimmed = line.trim();
|
|
248
|
+
if (!trimmed)
|
|
249
|
+
continue;
|
|
250
|
+
let entry;
|
|
251
|
+
try {
|
|
252
|
+
entry = JSON.parse(trimmed);
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
const payload = asObject(entry.payload);
|
|
258
|
+
const ts = typeof entry.timestamp === 'string' ? entry.timestamp : '';
|
|
259
|
+
if (entry.type === 'session_meta') {
|
|
260
|
+
if (typeof payload.session_id === 'string')
|
|
261
|
+
sessionId = payload.session_id;
|
|
262
|
+
if (typeof payload.cli_version === 'string' && payload.cli_version && ts >= versionAt) {
|
|
263
|
+
version = payload.cli_version;
|
|
264
|
+
versionAt = ts;
|
|
265
|
+
}
|
|
266
|
+
if (typeof payload.model_provider === 'string' && payload.model_provider) {
|
|
267
|
+
providers.add(payload.model_provider);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
// Model id can appear on turn/response lines as payload.model.
|
|
271
|
+
if (typeof payload.model === 'string' && payload.model) {
|
|
272
|
+
models.add(payload.model);
|
|
273
|
+
if (!sessionModel)
|
|
274
|
+
sessionModel = payload.model;
|
|
275
|
+
}
|
|
276
|
+
if (payload.type === 'token_count') {
|
|
277
|
+
const usage = asObject(asObject(payload.info).total_token_usage);
|
|
278
|
+
if (Object.keys(usage).length) {
|
|
279
|
+
// Cumulative within the session — keep the latest seen.
|
|
280
|
+
sessionTotal = {
|
|
281
|
+
input: num(usage.input_tokens),
|
|
282
|
+
cached: num(usage.cached_input_tokens),
|
|
283
|
+
output: num(usage.output_tokens) + num(usage.reasoning_output_tokens),
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
const sessionKey = sessionId || file;
|
|
289
|
+
sessions.add(sessionKey);
|
|
290
|
+
if (sessionTotal) {
|
|
291
|
+
const sum = sessionTotal.input + sessionTotal.cached + sessionTotal.output;
|
|
292
|
+
const prev = sessionTotals.get(sessionKey);
|
|
293
|
+
// Keep the largest cumulative snapshot for a session so a duplicate file doesn't double-count.
|
|
294
|
+
if (!prev || sum > prev.sum) {
|
|
295
|
+
sessionTotals.set(sessionKey, {
|
|
296
|
+
model: sessionModel,
|
|
297
|
+
input: sessionTotal.input,
|
|
298
|
+
cached: sessionTotal.cached,
|
|
299
|
+
output: sessionTotal.output,
|
|
300
|
+
sum,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
for (const st of sessionTotals.values()) {
|
|
306
|
+
totals.input_tokens += st.input;
|
|
307
|
+
totals.cache_read_input_tokens += st.cached;
|
|
308
|
+
totals.output_tokens += st.output;
|
|
309
|
+
if (st.model && st.sum > 0) {
|
|
310
|
+
modelTokenSplit[st.model] = (modelTokenSplit[st.model] ?? 0) + st.sum;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const identity = readCodexIdentity(home);
|
|
314
|
+
const configuredModel = readConfiguredModel(home);
|
|
315
|
+
const routingConfig = readCodexRoutingConfig(home);
|
|
316
|
+
const sawUsage = files.length > 0;
|
|
317
|
+
if (!sawUsage && !identity.email)
|
|
318
|
+
return [];
|
|
319
|
+
return [
|
|
320
|
+
{
|
|
321
|
+
file_type: TOKEN_USAGE_FILE_TYPE,
|
|
322
|
+
file_path: codexDir, // stable synthetic path → one aggregate row per machine
|
|
323
|
+
raw_content: {
|
|
324
|
+
version,
|
|
325
|
+
session_count: sessions.size,
|
|
326
|
+
models: [...models].sort(),
|
|
327
|
+
providers: [...providers].sort(),
|
|
328
|
+
model_token_split: modelTokenSplit,
|
|
329
|
+
configured_model: configuredModel,
|
|
330
|
+
window_days: TOKEN_USAGE_RECENCY_DAYS,
|
|
331
|
+
totals,
|
|
332
|
+
model_provider: routingConfig.modelProvider,
|
|
333
|
+
openai_base_url: routingConfig.openaiBaseUrl,
|
|
334
|
+
oss_provider: routingConfig.ossProvider,
|
|
335
|
+
has_bedrock_config: routingConfig.hasBedrockConfig,
|
|
336
|
+
custom_model_providers: routingConfig.customProviders,
|
|
337
|
+
...identity, // auth_mode, email, name, plan, account_id, org_name (claims only)
|
|
338
|
+
},
|
|
339
|
+
},
|
|
340
|
+
];
|
|
341
|
+
}
|
|
342
|
+
export { collectCodexTokenUsage, TOKEN_USAGE_RECENCY_DAYS, TOKEN_USAGE_FILE_TYPE };
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { readJSONFile, readMCPConfig, readMarkdownFile, readInstalledExtensions } from '../readers/file_readers.js';
|
|
4
|
+
import { getExtensionsCachePath, getExtensionsCacheInstalledSuffix, getVscdbPath } from '../paths/path_constants_helpers.js';
|
|
5
|
+
import { resolvePatternToTargets, normalizePathSkipPrefixes } from '../paths/pattern_resolver.js';
|
|
6
|
+
import { enrichRawFromRecipe } from './enrichment_helpers.js';
|
|
7
|
+
import { applyCoworkSessionWhitelist } from './cowork_session_whitelist.js';
|
|
8
|
+
import { collectDirectoryEntries, collectDirectoryMetadata } from './directory_collector.js';
|
|
9
|
+
import { collectVscdbEntries } from '../readers/vscdb_config_builder.js';
|
|
10
|
+
import { pushDerivedFilesFromRecipe } from './openclaw_helpers.js';
|
|
11
|
+
import { collapseMetadataByFileType } from './metadata_merge.js';
|
|
12
|
+
function withOpencodeEnvPatterns(patterns) {
|
|
13
|
+
const out = [...patterns];
|
|
14
|
+
const seen = new Set(out.map((p) => `${p.type}:${p.file_type}:${p.path}`));
|
|
15
|
+
const add = (pattern) => {
|
|
16
|
+
const key = `${pattern.type}:${pattern.file_type}:${pattern.path}`;
|
|
17
|
+
if (seen.has(key))
|
|
18
|
+
return;
|
|
19
|
+
seen.add(key);
|
|
20
|
+
out.push(pattern);
|
|
21
|
+
};
|
|
22
|
+
const envConfig = (process.env.OPENCODE_CONFIG || '').trim();
|
|
23
|
+
if (envConfig) {
|
|
24
|
+
add({
|
|
25
|
+
path: envConfig,
|
|
26
|
+
type: 'file',
|
|
27
|
+
file_type: 'opencode_mcp',
|
|
28
|
+
content_format: 'json',
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
const envDirRaw = (process.env.OPENCODE_CONFIG_DIR || '').trim();
|
|
32
|
+
if (!envDirRaw)
|
|
33
|
+
return out;
|
|
34
|
+
const envDir = envDirRaw.replace(/[\\/]+$/, '');
|
|
35
|
+
add({ path: `${envDir}/opencode.json`, type: 'file', file_type: 'opencode_mcp', content_format: 'json' });
|
|
36
|
+
add({ path: `${envDir}/opencode.jsonc`, type: 'file', file_type: 'opencode_mcp', content_format: 'json' });
|
|
37
|
+
add({ path: `${envDir}/command-hooks.json`, type: 'file', file_type: 'opencode_command_hooks', content_format: 'json' });
|
|
38
|
+
add({ path: `${envDir}/command-hooks.jsonc`, type: 'file', file_type: 'opencode_command_hooks', content_format: 'json' });
|
|
39
|
+
add({ path: `${envDir}/AGENTS.md`, type: 'file', file_type: 'opencode_rule', content_format: 'markdown' });
|
|
40
|
+
add({ path: `${envDir}/agents/`, type: 'dir', file_type: 'opencode_subagent', content_format: 'markdown', dir_glob: '*.md' });
|
|
41
|
+
add({ path: `${envDir}/commands/`, type: 'dir', file_type: 'opencode_command', content_format: 'markdown', dir_glob: '*.md' });
|
|
42
|
+
add({ path: `${envDir}/skills/`, type: 'dir', file_type: 'opencode_skill', content_format: 'markdown', dir_glob: '*.md' });
|
|
43
|
+
add({ path: `${envDir}/mcp-auth.json`, type: 'file', file_type: 'opencode_mcp_auth', content_format: 'json' });
|
|
44
|
+
add({ path: `${envDir}/managed.json`, type: 'file', file_type: 'opencode_managed_preference', content_format: 'json' });
|
|
45
|
+
add({ path: `${envDir}/package.json`, type: 'file', file_type: 'opencode_plugin_manifest', content_format: 'json' });
|
|
46
|
+
add({ path: `${envDir}/plugins/`, type: 'dir', file_type: 'opencode_plugin', content_format: 'text', dir_glob: '*' });
|
|
47
|
+
add({ path: `${envDir}/plugin/`, type: 'dir', file_type: 'opencode_plugin', content_format: 'text', dir_glob: '*' });
|
|
48
|
+
add({ path: `${envDir}/plugins/*/package.json`, type: 'file', file_type: 'opencode_plugin_manifest', content_format: 'json' });
|
|
49
|
+
add({ path: `${envDir}/plugin/*/package.json`, type: 'file', file_type: 'opencode_plugin_manifest', content_format: 'json' });
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
function buildCollectionContext(patterns, projectRoot, home, homeRecurseSkipDirs = [], absolutePathPrefixes = [], mcpToolGlobSpec = null, clientPathConstants = null, pathResolutionSpecs = null) {
|
|
53
|
+
const seenPaths = new Set();
|
|
54
|
+
const targets = [];
|
|
55
|
+
const enrichByFileType = {};
|
|
56
|
+
const metadataOnlyFileTypes = new Set(patterns.filter((p) => p.collect_style === 'metadata').map((p) => p.file_type));
|
|
57
|
+
if (metadataOnlyFileTypes.size === 0)
|
|
58
|
+
metadataOnlyFileTypes.add('claude_log');
|
|
59
|
+
for (const p of patterns) {
|
|
60
|
+
if (p.enrich && !enrichByFileType[p.file_type])
|
|
61
|
+
enrichByFileType[p.file_type] = p.enrich;
|
|
62
|
+
}
|
|
63
|
+
for (const p of patterns) {
|
|
64
|
+
const pathSkipPrefixes = p.path_resolution === 'mcp_tool_glob' ? normalizePathSkipPrefixes(p.path_skip_prefixes) : [];
|
|
65
|
+
for (const t of resolvePatternToTargets(p.path, p.type, p.file_type, projectRoot, home, pathSkipPrefixes, p.collect_style, p.content_format, p.dir_glob, homeRecurseSkipDirs, p.path_resolution, p.resolve_under_home !== false, absolutePathPrefixes, mcpToolGlobSpec, clientPathConstants, pathResolutionSpecs)) {
|
|
66
|
+
if (p.raw_key)
|
|
67
|
+
t.raw_key = p.raw_key;
|
|
68
|
+
if (p.dir_subdir_filename)
|
|
69
|
+
t.dir_subdir_filename = p.dir_subdir_filename;
|
|
70
|
+
if (p.dir_subdir_source)
|
|
71
|
+
t.dir_subdir_source = p.dir_subdir_source;
|
|
72
|
+
const key = `${t.path}\t${t.file_type}\t${t.isDirectory ? 'dir' : 'file'}\t${t.dir_glob ?? ''}`;
|
|
73
|
+
if (!seenPaths.has(key)) {
|
|
74
|
+
seenPaths.add(key);
|
|
75
|
+
targets.push(t);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return { targets, enrichByFileType, metadataOnlyFileTypes };
|
|
80
|
+
}
|
|
81
|
+
function readContentByFormat(path, format) {
|
|
82
|
+
if (format === 'markdown' || format === 'text')
|
|
83
|
+
return readMarkdownFile(path);
|
|
84
|
+
if (format === 'sqlite_vscdb')
|
|
85
|
+
return null; // handled upstream by collectVscdbEntries
|
|
86
|
+
// json (default) — fall back to markdown for .md files from agents not yet annotated
|
|
87
|
+
return readJSONFile(path) ?? readMCPConfig(path) ?? (path.endsWith('.md') ? readMarkdownFile(path) : null);
|
|
88
|
+
}
|
|
89
|
+
function isPortableCursorUserSettingsTarget(t) {
|
|
90
|
+
const p = (t.logicalFilePath ?? t.path).replace(/\\/g, '/').toLowerCase();
|
|
91
|
+
return (p === 'cursor/user/settings.json' ||
|
|
92
|
+
p.endsWith('/cursor/user/settings.json') ||
|
|
93
|
+
p.includes('application support/cursor/user/settings.json'));
|
|
94
|
+
}
|
|
95
|
+
function collectRegularFileEntry(t, enrichByFileType) {
|
|
96
|
+
const format = t.content_format || 'json';
|
|
97
|
+
let content = readContentByFormat(t.path, format);
|
|
98
|
+
if (content === null)
|
|
99
|
+
return null;
|
|
100
|
+
// For JSON-format files, backend/policy engine expect raw_content to be the parsed object (e.g. permissions.allow)
|
|
101
|
+
if (format === 'json' && typeof content === 'string') {
|
|
102
|
+
try {
|
|
103
|
+
content = JSON.parse(content);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
// leave as string if parse fails
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
let raw;
|
|
110
|
+
if (t.raw_key && typeof content === 'string') {
|
|
111
|
+
raw = { [t.raw_key]: content };
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
raw = typeof content === 'string' ? { content, source: 'file' } : content;
|
|
115
|
+
}
|
|
116
|
+
const recipe = enrichByFileType[t.file_type];
|
|
117
|
+
if (recipe)
|
|
118
|
+
enrichRawFromRecipe(raw, recipe);
|
|
119
|
+
// Cowork session metadata: reduce to the PII whitelist (hash identity, drop
|
|
120
|
+
// server URLs, keep only the activity-signal keys) BEFORE it leaves the
|
|
121
|
+
// device. Drop the entry entirely if nothing whitelisted survives so raw
|
|
122
|
+
// session content can never be emitted (e.g. on a JSON parse failure).
|
|
123
|
+
if (t.file_type === 'cowork_session_metadata') {
|
|
124
|
+
const filtered = applyCoworkSessionWhitelist(raw);
|
|
125
|
+
if (filtered === null)
|
|
126
|
+
return null;
|
|
127
|
+
raw = filtered;
|
|
128
|
+
}
|
|
129
|
+
if (t.file_type === 'vscode_settings' &&
|
|
130
|
+
isPortableCursorUserSettingsTarget(t) &&
|
|
131
|
+
Object.keys(raw).length === 0) {
|
|
132
|
+
console.warn(`Skipping empty Cursor User/settings.json upload (path=${t.path}); global ignore would be invisible to the server.`);
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
return { file_type: t.file_type, file_path: t.logicalFilePath ?? t.path, raw_content: raw };
|
|
136
|
+
}
|
|
137
|
+
function collectMetadataEntry(t) {
|
|
138
|
+
try {
|
|
139
|
+
const stats = statSync(t.path);
|
|
140
|
+
if (stats.isDirectory?.()) {
|
|
141
|
+
try {
|
|
142
|
+
if (readdirSync(t.path).length === 0)
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
catch { /* unreadable — assume non-empty, allow */ }
|
|
146
|
+
}
|
|
147
|
+
return { file_type: t.file_type, file_path: t.logicalFilePath ?? t.path, raw_content: { filename: t.path, last_modified: stats.mtime.toISOString(), source: 'file_metadata' }, collect_style: 'metadata' };
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
console.warn(`Error stating ${t.path}:`, err instanceof Error ? err.message : String(err));
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function handleInstalledExtensions(handledSpecialPaths, configFiles, extensionsCachePath, pathConstants) {
|
|
155
|
+
if (handledSpecialPaths.has(extensionsCachePath))
|
|
156
|
+
return;
|
|
157
|
+
handledSpecialPaths.add(extensionsCachePath);
|
|
158
|
+
const installed = readInstalledExtensions(extensionsCachePath);
|
|
159
|
+
const suffix = getExtensionsCacheInstalledSuffix(pathConstants);
|
|
160
|
+
if (installed.length > 0)
|
|
161
|
+
configFiles.push({ file_type: 'cursor_extensions', file_path: `${extensionsCachePath}${suffix}`, raw_content: { installedExtensions: installed, source: 'extensions.user.cache', extracted_at: new Date().toISOString() } });
|
|
162
|
+
}
|
|
163
|
+
function collectConfigFilesFromPatterns(patterns, projectRoot, onProgress, options) {
|
|
164
|
+
// Prefer process.env.HOME so hook invoker (e.g. Cursor) can pass real home when it differs from os.homedir()
|
|
165
|
+
const home = (process.env.HOME && process.env.HOME.trim()) || homedir();
|
|
166
|
+
const pathConstants = options?.client_path_constants;
|
|
167
|
+
if (!pathConstants)
|
|
168
|
+
throw new Error('client_path_constants required from API response but not provided');
|
|
169
|
+
const vscdbPath = getVscdbPath(home, pathConstants);
|
|
170
|
+
const extensionsCachePath = getExtensionsCachePath(home, pathConstants);
|
|
171
|
+
const extensionsInstalledSuffix = getExtensionsCacheInstalledSuffix(pathConstants);
|
|
172
|
+
const effectivePatterns = withOpencodeEnvPatterns(patterns);
|
|
173
|
+
const { targets, enrichByFileType, metadataOnlyFileTypes } = buildCollectionContext(effectivePatterns, projectRoot, home, options?.home_recurse_skip_dirs ?? [], options?.absolute_path_prefixes ?? [], options?.mcp_tool_glob_spec ?? null, pathConstants, options?.path_resolution_specs ?? null);
|
|
174
|
+
const configFiles = [];
|
|
175
|
+
const handledSpecialPaths = new Set();
|
|
176
|
+
const loggedFileTypes = new Set();
|
|
177
|
+
const vscdbEntrySpecs = options?.vscdbEntrySpecs;
|
|
178
|
+
const vscdbReadQueries = options?.vscdb_read_queries;
|
|
179
|
+
const vscdbMergeFromComposerState = options?.vscdb_merge_from_composer_state;
|
|
180
|
+
for (const t of targets) {
|
|
181
|
+
if (onProgress && !loggedFileTypes.has(t.file_type)) {
|
|
182
|
+
loggedFileTypes.add(t.file_type);
|
|
183
|
+
onProgress(`scanning file_type=${t.file_type}`);
|
|
184
|
+
}
|
|
185
|
+
if (t.isDirectory) {
|
|
186
|
+
if (metadataOnlyFileTypes.has(t.file_type) || t.collect_style === 'metadata') {
|
|
187
|
+
const entry = collectDirectoryMetadata(t);
|
|
188
|
+
if (entry)
|
|
189
|
+
configFiles.push(entry);
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
configFiles.push(...collectDirectoryEntries(t));
|
|
193
|
+
}
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (extensionsInstalledSuffix && t.path.includes(extensionsInstalledSuffix)) {
|
|
197
|
+
handleInstalledExtensions(handledSpecialPaths, configFiles, extensionsCachePath, pathConstants);
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
if (t.path === vscdbPath || t.path.startsWith(vscdbPath + '#')) {
|
|
201
|
+
if (!handledSpecialPaths.has(vscdbPath)) {
|
|
202
|
+
handledSpecialPaths.add(vscdbPath);
|
|
203
|
+
configFiles.push(...collectVscdbEntries(vscdbPath, vscdbEntrySpecs, vscdbReadQueries, vscdbMergeFromComposerState));
|
|
204
|
+
}
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
if (!existsSync(t.path))
|
|
208
|
+
continue;
|
|
209
|
+
if ((metadataOnlyFileTypes.has(t.file_type) || t.collect_style === 'metadata') && !t.isDirectory) {
|
|
210
|
+
const entry = collectMetadataEntry(t);
|
|
211
|
+
if (entry)
|
|
212
|
+
configFiles.push(entry);
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
const entry = collectRegularFileEntry(t, enrichByFileType);
|
|
216
|
+
if (entry) {
|
|
217
|
+
configFiles.push(entry);
|
|
218
|
+
const recipe = enrichByFileType[t.file_type];
|
|
219
|
+
if (recipe?.derived_files?.length)
|
|
220
|
+
pushDerivedFilesFromRecipe(entry.raw_content, configFiles, recipe);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return collapseMetadataByFileType(configFiles);
|
|
224
|
+
}
|
|
225
|
+
// ─── Re-exports (barrel for public API) ──────────────────────────────────────
|
|
226
|
+
export { collectConfigFilesFromPatterns };
|
|
227
|
+
export { collectMcpToolFiles } from './mcp_tool_collector.js';
|
|
228
|
+
export { collectConfigFilesFromInstalledPlugins, collectPluginCacheMcpFiles } from './plugin_collector.js';
|
|
229
|
+
export { collectMcpFromClaudeJsonProjects } from './claude_known_projects_collector.js';
|
|
230
|
+
export { collectClaudeTokenUsage } from './claude_token_usage_collector.js';
|
|
231
|
+
export { collectCopilotTokenUsage } from './copilot_token_usage_collector.js';
|
|
232
|
+
export { collectOpencodeTokenUsage } from './opencode_token_usage_collector.js';
|
|
233
|
+
export { collectCodexTokenUsage } from './codex_token_usage_collector.js';
|
|
234
|
+
export { collectCursorTokenUsage } from './cursor_token_usage_collector.js';
|
|
235
|
+
export { collectHermesTokenUsage } from './hermes_token_usage_collector.js';
|
|
236
|
+
export { collectOpenclawTokenUsage } from './openclaw_token_usage_collector.js';
|
|
237
|
+
export { collectPiTokenUsage } from './pi_token_usage_collector.js';
|
|
238
|
+
export { collectCoworkDesktopVersion } from './cowork_desktop_version_collector.js';
|
|
239
|
+
export { collectGrokCliVersion } from './grok_cli_version_collector.js';
|
|
240
|
+
export { collectClaudeDesktopExtensionManifests, collectClaudeDesktopExtensionSettingsFiles, enrichClaudeDesktopExtensionsInstallationsUpload, mergeClaudeDesktopExtensionEnableSettings, } from './claude_desktop_extensions_collector.js';
|
|
241
|
+
export { collectCursorProjectWorkspaceMcpConfigs } from './cursor_project_mcp_collector.js';
|
|
242
|
+
export { determineFileTypeFromPath } from './file_type_rules.js';
|
|
243
|
+
export { enrichRawFromRecipe } from './enrichment_helpers.js';
|
|
244
|
+
export { pushDerivedFilesFromRecipe } from './openclaw_helpers.js';
|
|
245
|
+
export { getByPath } from './enrichment_helpers.js';
|