farming-code 2.2.14 → 2.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/README.zh_cn.md +3 -3
- package/THIRD_PARTY_NOTICES.md +17 -1
- package/backend/acp/packaged-codex-acp.js +13 -0
- package/backend/acp-runtime.js +413 -24
- package/backend/agent-manager.js +572 -557
- package/backend/agent-provider-session.js +15 -1
- package/backend/agent-runtime-binding.js +9 -55
- package/backend/chat-runtime.js +28 -0
- package/backend/codex-rollout-follower.js +83 -0
- package/backend/codex-session-archive.js +12 -1
- package/backend/codex-session-history.js +94 -0
- package/backend/codex-transcript.js +2 -1
- package/backend/config-manager.js +25 -17
- package/backend/control-api.js +3 -1
- package/backend/farming-app-cli.js +21 -0
- package/backend/farming-session-store.js +2 -1
- package/backend/native-pty-host-client.js +59 -2
- package/backend/npm-update-helper.js +65 -7
- package/backend/provider-adapters.js +123 -38
- package/backend/provider-session-service.js +138 -34
- package/backend/server.js +94 -105
- package/backend/storage-layout.js +5 -16
- package/backend/update-service.js +94 -5
- package/backend/usage-history-client.js +305 -0
- package/backend/usage-history-scanner.generated.js +2250 -0
- package/backend/usage-history-scanner.ts +2718 -0
- package/backend/usage-history-smoke.js +64 -0
- package/backend/usage-history-worker.js +34 -0
- package/backend/usage-monitor.js +188 -588
- package/backend/vendor/usage-parsers/CC_STATISTICS_LICENSE +21 -0
- package/backend/vendor/usage-parsers/CODEXBAR_LICENSE +21 -0
- package/backend/vendor/usage-parsers/VENDOR.json +45 -0
- package/dist/acp/LICENSE.codex-acp +190 -0
- package/dist/acp/codex-acp-1.1.4.js +30718 -0
- package/dist/assets/App-6yWdwzj8.js +210 -0
- package/dist/assets/{FileEditorMarkdownPreview-UlVd0O4D.js → FileEditorMarkdownPreview-R0jwwLME.js} +4 -4
- package/dist/assets/FileEditorPane-DNy3LAqS.js +2 -0
- package/dist/assets/{IconGlyphs-AU22gPDY.js → IconGlyphs-BsUfqpKN.js} +1 -1
- package/dist/assets/ProjectFilesSection-BhDEUVHA.js +12 -0
- package/dist/assets/{ReviewPage-CeDKDyZh.js → ReviewPage-ChtByBe3.js} +1 -1
- package/dist/assets/{abnfDiagram-VRR7QNED-BgawzDQT.js → abnfDiagram-VRR7QNED-nnglrXFi.js} +1 -1
- package/dist/assets/{architectureDiagram-ZJ3FMSHR-DUShwPg0.js → architectureDiagram-ZJ3FMSHR-BEXMaFfY.js} +1 -1
- package/dist/assets/{blockDiagram-677ZJIJ3-CeGZviXR.js → blockDiagram-677ZJIJ3-1mCezSBk.js} +1 -1
- package/dist/assets/{c4Diagram-LMCZKHZV-CLhYht6S.js → c4Diagram-LMCZKHZV-DhTiI6Uv.js} +1 -1
- package/dist/assets/channel-BVWNatUB.js +1 -0
- package/dist/assets/{chunk-32BRIVSS-LFYpT-Cu.js → chunk-32BRIVSS-CCdsKd73.js} +1 -1
- package/dist/assets/{chunk-52WLFC77-BH6jDw9J.js → chunk-52WLFC77-Gncjk4LF.js} +1 -1
- package/dist/assets/{chunk-C7G6YPKG-B_mIUlb7.js → chunk-C7G6YPKG-CiDVUP82.js} +1 -1
- package/dist/assets/{chunk-EX3LRPZG-C63hVuAJ.js → chunk-EX3LRPZG-z6gXP2_N.js} +1 -1
- package/dist/assets/{chunk-FWX5IMBZ-CboX-WF_.js → chunk-FWX5IMBZ-DMZBccQa.js} +2 -2
- package/dist/assets/{chunk-HOUHSVGY-BOgSd6Un.js → chunk-HOUHSVGY-D8tIeoPk.js} +1 -1
- package/dist/assets/{chunk-ICXQ74PX-C0K-pgZt.js → chunk-ICXQ74PX-b6oQ2Q17.js} +1 -1
- package/dist/assets/{chunk-MOJQB5TN-CVMfwN_7.js → chunk-MOJQB5TN-DAVH_p2a.js} +1 -1
- package/dist/assets/{chunk-OGEWGWER-DLT3pxyw.js → chunk-OGEWGWER-DxoiMCI-.js} +1 -1
- package/dist/assets/{chunk-PUDLZKDR-Cd079yIQ.js → chunk-PUDLZKDR-NsOWh7HM.js} +1 -1
- package/dist/assets/{chunk-Q4XR5HBZ-D0jHa6SO.js → chunk-Q4XR5HBZ-Di4H7L21.js} +1 -1
- package/dist/assets/{chunk-V7JOEXUC-CVDtR75H.js → chunk-V7JOEXUC-DttpZg9Y.js} +1 -1
- package/dist/assets/{chunk-VAUOI2AC-B-W2_9Ot.js → chunk-VAUOI2AC-UHNr_yhX.js} +1 -1
- package/dist/assets/{chunk-VR4S4FIN-FD3kWh32.js → chunk-VR4S4FIN-D-Fmtd4Z.js} +1 -1
- package/dist/assets/{chunk-WYO6CB5R-C2bGaS5r.js → chunk-WYO6CB5R-CUxMtfK2.js} +4 -4
- package/dist/assets/{chunk-ZGVPDNZ5-ymgts0zo.js → chunk-ZGVPDNZ5-CoV2QEl0.js} +1 -1
- package/dist/assets/classDiagram-OUVF2IWQ-Db0ib5iE.js +1 -0
- package/dist/assets/classDiagram-v2-EOCWNBFH-Bkf69XF9.js +1 -0
- package/dist/assets/code-dark-5esj0bRu.css +1 -0
- package/dist/assets/{cynefinDiagram-TSTJHNR4-CwpobXIl.js → cynefinDiagram-TSTJHNR4-0xPI92Lg.js} +1 -1
- package/dist/assets/{dagre-VKFMJZFB-Dkhlfjna.js → dagre-VKFMJZFB-DfuMG4gU.js} +1 -1
- package/dist/assets/{diagram-FQU43EPY-hA8dicp6.js → diagram-FQU43EPY-CT00bLZ6.js} +1 -1
- package/dist/assets/{diagram-G47NLZAW-BTY6JTzO.js → diagram-G47NLZAW-Xbh7zXUx.js} +1 -1
- package/dist/assets/{diagram-NH7WQ7WH-xkaHHX9M.js → diagram-NH7WQ7WH-SRDVPfn7.js} +1 -1
- package/dist/assets/{diagram-OA4YK3LP-gk_fg9UU.js → diagram-OA4YK3LP-CQRli4Np.js} +1 -1
- package/dist/assets/{diagram-WEI45ONY-BvC8xd6T.js → diagram-WEI45ONY-DMf17WQi.js} +1 -1
- package/dist/assets/{ebnfDiagram-CCIWWBDH-CR7RaNat.js → ebnfDiagram-CCIWWBDH-DhpFkgJ5.js} +1 -1
- package/dist/assets/{erDiagram-Q63AITRT-_M-JFPbj.js → erDiagram-Q63AITRT-1IwxWmJo.js} +1 -1
- package/dist/assets/file-icons-DpbrD5ZP.js +1 -0
- package/dist/assets/flowDiagram-23GEKE2U-DYFo2xmB.js +1 -0
- package/dist/assets/{ganttDiagram-NO4QXBWP-Bl8P1yf_.js → ganttDiagram-NO4QXBWP-BNCdfyr1.js} +1 -1
- package/dist/assets/{gitGraphDiagram-IHSO6WYX-DoHa5ysz.js → gitGraphDiagram-IHSO6WYX-pbyWZuiX.js} +1 -1
- package/dist/assets/{index-BA-iH2Kg.js → index-C8ZFKye1.js} +2 -2
- package/dist/assets/{infoDiagram-FWYZ7A6U-DSFcSuUs.js → infoDiagram-FWYZ7A6U-BcTOgDbe.js} +1 -1
- package/dist/assets/{ishikawaDiagram-FXEZZL3T-DdkqullI.js → ishikawaDiagram-FXEZZL3T-DCdvGKrn.js} +1 -1
- package/dist/assets/{journeyDiagram-5HDEW3XC-B4hRnQJr.js → journeyDiagram-5HDEW3XC-BZ-1q_IT.js} +1 -1
- package/dist/assets/{kanban-definition-HUTT4EX6-tm2FZB2E.js → kanban-definition-HUTT4EX6-BIgejCjj.js} +1 -1
- package/dist/assets/{line-CmsjK4kn.js → line-CfptFMmX.js} +1 -1
- package/dist/assets/main-DSO3i_Gs.css +1 -0
- package/dist/assets/{mermaid.core-C7UT_EEL.js → mermaid.core-gSpX30pV.js} +3 -3
- package/dist/assets/{mindmap-definition-LN4V7U3C-DplfjGms.js → mindmap-definition-LN4V7U3C-BKBaI4LW.js} +1 -1
- package/dist/assets/{pegDiagram-2B236MQR-lu3SdkQs.js → pegDiagram-2B236MQR-sBvDY3VX.js} +1 -1
- package/dist/assets/{pieDiagram-ENE6RG2P-3fHbmDf9.js → pieDiagram-ENE6RG2P-DWlWcu08.js} +1 -1
- package/dist/assets/{quadrantDiagram-ABIIQ3AL-a5qoWz-X.js → quadrantDiagram-ABIIQ3AL-DZutUd3J.js} +1 -1
- package/dist/assets/{railroadDiagram-RFXS5EU6-D2TOuXW9.js → railroadDiagram-RFXS5EU6-DQb3lXGM.js} +1 -1
- package/dist/assets/{requirementDiagram-TGXJPOKE-LQBWpUvd.js → requirementDiagram-TGXJPOKE-DqFGXu2e.js} +1 -1
- package/dist/assets/{sankeyDiagram-HTMAVEWB-B1ohYoyK.js → sankeyDiagram-HTMAVEWB-d7ECahnO.js} +1 -1
- package/dist/assets/{sequenceDiagram-DBY2YBRQ-C3vM-hMq.js → sequenceDiagram-DBY2YBRQ-DdsbKqct.js} +1 -1
- package/dist/assets/{stateDiagram-2N3HPSRC-BvFhOryI.js → stateDiagram-2N3HPSRC-P6iSrgwR.js} +1 -1
- package/dist/assets/stateDiagram-v2-6OUMAXLB-DIYSf_qh.js +1 -0
- package/dist/assets/{swimlanes-5IMT3BWC-DKF_b-2s.js → swimlanes-5IMT3BWC-BGnZJx1M.js} +1 -1
- package/dist/assets/swimlanesDiagram-G3AALYLV-BIkrlKrZ.js +8 -0
- package/dist/assets/{timeline-definition-FHXFAJF6-vAs11ux_.js → timeline-definition-FHXFAJF6-rA-r3aUR.js} +1 -1
- package/dist/assets/{vennDiagram-L72KCM5P-BybTX_9B.js → vennDiagram-L72KCM5P-DiVKUjA9.js} +1 -1
- package/dist/assets/{wardleyDiagram-EHGQE667-CicUmMBR.js → wardleyDiagram-EHGQE667-Cfxh7UOn.js} +1 -1
- package/dist/assets/workspace-editor-model-D0Im34S5.js +1 -0
- package/dist/assets/{workspace-editor-monaco-BICJESG0.js → workspace-editor-monaco-BIpqcWf1.js} +1 -1
- package/dist/assets/workspace-editor-monaco-DLlws5Ig.js +1 -0
- package/dist/assets/{workspace-view-state-DvYG_9PH.js → workspace-view-state-BpPINu84.js} +1 -1
- package/dist/assets/{xychartDiagram-FW5EYKEG-BhWDyCX-.js → xychartDiagram-FW5EYKEG--hmryQeL.js} +1 -1
- package/dist/crt-mermaid-renderer.js +313 -313
- package/dist/index.html +1 -1
- package/frontend/session-bridge.js +24 -2
- package/frontend/skins/crt/app.js +203 -85
- package/frontend/skins/crt/index.html +2 -11
- package/frontend/skins/crt/styles/effects.css +2 -34
- package/package.json +30 -12
- package/shared/backend-connection-status.d.ts +25 -0
- package/shared/backend-connection-status.js +40 -0
- package/shared/browser-protocol.js +3 -3
- package/backend/app-server-api.js +0 -917
- package/backend/codex-app-server-home.js +0 -39
- package/backend/codex-app-server-runtime.js +0 -901
- package/dist/assets/App-Bumi772v.js +0 -206
- package/dist/assets/FileEditorPane-Dxx4rKOg.js +0 -2
- package/dist/assets/ProjectFilesSection-DE7Nb9Jk.js +0 -12
- package/dist/assets/channel-Uw90AO0b.js +0 -1
- package/dist/assets/classDiagram-OUVF2IWQ-DKW_dtku.js +0 -1
- package/dist/assets/classDiagram-v2-EOCWNBFH-C8Fi7-yv.js +0 -1
- package/dist/assets/code-dark-DPiuyeSp.css +0 -1
- package/dist/assets/code-mobile-3C2Rr-CN.css +0 -1
- package/dist/assets/file-icons-Bw2qd5iT.js +0 -1
- package/dist/assets/flowDiagram-23GEKE2U-DAiS5Y7G.js +0 -1
- package/dist/assets/main-Bqtb5kuz.css +0 -1
- package/dist/assets/stateDiagram-v2-6OUMAXLB-WMrz4PNo.js +0 -1
- package/dist/assets/swimlanesDiagram-G3AALYLV-BGswC9oC.js +0 -8
- package/dist/assets/workspace-editor-model-BQol4qbA.js +0 -1
- package/dist/assets/workspace-editor-monaco-DRJ7IaXk.js +0 -1
- package/scripts/sync-ghostty-vendor.js +0 -66
- /package/dist/assets/{core-D0LFJkDt.js → core-B9MwRjDE.js} +0 -0
|
@@ -0,0 +1,2250 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// backend/usage-history-scanner.ts
|
|
31
|
+
var usage_history_scanner_exports = {};
|
|
32
|
+
__export(usage_history_scanner_exports, {
|
|
33
|
+
collectUsage: () => collectUsage
|
|
34
|
+
});
|
|
35
|
+
module.exports = __toCommonJS(usage_history_scanner_exports);
|
|
36
|
+
var import_node_fs = __toESM(require("node:fs"));
|
|
37
|
+
var import_node_path = __toESM(require("node:path"));
|
|
38
|
+
var import_node_crypto = __toESM(require("node:crypto"));
|
|
39
|
+
var import_node_sqlite = require("node:sqlite");
|
|
40
|
+
var SCHEMA_VERSION = 11;
|
|
41
|
+
var SOURCE_VERSION = "farming-usage-ts-1-codexbar-v0.45.2";
|
|
42
|
+
var PREFIX_BYTES = 64 * 1024;
|
|
43
|
+
var READ_CHUNK_BYTES = 1024 * 1024;
|
|
44
|
+
var MAX_LINE_BYTES = 32 * 1024 * 1024;
|
|
45
|
+
var TRUNCATED_EDGE_BYTES = 512 * 1024;
|
|
46
|
+
var DEFAULT_RETENTION_DAYS = 52 * 7;
|
|
47
|
+
var DEFAULT_RECENT_RAW_MS = 24 * 60 * 60 * 1e3;
|
|
48
|
+
var DEFAULT_SCAN_BUDGET_MS = 5e3;
|
|
49
|
+
var SEEN_TOTALS_LIMIT = 64;
|
|
50
|
+
var DIRECTORY_AUDIT_BATCH = 256;
|
|
51
|
+
var DIRECTORY_ENTRY_BATCH = 256;
|
|
52
|
+
var SOURCE_AUDIT_BATCH = 128;
|
|
53
|
+
var ZERO = { input: 0, cached: 0, output: 0, cacheWrite: 0 };
|
|
54
|
+
var TOKEN_FIELDS = ["input", "cached", "output", "cacheWrite"];
|
|
55
|
+
var SCHEMA_SQL = `
|
|
56
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
57
|
+
key TEXT PRIMARY KEY,
|
|
58
|
+
value TEXT NOT NULL
|
|
59
|
+
) WITHOUT ROWID;
|
|
60
|
+
CREATE TABLE IF NOT EXISTS source_files (
|
|
61
|
+
path TEXT PRIMARY KEY,
|
|
62
|
+
provider TEXT NOT NULL,
|
|
63
|
+
directory_path TEXT NOT NULL,
|
|
64
|
+
directory_generation INTEGER NOT NULL,
|
|
65
|
+
size INTEGER NOT NULL,
|
|
66
|
+
mtime_ms REAL NOT NULL,
|
|
67
|
+
committed_offset INTEGER NOT NULL,
|
|
68
|
+
file_dev INTEGER NOT NULL,
|
|
69
|
+
file_ino INTEGER NOT NULL,
|
|
70
|
+
prefix_sha256 TEXT NOT NULL,
|
|
71
|
+
suffix_sha256 TEXT NOT NULL,
|
|
72
|
+
parser_state TEXT NOT NULL,
|
|
73
|
+
session_id TEXT NOT NULL,
|
|
74
|
+
scan_complete INTEGER NOT NULL,
|
|
75
|
+
last_checked_ms INTEGER NOT NULL
|
|
76
|
+
) WITHOUT ROWID;
|
|
77
|
+
CREATE INDEX IF NOT EXISTS source_files_provider_session
|
|
78
|
+
ON source_files(provider, session_id);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS source_files_directory
|
|
80
|
+
ON source_files(directory_path);
|
|
81
|
+
CREATE INDEX IF NOT EXISTS source_files_audit
|
|
82
|
+
ON source_files(last_checked_ms, mtime_ms);
|
|
83
|
+
CREATE INDEX IF NOT EXISTS source_files_recent
|
|
84
|
+
ON source_files(mtime_ms DESC);
|
|
85
|
+
CREATE INDEX IF NOT EXISTS source_files_directory_generation
|
|
86
|
+
ON source_files(directory_path, directory_generation);
|
|
87
|
+
CREATE TABLE IF NOT EXISTS discovery_directories (
|
|
88
|
+
path TEXT PRIMARY KEY,
|
|
89
|
+
provider TEXT NOT NULL,
|
|
90
|
+
root_path TEXT NOT NULL,
|
|
91
|
+
parent_path TEXT NOT NULL,
|
|
92
|
+
is_root INTEGER NOT NULL,
|
|
93
|
+
present INTEGER NOT NULL,
|
|
94
|
+
pending INTEGER NOT NULL,
|
|
95
|
+
scan_generation INTEGER NOT NULL,
|
|
96
|
+
parent_generation INTEGER NOT NULL,
|
|
97
|
+
mtime_ms REAL NOT NULL,
|
|
98
|
+
file_dev INTEGER NOT NULL,
|
|
99
|
+
file_ino INTEGER NOT NULL,
|
|
100
|
+
last_checked_ms INTEGER NOT NULL
|
|
101
|
+
) WITHOUT ROWID;
|
|
102
|
+
CREATE INDEX IF NOT EXISTS discovery_directories_pending
|
|
103
|
+
ON discovery_directories(present, pending, is_root DESC, last_checked_ms);
|
|
104
|
+
CREATE INDEX IF NOT EXISTS discovery_directories_audit
|
|
105
|
+
ON discovery_directories(present, is_root DESC, last_checked_ms);
|
|
106
|
+
CREATE INDEX IF NOT EXISTS discovery_directories_parent_generation
|
|
107
|
+
ON discovery_directories(parent_path, present, parent_generation);
|
|
108
|
+
CREATE TABLE IF NOT EXISTS discovery_source_queue (
|
|
109
|
+
path TEXT PRIMARY KEY,
|
|
110
|
+
provider TEXT NOT NULL,
|
|
111
|
+
action TEXT NOT NULL,
|
|
112
|
+
priority REAL NOT NULL,
|
|
113
|
+
queued_at_ms INTEGER NOT NULL
|
|
114
|
+
) WITHOUT ROWID;
|
|
115
|
+
CREATE INDEX IF NOT EXISTS discovery_source_queue_priority
|
|
116
|
+
ON discovery_source_queue(action, priority, queued_at_ms);
|
|
117
|
+
CREATE TABLE IF NOT EXISTS codex_quota_candidates (
|
|
118
|
+
source_path TEXT NOT NULL,
|
|
119
|
+
limit_id TEXT NOT NULL,
|
|
120
|
+
timestamp_ms INTEGER,
|
|
121
|
+
rate_limits_json TEXT NOT NULL,
|
|
122
|
+
PRIMARY KEY(source_path, limit_id)
|
|
123
|
+
) WITHOUT ROWID;
|
|
124
|
+
CREATE INDEX IF NOT EXISTS codex_quota_candidates_latest
|
|
125
|
+
ON codex_quota_candidates(timestamp_ms DESC);
|
|
126
|
+
CREATE TABLE IF NOT EXISTS cache_stats (
|
|
127
|
+
singleton INTEGER PRIMARY KEY CHECK(singleton = 1),
|
|
128
|
+
codex_files INTEGER NOT NULL,
|
|
129
|
+
claude_files INTEGER NOT NULL,
|
|
130
|
+
codex_incomplete INTEGER NOT NULL,
|
|
131
|
+
claude_incomplete INTEGER NOT NULL,
|
|
132
|
+
committed_bytes INTEGER NOT NULL,
|
|
133
|
+
hourly_rows INTEGER NOT NULL,
|
|
134
|
+
recent_rows INTEGER NOT NULL,
|
|
135
|
+
dedupe_rows INTEGER NOT NULL,
|
|
136
|
+
pending_directories INTEGER NOT NULL,
|
|
137
|
+
queued_sources INTEGER NOT NULL
|
|
138
|
+
);
|
|
139
|
+
INSERT OR IGNORE INTO cache_stats VALUES(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
|
|
140
|
+
CREATE TABLE IF NOT EXISTS usage_hourly (
|
|
141
|
+
source_path TEXT NOT NULL,
|
|
142
|
+
provider TEXT NOT NULL,
|
|
143
|
+
session_id TEXT NOT NULL,
|
|
144
|
+
bucket_start_ms INTEGER NOT NULL,
|
|
145
|
+
input_tokens INTEGER NOT NULL,
|
|
146
|
+
output_tokens INTEGER NOT NULL,
|
|
147
|
+
cache_read_tokens INTEGER NOT NULL,
|
|
148
|
+
cache_write_tokens INTEGER NOT NULL,
|
|
149
|
+
event_count INTEGER NOT NULL,
|
|
150
|
+
PRIMARY KEY(source_path, provider, session_id, bucket_start_ms),
|
|
151
|
+
FOREIGN KEY(source_path) REFERENCES source_files(path) ON DELETE CASCADE
|
|
152
|
+
) WITHOUT ROWID;
|
|
153
|
+
CREATE INDEX IF NOT EXISTS usage_hourly_provider_time
|
|
154
|
+
ON usage_hourly(provider, bucket_start_ms);
|
|
155
|
+
CREATE TABLE IF NOT EXISTS usage_provider_hourly (
|
|
156
|
+
provider TEXT NOT NULL,
|
|
157
|
+
bucket_start_ms INTEGER NOT NULL,
|
|
158
|
+
input_tokens INTEGER NOT NULL,
|
|
159
|
+
output_tokens INTEGER NOT NULL,
|
|
160
|
+
cache_read_tokens INTEGER NOT NULL,
|
|
161
|
+
cache_write_tokens INTEGER NOT NULL,
|
|
162
|
+
event_count INTEGER NOT NULL,
|
|
163
|
+
PRIMARY KEY(provider, bucket_start_ms)
|
|
164
|
+
) WITHOUT ROWID;
|
|
165
|
+
CREATE TABLE IF NOT EXISTS recent_events (
|
|
166
|
+
source_path TEXT NOT NULL,
|
|
167
|
+
event_key TEXT NOT NULL,
|
|
168
|
+
provider TEXT NOT NULL,
|
|
169
|
+
session_id TEXT NOT NULL,
|
|
170
|
+
timestamp_ms INTEGER NOT NULL,
|
|
171
|
+
input_tokens INTEGER NOT NULL,
|
|
172
|
+
output_tokens INTEGER NOT NULL,
|
|
173
|
+
cache_read_tokens INTEGER NOT NULL,
|
|
174
|
+
cache_write_tokens INTEGER NOT NULL,
|
|
175
|
+
PRIMARY KEY(source_path, event_key),
|
|
176
|
+
FOREIGN KEY(source_path) REFERENCES source_files(path) ON DELETE CASCADE
|
|
177
|
+
) WITHOUT ROWID;
|
|
178
|
+
CREATE INDEX IF NOT EXISTS recent_events_provider_time
|
|
179
|
+
ON recent_events(provider, timestamp_ms);
|
|
180
|
+
CREATE TABLE IF NOT EXISTS claude_messages (
|
|
181
|
+
owner_session_id TEXT NOT NULL,
|
|
182
|
+
message_id TEXT NOT NULL,
|
|
183
|
+
source_path TEXT NOT NULL,
|
|
184
|
+
timestamp_ms INTEGER NOT NULL,
|
|
185
|
+
input_tokens INTEGER NOT NULL,
|
|
186
|
+
output_tokens INTEGER NOT NULL,
|
|
187
|
+
cache_read_tokens INTEGER NOT NULL,
|
|
188
|
+
cache_write_tokens INTEGER NOT NULL,
|
|
189
|
+
PRIMARY KEY(owner_session_id, message_id)
|
|
190
|
+
) WITHOUT ROWID;
|
|
191
|
+
CREATE INDEX IF NOT EXISTS claude_messages_time
|
|
192
|
+
ON claude_messages(timestamp_ms);
|
|
193
|
+
CREATE TRIGGER IF NOT EXISTS source_files_stats_insert
|
|
194
|
+
AFTER INSERT ON source_files BEGIN
|
|
195
|
+
UPDATE cache_stats SET
|
|
196
|
+
codex_files = codex_files + (NEW.provider = 'codex'),
|
|
197
|
+
claude_files = claude_files + (NEW.provider = 'claude'),
|
|
198
|
+
codex_incomplete = codex_incomplete + (NEW.provider = 'codex' AND NEW.scan_complete = 0),
|
|
199
|
+
claude_incomplete = claude_incomplete + (NEW.provider = 'claude' AND NEW.scan_complete = 0),
|
|
200
|
+
committed_bytes = committed_bytes + NEW.committed_offset
|
|
201
|
+
WHERE singleton = 1;
|
|
202
|
+
END;
|
|
203
|
+
CREATE TRIGGER IF NOT EXISTS source_files_stats_delete
|
|
204
|
+
AFTER DELETE ON source_files BEGIN
|
|
205
|
+
UPDATE cache_stats SET
|
|
206
|
+
codex_files = codex_files - (OLD.provider = 'codex'),
|
|
207
|
+
claude_files = claude_files - (OLD.provider = 'claude'),
|
|
208
|
+
codex_incomplete = codex_incomplete - (OLD.provider = 'codex' AND OLD.scan_complete = 0),
|
|
209
|
+
claude_incomplete = claude_incomplete - (OLD.provider = 'claude' AND OLD.scan_complete = 0),
|
|
210
|
+
committed_bytes = committed_bytes - OLD.committed_offset
|
|
211
|
+
WHERE singleton = 1;
|
|
212
|
+
END;
|
|
213
|
+
CREATE TRIGGER IF NOT EXISTS source_files_stats_update
|
|
214
|
+
AFTER UPDATE OF provider, scan_complete, committed_offset ON source_files BEGIN
|
|
215
|
+
UPDATE cache_stats SET
|
|
216
|
+
codex_files = codex_files - (OLD.provider = 'codex') + (NEW.provider = 'codex'),
|
|
217
|
+
claude_files = claude_files - (OLD.provider = 'claude') + (NEW.provider = 'claude'),
|
|
218
|
+
codex_incomplete = codex_incomplete
|
|
219
|
+
- (OLD.provider = 'codex' AND OLD.scan_complete = 0)
|
|
220
|
+
+ (NEW.provider = 'codex' AND NEW.scan_complete = 0),
|
|
221
|
+
claude_incomplete = claude_incomplete
|
|
222
|
+
- (OLD.provider = 'claude' AND OLD.scan_complete = 0)
|
|
223
|
+
+ (NEW.provider = 'claude' AND NEW.scan_complete = 0),
|
|
224
|
+
committed_bytes = committed_bytes - OLD.committed_offset + NEW.committed_offset
|
|
225
|
+
WHERE singleton = 1;
|
|
226
|
+
END;
|
|
227
|
+
CREATE TRIGGER IF NOT EXISTS usage_hourly_stats_insert
|
|
228
|
+
AFTER INSERT ON usage_hourly BEGIN
|
|
229
|
+
UPDATE cache_stats SET hourly_rows = hourly_rows + 1 WHERE singleton = 1;
|
|
230
|
+
END;
|
|
231
|
+
CREATE TRIGGER IF NOT EXISTS usage_hourly_stats_delete
|
|
232
|
+
AFTER DELETE ON usage_hourly BEGIN
|
|
233
|
+
UPDATE cache_stats SET hourly_rows = hourly_rows - 1 WHERE singleton = 1;
|
|
234
|
+
END;
|
|
235
|
+
CREATE TRIGGER IF NOT EXISTS recent_events_stats_insert
|
|
236
|
+
AFTER INSERT ON recent_events BEGIN
|
|
237
|
+
UPDATE cache_stats SET recent_rows = recent_rows + 1 WHERE singleton = 1;
|
|
238
|
+
END;
|
|
239
|
+
CREATE TRIGGER IF NOT EXISTS recent_events_stats_delete
|
|
240
|
+
AFTER DELETE ON recent_events BEGIN
|
|
241
|
+
UPDATE cache_stats SET recent_rows = recent_rows - 1 WHERE singleton = 1;
|
|
242
|
+
END;
|
|
243
|
+
CREATE TRIGGER IF NOT EXISTS claude_messages_stats_insert
|
|
244
|
+
AFTER INSERT ON claude_messages BEGIN
|
|
245
|
+
UPDATE cache_stats SET dedupe_rows = dedupe_rows + 1 WHERE singleton = 1;
|
|
246
|
+
END;
|
|
247
|
+
CREATE TRIGGER IF NOT EXISTS claude_messages_stats_delete
|
|
248
|
+
AFTER DELETE ON claude_messages BEGIN
|
|
249
|
+
UPDATE cache_stats SET dedupe_rows = dedupe_rows - 1 WHERE singleton = 1;
|
|
250
|
+
END;
|
|
251
|
+
CREATE TRIGGER IF NOT EXISTS discovery_directories_stats_insert
|
|
252
|
+
AFTER INSERT ON discovery_directories WHEN NEW.present = 1 AND NEW.pending = 1 BEGIN
|
|
253
|
+
UPDATE cache_stats SET pending_directories = pending_directories + 1 WHERE singleton = 1;
|
|
254
|
+
END;
|
|
255
|
+
CREATE TRIGGER IF NOT EXISTS discovery_directories_stats_delete
|
|
256
|
+
AFTER DELETE ON discovery_directories WHEN OLD.present = 1 AND OLD.pending = 1 BEGIN
|
|
257
|
+
UPDATE cache_stats SET pending_directories = pending_directories - 1 WHERE singleton = 1;
|
|
258
|
+
END;
|
|
259
|
+
CREATE TRIGGER IF NOT EXISTS discovery_directories_stats_update
|
|
260
|
+
AFTER UPDATE OF present, pending ON discovery_directories
|
|
261
|
+
WHEN (OLD.present = 1 AND OLD.pending = 1) != (NEW.present = 1 AND NEW.pending = 1)
|
|
262
|
+
BEGIN
|
|
263
|
+
UPDATE cache_stats SET pending_directories = pending_directories
|
|
264
|
+
- (OLD.present = 1 AND OLD.pending = 1)
|
|
265
|
+
+ (NEW.present = 1 AND NEW.pending = 1)
|
|
266
|
+
WHERE singleton = 1;
|
|
267
|
+
END;
|
|
268
|
+
CREATE TRIGGER IF NOT EXISTS discovery_source_queue_stats_insert
|
|
269
|
+
AFTER INSERT ON discovery_source_queue BEGIN
|
|
270
|
+
UPDATE cache_stats SET queued_sources = queued_sources + 1 WHERE singleton = 1;
|
|
271
|
+
END;
|
|
272
|
+
CREATE TRIGGER IF NOT EXISTS discovery_source_queue_stats_delete
|
|
273
|
+
AFTER DELETE ON discovery_source_queue BEGIN
|
|
274
|
+
UPDATE cache_stats SET queued_sources = queued_sources - 1 WHERE singleton = 1;
|
|
275
|
+
END;
|
|
276
|
+
`;
|
|
277
|
+
function integer(value) {
|
|
278
|
+
const parsed = Number(value);
|
|
279
|
+
return Number.isFinite(parsed) ? Math.max(0, Math.trunc(parsed)) : 0;
|
|
280
|
+
}
|
|
281
|
+
function timestampMs(value) {
|
|
282
|
+
if (typeof value === "number") {
|
|
283
|
+
if (!Number.isFinite(value) || value <= 0) return null;
|
|
284
|
+
return Math.trunc(value < 1e10 ? value * 1e3 : value);
|
|
285
|
+
}
|
|
286
|
+
if (typeof value !== "string" || !value) return null;
|
|
287
|
+
if (/^\d+$/.test(value)) return timestampMs(Number(value));
|
|
288
|
+
const parsed = Date.parse(value);
|
|
289
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
290
|
+
}
|
|
291
|
+
function cloneTotals(value) {
|
|
292
|
+
return value ? { ...value } : null;
|
|
293
|
+
}
|
|
294
|
+
function addTotals(left, right) {
|
|
295
|
+
const base = left || ZERO;
|
|
296
|
+
return {
|
|
297
|
+
input: base.input + right.input,
|
|
298
|
+
cached: base.cached + right.cached,
|
|
299
|
+
output: base.output + right.output,
|
|
300
|
+
cacheWrite: base.cacheWrite + right.cacheWrite
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
function minTotals(left, right) {
|
|
304
|
+
return {
|
|
305
|
+
input: Math.min(left.input, right.input),
|
|
306
|
+
cached: Math.min(left.cached, right.cached),
|
|
307
|
+
output: Math.min(left.output, right.output),
|
|
308
|
+
cacheWrite: Math.min(left.cacheWrite, right.cacheWrite)
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
function maxTotals(left, right) {
|
|
312
|
+
if (!left) return { ...right };
|
|
313
|
+
return {
|
|
314
|
+
input: Math.max(left.input, right.input),
|
|
315
|
+
cached: Math.max(left.cached, right.cached),
|
|
316
|
+
output: Math.max(left.output, right.output),
|
|
317
|
+
cacheWrite: Math.max(left.cacheWrite, right.cacheWrite)
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
function subtractTotals(current, baseline) {
|
|
321
|
+
const base = baseline || ZERO;
|
|
322
|
+
return {
|
|
323
|
+
input: Math.max(0, current.input - base.input),
|
|
324
|
+
cached: Math.max(0, current.cached - base.cached),
|
|
325
|
+
output: Math.max(0, current.output - base.output),
|
|
326
|
+
cacheWrite: Math.max(0, current.cacheWrite - base.cacheWrite)
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
function totalsEqual(left, right) {
|
|
330
|
+
if (!left || !right) return left === right;
|
|
331
|
+
return TOKEN_FIELDS.every((field) => left[field] === right[field]);
|
|
332
|
+
}
|
|
333
|
+
function totalsAtLeast(left, right) {
|
|
334
|
+
return TOKEN_FIELDS.every((field) => left[field] >= right[field]);
|
|
335
|
+
}
|
|
336
|
+
function totalsAtMost(left, right) {
|
|
337
|
+
return TOKEN_FIELDS.every((field) => left[field] <= right[field]);
|
|
338
|
+
}
|
|
339
|
+
function hasUsage(value) {
|
|
340
|
+
return Boolean(value && TOKEN_FIELDS.some((field) => value[field] > 0));
|
|
341
|
+
}
|
|
342
|
+
function localHourMs(value) {
|
|
343
|
+
const date = new Date(value);
|
|
344
|
+
date.setMinutes(0, 0, 0);
|
|
345
|
+
return date.getTime();
|
|
346
|
+
}
|
|
347
|
+
function removeDatabaseFiles(filePath) {
|
|
348
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
349
|
+
try {
|
|
350
|
+
import_node_fs.default.rmSync(`${filePath}${suffix}`, { force: true });
|
|
351
|
+
} catch (error) {
|
|
352
|
+
if (suffix === "") throw error;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
function databaseCompatible(filePath) {
|
|
357
|
+
if (!import_node_fs.default.existsSync(filePath)) return true;
|
|
358
|
+
try {
|
|
359
|
+
const database = new import_node_sqlite.DatabaseSync(filePath, { readOnly: true });
|
|
360
|
+
try {
|
|
361
|
+
const tables = database.prepare(
|
|
362
|
+
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
|
|
363
|
+
).all();
|
|
364
|
+
if (tables.length === 0) return true;
|
|
365
|
+
const schema = database.prepare(
|
|
366
|
+
"SELECT value FROM metadata WHERE key = 'schema_version'"
|
|
367
|
+
).get();
|
|
368
|
+
const source = database.prepare(
|
|
369
|
+
"SELECT value FROM metadata WHERE key = 'source_version'"
|
|
370
|
+
).get();
|
|
371
|
+
const names = new Set(tables.map((row) => row.name));
|
|
372
|
+
const triggers = new Set(database.prepare(
|
|
373
|
+
"SELECT name FROM sqlite_master WHERE type = 'trigger'"
|
|
374
|
+
).all().map((row) => row.name));
|
|
375
|
+
const stats = database.prepare(
|
|
376
|
+
"SELECT singleton FROM cache_stats WHERE singleton = 1"
|
|
377
|
+
).get();
|
|
378
|
+
return schema?.value === String(SCHEMA_VERSION) && source?.value === SOURCE_VERSION && [
|
|
379
|
+
"source_files",
|
|
380
|
+
"usage_hourly",
|
|
381
|
+
"usage_provider_hourly",
|
|
382
|
+
"recent_events",
|
|
383
|
+
"claude_messages",
|
|
384
|
+
"discovery_directories",
|
|
385
|
+
"discovery_source_queue",
|
|
386
|
+
"codex_quota_candidates",
|
|
387
|
+
"cache_stats"
|
|
388
|
+
].every((name) => names.has(name)) && [
|
|
389
|
+
"source_files_stats_insert",
|
|
390
|
+
"source_files_stats_delete",
|
|
391
|
+
"source_files_stats_update",
|
|
392
|
+
"usage_hourly_stats_insert",
|
|
393
|
+
"usage_hourly_stats_delete",
|
|
394
|
+
"recent_events_stats_insert",
|
|
395
|
+
"recent_events_stats_delete",
|
|
396
|
+
"claude_messages_stats_insert",
|
|
397
|
+
"claude_messages_stats_delete",
|
|
398
|
+
"discovery_directories_stats_insert",
|
|
399
|
+
"discovery_directories_stats_delete",
|
|
400
|
+
"discovery_directories_stats_update",
|
|
401
|
+
"discovery_source_queue_stats_insert",
|
|
402
|
+
"discovery_source_queue_stats_delete"
|
|
403
|
+
].every((name) => triggers.has(name)) && stats?.singleton === 1;
|
|
404
|
+
} finally {
|
|
405
|
+
database.close();
|
|
406
|
+
}
|
|
407
|
+
} catch {
|
|
408
|
+
return false;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
function openDatabase(filePath) {
|
|
412
|
+
import_node_fs.default.mkdirSync(import_node_path.default.dirname(filePath), { recursive: true, mode: 448 });
|
|
413
|
+
const compatible = databaseCompatible(filePath);
|
|
414
|
+
if (!compatible) {
|
|
415
|
+
closeCacheDirectoryCursors(filePath);
|
|
416
|
+
removeDatabaseFiles(filePath);
|
|
417
|
+
}
|
|
418
|
+
const database = new import_node_sqlite.DatabaseSync(filePath);
|
|
419
|
+
database.exec("PRAGMA journal_mode=WAL");
|
|
420
|
+
database.exec("PRAGMA synchronous=NORMAL");
|
|
421
|
+
database.exec("PRAGMA foreign_keys=ON");
|
|
422
|
+
database.exec("PRAGMA busy_timeout=30000");
|
|
423
|
+
database.exec(SCHEMA_SQL);
|
|
424
|
+
database.prepare(
|
|
425
|
+
"INSERT OR REPLACE INTO metadata(key, value) VALUES('schema_version', ?)"
|
|
426
|
+
).run(String(SCHEMA_VERSION));
|
|
427
|
+
database.prepare(
|
|
428
|
+
"INSERT OR REPLACE INTO metadata(key, value) VALUES('source_version', ?)"
|
|
429
|
+
).run(SOURCE_VERSION);
|
|
430
|
+
try {
|
|
431
|
+
import_node_fs.default.chmodSync(filePath, 384);
|
|
432
|
+
} catch {
|
|
433
|
+
}
|
|
434
|
+
return { database, rebuilt: !compatible };
|
|
435
|
+
}
|
|
436
|
+
function prefixHash(filePath, size) {
|
|
437
|
+
const descriptor = import_node_fs.default.openSync(filePath, "r");
|
|
438
|
+
try {
|
|
439
|
+
const lengthLimit = Math.min(PREFIX_BYTES, size ?? PREFIX_BYTES);
|
|
440
|
+
const buffer = Buffer.allocUnsafe(lengthLimit);
|
|
441
|
+
const length = import_node_fs.default.readSync(descriptor, buffer, 0, buffer.length, 0);
|
|
442
|
+
return import_node_crypto.default.createHash("sha256").update(buffer.subarray(0, length)).digest("hex");
|
|
443
|
+
} finally {
|
|
444
|
+
import_node_fs.default.closeSync(descriptor);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
function suffixHash(filePath, size) {
|
|
448
|
+
const descriptor = import_node_fs.default.openSync(filePath, "r");
|
|
449
|
+
try {
|
|
450
|
+
const fileSize = size ?? import_node_fs.default.fstatSync(descriptor).size;
|
|
451
|
+
const length = Math.min(fileSize, PREFIX_BYTES);
|
|
452
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
453
|
+
const read = import_node_fs.default.readSync(
|
|
454
|
+
descriptor,
|
|
455
|
+
buffer,
|
|
456
|
+
0,
|
|
457
|
+
length,
|
|
458
|
+
Math.max(0, fileSize - length)
|
|
459
|
+
);
|
|
460
|
+
return import_node_crypto.default.createHash("sha256").update(buffer.subarray(0, read)).digest("hex");
|
|
461
|
+
} finally {
|
|
462
|
+
import_node_fs.default.closeSync(descriptor);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
function claudeOwnerSession(filePath) {
|
|
466
|
+
const parent = import_node_path.default.dirname(filePath);
|
|
467
|
+
if (import_node_path.default.basename(parent) === "subagents") {
|
|
468
|
+
return import_node_path.default.basename(import_node_path.default.dirname(parent));
|
|
469
|
+
}
|
|
470
|
+
return import_node_path.default.basename(filePath, ".jsonl");
|
|
471
|
+
}
|
|
472
|
+
function defaultState(provider, filePath) {
|
|
473
|
+
if (provider === "claude") {
|
|
474
|
+
return {
|
|
475
|
+
sessionId: claudeOwnerSession(filePath),
|
|
476
|
+
projectPath: ""
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
return {
|
|
480
|
+
sessionId: import_node_path.default.basename(filePath, ".jsonl"),
|
|
481
|
+
leafSessionId: null,
|
|
482
|
+
parentSessionId: null,
|
|
483
|
+
forkTimestamp: null,
|
|
484
|
+
projectPath: "",
|
|
485
|
+
lastAssistantTimestamp: null,
|
|
486
|
+
copiedPrefix: false,
|
|
487
|
+
isSubagent: false,
|
|
488
|
+
ownBoundarySeen: false,
|
|
489
|
+
pendingOwnedBoundary: false,
|
|
490
|
+
inheritedRaw: null,
|
|
491
|
+
counted: null,
|
|
492
|
+
rawBaseline: null,
|
|
493
|
+
watermark: null,
|
|
494
|
+
seenRawTotals: [],
|
|
495
|
+
divergent: false,
|
|
496
|
+
interleaved: false,
|
|
497
|
+
skippedUnresolvedForkTotal: false,
|
|
498
|
+
quotas: {}
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
function normalizedCodexTotals(value) {
|
|
502
|
+
if (!value || typeof value !== "object") return null;
|
|
503
|
+
const usage = value;
|
|
504
|
+
const cached = integer(usage.cached_input_tokens ?? usage.cache_read_input_tokens);
|
|
505
|
+
const rawInput = integer(usage.input_tokens);
|
|
506
|
+
const result = {
|
|
507
|
+
// Codex input_tokens includes cached_input_tokens. Farming displays the
|
|
508
|
+
// mutually-exclusive breakdown while retaining CodexBar's cumulative math.
|
|
509
|
+
input: Math.max(0, rawInput - cached),
|
|
510
|
+
cached,
|
|
511
|
+
output: integer(usage.output_tokens),
|
|
512
|
+
cacheWrite: integer(usage.cache_creation_input_tokens)
|
|
513
|
+
};
|
|
514
|
+
return hasUsage(result) ? result : null;
|
|
515
|
+
}
|
|
516
|
+
function claudeTotals(value) {
|
|
517
|
+
if (!value || typeof value !== "object") return null;
|
|
518
|
+
const usage = value;
|
|
519
|
+
const result = {
|
|
520
|
+
input: integer(usage.input_tokens),
|
|
521
|
+
cached: integer(usage.cache_read_input_tokens),
|
|
522
|
+
output: integer(usage.output_tokens),
|
|
523
|
+
cacheWrite: integer(usage.cache_creation_input_tokens)
|
|
524
|
+
};
|
|
525
|
+
return hasUsage(result) ? result : null;
|
|
526
|
+
}
|
|
527
|
+
function parseLargeJsonEdge(first, tail) {
|
|
528
|
+
const text = `${first.toString("utf8")}
|
|
529
|
+
${tail.toString("utf8")}`;
|
|
530
|
+
const timestamp = /"timestamp"\s*:\s*"([^"]+)"/.exec(text)?.[1];
|
|
531
|
+
const topType = /"type"\s*:\s*"(assistant|event_msg|session_meta|response_item|turn_context)"/.exec(text)?.[1];
|
|
532
|
+
if (!topType) return null;
|
|
533
|
+
if (topType === "assistant") {
|
|
534
|
+
const sessionId = /"sessionId"\s*:\s*"([^"]+)"/.exec(text)?.[1];
|
|
535
|
+
const messageId = /"id"\s*:\s*"([^"]+)"/.exec(text)?.[1];
|
|
536
|
+
const usageStart = text.lastIndexOf('"usage"');
|
|
537
|
+
const usageText = usageStart >= 0 ? text.slice(usageStart, usageStart + 2048) : "";
|
|
538
|
+
return {
|
|
539
|
+
type: "assistant",
|
|
540
|
+
timestamp,
|
|
541
|
+
sessionId,
|
|
542
|
+
message: {
|
|
543
|
+
id: messageId,
|
|
544
|
+
usage: {
|
|
545
|
+
input_tokens: integer(/"input_tokens"\s*:\s*(\d+)/.exec(usageText)?.[1]),
|
|
546
|
+
output_tokens: integer(/"output_tokens"\s*:\s*(\d+)/.exec(usageText)?.[1]),
|
|
547
|
+
cache_read_input_tokens: integer(
|
|
548
|
+
/"cache_read_input_tokens"\s*:\s*(\d+)/.exec(usageText)?.[1]
|
|
549
|
+
),
|
|
550
|
+
cache_creation_input_tokens: integer(
|
|
551
|
+
/"cache_creation_input_tokens"\s*:\s*(\d+)/.exec(usageText)?.[1]
|
|
552
|
+
)
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
if (topType !== "event_msg" || !/"payload"\s*:\s*\{\s*"type"\s*:\s*"token_count"/.test(text)) {
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
const totalStart = text.indexOf('"total_token_usage"');
|
|
561
|
+
const lastStart = text.indexOf('"last_token_usage"');
|
|
562
|
+
const totalsText = totalStart >= 0 ? text.slice(totalStart, totalStart + 1024) : "";
|
|
563
|
+
const lastText = lastStart >= 0 ? text.slice(lastStart, lastStart + 1024) : "";
|
|
564
|
+
const fields = (source) => ({
|
|
565
|
+
input_tokens: integer(/"input_tokens"\s*:\s*(\d+)/.exec(source)?.[1]),
|
|
566
|
+
cached_input_tokens: integer(/"cached_input_tokens"\s*:\s*(\d+)/.exec(source)?.[1]),
|
|
567
|
+
output_tokens: integer(/"output_tokens"\s*:\s*(\d+)/.exec(source)?.[1])
|
|
568
|
+
});
|
|
569
|
+
return {
|
|
570
|
+
type: "event_msg",
|
|
571
|
+
timestamp,
|
|
572
|
+
payload: {
|
|
573
|
+
type: "token_count",
|
|
574
|
+
info: {
|
|
575
|
+
total_token_usage: fields(totalsText),
|
|
576
|
+
last_token_usage: fields(lastText)
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
function interesting(provider, buffer) {
|
|
582
|
+
if (provider === "claude") return buffer.includes(Buffer.from('"assistant"'));
|
|
583
|
+
return [
|
|
584
|
+
'"session_meta"',
|
|
585
|
+
'"token_count"',
|
|
586
|
+
'"agent_message"',
|
|
587
|
+
'"response_item"',
|
|
588
|
+
'"turn_context"',
|
|
589
|
+
'"task_started"',
|
|
590
|
+
'"inter_agent_communication_metadata"'
|
|
591
|
+
].some((marker) => buffer.includes(Buffer.from(marker)));
|
|
592
|
+
}
|
|
593
|
+
function parseRecord(provider, fullLine, first, tail) {
|
|
594
|
+
const searchable = fullLine || Buffer.concat([first, tail]);
|
|
595
|
+
if (!interesting(provider, searchable)) return null;
|
|
596
|
+
if (fullLine) {
|
|
597
|
+
try {
|
|
598
|
+
const parsed = JSON.parse(fullLine.toString("utf8"));
|
|
599
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
600
|
+
} catch {
|
|
601
|
+
return null;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return parseLargeJsonEdge(first, tail);
|
|
605
|
+
}
|
|
606
|
+
function divergentDelta(rawBaseline, countedBaseline, current) {
|
|
607
|
+
const raw = rawBaseline || ZERO;
|
|
608
|
+
const counted = countedBaseline || ZERO;
|
|
609
|
+
const component = (rawValue, countedValue, currentValue) => currentValue >= rawValue ? Math.max(0, currentValue - rawValue) : Math.max(0, currentValue - countedValue);
|
|
610
|
+
return {
|
|
611
|
+
input: component(raw.input, counted.input, current.input),
|
|
612
|
+
cached: component(raw.cached, counted.cached, current.cached),
|
|
613
|
+
output: component(raw.output, counted.output, current.output),
|
|
614
|
+
cacheWrite: component(raw.cacheWrite, counted.cacheWrite, current.cacheWrite)
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
function containedDelta(watermark, counted, current) {
|
|
618
|
+
const water = watermark || ZERO;
|
|
619
|
+
const base = counted || ZERO;
|
|
620
|
+
const component = (waterValue, countedValue, currentValue) => currentValue >= waterValue ? Math.max(0, currentValue - Math.max(waterValue, countedValue)) : Math.max(0, currentValue - countedValue);
|
|
621
|
+
return {
|
|
622
|
+
input: component(water.input, base.input, current.input),
|
|
623
|
+
cached: component(water.cached, base.cached, current.cached),
|
|
624
|
+
output: component(water.output, base.output, current.output),
|
|
625
|
+
cacheWrite: component(water.cacheWrite, base.cacheWrite, current.cacheWrite)
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
function applyCodexTotals(state, last, total) {
|
|
629
|
+
if (total && state.seenRawTotals.some((value) => totalsEqual(value, total))) {
|
|
630
|
+
return null;
|
|
631
|
+
}
|
|
632
|
+
if (total && state.watermark && TOKEN_FIELDS.some(
|
|
633
|
+
(field) => total[field] < state.watermark[field]
|
|
634
|
+
)) {
|
|
635
|
+
state.interleaved = true;
|
|
636
|
+
}
|
|
637
|
+
const watermarkBaseline = state.watermark || state.rawBaseline;
|
|
638
|
+
let delta = null;
|
|
639
|
+
if (last) {
|
|
640
|
+
delta = last;
|
|
641
|
+
if (total) {
|
|
642
|
+
if (state.interleaved) {
|
|
643
|
+
delta = minTotals(last, containedDelta(watermarkBaseline, state.counted, total));
|
|
644
|
+
} else {
|
|
645
|
+
const totalDelta = subtractTotals(total, watermarkBaseline);
|
|
646
|
+
if (!state.divergent && totalsAtLeast(total, watermarkBaseline || ZERO) && totalsAtMost(totalDelta, last)) {
|
|
647
|
+
delta = totalDelta;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
state.counted = addTotals(state.counted, delta);
|
|
651
|
+
state.rawBaseline = { ...total };
|
|
652
|
+
if (!totalsEqual(state.rawBaseline, state.counted)) state.divergent = true;
|
|
653
|
+
} else {
|
|
654
|
+
state.counted = addTotals(state.counted, delta);
|
|
655
|
+
state.rawBaseline = cloneTotals(state.counted);
|
|
656
|
+
state.watermark = maxTotals(state.watermark, state.counted);
|
|
657
|
+
}
|
|
658
|
+
} else if (total) {
|
|
659
|
+
delta = state.interleaved ? containedDelta(watermarkBaseline, state.counted, total) : state.divergent ? divergentDelta(watermarkBaseline, state.counted, total) : subtractTotals(total, watermarkBaseline);
|
|
660
|
+
state.counted = addTotals(state.counted, delta);
|
|
661
|
+
state.rawBaseline = { ...total };
|
|
662
|
+
if (!totalsEqual(state.rawBaseline, state.counted)) state.divergent = true;
|
|
663
|
+
}
|
|
664
|
+
if (total) {
|
|
665
|
+
state.watermark = maxTotals(state.watermark, total);
|
|
666
|
+
if (!state.seenRawTotals.some((value) => totalsEqual(value, total))) {
|
|
667
|
+
state.seenRawTotals.push({ ...total });
|
|
668
|
+
state.seenRawTotals = state.seenRawTotals.slice(-SEEN_TOTALS_LIMIT);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
return hasUsage(delta) ? delta : null;
|
|
672
|
+
}
|
|
673
|
+
function parseCodexMeta(record) {
|
|
674
|
+
if (record.type !== "session_meta") return null;
|
|
675
|
+
const payload = record.payload && typeof record.payload === "object" ? record.payload : {};
|
|
676
|
+
const source = payload.source && typeof payload.source === "object" ? payload.source : {};
|
|
677
|
+
const subagent = source.subagent && typeof source.subagent === "object" ? source.subagent : {};
|
|
678
|
+
const spawn = subagent.thread_spawn && typeof subagent.thread_spawn === "object" ? subagent.thread_spawn : {};
|
|
679
|
+
const id = typeof payload.id === "string" && payload.id.trim() ? payload.id.trim() : null;
|
|
680
|
+
const explicitParent = typeof payload.forked_from_id === "string" ? payload.forked_from_id.trim() : "";
|
|
681
|
+
const spawnedParent = typeof spawn.parent_thread_id === "string" ? spawn.parent_thread_id.trim() : "";
|
|
682
|
+
return {
|
|
683
|
+
id,
|
|
684
|
+
parentId: explicitParent || spawnedParent || null,
|
|
685
|
+
timestamp: timestampMs(payload.timestamp) || timestampMs(record.timestamp),
|
|
686
|
+
cwd: typeof payload.cwd === "string" ? payload.cwd : "",
|
|
687
|
+
isSubagent: Boolean(source.subagent)
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
function updateCodexMetadata(state, record) {
|
|
691
|
+
const metadata = parseCodexMeta(record);
|
|
692
|
+
if (!metadata) return;
|
|
693
|
+
if (!state.leafSessionId) {
|
|
694
|
+
state.leafSessionId = metadata.id;
|
|
695
|
+
if (metadata.id) state.sessionId = metadata.id;
|
|
696
|
+
state.parentSessionId = metadata.parentId;
|
|
697
|
+
state.forkTimestamp = metadata.timestamp;
|
|
698
|
+
state.isSubagent = metadata.isSubagent;
|
|
699
|
+
if (metadata.cwd) state.projectPath = metadata.cwd;
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
if (metadata.id && metadata.id !== state.leafSessionId) {
|
|
703
|
+
state.copiedPrefix = true;
|
|
704
|
+
state.ownBoundarySeen = false;
|
|
705
|
+
state.pendingOwnedBoundary = false;
|
|
706
|
+
if (!state.parentSessionId) state.parentSessionId = metadata.id;
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
if (!state.parentSessionId && metadata.parentId) {
|
|
710
|
+
state.parentSessionId = metadata.parentId;
|
|
711
|
+
state.forkTimestamp = metadata.timestamp || state.forkTimestamp;
|
|
712
|
+
}
|
|
713
|
+
if (!state.projectPath && metadata.cwd) state.projectPath = metadata.cwd;
|
|
714
|
+
}
|
|
715
|
+
function codexRecord(record, state) {
|
|
716
|
+
updateCodexMetadata(state, record);
|
|
717
|
+
const recordTimestamp = timestampMs(record.timestamp);
|
|
718
|
+
const payload = record.payload && typeof record.payload === "object" ? record.payload : {};
|
|
719
|
+
if (record.type === "event_msg" && payload.type === "agent_message" && typeof payload.message === "string") {
|
|
720
|
+
state.pendingOwnedBoundary = false;
|
|
721
|
+
state.lastAssistantTimestamp = recordTimestamp;
|
|
722
|
+
return { usage: null, timestamp: recordTimestamp, quota: null };
|
|
723
|
+
}
|
|
724
|
+
if (record.type === "response_item") {
|
|
725
|
+
state.pendingOwnedBoundary = false;
|
|
726
|
+
const visibleAssistant = payload.type === "function_call" || payload.type === "web_search_call" || payload.type === "message" && payload.role === "assistant";
|
|
727
|
+
if (visibleAssistant) state.lastAssistantTimestamp = recordTimestamp;
|
|
728
|
+
return { usage: null, timestamp: recordTimestamp, quota: null };
|
|
729
|
+
}
|
|
730
|
+
if (record.type === "turn_context") {
|
|
731
|
+
if (state.copiedPrefix && state.isSubagent) state.pendingOwnedBoundary = true;
|
|
732
|
+
return { usage: null, timestamp: recordTimestamp, quota: null };
|
|
733
|
+
}
|
|
734
|
+
if (payload.type === "inter_agent_communication_metadata") {
|
|
735
|
+
if (state.copiedPrefix && state.isSubagent && state.pendingOwnedBoundary) {
|
|
736
|
+
state.ownBoundarySeen = true;
|
|
737
|
+
}
|
|
738
|
+
state.pendingOwnedBoundary = false;
|
|
739
|
+
return { usage: null, timestamp: recordTimestamp, quota: null };
|
|
740
|
+
}
|
|
741
|
+
if (payload.type === "task_started") {
|
|
742
|
+
if (state.copiedPrefix && !state.isSubagent) state.ownBoundarySeen = true;
|
|
743
|
+
return { usage: null, timestamp: recordTimestamp, quota: null };
|
|
744
|
+
}
|
|
745
|
+
if (record.type !== "event_msg" || payload.type !== "token_count") {
|
|
746
|
+
state.pendingOwnedBoundary = false;
|
|
747
|
+
return { usage: null, timestamp: recordTimestamp, quota: null };
|
|
748
|
+
}
|
|
749
|
+
state.pendingOwnedBoundary = false;
|
|
750
|
+
const info = payload.info && typeof payload.info === "object" ? payload.info : {};
|
|
751
|
+
const total = normalizedCodexTotals(info.total_token_usage);
|
|
752
|
+
const last = normalizedCodexTotals(info.last_token_usage);
|
|
753
|
+
const quota = payload.rate_limits && typeof payload.rate_limits === "object" ? payload.rate_limits : null;
|
|
754
|
+
if (state.copiedPrefix && !state.inheritedRaw) {
|
|
755
|
+
if (!state.ownBoundarySeen) {
|
|
756
|
+
if (total) state.inheritedRaw = maxTotals(state.inheritedRaw, total);
|
|
757
|
+
return { usage: null, timestamp: recordTimestamp, quota };
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
let adjustedTotal = total;
|
|
761
|
+
if (state.parentSessionId) {
|
|
762
|
+
if (state.inheritedRaw && total) {
|
|
763
|
+
adjustedTotal = subtractTotals(total, state.inheritedRaw);
|
|
764
|
+
} else if (!state.copiedPrefix && !state.skippedUnresolvedForkTotal) {
|
|
765
|
+
state.skippedUnresolvedForkTotal = true;
|
|
766
|
+
if (last) {
|
|
767
|
+
if (total) state.inheritedRaw = subtractTotals(total, last);
|
|
768
|
+
adjustedTotal = { ...last };
|
|
769
|
+
} else {
|
|
770
|
+
state.inheritedRaw = cloneTotals(total);
|
|
771
|
+
return { usage: null, timestamp: recordTimestamp, quota };
|
|
772
|
+
}
|
|
773
|
+
} else if (!state.copiedPrefix && !state.inheritedRaw) {
|
|
774
|
+
if (!last) return { usage: null, timestamp: recordTimestamp, quota };
|
|
775
|
+
if (total) state.inheritedRaw = subtractTotals(total, last);
|
|
776
|
+
adjustedTotal = { ...last };
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
const usage = applyCodexTotals(state, last, adjustedTotal);
|
|
780
|
+
return {
|
|
781
|
+
usage,
|
|
782
|
+
timestamp: state.lastAssistantTimestamp || recordTimestamp,
|
|
783
|
+
quota
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
function claudeRecord(record, state) {
|
|
787
|
+
if (record.type !== "assistant") {
|
|
788
|
+
return { usage: null, timestamp: null, messageId: null };
|
|
789
|
+
}
|
|
790
|
+
const message = record.message && typeof record.message === "object" ? record.message : {};
|
|
791
|
+
const usage = claudeTotals(message.usage);
|
|
792
|
+
if (typeof record.cwd === "string" && record.cwd) state.projectPath = record.cwd;
|
|
793
|
+
if (typeof record.sessionId === "string" && record.sessionId && !state.sessionId) {
|
|
794
|
+
state.sessionId = record.sessionId;
|
|
795
|
+
}
|
|
796
|
+
return {
|
|
797
|
+
usage,
|
|
798
|
+
timestamp: timestampMs(record.timestamp),
|
|
799
|
+
messageId: typeof message.id === "string" && message.id ? message.id : null
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
function hourlyAdd(database, sourcePath, provider, sessionId, timestamp, usage, eventCount = 1) {
|
|
803
|
+
database.prepare(`
|
|
804
|
+
INSERT INTO usage_hourly(
|
|
805
|
+
source_path, provider, session_id, bucket_start_ms,
|
|
806
|
+
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, event_count
|
|
807
|
+
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
808
|
+
ON CONFLICT(source_path, provider, session_id, bucket_start_ms) DO UPDATE SET
|
|
809
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
810
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
811
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
812
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
813
|
+
event_count = event_count + excluded.event_count
|
|
814
|
+
`).run(
|
|
815
|
+
sourcePath,
|
|
816
|
+
provider,
|
|
817
|
+
sessionId,
|
|
818
|
+
localHourMs(timestamp),
|
|
819
|
+
usage.input,
|
|
820
|
+
usage.output,
|
|
821
|
+
usage.cached,
|
|
822
|
+
usage.cacheWrite,
|
|
823
|
+
eventCount
|
|
824
|
+
);
|
|
825
|
+
database.prepare(`
|
|
826
|
+
INSERT INTO usage_provider_hourly(
|
|
827
|
+
provider, bucket_start_ms, input_tokens, output_tokens,
|
|
828
|
+
cache_read_tokens, cache_write_tokens, event_count
|
|
829
|
+
) VALUES(?, ?, ?, ?, ?, ?, ?)
|
|
830
|
+
ON CONFLICT(provider, bucket_start_ms) DO UPDATE SET
|
|
831
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
832
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
833
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
834
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
835
|
+
event_count = event_count + excluded.event_count
|
|
836
|
+
`).run(
|
|
837
|
+
provider,
|
|
838
|
+
localHourMs(timestamp),
|
|
839
|
+
usage.input,
|
|
840
|
+
usage.output,
|
|
841
|
+
usage.cached,
|
|
842
|
+
usage.cacheWrite,
|
|
843
|
+
eventCount
|
|
844
|
+
);
|
|
845
|
+
if (eventCount < 0) {
|
|
846
|
+
database.prepare(`
|
|
847
|
+
DELETE FROM usage_provider_hourly
|
|
848
|
+
WHERE provider = ? AND bucket_start_ms = ? AND event_count <= 0
|
|
849
|
+
`).run(provider, localHourMs(timestamp));
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
function hourlyDelta(database, sourcePath, provider, sessionId, timestamp, usage, direction) {
|
|
853
|
+
const scaled = {
|
|
854
|
+
input: usage.input * direction,
|
|
855
|
+
cached: usage.cached * direction,
|
|
856
|
+
output: usage.output * direction,
|
|
857
|
+
cacheWrite: usage.cacheWrite * direction
|
|
858
|
+
};
|
|
859
|
+
hourlyAdd(database, sourcePath, provider, sessionId, timestamp, scaled, direction);
|
|
860
|
+
if (direction < 0) {
|
|
861
|
+
database.prepare(`
|
|
862
|
+
DELETE FROM usage_hourly
|
|
863
|
+
WHERE source_path = ? AND provider = ? AND session_id = ?
|
|
864
|
+
AND bucket_start_ms = ? AND event_count <= 0
|
|
865
|
+
`).run(sourcePath, provider, sessionId, localHourMs(timestamp));
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
function recentUpsert(database, sourcePath, eventKey, provider, sessionId, timestamp, usage) {
|
|
869
|
+
database.prepare(`
|
|
870
|
+
INSERT INTO recent_events(
|
|
871
|
+
source_path, event_key, provider, session_id, timestamp_ms,
|
|
872
|
+
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
873
|
+
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
874
|
+
ON CONFLICT(source_path, event_key) DO UPDATE SET
|
|
875
|
+
provider = excluded.provider,
|
|
876
|
+
session_id = excluded.session_id,
|
|
877
|
+
timestamp_ms = excluded.timestamp_ms,
|
|
878
|
+
input_tokens = excluded.input_tokens,
|
|
879
|
+
output_tokens = excluded.output_tokens,
|
|
880
|
+
cache_read_tokens = excluded.cache_read_tokens,
|
|
881
|
+
cache_write_tokens = excluded.cache_write_tokens
|
|
882
|
+
`).run(
|
|
883
|
+
sourcePath,
|
|
884
|
+
eventKey,
|
|
885
|
+
provider,
|
|
886
|
+
sessionId,
|
|
887
|
+
timestamp,
|
|
888
|
+
usage.input,
|
|
889
|
+
usage.output,
|
|
890
|
+
usage.cached,
|
|
891
|
+
usage.cacheWrite
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
function claudeMessageUpsert(database, sourcePath, ownerSessionId, messageId, timestamp, usage, recentBoundary) {
|
|
895
|
+
const existing = database.prepare(`
|
|
896
|
+
SELECT source_path, timestamp_ms, input_tokens, output_tokens,
|
|
897
|
+
cache_read_tokens, cache_write_tokens
|
|
898
|
+
FROM claude_messages
|
|
899
|
+
WHERE owner_session_id = ? AND message_id = ?
|
|
900
|
+
`).get(ownerSessionId, messageId);
|
|
901
|
+
if (existing && (Number(existing.output_tokens) > usage.output || Number(existing.output_tokens) === usage.output && String(existing.source_path) <= sourcePath)) {
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
904
|
+
if (existing) {
|
|
905
|
+
const oldUsage = {
|
|
906
|
+
input: Number(existing.input_tokens),
|
|
907
|
+
output: Number(existing.output_tokens),
|
|
908
|
+
cached: Number(existing.cache_read_tokens),
|
|
909
|
+
cacheWrite: Number(existing.cache_write_tokens)
|
|
910
|
+
};
|
|
911
|
+
hourlyDelta(
|
|
912
|
+
database,
|
|
913
|
+
String(existing.source_path),
|
|
914
|
+
"claude",
|
|
915
|
+
ownerSessionId,
|
|
916
|
+
Number(existing.timestamp_ms),
|
|
917
|
+
oldUsage,
|
|
918
|
+
-1
|
|
919
|
+
);
|
|
920
|
+
database.prepare(
|
|
921
|
+
"DELETE FROM recent_events WHERE source_path = ? AND event_key = ?"
|
|
922
|
+
).run(String(existing.source_path), `message:${messageId}`);
|
|
923
|
+
}
|
|
924
|
+
database.prepare(`
|
|
925
|
+
INSERT INTO claude_messages(
|
|
926
|
+
owner_session_id, message_id, source_path, timestamp_ms,
|
|
927
|
+
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
928
|
+
) VALUES(?, ?, ?, ?, ?, ?, ?, ?)
|
|
929
|
+
ON CONFLICT(owner_session_id, message_id) DO UPDATE SET
|
|
930
|
+
source_path = excluded.source_path,
|
|
931
|
+
timestamp_ms = excluded.timestamp_ms,
|
|
932
|
+
input_tokens = excluded.input_tokens,
|
|
933
|
+
output_tokens = excluded.output_tokens,
|
|
934
|
+
cache_read_tokens = excluded.cache_read_tokens,
|
|
935
|
+
cache_write_tokens = excluded.cache_write_tokens
|
|
936
|
+
WHERE excluded.output_tokens > claude_messages.output_tokens
|
|
937
|
+
OR (
|
|
938
|
+
excluded.output_tokens = claude_messages.output_tokens
|
|
939
|
+
AND excluded.source_path < claude_messages.source_path
|
|
940
|
+
)
|
|
941
|
+
`).run(
|
|
942
|
+
ownerSessionId,
|
|
943
|
+
messageId,
|
|
944
|
+
sourcePath,
|
|
945
|
+
timestamp,
|
|
946
|
+
usage.input,
|
|
947
|
+
usage.output,
|
|
948
|
+
usage.cached,
|
|
949
|
+
usage.cacheWrite
|
|
950
|
+
);
|
|
951
|
+
hourlyDelta(
|
|
952
|
+
database,
|
|
953
|
+
sourcePath,
|
|
954
|
+
"claude",
|
|
955
|
+
ownerSessionId,
|
|
956
|
+
timestamp,
|
|
957
|
+
usage,
|
|
958
|
+
1
|
|
959
|
+
);
|
|
960
|
+
if (timestamp >= recentBoundary) {
|
|
961
|
+
recentUpsert(
|
|
962
|
+
database,
|
|
963
|
+
sourcePath,
|
|
964
|
+
`message:${messageId}`,
|
|
965
|
+
"claude",
|
|
966
|
+
ownerSessionId,
|
|
967
|
+
timestamp,
|
|
968
|
+
usage
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
function sourceRow(database, filePath) {
|
|
973
|
+
return database.prepare("SELECT * FROM source_files WHERE path = ?").get(filePath);
|
|
974
|
+
}
|
|
975
|
+
function directoryGeneration(database, directoryPath) {
|
|
976
|
+
const row = database.prepare(
|
|
977
|
+
"SELECT scan_generation FROM discovery_directories WHERE path = ?"
|
|
978
|
+
).get(directoryPath);
|
|
979
|
+
return Number(row?.scan_generation || 0);
|
|
980
|
+
}
|
|
981
|
+
function insertSource(database, candidate, state, hash) {
|
|
982
|
+
database.prepare(`
|
|
983
|
+
INSERT INTO source_files(
|
|
984
|
+
path, provider, directory_path, directory_generation,
|
|
985
|
+
size, mtime_ms, committed_offset, file_dev, file_ino,
|
|
986
|
+
prefix_sha256, suffix_sha256, parser_state, session_id, scan_complete,
|
|
987
|
+
last_checked_ms
|
|
988
|
+
) VALUES(?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, 0, ?)
|
|
989
|
+
`).run(
|
|
990
|
+
candidate.filePath,
|
|
991
|
+
candidate.provider,
|
|
992
|
+
import_node_path.default.dirname(candidate.filePath),
|
|
993
|
+
directoryGeneration(database, import_node_path.default.dirname(candidate.filePath)),
|
|
994
|
+
candidate.stat.size,
|
|
995
|
+
candidate.stat.mtimeMs,
|
|
996
|
+
Number(candidate.stat.dev),
|
|
997
|
+
Number(candidate.stat.ino),
|
|
998
|
+
hash,
|
|
999
|
+
suffixHash(candidate.filePath, candidate.stat.size),
|
|
1000
|
+
JSON.stringify(state),
|
|
1001
|
+
state.sessionId,
|
|
1002
|
+
Date.now()
|
|
1003
|
+
);
|
|
1004
|
+
}
|
|
1005
|
+
function deleteSource(database, filePath) {
|
|
1006
|
+
database.prepare(`
|
|
1007
|
+
INSERT INTO usage_provider_hourly(
|
|
1008
|
+
provider, bucket_start_ms, input_tokens, output_tokens,
|
|
1009
|
+
cache_read_tokens, cache_write_tokens, event_count
|
|
1010
|
+
)
|
|
1011
|
+
SELECT provider, bucket_start_ms, -input_tokens, -output_tokens,
|
|
1012
|
+
-cache_read_tokens, -cache_write_tokens, -event_count
|
|
1013
|
+
FROM usage_hourly
|
|
1014
|
+
WHERE source_path = ?
|
|
1015
|
+
ON CONFLICT(provider, bucket_start_ms) DO UPDATE SET
|
|
1016
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
1017
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
1018
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
1019
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
1020
|
+
event_count = event_count + excluded.event_count
|
|
1021
|
+
`).run(filePath);
|
|
1022
|
+
database.prepare(
|
|
1023
|
+
"DELETE FROM usage_provider_hourly WHERE event_count <= 0"
|
|
1024
|
+
).run();
|
|
1025
|
+
database.prepare("DELETE FROM claude_messages WHERE source_path = ?").run(filePath);
|
|
1026
|
+
database.prepare("DELETE FROM codex_quota_candidates WHERE source_path = ?").run(filePath);
|
|
1027
|
+
database.prepare("DELETE FROM source_files WHERE path = ?").run(filePath);
|
|
1028
|
+
}
|
|
1029
|
+
function processRecord(database, candidate, state, record, lineEnd, recentBoundary, metrics) {
|
|
1030
|
+
if (candidate.provider === "codex") {
|
|
1031
|
+
const codexState = state;
|
|
1032
|
+
const result2 = codexRecord(record, codexState);
|
|
1033
|
+
if (result2.quota && typeof result2.quota === "object") {
|
|
1034
|
+
const quota = result2.quota;
|
|
1035
|
+
const key = String(quota.limit_id || quota.limitId || "");
|
|
1036
|
+
const previous = codexState.quotas[key];
|
|
1037
|
+
const observedAt = timestampMs(record.timestamp);
|
|
1038
|
+
if (!previous || observedAt === null || previous.timestamp === null || observedAt >= previous.timestamp) {
|
|
1039
|
+
codexState.quotas[key] = { timestamp: observedAt, rateLimits: result2.quota };
|
|
1040
|
+
database.prepare(`
|
|
1041
|
+
INSERT INTO codex_quota_candidates(
|
|
1042
|
+
source_path, limit_id, timestamp_ms, rate_limits_json
|
|
1043
|
+
) VALUES(?, ?, ?, ?)
|
|
1044
|
+
ON CONFLICT(source_path, limit_id) DO UPDATE SET
|
|
1045
|
+
timestamp_ms = excluded.timestamp_ms,
|
|
1046
|
+
rate_limits_json = excluded.rate_limits_json
|
|
1047
|
+
WHERE excluded.timestamp_ms IS NULL
|
|
1048
|
+
OR codex_quota_candidates.timestamp_ms IS NULL
|
|
1049
|
+
OR excluded.timestamp_ms >= codex_quota_candidates.timestamp_ms
|
|
1050
|
+
`).run(
|
|
1051
|
+
candidate.filePath,
|
|
1052
|
+
key,
|
|
1053
|
+
observedAt,
|
|
1054
|
+
JSON.stringify(result2.quota)
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
if (!result2.usage || !result2.timestamp) return;
|
|
1059
|
+
hourlyAdd(
|
|
1060
|
+
database,
|
|
1061
|
+
candidate.filePath,
|
|
1062
|
+
"codex",
|
|
1063
|
+
codexState.sessionId,
|
|
1064
|
+
result2.timestamp,
|
|
1065
|
+
result2.usage
|
|
1066
|
+
);
|
|
1067
|
+
if (result2.timestamp >= recentBoundary) {
|
|
1068
|
+
recentUpsert(
|
|
1069
|
+
database,
|
|
1070
|
+
candidate.filePath,
|
|
1071
|
+
`offset:${lineEnd}`,
|
|
1072
|
+
"codex",
|
|
1073
|
+
codexState.sessionId,
|
|
1074
|
+
result2.timestamp,
|
|
1075
|
+
result2.usage
|
|
1076
|
+
);
|
|
1077
|
+
}
|
|
1078
|
+
metrics.parsed_events = Number(metrics.parsed_events) + 1;
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
const claudeState = state;
|
|
1082
|
+
const result = claudeRecord(record, claudeState);
|
|
1083
|
+
if (!result.usage || !result.timestamp) return;
|
|
1084
|
+
if (result.messageId) {
|
|
1085
|
+
claudeMessageUpsert(
|
|
1086
|
+
database,
|
|
1087
|
+
candidate.filePath,
|
|
1088
|
+
claudeState.sessionId,
|
|
1089
|
+
result.messageId,
|
|
1090
|
+
result.timestamp,
|
|
1091
|
+
result.usage,
|
|
1092
|
+
recentBoundary
|
|
1093
|
+
);
|
|
1094
|
+
} else {
|
|
1095
|
+
hourlyAdd(
|
|
1096
|
+
database,
|
|
1097
|
+
candidate.filePath,
|
|
1098
|
+
"claude",
|
|
1099
|
+
claudeState.sessionId,
|
|
1100
|
+
result.timestamp,
|
|
1101
|
+
result.usage
|
|
1102
|
+
);
|
|
1103
|
+
if (result.timestamp >= recentBoundary) {
|
|
1104
|
+
recentUpsert(
|
|
1105
|
+
database,
|
|
1106
|
+
candidate.filePath,
|
|
1107
|
+
`offset:${lineEnd}`,
|
|
1108
|
+
"claude",
|
|
1109
|
+
claudeState.sessionId,
|
|
1110
|
+
result.timestamp,
|
|
1111
|
+
result.usage
|
|
1112
|
+
);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
metrics.parsed_events = Number(metrics.parsed_events) + 1;
|
|
1116
|
+
}
|
|
1117
|
+
function isCompleteJsonValue(filePath, startOffset, endOffset) {
|
|
1118
|
+
const descriptor = import_node_fs.default.openSync(filePath, "r");
|
|
1119
|
+
let physicalOffset = startOffset;
|
|
1120
|
+
let started = false;
|
|
1121
|
+
let finished = false;
|
|
1122
|
+
let inString = false;
|
|
1123
|
+
let escaped = false;
|
|
1124
|
+
let invalid = false;
|
|
1125
|
+
const stack = [];
|
|
1126
|
+
try {
|
|
1127
|
+
const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES);
|
|
1128
|
+
while (physicalOffset < endOffset && !invalid) {
|
|
1129
|
+
const length = import_node_fs.default.readSync(
|
|
1130
|
+
descriptor,
|
|
1131
|
+
buffer,
|
|
1132
|
+
0,
|
|
1133
|
+
Math.min(buffer.length, endOffset - physicalOffset),
|
|
1134
|
+
physicalOffset
|
|
1135
|
+
);
|
|
1136
|
+
if (length === 0) break;
|
|
1137
|
+
physicalOffset += length;
|
|
1138
|
+
for (const byte of buffer.subarray(0, length)) {
|
|
1139
|
+
if (!started) {
|
|
1140
|
+
if (byte === 32 || byte === 9 || byte === 13 || byte === 10) continue;
|
|
1141
|
+
if (byte !== 123 && byte !== 91) {
|
|
1142
|
+
invalid = true;
|
|
1143
|
+
break;
|
|
1144
|
+
}
|
|
1145
|
+
started = true;
|
|
1146
|
+
stack.push(byte);
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
if (finished) {
|
|
1150
|
+
if (byte !== 32 && byte !== 9 && byte !== 13 && byte !== 10) {
|
|
1151
|
+
invalid = true;
|
|
1152
|
+
break;
|
|
1153
|
+
}
|
|
1154
|
+
continue;
|
|
1155
|
+
}
|
|
1156
|
+
if (inString) {
|
|
1157
|
+
if (escaped) {
|
|
1158
|
+
escaped = false;
|
|
1159
|
+
} else if (byte === 92) {
|
|
1160
|
+
escaped = true;
|
|
1161
|
+
} else if (byte === 34) {
|
|
1162
|
+
inString = false;
|
|
1163
|
+
}
|
|
1164
|
+
continue;
|
|
1165
|
+
}
|
|
1166
|
+
if (byte === 34) {
|
|
1167
|
+
inString = true;
|
|
1168
|
+
} else if (byte === 123 || byte === 91) {
|
|
1169
|
+
stack.push(byte);
|
|
1170
|
+
} else if (byte === 125 || byte === 93) {
|
|
1171
|
+
const expected = byte === 125 ? 123 : 91;
|
|
1172
|
+
if (stack.pop() !== expected) {
|
|
1173
|
+
invalid = true;
|
|
1174
|
+
break;
|
|
1175
|
+
}
|
|
1176
|
+
if (stack.length === 0) finished = true;
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
return physicalOffset >= endOffset && started && finished && !inString && !escaped && !invalid && stack.length === 0;
|
|
1181
|
+
} finally {
|
|
1182
|
+
import_node_fs.default.closeSync(descriptor);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
function scanFile(database, candidate, recentBoundary, deadline, metrics) {
|
|
1186
|
+
let row = sourceRow(database, candidate.filePath);
|
|
1187
|
+
if (row && row.provider === candidate.provider && Number(row.size) === candidate.stat.size && Number(row.mtime_ms) === candidate.stat.mtimeMs && Number(row.scan_complete) === 1) {
|
|
1188
|
+
metrics.reused_files = Number(metrics.reused_files) + 1;
|
|
1189
|
+
return true;
|
|
1190
|
+
}
|
|
1191
|
+
let rebuild = !row || row.provider !== candidate.provider;
|
|
1192
|
+
if (row && !rebuild) {
|
|
1193
|
+
const sameIdentity = Number(row.file_dev) === Number(candidate.stat.dev) && Number(row.file_ino) === Number(candidate.stat.ino);
|
|
1194
|
+
if (candidate.stat.size < Number(row.committed_offset) || !sameIdentity) {
|
|
1195
|
+
rebuild = true;
|
|
1196
|
+
} else if (candidate.stat.mtimeMs !== Number(row.mtime_ms)) {
|
|
1197
|
+
if (prefixHash(candidate.filePath, Number(row.size)) !== row.prefix_sha256 || suffixHash(candidate.filePath, Number(row.size)) !== row.suffix_sha256) {
|
|
1198
|
+
rebuild = true;
|
|
1199
|
+
} else if (candidate.stat.size === Number(row.size) && Number(row.scan_complete) === 1) {
|
|
1200
|
+
database.prepare(
|
|
1201
|
+
"UPDATE source_files SET mtime_ms = ?, file_dev = ?, file_ino = ?, last_checked_ms = ? WHERE path = ?"
|
|
1202
|
+
).run(
|
|
1203
|
+
candidate.stat.mtimeMs,
|
|
1204
|
+
Number(candidate.stat.dev),
|
|
1205
|
+
Number(candidate.stat.ino),
|
|
1206
|
+
Date.now(),
|
|
1207
|
+
candidate.filePath
|
|
1208
|
+
);
|
|
1209
|
+
metrics.reused_files = Number(metrics.reused_files) + 1;
|
|
1210
|
+
return true;
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
if (rebuild) {
|
|
1215
|
+
if (row) {
|
|
1216
|
+
deleteSource(database, candidate.filePath);
|
|
1217
|
+
metrics[`${candidate.provider}_cache_rebuilt`] = true;
|
|
1218
|
+
}
|
|
1219
|
+
const state2 = defaultState(candidate.provider, candidate.filePath);
|
|
1220
|
+
if (candidate.provider === "codex") {
|
|
1221
|
+
const codexState = state2;
|
|
1222
|
+
codexState.sessionId = candidate.sessionId;
|
|
1223
|
+
codexState.parentSessionId = candidate.parentSessionId;
|
|
1224
|
+
codexState.forkTimestamp = candidate.forkTimestamp;
|
|
1225
|
+
codexState.copiedPrefix = candidate.copiedPrefix;
|
|
1226
|
+
codexState.inheritedRaw = cloneTotals(candidate.parentBaseline);
|
|
1227
|
+
}
|
|
1228
|
+
insertSource(database, candidate, state2, prefixHash(candidate.filePath));
|
|
1229
|
+
row = sourceRow(database, candidate.filePath);
|
|
1230
|
+
metrics.rebuilt_files = Number(metrics.rebuilt_files) + 1;
|
|
1231
|
+
} else {
|
|
1232
|
+
metrics.appended_files = Number(metrics.appended_files) + 1;
|
|
1233
|
+
}
|
|
1234
|
+
if (!row) throw new Error(`Unable to create usage checkpoint for ${candidate.filePath}`);
|
|
1235
|
+
const state = JSON.parse(String(row.parser_state));
|
|
1236
|
+
let committedOffset = Number(row.committed_offset);
|
|
1237
|
+
let physicalOffset = committedOffset;
|
|
1238
|
+
let fullChunks = [];
|
|
1239
|
+
let lineBytes = 0;
|
|
1240
|
+
let first = Buffer.alloc(0);
|
|
1241
|
+
let tail = Buffer.alloc(0);
|
|
1242
|
+
let truncated = false;
|
|
1243
|
+
let stoppedForBudget = false;
|
|
1244
|
+
const resetLine = () => {
|
|
1245
|
+
fullChunks = [];
|
|
1246
|
+
lineBytes = 0;
|
|
1247
|
+
first = Buffer.alloc(0);
|
|
1248
|
+
tail = Buffer.alloc(0);
|
|
1249
|
+
truncated = false;
|
|
1250
|
+
};
|
|
1251
|
+
const appendPart = (part) => {
|
|
1252
|
+
lineBytes += part.length;
|
|
1253
|
+
if (!truncated && lineBytes <= MAX_LINE_BYTES) {
|
|
1254
|
+
fullChunks.push(part);
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
if (!truncated) {
|
|
1258
|
+
const combined = Buffer.concat(fullChunks);
|
|
1259
|
+
first = combined.subarray(0, TRUNCATED_EDGE_BYTES);
|
|
1260
|
+
tail = combined.subarray(Math.max(0, combined.length - TRUNCATED_EDGE_BYTES));
|
|
1261
|
+
fullChunks = [];
|
|
1262
|
+
truncated = true;
|
|
1263
|
+
}
|
|
1264
|
+
if (first.length < TRUNCATED_EDGE_BYTES) {
|
|
1265
|
+
const needed = TRUNCATED_EDGE_BYTES - first.length;
|
|
1266
|
+
first = Buffer.concat([first, part.subarray(0, needed)]);
|
|
1267
|
+
}
|
|
1268
|
+
tail = Buffer.concat([tail, part]);
|
|
1269
|
+
if (tail.length > TRUNCATED_EDGE_BYTES) {
|
|
1270
|
+
tail = tail.subarray(tail.length - TRUNCATED_EDGE_BYTES);
|
|
1271
|
+
}
|
|
1272
|
+
};
|
|
1273
|
+
const finishLine = (lineEnd, requireCompleteJson = false) => {
|
|
1274
|
+
const fullLine = truncated ? null : Buffer.concat(fullChunks);
|
|
1275
|
+
if (requireCompleteJson) {
|
|
1276
|
+
if (fullLine) {
|
|
1277
|
+
try {
|
|
1278
|
+
JSON.parse(fullLine.toString("utf8"));
|
|
1279
|
+
} catch {
|
|
1280
|
+
return false;
|
|
1281
|
+
}
|
|
1282
|
+
} else if (!isCompleteJsonValue(candidate.filePath, committedOffset, lineEnd)) {
|
|
1283
|
+
return false;
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
const record = parseRecord(candidate.provider, fullLine, first, tail);
|
|
1287
|
+
if (record) {
|
|
1288
|
+
processRecord(
|
|
1289
|
+
database,
|
|
1290
|
+
candidate,
|
|
1291
|
+
state,
|
|
1292
|
+
record,
|
|
1293
|
+
lineEnd,
|
|
1294
|
+
recentBoundary,
|
|
1295
|
+
metrics
|
|
1296
|
+
);
|
|
1297
|
+
}
|
|
1298
|
+
committedOffset = lineEnd;
|
|
1299
|
+
resetLine();
|
|
1300
|
+
return true;
|
|
1301
|
+
};
|
|
1302
|
+
const descriptor = import_node_fs.default.openSync(candidate.filePath, "r");
|
|
1303
|
+
try {
|
|
1304
|
+
const chunk = Buffer.allocUnsafe(READ_CHUNK_BYTES);
|
|
1305
|
+
while (true) {
|
|
1306
|
+
const length = import_node_fs.default.readSync(
|
|
1307
|
+
descriptor,
|
|
1308
|
+
chunk,
|
|
1309
|
+
0,
|
|
1310
|
+
chunk.length,
|
|
1311
|
+
physicalOffset
|
|
1312
|
+
);
|
|
1313
|
+
if (length === 0) break;
|
|
1314
|
+
metrics.bytes_read = Number(metrics.bytes_read) + length;
|
|
1315
|
+
const block = chunk.subarray(0, length);
|
|
1316
|
+
let cursor = 0;
|
|
1317
|
+
while (cursor < block.length) {
|
|
1318
|
+
const newline = block.indexOf(10, cursor);
|
|
1319
|
+
const end = newline < 0 ? block.length : newline + 1;
|
|
1320
|
+
appendPart(Buffer.from(block.subarray(cursor, end)));
|
|
1321
|
+
physicalOffset += end - cursor;
|
|
1322
|
+
cursor = end;
|
|
1323
|
+
if (newline >= 0) finishLine(physicalOffset);
|
|
1324
|
+
}
|
|
1325
|
+
if (Date.now() >= deadline && lineBytes === 0 && physicalOffset < candidate.stat.size) {
|
|
1326
|
+
stoppedForBudget = true;
|
|
1327
|
+
break;
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
} finally {
|
|
1331
|
+
import_node_fs.default.closeSync(descriptor);
|
|
1332
|
+
}
|
|
1333
|
+
const finalStat = import_node_fs.default.statSync(candidate.filePath);
|
|
1334
|
+
if (!stoppedForBudget && physicalOffset >= finalStat.size && lineBytes > 0) {
|
|
1335
|
+
finishLine(physicalOffset, true);
|
|
1336
|
+
}
|
|
1337
|
+
const scanComplete = !stoppedForBudget && physicalOffset >= finalStat.size && committedOffset >= finalStat.size;
|
|
1338
|
+
database.prepare(`
|
|
1339
|
+
UPDATE source_files SET
|
|
1340
|
+
size = ?, mtime_ms = ?, committed_offset = ?, file_dev = ?, file_ino = ?,
|
|
1341
|
+
prefix_sha256 = ?, suffix_sha256 = ?, parser_state = ?, session_id = ?,
|
|
1342
|
+
scan_complete = ?, last_checked_ms = ?
|
|
1343
|
+
WHERE path = ?
|
|
1344
|
+
`).run(
|
|
1345
|
+
finalStat.size,
|
|
1346
|
+
finalStat.mtimeMs,
|
|
1347
|
+
committedOffset,
|
|
1348
|
+
Number(finalStat.dev),
|
|
1349
|
+
Number(finalStat.ino),
|
|
1350
|
+
prefixHash(candidate.filePath, finalStat.size),
|
|
1351
|
+
suffixHash(candidate.filePath, finalStat.size),
|
|
1352
|
+
JSON.stringify(state),
|
|
1353
|
+
state.sessionId,
|
|
1354
|
+
scanComplete ? 1 : 0,
|
|
1355
|
+
Date.now(),
|
|
1356
|
+
candidate.filePath
|
|
1357
|
+
);
|
|
1358
|
+
metrics[scanComplete ? "scanned_files" : "partial_files"] = Number(metrics[scanComplete ? "scanned_files" : "partial_files"]) + 1;
|
|
1359
|
+
return scanComplete;
|
|
1360
|
+
}
|
|
1361
|
+
function probeCodex(filePath) {
|
|
1362
|
+
const fallback = import_node_path.default.basename(filePath, ".jsonl");
|
|
1363
|
+
let descriptor = null;
|
|
1364
|
+
try {
|
|
1365
|
+
descriptor = import_node_fs.default.openSync(filePath, "r");
|
|
1366
|
+
const buffer = Buffer.allocUnsafe(PREFIX_BYTES);
|
|
1367
|
+
const length = import_node_fs.default.readSync(descriptor, buffer, 0, buffer.length, 0);
|
|
1368
|
+
const lines = buffer.subarray(0, length).toString("utf8").split("\n");
|
|
1369
|
+
let leaf = null;
|
|
1370
|
+
let copiedPrefix = false;
|
|
1371
|
+
for (const line of lines) {
|
|
1372
|
+
if (!line.includes("session_meta")) continue;
|
|
1373
|
+
try {
|
|
1374
|
+
const parsed = JSON.parse(line);
|
|
1375
|
+
const metadata = parseCodexMeta(parsed);
|
|
1376
|
+
if (!metadata) continue;
|
|
1377
|
+
if (!leaf) {
|
|
1378
|
+
leaf = metadata;
|
|
1379
|
+
} else if (metadata.id && metadata.id !== leaf.id) {
|
|
1380
|
+
copiedPrefix = true;
|
|
1381
|
+
}
|
|
1382
|
+
} catch {
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
return {
|
|
1386
|
+
sessionId: leaf?.id || fallback,
|
|
1387
|
+
parentSessionId: leaf?.parentId || null,
|
|
1388
|
+
forkTimestamp: leaf?.timestamp || null,
|
|
1389
|
+
copiedPrefix
|
|
1390
|
+
};
|
|
1391
|
+
} catch {
|
|
1392
|
+
return {
|
|
1393
|
+
sessionId: fallback,
|
|
1394
|
+
parentSessionId: null,
|
|
1395
|
+
forkTimestamp: null,
|
|
1396
|
+
copiedPrefix: false
|
|
1397
|
+
};
|
|
1398
|
+
} finally {
|
|
1399
|
+
if (descriptor !== null) import_node_fs.default.closeSync(descriptor);
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
function codexRawTotalsAt(filePath, cutoff, metrics) {
|
|
1403
|
+
const descriptor = import_node_fs.default.openSync(filePath, "r");
|
|
1404
|
+
let physicalOffset = 0;
|
|
1405
|
+
let chunks = [];
|
|
1406
|
+
let lineBytes = 0;
|
|
1407
|
+
let latest = null;
|
|
1408
|
+
let accumulated = null;
|
|
1409
|
+
let done = false;
|
|
1410
|
+
const consume = () => {
|
|
1411
|
+
if (lineBytes === 0 || lineBytes > 512 * 1024) {
|
|
1412
|
+
chunks = [];
|
|
1413
|
+
lineBytes = 0;
|
|
1414
|
+
return;
|
|
1415
|
+
}
|
|
1416
|
+
try {
|
|
1417
|
+
const record = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
1418
|
+
const observedAt = timestampMs(record.timestamp);
|
|
1419
|
+
if (observedAt && observedAt > cutoff) {
|
|
1420
|
+
done = true;
|
|
1421
|
+
return;
|
|
1422
|
+
}
|
|
1423
|
+
const payload = record.payload && typeof record.payload === "object" ? record.payload : {};
|
|
1424
|
+
if (record.type !== "event_msg" || payload.type !== "token_count") return;
|
|
1425
|
+
const info = payload.info && typeof payload.info === "object" ? payload.info : {};
|
|
1426
|
+
const total = normalizedCodexTotals(info.total_token_usage);
|
|
1427
|
+
const last = normalizedCodexTotals(info.last_token_usage);
|
|
1428
|
+
if (total) {
|
|
1429
|
+
latest = total;
|
|
1430
|
+
accumulated = total;
|
|
1431
|
+
} else if (last) {
|
|
1432
|
+
accumulated = addTotals(accumulated, last);
|
|
1433
|
+
latest = accumulated;
|
|
1434
|
+
}
|
|
1435
|
+
} catch {
|
|
1436
|
+
} finally {
|
|
1437
|
+
chunks = [];
|
|
1438
|
+
lineBytes = 0;
|
|
1439
|
+
}
|
|
1440
|
+
};
|
|
1441
|
+
try {
|
|
1442
|
+
const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES);
|
|
1443
|
+
while (!done) {
|
|
1444
|
+
const length = import_node_fs.default.readSync(
|
|
1445
|
+
descriptor,
|
|
1446
|
+
buffer,
|
|
1447
|
+
0,
|
|
1448
|
+
buffer.length,
|
|
1449
|
+
physicalOffset
|
|
1450
|
+
);
|
|
1451
|
+
if (length === 0) break;
|
|
1452
|
+
metrics.bytes_read = Number(metrics.bytes_read) + length;
|
|
1453
|
+
const block = buffer.subarray(0, length);
|
|
1454
|
+
let cursor = 0;
|
|
1455
|
+
while (cursor < block.length && !done) {
|
|
1456
|
+
const newline = block.indexOf(10, cursor);
|
|
1457
|
+
const end = newline < 0 ? block.length : newline + 1;
|
|
1458
|
+
const part = Buffer.from(block.subarray(cursor, end));
|
|
1459
|
+
lineBytes += part.length;
|
|
1460
|
+
if (lineBytes <= 512 * 1024) chunks.push(part);
|
|
1461
|
+
physicalOffset += part.length;
|
|
1462
|
+
cursor = end;
|
|
1463
|
+
if (newline >= 0) consume();
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
if (!done && lineBytes > 0) consume();
|
|
1467
|
+
return cloneTotals(latest);
|
|
1468
|
+
} finally {
|
|
1469
|
+
import_node_fs.default.closeSync(descriptor);
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
function rowEvent(row) {
|
|
1473
|
+
const input = Number(row.input_tokens);
|
|
1474
|
+
const output = Number(row.output_tokens);
|
|
1475
|
+
const cached = Number(row.cache_read_tokens);
|
|
1476
|
+
const cacheWrite = Number(row.cache_write_tokens);
|
|
1477
|
+
return {
|
|
1478
|
+
timestamp: Number(row.timestamp_ms),
|
|
1479
|
+
sessionId: String(row.session_id),
|
|
1480
|
+
totalTokens: input + output + cached + cacheWrite,
|
|
1481
|
+
inputTokens: input,
|
|
1482
|
+
outputTokens: output,
|
|
1483
|
+
cacheReadTokens: cached,
|
|
1484
|
+
cacheWriteTokens: cacheWrite,
|
|
1485
|
+
unattributedTokens: 0
|
|
1486
|
+
};
|
|
1487
|
+
}
|
|
1488
|
+
function providerEvents(database, provider, retentionCutoff, recentBoundary) {
|
|
1489
|
+
const rows = database.prepare(`
|
|
1490
|
+
SELECT provider || '-aggregate' AS session_id, bucket_start_ms AS timestamp_ms,
|
|
1491
|
+
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
1492
|
+
FROM usage_provider_hourly
|
|
1493
|
+
WHERE provider = ? AND bucket_start_ms >= ? AND bucket_start_ms < ?
|
|
1494
|
+
UNION ALL
|
|
1495
|
+
SELECT session_id, timestamp_ms,
|
|
1496
|
+
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
1497
|
+
FROM recent_events
|
|
1498
|
+
WHERE provider = ? AND timestamp_ms >= ?
|
|
1499
|
+
ORDER BY timestamp_ms, session_id
|
|
1500
|
+
`).all(
|
|
1501
|
+
provider,
|
|
1502
|
+
localHourMs(retentionCutoff),
|
|
1503
|
+
recentBoundary,
|
|
1504
|
+
provider,
|
|
1505
|
+
recentBoundary
|
|
1506
|
+
);
|
|
1507
|
+
return rows.map(rowEvent);
|
|
1508
|
+
}
|
|
1509
|
+
function latestQuotas(database) {
|
|
1510
|
+
const rows = database.prepare(`
|
|
1511
|
+
SELECT timestamp_ms, rate_limits_json
|
|
1512
|
+
FROM codex_quota_candidates
|
|
1513
|
+
ORDER BY timestamp_ms DESC
|
|
1514
|
+
LIMIT 32
|
|
1515
|
+
`).all();
|
|
1516
|
+
const candidates = [];
|
|
1517
|
+
for (const row of rows) {
|
|
1518
|
+
try {
|
|
1519
|
+
candidates.push({
|
|
1520
|
+
timestamp: row.timestamp_ms === null ? null : Number(row.timestamp_ms),
|
|
1521
|
+
rateLimits: JSON.parse(row.rate_limits_json)
|
|
1522
|
+
});
|
|
1523
|
+
} catch {
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
return candidates;
|
|
1527
|
+
}
|
|
1528
|
+
function metadataValue(database, key) {
|
|
1529
|
+
const row = database.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
1530
|
+
return String(row?.value || "");
|
|
1531
|
+
}
|
|
1532
|
+
function setMetadataValue(database, key, value) {
|
|
1533
|
+
database.prepare(
|
|
1534
|
+
"INSERT OR REPLACE INTO metadata(key, value) VALUES(?, ?)"
|
|
1535
|
+
).run(key, value);
|
|
1536
|
+
}
|
|
1537
|
+
function configuredRoots(roots) {
|
|
1538
|
+
const configured = [];
|
|
1539
|
+
for (const provider of ["codex", "claude"]) {
|
|
1540
|
+
for (const root of Array.from(new Set(roots[provider] || [])).sort()) {
|
|
1541
|
+
configured.push({ provider, rootPath: import_node_path.default.resolve(root) });
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
return configured;
|
|
1545
|
+
}
|
|
1546
|
+
function resetDiscovery(database, cacheFile) {
|
|
1547
|
+
closeCacheDirectoryCursors(cacheFile);
|
|
1548
|
+
database.exec(`
|
|
1549
|
+
DELETE FROM discovery_source_queue;
|
|
1550
|
+
DELETE FROM discovery_directories;
|
|
1551
|
+
DELETE FROM codex_quota_candidates;
|
|
1552
|
+
DELETE FROM claude_messages;
|
|
1553
|
+
DELETE FROM usage_provider_hourly;
|
|
1554
|
+
DELETE FROM source_files;
|
|
1555
|
+
`);
|
|
1556
|
+
}
|
|
1557
|
+
function ensureDiscoveryRoots(database, roots, nowMs, cacheFile) {
|
|
1558
|
+
const configured = configuredRoots(roots);
|
|
1559
|
+
const signature = JSON.stringify(configured);
|
|
1560
|
+
if (metadataValue(database, "roots_signature") !== signature) {
|
|
1561
|
+
resetDiscovery(database, cacheFile);
|
|
1562
|
+
setMetadataValue(database, "roots_signature", signature);
|
|
1563
|
+
setMetadataValue(database, "discovery_ready", "0");
|
|
1564
|
+
}
|
|
1565
|
+
const configuredPaths = new Set(configured.map((entry) => entry.rootPath));
|
|
1566
|
+
database.prepare(
|
|
1567
|
+
"DELETE FROM discovery_directories WHERE is_root = 1 AND path NOT IN (" + (configured.length ? configured.map(() => "?").join(",") : "''") + ")"
|
|
1568
|
+
).run(...configured.map((entry) => entry.rootPath));
|
|
1569
|
+
for (const { provider, rootPath } of configured) {
|
|
1570
|
+
database.prepare(`
|
|
1571
|
+
INSERT INTO discovery_directories(
|
|
1572
|
+
path, provider, root_path, parent_path, is_root, present, pending,
|
|
1573
|
+
scan_generation, parent_generation, mtime_ms, file_dev, file_ino,
|
|
1574
|
+
last_checked_ms
|
|
1575
|
+
) VALUES(?, ?, ?, '', 1, 1, 1, 0, 0, -1, -1, -1, 0)
|
|
1576
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
1577
|
+
provider = excluded.provider,
|
|
1578
|
+
root_path = excluded.root_path,
|
|
1579
|
+
is_root = 1,
|
|
1580
|
+
present = 1,
|
|
1581
|
+
last_checked_ms = MIN(discovery_directories.last_checked_ms, ?)
|
|
1582
|
+
`).run(rootPath, provider, rootPath, nowMs);
|
|
1583
|
+
}
|
|
1584
|
+
if (configuredPaths.size === 0) setMetadataValue(database, "discovery_ready", "1");
|
|
1585
|
+
}
|
|
1586
|
+
function queueSource(database, provider, filePath, action, priority, nowMs) {
|
|
1587
|
+
database.prepare(`
|
|
1588
|
+
INSERT INTO discovery_source_queue(path, provider, action, priority, queued_at_ms)
|
|
1589
|
+
VALUES(?, ?, ?, ?, ?)
|
|
1590
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
1591
|
+
provider = excluded.provider,
|
|
1592
|
+
action = excluded.action,
|
|
1593
|
+
priority = excluded.priority,
|
|
1594
|
+
queued_at_ms = excluded.queued_at_ms
|
|
1595
|
+
`).run(filePath, provider, action, priority, nowMs);
|
|
1596
|
+
}
|
|
1597
|
+
var directoryCursors = /* @__PURE__ */ new Map();
|
|
1598
|
+
function directoryCursorKey(cacheFile, directoryPath) {
|
|
1599
|
+
return `${import_node_path.default.resolve(cacheFile)}\0${directoryPath}`;
|
|
1600
|
+
}
|
|
1601
|
+
function closeDirectoryCursor(cacheFile, directoryPath) {
|
|
1602
|
+
const key = directoryCursorKey(cacheFile, directoryPath);
|
|
1603
|
+
const cursor = directoryCursors.get(key);
|
|
1604
|
+
if (!cursor) return;
|
|
1605
|
+
directoryCursors.delete(key);
|
|
1606
|
+
try {
|
|
1607
|
+
cursor.directory.closeSync();
|
|
1608
|
+
} catch {
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
function closeCacheDirectoryCursors(cacheFile) {
|
|
1612
|
+
const prefix = `${import_node_path.default.resolve(cacheFile)}\0`;
|
|
1613
|
+
for (const [key, cursor] of directoryCursors) {
|
|
1614
|
+
if (!key.startsWith(prefix)) continue;
|
|
1615
|
+
directoryCursors.delete(key);
|
|
1616
|
+
try {
|
|
1617
|
+
cursor.directory.closeSync();
|
|
1618
|
+
} catch {
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
function markDirectoryTreeMissing(database, directoryPath, nowMs, cacheFile) {
|
|
1623
|
+
closeDirectoryCursor(cacheFile, directoryPath);
|
|
1624
|
+
const prefix = `${directoryPath}${import_node_path.default.sep}%`;
|
|
1625
|
+
database.prepare(
|
|
1626
|
+
"UPDATE discovery_directories SET present = 0, pending = 0, last_checked_ms = ? WHERE path = ? OR path LIKE ?"
|
|
1627
|
+
).run(nowMs, directoryPath, prefix);
|
|
1628
|
+
database.prepare(`
|
|
1629
|
+
INSERT INTO discovery_source_queue(path, provider, action, priority, queued_at_ms)
|
|
1630
|
+
SELECT path, provider, 'remove', -1, ?
|
|
1631
|
+
FROM source_files
|
|
1632
|
+
WHERE directory_path = ? OR directory_path LIKE ?
|
|
1633
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
1634
|
+
action = CASE
|
|
1635
|
+
WHEN discovery_source_queue.action = 'upsert' THEN 'upsert'
|
|
1636
|
+
ELSE 'remove'
|
|
1637
|
+
END,
|
|
1638
|
+
queued_at_ms = MIN(discovery_source_queue.queued_at_ms, excluded.queued_at_ms)
|
|
1639
|
+
`).run(nowMs, directoryPath, prefix);
|
|
1640
|
+
}
|
|
1641
|
+
function auditDirectories(database, nowMs, deadline, metrics, cacheFile) {
|
|
1642
|
+
const rows = database.prepare(`
|
|
1643
|
+
SELECT * FROM discovery_directories
|
|
1644
|
+
WHERE present = 1
|
|
1645
|
+
ORDER BY is_root DESC, last_checked_ms ASC
|
|
1646
|
+
LIMIT ?
|
|
1647
|
+
`).all(DIRECTORY_AUDIT_BATCH);
|
|
1648
|
+
for (const row of rows) {
|
|
1649
|
+
if (Date.now() >= deadline) break;
|
|
1650
|
+
const directoryPath = String(row.path);
|
|
1651
|
+
try {
|
|
1652
|
+
const stat = import_node_fs.default.statSync(directoryPath);
|
|
1653
|
+
const changed = !stat.isDirectory() || Number(row.file_dev) !== Number(stat.dev) || Number(row.file_ino) !== Number(stat.ino) || Number(row.mtime_ms) !== stat.mtimeMs;
|
|
1654
|
+
database.prepare(`
|
|
1655
|
+
UPDATE discovery_directories SET
|
|
1656
|
+
present = ?, pending = CASE WHEN ? THEN 1 ELSE pending END,
|
|
1657
|
+
mtime_ms = ?, file_dev = ?, file_ino = ?, last_checked_ms = ?
|
|
1658
|
+
WHERE path = ?
|
|
1659
|
+
`).run(
|
|
1660
|
+
stat.isDirectory() ? 1 : 0,
|
|
1661
|
+
changed ? 1 : 0,
|
|
1662
|
+
stat.mtimeMs,
|
|
1663
|
+
Number(stat.dev),
|
|
1664
|
+
Number(stat.ino),
|
|
1665
|
+
nowMs,
|
|
1666
|
+
directoryPath
|
|
1667
|
+
);
|
|
1668
|
+
if (!stat.isDirectory()) {
|
|
1669
|
+
markDirectoryTreeMissing(database, directoryPath, nowMs, cacheFile);
|
|
1670
|
+
}
|
|
1671
|
+
} catch {
|
|
1672
|
+
markDirectoryTreeMissing(database, directoryPath, nowMs, cacheFile);
|
|
1673
|
+
}
|
|
1674
|
+
metrics.audited_directories = Number(metrics.audited_directories) + 1;
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
function scanPendingDirectories(database, nowMs, deadline, metrics, cacheFile) {
|
|
1678
|
+
const remainingEntryBudget = DIRECTORY_ENTRY_BATCH - Number(metrics.enumerated_entries);
|
|
1679
|
+
if (remainingEntryBudget <= 0) return;
|
|
1680
|
+
const row = database.prepare(`
|
|
1681
|
+
SELECT * FROM discovery_directories
|
|
1682
|
+
WHERE present = 1 AND pending = 1
|
|
1683
|
+
ORDER BY is_root DESC, last_checked_ms ASC
|
|
1684
|
+
LIMIT 1
|
|
1685
|
+
`).get();
|
|
1686
|
+
if (!row || Date.now() >= deadline) return;
|
|
1687
|
+
const directoryPath = String(row.path);
|
|
1688
|
+
const provider = String(row.provider);
|
|
1689
|
+
const cursorKey = directoryCursorKey(cacheFile, directoryPath);
|
|
1690
|
+
let cursor = directoryCursors.get(cursorKey);
|
|
1691
|
+
try {
|
|
1692
|
+
if (!cursor) {
|
|
1693
|
+
const startStat = import_node_fs.default.statSync(directoryPath);
|
|
1694
|
+
const generation = Number(row.scan_generation) + 1;
|
|
1695
|
+
cursor = {
|
|
1696
|
+
directory: import_node_fs.default.opendirSync(directoryPath),
|
|
1697
|
+
startStat,
|
|
1698
|
+
generation
|
|
1699
|
+
};
|
|
1700
|
+
directoryCursors.set(cursorKey, cursor);
|
|
1701
|
+
database.prepare(
|
|
1702
|
+
"UPDATE discovery_directories SET scan_generation = ?, last_checked_ms = ? WHERE path = ?"
|
|
1703
|
+
).run(generation, nowMs, directoryPath);
|
|
1704
|
+
}
|
|
1705
|
+
} catch {
|
|
1706
|
+
markDirectoryTreeMissing(database, directoryPath, nowMs, cacheFile);
|
|
1707
|
+
return;
|
|
1708
|
+
}
|
|
1709
|
+
const entries = [];
|
|
1710
|
+
let complete = false;
|
|
1711
|
+
try {
|
|
1712
|
+
while (entries.length < remainingEntryBudget && Date.now() < deadline) {
|
|
1713
|
+
const entry = cursor.directory.readSync();
|
|
1714
|
+
if (!entry) {
|
|
1715
|
+
complete = true;
|
|
1716
|
+
break;
|
|
1717
|
+
}
|
|
1718
|
+
entries.push(entry);
|
|
1719
|
+
}
|
|
1720
|
+
} catch {
|
|
1721
|
+
closeDirectoryCursor(cacheFile, directoryPath);
|
|
1722
|
+
return;
|
|
1723
|
+
}
|
|
1724
|
+
database.exec("BEGIN IMMEDIATE");
|
|
1725
|
+
try {
|
|
1726
|
+
for (const entry of entries) {
|
|
1727
|
+
const child = import_node_path.default.join(directoryPath, entry.name);
|
|
1728
|
+
if (entry.isDirectory()) {
|
|
1729
|
+
let childStat;
|
|
1730
|
+
try {
|
|
1731
|
+
childStat = import_node_fs.default.statSync(child);
|
|
1732
|
+
} catch {
|
|
1733
|
+
continue;
|
|
1734
|
+
}
|
|
1735
|
+
database.prepare(`
|
|
1736
|
+
INSERT INTO discovery_directories(
|
|
1737
|
+
path, provider, root_path, parent_path, is_root, present, pending,
|
|
1738
|
+
scan_generation, parent_generation, mtime_ms, file_dev, file_ino,
|
|
1739
|
+
last_checked_ms
|
|
1740
|
+
) VALUES(?, ?, ?, ?, 0, 1, 1, 0, ?, ?, ?, ?, 0)
|
|
1741
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
1742
|
+
present = 1,
|
|
1743
|
+
parent_generation = excluded.parent_generation,
|
|
1744
|
+
pending = CASE
|
|
1745
|
+
WHEN discovery_directories.file_dev != excluded.file_dev
|
|
1746
|
+
OR discovery_directories.file_ino != excluded.file_ino
|
|
1747
|
+
OR discovery_directories.mtime_ms != excluded.mtime_ms
|
|
1748
|
+
THEN 1 ELSE discovery_directories.pending
|
|
1749
|
+
END,
|
|
1750
|
+
mtime_ms = excluded.mtime_ms,
|
|
1751
|
+
file_dev = excluded.file_dev,
|
|
1752
|
+
file_ino = excluded.file_ino
|
|
1753
|
+
`).run(
|
|
1754
|
+
child,
|
|
1755
|
+
provider,
|
|
1756
|
+
String(row.root_path),
|
|
1757
|
+
directoryPath,
|
|
1758
|
+
cursor.generation,
|
|
1759
|
+
childStat.mtimeMs,
|
|
1760
|
+
Number(childStat.dev),
|
|
1761
|
+
Number(childStat.ino)
|
|
1762
|
+
);
|
|
1763
|
+
} else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
1764
|
+
try {
|
|
1765
|
+
const stat = import_node_fs.default.statSync(child);
|
|
1766
|
+
const cached = sourceRow(database, child);
|
|
1767
|
+
if (cached) {
|
|
1768
|
+
database.prepare(
|
|
1769
|
+
"UPDATE source_files SET directory_generation = ? WHERE path = ?"
|
|
1770
|
+
).run(cursor.generation, child);
|
|
1771
|
+
}
|
|
1772
|
+
if (!cached || Number(cached.size) !== stat.size || Number(cached.mtime_ms) !== stat.mtimeMs || Number(cached.file_dev) !== Number(stat.dev) || Number(cached.file_ino) !== Number(stat.ino) || Number(cached.scan_complete) !== 1) {
|
|
1773
|
+
queueSource(database, provider, child, "upsert", stat.mtimeMs, nowMs);
|
|
1774
|
+
} else {
|
|
1775
|
+
database.prepare(
|
|
1776
|
+
"UPDATE source_files SET last_checked_ms = ? WHERE path = ?"
|
|
1777
|
+
).run(nowMs, child);
|
|
1778
|
+
metrics.reused_files = Number(metrics.reused_files) + 1;
|
|
1779
|
+
}
|
|
1780
|
+
} catch {
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
if (complete) {
|
|
1785
|
+
const endStat = import_node_fs.default.statSync(directoryPath);
|
|
1786
|
+
const stable = Number(cursor.startStat.dev) === Number(endStat.dev) && Number(cursor.startStat.ino) === Number(endStat.ino) && cursor.startStat.mtimeMs === endStat.mtimeMs;
|
|
1787
|
+
if (stable) {
|
|
1788
|
+
const missingChildren = database.prepare(`
|
|
1789
|
+
SELECT path FROM discovery_directories
|
|
1790
|
+
WHERE parent_path = ? AND present = 1 AND parent_generation < ?
|
|
1791
|
+
`).all(directoryPath, cursor.generation);
|
|
1792
|
+
for (const child of missingChildren) {
|
|
1793
|
+
markDirectoryTreeMissing(database, child.path, nowMs, cacheFile);
|
|
1794
|
+
}
|
|
1795
|
+
database.prepare(`
|
|
1796
|
+
INSERT INTO discovery_source_queue(path, provider, action, priority, queued_at_ms)
|
|
1797
|
+
SELECT path, provider, 'remove', -1, ?
|
|
1798
|
+
FROM source_files
|
|
1799
|
+
WHERE directory_path = ? AND directory_generation < ?
|
|
1800
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
1801
|
+
action = 'remove',
|
|
1802
|
+
priority = -1,
|
|
1803
|
+
queued_at_ms = excluded.queued_at_ms
|
|
1804
|
+
`).run(nowMs, directoryPath, cursor.generation);
|
|
1805
|
+
}
|
|
1806
|
+
database.prepare(`
|
|
1807
|
+
UPDATE discovery_directories SET
|
|
1808
|
+
present = 1, pending = ?, mtime_ms = ?, file_dev = ?, file_ino = ?,
|
|
1809
|
+
last_checked_ms = ?
|
|
1810
|
+
WHERE path = ?
|
|
1811
|
+
`).run(
|
|
1812
|
+
stable ? 0 : 1,
|
|
1813
|
+
endStat.mtimeMs,
|
|
1814
|
+
Number(endStat.dev),
|
|
1815
|
+
Number(endStat.ino),
|
|
1816
|
+
nowMs,
|
|
1817
|
+
directoryPath
|
|
1818
|
+
);
|
|
1819
|
+
}
|
|
1820
|
+
database.exec("COMMIT");
|
|
1821
|
+
} catch (error) {
|
|
1822
|
+
database.exec("ROLLBACK");
|
|
1823
|
+
closeDirectoryCursor(cacheFile, directoryPath);
|
|
1824
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
|
1825
|
+
markDirectoryTreeMissing(database, directoryPath, nowMs, cacheFile);
|
|
1826
|
+
return;
|
|
1827
|
+
}
|
|
1828
|
+
throw error;
|
|
1829
|
+
} finally {
|
|
1830
|
+
if (complete) closeDirectoryCursor(cacheFile, directoryPath);
|
|
1831
|
+
}
|
|
1832
|
+
metrics.enumerated_entries = Number(metrics.enumerated_entries) + entries.length;
|
|
1833
|
+
if (complete) {
|
|
1834
|
+
metrics.scanned_directories = Number(metrics.scanned_directories) + 1;
|
|
1835
|
+
if (Number(metrics.enumerated_entries) < DIRECTORY_ENTRY_BATCH && Date.now() < deadline) {
|
|
1836
|
+
scanPendingDirectories(database, nowMs, deadline, metrics, cacheFile);
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
function auditSources(database, nowMs, deadline, metrics) {
|
|
1841
|
+
const rows = database.prepare(`
|
|
1842
|
+
SELECT * FROM source_files
|
|
1843
|
+
ORDER BY last_checked_ms ASC
|
|
1844
|
+
LIMIT ?
|
|
1845
|
+
`).all(SOURCE_AUDIT_BATCH);
|
|
1846
|
+
const recent = database.prepare(`
|
|
1847
|
+
SELECT * FROM source_files
|
|
1848
|
+
ORDER BY mtime_ms DESC
|
|
1849
|
+
LIMIT ?
|
|
1850
|
+
`).all(Math.min(32, SOURCE_AUDIT_BATCH));
|
|
1851
|
+
const byPath = new Map([...recent, ...rows].map((row) => [String(row.path), row]));
|
|
1852
|
+
for (const row of byPath.values()) {
|
|
1853
|
+
if (Date.now() >= deadline) break;
|
|
1854
|
+
const filePath = String(row.path);
|
|
1855
|
+
try {
|
|
1856
|
+
const stat = import_node_fs.default.statSync(filePath);
|
|
1857
|
+
const changed = stat.size !== Number(row.size) || stat.mtimeMs !== Number(row.mtime_ms) || Number(stat.dev) !== Number(row.file_dev) || Number(stat.ino) !== Number(row.file_ino) || Number(row.scan_complete) !== 1;
|
|
1858
|
+
if (changed) {
|
|
1859
|
+
queueSource(
|
|
1860
|
+
database,
|
|
1861
|
+
String(row.provider),
|
|
1862
|
+
filePath,
|
|
1863
|
+
"upsert",
|
|
1864
|
+
stat.mtimeMs,
|
|
1865
|
+
nowMs
|
|
1866
|
+
);
|
|
1867
|
+
} else {
|
|
1868
|
+
database.prepare(
|
|
1869
|
+
"UPDATE source_files SET last_checked_ms = ? WHERE path = ?"
|
|
1870
|
+
).run(nowMs, filePath);
|
|
1871
|
+
metrics.reused_files = Number(metrics.reused_files) + 1;
|
|
1872
|
+
}
|
|
1873
|
+
} catch {
|
|
1874
|
+
queueSource(
|
|
1875
|
+
database,
|
|
1876
|
+
String(row.provider),
|
|
1877
|
+
filePath,
|
|
1878
|
+
"remove",
|
|
1879
|
+
-1,
|
|
1880
|
+
nowMs
|
|
1881
|
+
);
|
|
1882
|
+
}
|
|
1883
|
+
metrics.audited_files = Number(metrics.audited_files) + 1;
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
function candidateFromPath(database, provider, filePath) {
|
|
1887
|
+
const stat = import_node_fs.default.statSync(filePath);
|
|
1888
|
+
const cached = sourceRow(database, filePath);
|
|
1889
|
+
if (provider === "codex") {
|
|
1890
|
+
let probe;
|
|
1891
|
+
if (cached?.provider === "codex") {
|
|
1892
|
+
try {
|
|
1893
|
+
const state = JSON.parse(String(cached.parser_state));
|
|
1894
|
+
probe = {
|
|
1895
|
+
sessionId: state.sessionId,
|
|
1896
|
+
parentSessionId: state.parentSessionId,
|
|
1897
|
+
forkTimestamp: state.forkTimestamp,
|
|
1898
|
+
copiedPrefix: state.copiedPrefix
|
|
1899
|
+
};
|
|
1900
|
+
} catch {
|
|
1901
|
+
probe = probeCodex(filePath);
|
|
1902
|
+
}
|
|
1903
|
+
} else {
|
|
1904
|
+
probe = probeCodex(filePath);
|
|
1905
|
+
}
|
|
1906
|
+
return { provider, filePath, stat, ...probe, parentBaseline: null };
|
|
1907
|
+
}
|
|
1908
|
+
return {
|
|
1909
|
+
provider,
|
|
1910
|
+
filePath,
|
|
1911
|
+
stat,
|
|
1912
|
+
sessionId: claudeOwnerSession(filePath),
|
|
1913
|
+
parentSessionId: null,
|
|
1914
|
+
forkTimestamp: null,
|
|
1915
|
+
copiedPrefix: false,
|
|
1916
|
+
parentBaseline: null
|
|
1917
|
+
};
|
|
1918
|
+
}
|
|
1919
|
+
function moveSource(database, row, candidate) {
|
|
1920
|
+
const oldPath = String(row.path);
|
|
1921
|
+
database.prepare(`
|
|
1922
|
+
INSERT INTO source_files(
|
|
1923
|
+
path, provider, directory_path, directory_generation,
|
|
1924
|
+
size, mtime_ms, committed_offset, file_dev, file_ino,
|
|
1925
|
+
prefix_sha256, suffix_sha256, parser_state, session_id, scan_complete,
|
|
1926
|
+
last_checked_ms
|
|
1927
|
+
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1928
|
+
`).run(
|
|
1929
|
+
candidate.filePath,
|
|
1930
|
+
candidate.provider,
|
|
1931
|
+
import_node_path.default.dirname(candidate.filePath),
|
|
1932
|
+
directoryGeneration(database, import_node_path.default.dirname(candidate.filePath)),
|
|
1933
|
+
candidate.stat.size,
|
|
1934
|
+
candidate.stat.mtimeMs,
|
|
1935
|
+
Number(row.committed_offset),
|
|
1936
|
+
Number(candidate.stat.dev),
|
|
1937
|
+
Number(candidate.stat.ino),
|
|
1938
|
+
String(row.prefix_sha256),
|
|
1939
|
+
suffixHash(candidate.filePath, candidate.stat.size),
|
|
1940
|
+
String(row.parser_state),
|
|
1941
|
+
String(row.session_id),
|
|
1942
|
+
candidate.stat.size === Number(row.size) && Number(row.scan_complete) === 1 ? 1 : 0,
|
|
1943
|
+
Date.now()
|
|
1944
|
+
);
|
|
1945
|
+
database.prepare(
|
|
1946
|
+
"UPDATE usage_hourly SET source_path = ? WHERE source_path = ?"
|
|
1947
|
+
).run(candidate.filePath, oldPath);
|
|
1948
|
+
database.prepare(
|
|
1949
|
+
"UPDATE recent_events SET source_path = ? WHERE source_path = ?"
|
|
1950
|
+
).run(candidate.filePath, oldPath);
|
|
1951
|
+
database.prepare(
|
|
1952
|
+
"UPDATE claude_messages SET source_path = ? WHERE source_path = ?"
|
|
1953
|
+
).run(candidate.filePath, oldPath);
|
|
1954
|
+
database.prepare(
|
|
1955
|
+
"UPDATE codex_quota_candidates SET source_path = ? WHERE source_path = ?"
|
|
1956
|
+
).run(candidate.filePath, oldPath);
|
|
1957
|
+
database.prepare("DELETE FROM source_files WHERE path = ?").run(oldPath);
|
|
1958
|
+
}
|
|
1959
|
+
function prepareCodexCandidate(database, candidate, metrics) {
|
|
1960
|
+
if (sourceRow(database, candidate.filePath)) return true;
|
|
1961
|
+
const existing = database.prepare(
|
|
1962
|
+
"SELECT * FROM source_files WHERE provider = 'codex' AND session_id = ? AND path != ? ORDER BY mtime_ms DESC LIMIT 1"
|
|
1963
|
+
).get(candidate.sessionId, candidate.filePath);
|
|
1964
|
+
if (!existing) return true;
|
|
1965
|
+
const sameIdentity = Number(existing.file_dev) === Number(candidate.stat.dev) && Number(existing.file_ino) === Number(candidate.stat.ino);
|
|
1966
|
+
if (sameIdentity && !import_node_fs.default.existsSync(String(existing.path))) {
|
|
1967
|
+
moveSource(database, existing, candidate);
|
|
1968
|
+
metrics.moved_files = Number(metrics.moved_files) + 1;
|
|
1969
|
+
return true;
|
|
1970
|
+
}
|
|
1971
|
+
const existingArchived = String(existing.path).includes(`${import_node_path.default.sep}archived_sessions${import_node_path.default.sep}`);
|
|
1972
|
+
const candidateArchived = candidate.filePath.includes(`${import_node_path.default.sep}archived_sessions${import_node_path.default.sep}`);
|
|
1973
|
+
const candidateWins = existingArchived && !candidateArchived || existingArchived === candidateArchived && candidate.stat.mtimeMs > Number(existing.mtime_ms);
|
|
1974
|
+
if (!candidateWins) return false;
|
|
1975
|
+
deleteSource(database, String(existing.path));
|
|
1976
|
+
return true;
|
|
1977
|
+
}
|
|
1978
|
+
function resolveParentBaselineFromDatabase(database, candidate, cache, metrics) {
|
|
1979
|
+
if (!candidate.parentSessionId || !candidate.forkTimestamp) return null;
|
|
1980
|
+
const key = `${candidate.parentSessionId}:${candidate.forkTimestamp}`;
|
|
1981
|
+
if (cache.has(key)) return cloneTotals(cache.get(key));
|
|
1982
|
+
const row = database.prepare(
|
|
1983
|
+
"SELECT path FROM source_files WHERE provider = 'codex' AND session_id = ? LIMIT 1"
|
|
1984
|
+
).get(candidate.parentSessionId);
|
|
1985
|
+
const baseline = row?.path && row.path !== candidate.filePath ? codexRawTotalsAt(row.path, candidate.forkTimestamp, metrics) : null;
|
|
1986
|
+
cache.set(key, cloneTotals(baseline));
|
|
1987
|
+
return baseline;
|
|
1988
|
+
}
|
|
1989
|
+
function processSourceQueue(database, recentBoundary, deadline, metrics) {
|
|
1990
|
+
const parentCache = /* @__PURE__ */ new Map();
|
|
1991
|
+
while (Date.now() < deadline) {
|
|
1992
|
+
const task = database.prepare(`
|
|
1993
|
+
SELECT * FROM discovery_source_queue
|
|
1994
|
+
WHERE action = 'upsert'
|
|
1995
|
+
ORDER BY priority ASC, queued_at_ms ASC
|
|
1996
|
+
LIMIT 1
|
|
1997
|
+
`).get() || database.prepare(`
|
|
1998
|
+
SELECT * FROM discovery_source_queue
|
|
1999
|
+
WHERE action = 'remove'
|
|
2000
|
+
ORDER BY priority ASC, queued_at_ms ASC
|
|
2001
|
+
LIMIT 1
|
|
2002
|
+
`).get();
|
|
2003
|
+
if (!task) return;
|
|
2004
|
+
const filePath = String(task.path);
|
|
2005
|
+
if (String(task.action) === "remove") {
|
|
2006
|
+
database.exec("BEGIN IMMEDIATE");
|
|
2007
|
+
try {
|
|
2008
|
+
database.prepare(
|
|
2009
|
+
"DELETE FROM discovery_source_queue WHERE path = ?"
|
|
2010
|
+
).run(filePath);
|
|
2011
|
+
try {
|
|
2012
|
+
const stat = import_node_fs.default.statSync(filePath);
|
|
2013
|
+
queueSource(
|
|
2014
|
+
database,
|
|
2015
|
+
String(task.provider),
|
|
2016
|
+
filePath,
|
|
2017
|
+
"upsert",
|
|
2018
|
+
stat.mtimeMs,
|
|
2019
|
+
Date.now()
|
|
2020
|
+
);
|
|
2021
|
+
} catch (error) {
|
|
2022
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") {
|
|
2023
|
+
throw error;
|
|
2024
|
+
}
|
|
2025
|
+
deleteSource(database, filePath);
|
|
2026
|
+
metrics.removed_files = Number(metrics.removed_files) + 1;
|
|
2027
|
+
}
|
|
2028
|
+
database.exec("COMMIT");
|
|
2029
|
+
} catch (error) {
|
|
2030
|
+
database.exec("ROLLBACK");
|
|
2031
|
+
throw error;
|
|
2032
|
+
}
|
|
2033
|
+
continue;
|
|
2034
|
+
}
|
|
2035
|
+
try {
|
|
2036
|
+
const candidate = candidateFromPath(
|
|
2037
|
+
database,
|
|
2038
|
+
String(task.provider),
|
|
2039
|
+
filePath
|
|
2040
|
+
);
|
|
2041
|
+
database.exec("BEGIN IMMEDIATE");
|
|
2042
|
+
let complete = false;
|
|
2043
|
+
try {
|
|
2044
|
+
if (candidate.provider === "codex") {
|
|
2045
|
+
if (!prepareCodexCandidate(database, candidate, metrics)) {
|
|
2046
|
+
database.prepare("DELETE FROM discovery_source_queue WHERE path = ?").run(filePath);
|
|
2047
|
+
database.exec("COMMIT");
|
|
2048
|
+
continue;
|
|
2049
|
+
}
|
|
2050
|
+
if (!sourceRow(database, filePath) && candidate.parentSessionId && !database.prepare(
|
|
2051
|
+
"SELECT 1 FROM source_files WHERE provider = 'codex' AND session_id = ? LIMIT 1"
|
|
2052
|
+
).get(candidate.parentSessionId)) {
|
|
2053
|
+
const otherUpsert = database.prepare(
|
|
2054
|
+
"SELECT 1 FROM discovery_source_queue WHERE action = 'upsert' AND path != ? LIMIT 1"
|
|
2055
|
+
).get(filePath);
|
|
2056
|
+
if (otherUpsert) {
|
|
2057
|
+
const row = database.prepare(
|
|
2058
|
+
"SELECT priority + 1 AS priority FROM discovery_source_queue WHERE action = 'upsert' ORDER BY priority DESC LIMIT 1"
|
|
2059
|
+
).get();
|
|
2060
|
+
database.prepare(
|
|
2061
|
+
"UPDATE discovery_source_queue SET priority = ?, queued_at_ms = ? WHERE path = ?"
|
|
2062
|
+
).run(Number(row.priority), Date.now(), filePath);
|
|
2063
|
+
database.exec("COMMIT");
|
|
2064
|
+
continue;
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
if (!sourceRow(database, filePath) && candidate.parentSessionId) {
|
|
2068
|
+
candidate.parentBaseline = resolveParentBaselineFromDatabase(
|
|
2069
|
+
database,
|
|
2070
|
+
candidate,
|
|
2071
|
+
parentCache,
|
|
2072
|
+
metrics
|
|
2073
|
+
);
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
complete = scanFile(database, candidate, recentBoundary, deadline, metrics);
|
|
2077
|
+
database.exec("COMMIT");
|
|
2078
|
+
} catch (error) {
|
|
2079
|
+
database.exec("ROLLBACK");
|
|
2080
|
+
throw error;
|
|
2081
|
+
}
|
|
2082
|
+
if (complete) {
|
|
2083
|
+
database.prepare("DELETE FROM discovery_source_queue WHERE path = ?").run(filePath);
|
|
2084
|
+
} else {
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
2087
|
+
} catch (error) {
|
|
2088
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
|
2089
|
+
database.exec("BEGIN IMMEDIATE");
|
|
2090
|
+
try {
|
|
2091
|
+
deleteSource(database, filePath);
|
|
2092
|
+
database.prepare(
|
|
2093
|
+
"DELETE FROM discovery_source_queue WHERE path = ?"
|
|
2094
|
+
).run(filePath);
|
|
2095
|
+
database.exec("COMMIT");
|
|
2096
|
+
} catch (cleanupError) {
|
|
2097
|
+
database.exec("ROLLBACK");
|
|
2098
|
+
throw cleanupError;
|
|
2099
|
+
}
|
|
2100
|
+
metrics.removed_files = Number(metrics.removed_files) + 1;
|
|
2101
|
+
continue;
|
|
2102
|
+
}
|
|
2103
|
+
metrics.errors = Number(metrics.errors) + 1;
|
|
2104
|
+
const errors = metrics.errors_by_provider;
|
|
2105
|
+
errors[String(task.provider)] += 1;
|
|
2106
|
+
return;
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
function collectUsage(request) {
|
|
2111
|
+
const cacheFile = import_node_path.default.resolve(request.cacheFile);
|
|
2112
|
+
const legacyCacheFile = request.legacyCacheFile ? import_node_path.default.resolve(request.legacyCacheFile) : import_node_path.default.join(import_node_path.default.dirname(cacheFile), "cc-statistics-usage-v1.sqlite3");
|
|
2113
|
+
if (legacyCacheFile !== cacheFile && import_node_fs.default.existsSync(legacyCacheFile)) {
|
|
2114
|
+
removeDatabaseFiles(legacyCacheFile);
|
|
2115
|
+
}
|
|
2116
|
+
const nowMs = request.nowMs || Date.now();
|
|
2117
|
+
const retentionDays = Math.max(1, request.retentionDays || DEFAULT_RETENTION_DAYS);
|
|
2118
|
+
const recentRawMs = Math.max(6e4, request.recentRawMs || DEFAULT_RECENT_RAW_MS);
|
|
2119
|
+
const scanBudgetMs = Math.max(100, request.scanBudgetMs || DEFAULT_SCAN_BUDGET_MS);
|
|
2120
|
+
const { database, rebuilt } = openDatabase(cacheFile);
|
|
2121
|
+
const deadline = Date.now() + scanBudgetMs;
|
|
2122
|
+
const metrics = {
|
|
2123
|
+
discovered_files: 0,
|
|
2124
|
+
scanned_directories: 0,
|
|
2125
|
+
audited_directories: 0,
|
|
2126
|
+
enumerated_entries: 0,
|
|
2127
|
+
audited_files: 0,
|
|
2128
|
+
scanned_files: 0,
|
|
2129
|
+
partial_files: 0,
|
|
2130
|
+
reused_files: 0,
|
|
2131
|
+
appended_files: 0,
|
|
2132
|
+
rebuilt_files: 0,
|
|
2133
|
+
parsed_events: 0,
|
|
2134
|
+
bytes_read: 0,
|
|
2135
|
+
pruned_events: 0,
|
|
2136
|
+
removed_files: 0,
|
|
2137
|
+
moved_files: 0,
|
|
2138
|
+
errors: 0,
|
|
2139
|
+
errors_by_provider: { codex: 0, claude: 0 },
|
|
2140
|
+
cache_rebuilt: rebuilt,
|
|
2141
|
+
codex_cache_rebuilt: false,
|
|
2142
|
+
claude_cache_rebuilt: false
|
|
2143
|
+
};
|
|
2144
|
+
let completed = false;
|
|
2145
|
+
try {
|
|
2146
|
+
const retentionCutoff = nowMs - retentionDays * 24 * 60 * 60 * 1e3;
|
|
2147
|
+
const recentCutoff = Math.max(retentionCutoff, nowMs - recentRawMs);
|
|
2148
|
+
const recentBoundary = localHourMs(recentCutoff);
|
|
2149
|
+
database.exec("BEGIN IMMEDIATE");
|
|
2150
|
+
try {
|
|
2151
|
+
ensureDiscoveryRoots(database, request.roots || {}, nowMs, cacheFile);
|
|
2152
|
+
auditDirectories(database, nowMs, deadline, metrics, cacheFile);
|
|
2153
|
+
database.exec("COMMIT");
|
|
2154
|
+
} catch (error) {
|
|
2155
|
+
database.exec("ROLLBACK");
|
|
2156
|
+
throw error;
|
|
2157
|
+
}
|
|
2158
|
+
scanPendingDirectories(database, nowMs, deadline, metrics, cacheFile);
|
|
2159
|
+
if (Date.now() < deadline) auditSources(database, nowMs, deadline, metrics);
|
|
2160
|
+
if (Date.now() < deadline) {
|
|
2161
|
+
processSourceQueue(database, recentBoundary, deadline, metrics);
|
|
2162
|
+
}
|
|
2163
|
+
database.exec("BEGIN IMMEDIATE");
|
|
2164
|
+
try {
|
|
2165
|
+
let pruned = 0;
|
|
2166
|
+
pruned += Number(database.prepare(
|
|
2167
|
+
"DELETE FROM usage_hourly WHERE bucket_start_ms < ?"
|
|
2168
|
+
).run(localHourMs(retentionCutoff)).changes);
|
|
2169
|
+
database.prepare(
|
|
2170
|
+
"DELETE FROM usage_provider_hourly WHERE bucket_start_ms < ?"
|
|
2171
|
+
).run(localHourMs(retentionCutoff));
|
|
2172
|
+
pruned += Number(database.prepare(
|
|
2173
|
+
"DELETE FROM recent_events WHERE timestamp_ms < ?"
|
|
2174
|
+
).run(recentBoundary).changes);
|
|
2175
|
+
pruned += Number(database.prepare(
|
|
2176
|
+
"DELETE FROM claude_messages WHERE timestamp_ms < ?"
|
|
2177
|
+
).run(retentionCutoff).changes);
|
|
2178
|
+
metrics.pruned_events = pruned;
|
|
2179
|
+
database.exec("COMMIT");
|
|
2180
|
+
} catch (error) {
|
|
2181
|
+
database.exec("ROLLBACK");
|
|
2182
|
+
throw error;
|
|
2183
|
+
}
|
|
2184
|
+
const stats = database.prepare(
|
|
2185
|
+
"SELECT * FROM cache_stats WHERE singleton = 1"
|
|
2186
|
+
).get();
|
|
2187
|
+
const pendingDirectories = Number(stats.pending_directories);
|
|
2188
|
+
const pendingFiles = Number(stats.queued_sources);
|
|
2189
|
+
const incompleteFiles = Number(stats.codex_incomplete) + Number(stats.claude_incomplete);
|
|
2190
|
+
const scanComplete = pendingDirectories === 0 && pendingFiles === 0 && incompleteFiles === 0 && Number(metrics.errors) === 0;
|
|
2191
|
+
if (scanComplete) setMetadataValue(database, "discovery_ready", "1");
|
|
2192
|
+
const discoveryReady = metadataValue(database, "discovery_ready") === "1";
|
|
2193
|
+
const discovered = {
|
|
2194
|
+
codex: Number(stats.codex_files),
|
|
2195
|
+
claude: Number(stats.claude_files)
|
|
2196
|
+
};
|
|
2197
|
+
metrics.discovered_files = discovered.codex + discovered.claude;
|
|
2198
|
+
metrics.scan_complete = scanComplete;
|
|
2199
|
+
metrics.pending_files = pendingFiles;
|
|
2200
|
+
metrics.pending_directories = pendingDirectories;
|
|
2201
|
+
metrics.discovery_ready = discoveryReady;
|
|
2202
|
+
metrics.hourly_rows = Number(stats.hourly_rows);
|
|
2203
|
+
metrics.recent_rows = Number(stats.recent_rows);
|
|
2204
|
+
metrics.committed_bytes = Number(stats.committed_bytes);
|
|
2205
|
+
metrics.dedupe_rows = Number(stats.dedupe_rows);
|
|
2206
|
+
metrics.codex_fingerprint_rows = 0;
|
|
2207
|
+
metrics.codex_session_rows = discovered.codex;
|
|
2208
|
+
metrics.codex_session_cache_rows = 0;
|
|
2209
|
+
const providerComplete = {
|
|
2210
|
+
codex: discoveryReady && Number(stats.codex_incomplete) === 0,
|
|
2211
|
+
claude: discoveryReady && Number(stats.claude_incomplete) === 0
|
|
2212
|
+
};
|
|
2213
|
+
const providers = {
|
|
2214
|
+
codex: {},
|
|
2215
|
+
claude: {}
|
|
2216
|
+
};
|
|
2217
|
+
for (const provider of ["codex", "claude"]) {
|
|
2218
|
+
const providerErrors = metrics.errors_by_provider[provider];
|
|
2219
|
+
const available = discovered[provider] > 0 && providerErrors === 0;
|
|
2220
|
+
providers[provider] = {
|
|
2221
|
+
source: provider === "codex" ? "Farming local history \xB7 CodexBar 0.45.2-derived" : "Farming local history \xB7 cc-statistics 1.1.0-derived",
|
|
2222
|
+
events: !providerComplete[provider] ? [] : providerEvents(database, provider, retentionCutoff, recentBoundary),
|
|
2223
|
+
quotaCandidates: provider === "codex" ? latestQuotas(database) : [],
|
|
2224
|
+
fileCount: discovered[provider],
|
|
2225
|
+
available,
|
|
2226
|
+
partial: !providerComplete[provider],
|
|
2227
|
+
...!available ? {
|
|
2228
|
+
reason: providerErrors ? `${providerErrors} session file(s) could not be parsed.` : "No local session files were found."
|
|
2229
|
+
} : {}
|
|
2230
|
+
};
|
|
2231
|
+
}
|
|
2232
|
+
const result = {
|
|
2233
|
+
schemaVersion: SCHEMA_VERSION,
|
|
2234
|
+
source: SOURCE_VERSION,
|
|
2235
|
+
sampledAt: nowMs,
|
|
2236
|
+
retentionDays,
|
|
2237
|
+
providers,
|
|
2238
|
+
cache: { ...metrics, path: cacheFile }
|
|
2239
|
+
};
|
|
2240
|
+
completed = true;
|
|
2241
|
+
return result;
|
|
2242
|
+
} finally {
|
|
2243
|
+
if (!completed) closeCacheDirectoryCursors(cacheFile);
|
|
2244
|
+
database.close();
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
2248
|
+
0 && (module.exports = {
|
|
2249
|
+
collectUsage
|
|
2250
|
+
});
|