@tokenwarden/opencode 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/dist/src/cli/index.js +105 -0
- package/dist/src/core/billing.js +252 -0
- package/dist/src/core/build-config.js +45 -0
- package/dist/src/core/code-summary.js +162 -0
- package/dist/src/core/format.js +71 -0
- package/dist/src/core/log-reducer.js +97 -0
- package/dist/src/core/protection/engine.js +226 -0
- package/dist/src/core/protection/license-public-key.generated.js +2 -0
- package/dist/src/core/protection/license.js +27 -0
- package/dist/src/core/protection/rule-pack.js +38 -0
- package/dist/src/core/protection/sidecar-public-key.generated.js +2 -0
- package/dist/src/core/protection/sidecar.js +197 -0
- package/dist/src/core/protection/signing.js +28 -0
- package/dist/src/core/protection/tamper.js +18 -0
- package/dist/src/core/savings.js +91 -0
- package/dist/src/core/storage.js +123 -0
- package/dist/src/core/tokens.js +40 -0
- package/dist/src/plugin/index.js +492 -0
- package/native/bin/tokenwarden-build-config.json +4 -0
- package/native/bin/tokenwarden-engine-darwin-arm64 +0 -0
- package/native/bin/tokenwarden-engine-darwin-arm64.manifest.json +17 -0
- package/native/bin/tokenwarden-engine-darwin-x64 +0 -0
- package/native/bin/tokenwarden-engine-darwin-x64.manifest.json +17 -0
- package/native/bin/tokenwarden-engine-linux-arm64 +0 -0
- package/native/bin/tokenwarden-engine-linux-arm64.manifest.json +17 -0
- package/native/bin/tokenwarden-engine-linux-x64 +0 -0
- package/native/bin/tokenwarden-engine-linux-x64.manifest.json +17 -0
- package/native/bin/tokenwarden-engine-win32-x64.exe +0 -0
- package/native/bin/tokenwarden-engine-win32-x64.exe.manifest.json +17 -0
- package/package.json +33 -0
package/README.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# @tokenwarden/opencode
|
|
2
|
+
|
|
3
|
+
Local-first opencode cost and context control plugin.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
npm install -g @tokenwarden/opencode
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Add the plugin to `opencode.json` or `~/.config/opencode/opencode.jsonc`:
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{
|
|
15
|
+
"$schema": "https://opencode.ai/config.json",
|
|
16
|
+
"plugin": ["tokenwarden"]
|
|
17
|
+
}
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Restart opencode after changing config.
|
|
21
|
+
|
|
22
|
+
## Local Slash Commands
|
|
23
|
+
|
|
24
|
+
- `/tokenwarden-report`: show observed savings from local usage data.
|
|
25
|
+
- `/tokenwarden-status`: show status in the opencode TUI only.
|
|
26
|
+
- `/tokenwarden-account`: open your hosted TokenWarden account page.
|
|
27
|
+
- `/tokenwarden-connect`: connect this machine to a hosted account.
|
|
28
|
+
- `/tokenwarden-disconnect`: remove the local paid license and revert to the capped free seat.
|
|
29
|
+
|
|
30
|
+
## CLI
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
tokenwarden report
|
|
34
|
+
tokenwarden status
|
|
35
|
+
tokenwarden account
|
|
36
|
+
tokenwarden connect --email you@example.com
|
|
37
|
+
tokenwarden disconnect
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
See the repository root `README.md` and website `/readme` page for product, billing, and deployment notes.
|
|
41
|
+
|
|
42
|
+
## Native Sidecar Support
|
|
43
|
+
|
|
44
|
+
The published package includes signed native sidecar binaries for macOS arm64, macOS x64, Linux x64 musl, Linux ARM64 musl, and Windows x64. TokenWarden selects the matching binary locally and verifies its signed manifest before using native optimization.
|
|
45
|
+
|
|
46
|
+
Usage, account, license, and raw-output records are stored under `~/.cache/tokenwarden/encrypted/`. The TypeScript plugin asks the verified native sidecar to read and write those encrypted files instead of storing plain JSON/JSONL directly. Local failure diagnostics are written to `~/.cache/tokenwarden/logs/failures.jsonl` without tool output or source content.
|
|
47
|
+
|
|
48
|
+
Development and production plugin builds always compile every packaged sidecar target. On macOS, install the native cross-compilers before running `npm run build:plugin` or `npm run build:plugin:prod` from the repository root:
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
brew install mingw-w64
|
|
52
|
+
brew install FiloSottile/musl-cross/musl-cross
|
|
53
|
+
```
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { formatTokenReport } from "../core/format.js";
|
|
4
|
+
import { createProtectedEngine } from "../core/protection/engine.js";
|
|
5
|
+
import { formatBillingStatus, tokenWardenSiteURL } from "../core/billing.js";
|
|
6
|
+
import { createStore } from "../core/storage.js";
|
|
7
|
+
async function main() {
|
|
8
|
+
const [, , command = "report", ...args] = process.argv;
|
|
9
|
+
if (command === "report") {
|
|
10
|
+
const sessionID = valueAfter(args, "--session");
|
|
11
|
+
const color = !args.includes("--no-color");
|
|
12
|
+
const engine = await createProtectedEngine({ dataDir: valueAfter(args, "--data-dir") });
|
|
13
|
+
const summary = await engine.summarize(sessionID);
|
|
14
|
+
process.stdout.write(`${formatTokenReport(summary, { color })}\n`);
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
if (command === "status") {
|
|
18
|
+
const store = createStore(valueAfter(args, "--data-dir"));
|
|
19
|
+
const engine = await createProtectedEngine({ dataDir: store.dataDir });
|
|
20
|
+
const protection = await engine.protectionStatus();
|
|
21
|
+
process.stdout.write(`TokenWarden data dir: ${store.dataDir}\n`);
|
|
22
|
+
process.stdout.write(`Usage log: ${store.eventsPath}\n`);
|
|
23
|
+
process.stdout.write(`Engine mode: ${protection.mode}\n`);
|
|
24
|
+
process.stdout.write(`License: ${protection.license}\n`);
|
|
25
|
+
process.stdout.write(`Rule pack: ${protection.rulePack}\n`);
|
|
26
|
+
process.stdout.write(`Tamper: ${protection.tamper}\n`);
|
|
27
|
+
process.stdout.write(`${formatBillingStatus(await engine.billing())}\n`);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (command === "account") {
|
|
31
|
+
const engine = await createProtectedEngine({ dataDir: valueAfter(args, "--data-dir") });
|
|
32
|
+
const entitlement = await engine.billing();
|
|
33
|
+
const url = accountPageURL(entitlement);
|
|
34
|
+
const opened = openBrowser(url);
|
|
35
|
+
process.stdout.write(opened ? `Opened: ${url}\n` : `Open: ${url}\n`);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (command === "disconnect") {
|
|
39
|
+
const engine = await createProtectedEngine({ dataDir: valueAfter(args, "--data-dir") });
|
|
40
|
+
await engine.disconnect();
|
|
41
|
+
process.stdout.write("TokenWarden disconnected. Reverted to the free seat.\n");
|
|
42
|
+
process.stdout.write(`${formatBillingStatus(await engine.billing())}\n`);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (command === "connect" || command === "activate") {
|
|
46
|
+
const store = createStore(valueAfter(args, "--data-dir"));
|
|
47
|
+
const engine = await createProtectedEngine({ dataDir: store.dataDir });
|
|
48
|
+
let openedURL;
|
|
49
|
+
const activation = await engine.activate({
|
|
50
|
+
email: valueAfter(args, "--email"),
|
|
51
|
+
deviceStartURL: valueAfter(args, "--device-start-url"),
|
|
52
|
+
devicePollURL: valueAfter(args, "--device-poll-url"),
|
|
53
|
+
deviceAckURL: valueAfter(args, "--device-ack-url"),
|
|
54
|
+
onVerificationURL(url) {
|
|
55
|
+
openedURL = url;
|
|
56
|
+
const opened = openBrowser(url);
|
|
57
|
+
process.stdout.write(opened ? `Opened: ${url}\n` : `Open: ${url}\n`);
|
|
58
|
+
process.stdout.write("Waiting for browser authorization...\n");
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
writeActivationResult(activation, Boolean(openedURL));
|
|
62
|
+
process.stdout.write(`${formatBillingStatus(await engine.billing())}\n`);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
process.stderr.write(`Unknown command: ${command}\n`);
|
|
66
|
+
process.stderr.write("Usage: tokenwarden report|status|account|connect|activate|disconnect [--session SESSION_ID] [--data-dir DIR] [--email EMAIL] [--device-start-url URL] [--device-poll-url URL] [--device-ack-url URL] [--no-color]\n");
|
|
67
|
+
process.stderr.write("report defaults to all locally recorded sessions; pass --session to filter one session.\n");
|
|
68
|
+
process.exitCode = 1;
|
|
69
|
+
}
|
|
70
|
+
function valueAfter(args, flag) {
|
|
71
|
+
const index = args.indexOf(flag);
|
|
72
|
+
return index >= 0 ? args[index + 1] : undefined;
|
|
73
|
+
}
|
|
74
|
+
function writeActivationResult(activation, browserOpened) {
|
|
75
|
+
process.stdout.write("TokenWarden hosted license activated.\n");
|
|
76
|
+
if (browserOpened)
|
|
77
|
+
process.stdout.write("Browser authorization complete.\n");
|
|
78
|
+
process.stdout.write(`License: ${activation.licensePath}\n`);
|
|
79
|
+
process.stdout.write(`Offline grace until: ${activation.offlineGraceUntil}\n`);
|
|
80
|
+
process.stdout.write(`Verification: ${activation.verified ? "passed" : "failed"}\n`);
|
|
81
|
+
}
|
|
82
|
+
function accountPageURL(entitlement) {
|
|
83
|
+
const url = new URL("/account", tokenWardenSiteURL());
|
|
84
|
+
if (entitlement.monthlyCapTokens === null) {
|
|
85
|
+
url.searchParams.set("optimization", "unlimited");
|
|
86
|
+
return url.toString();
|
|
87
|
+
}
|
|
88
|
+
url.searchParams.set("remainingOptimizableTokens", String(entitlement.remainingFreeTokens ?? 0));
|
|
89
|
+
url.searchParams.set("monthlyOptimizedTokens", String(entitlement.monthlySavedTokens));
|
|
90
|
+
url.searchParams.set("monthlyCapTokens", String(entitlement.monthlyCapTokens));
|
|
91
|
+
return url.toString();
|
|
92
|
+
}
|
|
93
|
+
function openBrowser(url) {
|
|
94
|
+
const command = process.platform === "darwin" ? "open" : process.platform === "win32" ? "cmd" : "xdg-open";
|
|
95
|
+
const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
|
|
96
|
+
try {
|
|
97
|
+
const child = spawn(command, args, { detached: true, stdio: "ignore" });
|
|
98
|
+
child.unref();
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
await main();
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { createSavingsEvent } from "./savings.js";
|
|
4
|
+
import { readSavingsEvents } from "./storage.js";
|
|
5
|
+
import { getMachineHash, verifyLicense } from "./protection/license.js";
|
|
6
|
+
import { LICENSE_PUBLIC_KEY } from "./protection/license-public-key.generated.js";
|
|
7
|
+
import { DEFAULT_FREE_MONTHLY_SAVED_TOKEN_CAP, DEFAULT_TOKENWARDEN_SITE_URL, tokenWardenSiteURL } from "./build-config.js";
|
|
8
|
+
import { encryptedDeleteFile, encryptedReadFile, encryptedWriteFile } from "./protection/sidecar.js";
|
|
9
|
+
export const FREE_MONTHLY_SAVED_TOKEN_CAP = DEFAULT_FREE_MONTHLY_SAVED_TOKEN_CAP;
|
|
10
|
+
export { DEFAULT_TOKENWARDEN_SITE_URL, tokenWardenSiteURL };
|
|
11
|
+
export function freeMonthlySavedTokenCap() {
|
|
12
|
+
return FREE_MONTHLY_SAVED_TOKEN_CAP;
|
|
13
|
+
}
|
|
14
|
+
export function accountPath(store) {
|
|
15
|
+
return join(store.dataDir, "encrypted", "account.json.enc");
|
|
16
|
+
}
|
|
17
|
+
export function licensePath(store) {
|
|
18
|
+
return join(store.dataDir, "encrypted", "license.json.enc");
|
|
19
|
+
}
|
|
20
|
+
export async function readBillingAccount(store) {
|
|
21
|
+
try {
|
|
22
|
+
const content = await encryptedReadFile(accountPath(store));
|
|
23
|
+
if (!content)
|
|
24
|
+
return createFreeAccount();
|
|
25
|
+
return JSON.parse(content);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return createFreeAccount();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export async function connectBillingClient(store, input = {}) {
|
|
32
|
+
const existing = await readBillingAccount(store);
|
|
33
|
+
const email = input.email ?? existing.email;
|
|
34
|
+
const account = {
|
|
35
|
+
...existing,
|
|
36
|
+
email,
|
|
37
|
+
connectedAt: new Date().toISOString(),
|
|
38
|
+
authURL: createAuthURL(existing.accountID, email),
|
|
39
|
+
};
|
|
40
|
+
await encryptedWriteFile(accountPath(store), `${JSON.stringify(account, null, 2)}\n`);
|
|
41
|
+
return account;
|
|
42
|
+
}
|
|
43
|
+
export async function disconnectBillingClient(store) {
|
|
44
|
+
const account = createFreeAccount();
|
|
45
|
+
await encryptedWriteFile(accountPath(store), `${JSON.stringify(account, null, 2)}\n`);
|
|
46
|
+
await encryptedDeleteFile(licensePath(store));
|
|
47
|
+
return account;
|
|
48
|
+
}
|
|
49
|
+
export async function activateHostedLicense(store, input = {}) {
|
|
50
|
+
const fetchImpl = input.fetchImpl ?? fetch;
|
|
51
|
+
const existing = await readBillingAccount(store);
|
|
52
|
+
const email = input.email ?? existing.email;
|
|
53
|
+
const machineHash = input.machineHash ?? getMachineHash();
|
|
54
|
+
const start = await postJson(fetchImpl, input.deviceStartURL ?? createDeviceStartURL(), {
|
|
55
|
+
machineHash,
|
|
56
|
+
email,
|
|
57
|
+
localOpencodeAccountId: existing.accountID,
|
|
58
|
+
});
|
|
59
|
+
input.onVerificationURL?.(start.verificationUrl);
|
|
60
|
+
const expiresAt = new Date(start.expiresAt).getTime();
|
|
61
|
+
const deadline = Math.min(Number.isFinite(expiresAt) ? expiresAt : Date.now() + 10 * 60 * 1000, Date.now() + (input.maxWaitMs ?? 10 * 60 * 1000));
|
|
62
|
+
const pollIntervalMs = Math.max(500, input.pollIntervalMs ?? start.pollIntervalSeconds * 1000);
|
|
63
|
+
while (Date.now() < deadline) {
|
|
64
|
+
await sleep(pollIntervalMs);
|
|
65
|
+
const poll = await postJson(fetchImpl, input.devicePollURL ?? createDevicePollURL(), { deviceCode: start.deviceCode });
|
|
66
|
+
if (poll.status === "pending") {
|
|
67
|
+
input.onPoll?.("pending");
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (poll.status === "authorized" && poll.license) {
|
|
71
|
+
const activation = await installHostedLicense(store, poll.license, machineHash, email);
|
|
72
|
+
await acknowledgeDeviceAuthorization(fetchImpl, input.deviceAckURL ?? createDeviceAckURL(), start.deviceCode);
|
|
73
|
+
return activation;
|
|
74
|
+
}
|
|
75
|
+
throw new Error(poll.error || `Device authorization failed with status ${poll.status}`);
|
|
76
|
+
}
|
|
77
|
+
throw new Error("Device authorization timed out before browser sign-in completed");
|
|
78
|
+
}
|
|
79
|
+
async function acknowledgeDeviceAuthorization(fetchImpl, url, deviceCode) {
|
|
80
|
+
try {
|
|
81
|
+
await postJson(fetchImpl, url, { deviceCode });
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// The license is already verified and installed locally. A later cleanup job can expire the authorization row.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
async function postJson(fetchImpl, url, body) {
|
|
88
|
+
const response = await fetchImpl(url, {
|
|
89
|
+
method: "POST",
|
|
90
|
+
headers: { "content-type": "application/json" },
|
|
91
|
+
body: JSON.stringify(body),
|
|
92
|
+
});
|
|
93
|
+
const text = await response.text();
|
|
94
|
+
let parsed;
|
|
95
|
+
try {
|
|
96
|
+
parsed = JSON.parse(text);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
throw new Error(`Hosted activation request failed (${response.status}): ${text.slice(0, 240)}`);
|
|
100
|
+
}
|
|
101
|
+
if (!response.ok) {
|
|
102
|
+
throw new Error(parsed.error || `Hosted activation request failed (${response.status})`);
|
|
103
|
+
}
|
|
104
|
+
return parsed;
|
|
105
|
+
}
|
|
106
|
+
async function installHostedLicense(store, license, machineHash = getMachineHash(), email) {
|
|
107
|
+
const check = verifyLicense({ token: license, publicKeyPem: LICENSE_PUBLIC_KEY, machineHash });
|
|
108
|
+
if (!check.ok)
|
|
109
|
+
throw new Error(`Hosted license verification failed: ${check.reason}`);
|
|
110
|
+
const { payload } = license;
|
|
111
|
+
const existing = await readBillingAccount(store);
|
|
112
|
+
const account = {
|
|
113
|
+
accountID: payload.accountID,
|
|
114
|
+
seatID: payload.seatID,
|
|
115
|
+
email: email ?? existing.email,
|
|
116
|
+
plan: "standard",
|
|
117
|
+
status: "active",
|
|
118
|
+
connectedAt: new Date().toISOString(),
|
|
119
|
+
authURL: existing.authURL,
|
|
120
|
+
};
|
|
121
|
+
await encryptedWriteFile(licensePath(store), `${JSON.stringify({ installedAt: new Date().toISOString(), license }, null, 2)}\n`);
|
|
122
|
+
await encryptedWriteFile(accountPath(store), `${JSON.stringify(account, null, 2)}\n`);
|
|
123
|
+
return {
|
|
124
|
+
account,
|
|
125
|
+
licensePath: licensePath(store),
|
|
126
|
+
verified: true,
|
|
127
|
+
offlineGraceUntil: payload.offlineGraceUntil,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
export async function getBillingEntitlement(store, now = new Date(), freeCapTokens = FREE_MONTHLY_SAVED_TOKEN_CAP) {
|
|
131
|
+
const account = await readBillingAccount(store);
|
|
132
|
+
const monthlySavedTokens = (await readSavingsEvents(store))
|
|
133
|
+
.filter((event) => isSameUtcMonth(new Date(event.timestamp), now))
|
|
134
|
+
.reduce((total, event) => total + event.savedTokens, 0);
|
|
135
|
+
if (account.plan === "standard" &&
|
|
136
|
+
account.status === "active" &&
|
|
137
|
+
(await hasValidStoredLicense(store, now))) {
|
|
138
|
+
return {
|
|
139
|
+
account,
|
|
140
|
+
monthlySavedTokens,
|
|
141
|
+
monthlyCapTokens: null,
|
|
142
|
+
remainingFreeTokens: null,
|
|
143
|
+
canOptimize: true,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
const monthlyCapTokens = positiveInteger(freeCapTokens, FREE_MONTHLY_SAVED_TOKEN_CAP);
|
|
147
|
+
const remainingFreeTokens = Math.max(0, monthlyCapTokens - monthlySavedTokens);
|
|
148
|
+
return {
|
|
149
|
+
account,
|
|
150
|
+
monthlySavedTokens,
|
|
151
|
+
monthlyCapTokens,
|
|
152
|
+
remainingFreeTokens,
|
|
153
|
+
canOptimize: remainingFreeTokens > 0,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
async function hasValidStoredLicense(store, now) {
|
|
157
|
+
try {
|
|
158
|
+
const content = await encryptedReadFile(licensePath(store));
|
|
159
|
+
if (!content)
|
|
160
|
+
return false;
|
|
161
|
+
const stored = JSON.parse(content);
|
|
162
|
+
if (!stored.license)
|
|
163
|
+
return false;
|
|
164
|
+
return verifyLicense({ token: stored.license, publicKeyPem: LICENSE_PUBLIC_KEY, now }).ok;
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
export async function applySavingsCap(store, event, freeCapTokens = FREE_MONTHLY_SAVED_TOKEN_CAP) {
|
|
171
|
+
if (event.metadata?.capCheckedBy === "native-sidecar")
|
|
172
|
+
return event;
|
|
173
|
+
const entitlement = await getBillingEntitlement(store, new Date(event.timestamp), freeCapTokens);
|
|
174
|
+
if (entitlement.monthlyCapTokens === null)
|
|
175
|
+
return event;
|
|
176
|
+
if (event.savedTokens <= 0)
|
|
177
|
+
return event;
|
|
178
|
+
const remaining = entitlement.remainingFreeTokens ?? 0;
|
|
179
|
+
if (remaining <= 0) {
|
|
180
|
+
return createSavingsEvent({
|
|
181
|
+
sessionID: event.sessionID,
|
|
182
|
+
source: event.source,
|
|
183
|
+
label: `${event.label} (free cap reached)`,
|
|
184
|
+
wouldHaveUsedTokens: event.wouldHaveUsedTokens,
|
|
185
|
+
usedTokens: event.wouldHaveUsedTokens,
|
|
186
|
+
metadata: {
|
|
187
|
+
...event.metadata,
|
|
188
|
+
capped: true,
|
|
189
|
+
freeMonthlyCapTokens: entitlement.monthlyCapTokens,
|
|
190
|
+
},
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
if (event.savedTokens <= remaining)
|
|
194
|
+
return event;
|
|
195
|
+
return createSavingsEvent({
|
|
196
|
+
sessionID: event.sessionID,
|
|
197
|
+
source: event.source,
|
|
198
|
+
label: `${event.label} (free cap applied)`,
|
|
199
|
+
wouldHaveUsedTokens: event.wouldHaveUsedTokens,
|
|
200
|
+
usedTokens: event.wouldHaveUsedTokens - remaining,
|
|
201
|
+
metadata: {
|
|
202
|
+
...event.metadata,
|
|
203
|
+
capped: true,
|
|
204
|
+
freeMonthlyCapTokens: entitlement.monthlyCapTokens,
|
|
205
|
+
},
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
export function formatBillingStatus(entitlement) {
|
|
209
|
+
if (entitlement.monthlyCapTokens === null) {
|
|
210
|
+
return `Account: ${entitlement.account.plan} (${entitlement.account.status})`;
|
|
211
|
+
}
|
|
212
|
+
const credited = Math.min(entitlement.monthlySavedTokens, entitlement.monthlyCapTokens);
|
|
213
|
+
return `Account: free seat | credited ${credited.toLocaleString()}/${entitlement.monthlyCapTokens.toLocaleString()} observed saved tokens this month | observed ${entitlement.monthlySavedTokens.toLocaleString()} | remaining ${(entitlement.remainingFreeTokens ?? 0).toLocaleString()}`;
|
|
214
|
+
}
|
|
215
|
+
function createFreeAccount() {
|
|
216
|
+
const accountID = `free_${randomUUID()}`;
|
|
217
|
+
return {
|
|
218
|
+
accountID,
|
|
219
|
+
seatID: `seat_${randomUUID()}`,
|
|
220
|
+
plan: "free",
|
|
221
|
+
status: "free",
|
|
222
|
+
connectedAt: new Date().toISOString(),
|
|
223
|
+
authURL: createAuthURL(accountID),
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
function createAuthURL(accountID, email) {
|
|
227
|
+
const baseURL = tokenWardenSiteURL();
|
|
228
|
+
const url = new URL("/auth/device", baseURL);
|
|
229
|
+
url.searchParams.set("account", accountID);
|
|
230
|
+
if (email)
|
|
231
|
+
url.searchParams.set("email", email);
|
|
232
|
+
return url.toString();
|
|
233
|
+
}
|
|
234
|
+
function createDeviceStartURL() {
|
|
235
|
+
return new URL("/api/device/start", tokenWardenSiteURL()).toString();
|
|
236
|
+
}
|
|
237
|
+
function createDevicePollURL() {
|
|
238
|
+
return new URL("/api/device/poll", tokenWardenSiteURL()).toString();
|
|
239
|
+
}
|
|
240
|
+
function createDeviceAckURL() {
|
|
241
|
+
return new URL("/api/device/ack", tokenWardenSiteURL()).toString();
|
|
242
|
+
}
|
|
243
|
+
function sleep(ms) {
|
|
244
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
245
|
+
}
|
|
246
|
+
function positiveInteger(value, fallback) {
|
|
247
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
|
|
248
|
+
}
|
|
249
|
+
function isSameUtcMonth(left, right) {
|
|
250
|
+
return (left.getUTCFullYear() === right.getUTCFullYear() &&
|
|
251
|
+
left.getUTCMonth() === right.getUTCMonth());
|
|
252
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
export const DEFAULT_FREE_MONTHLY_SAVED_TOKEN_CAP = 100000;
|
|
4
|
+
export const DEFAULT_TOKENWARDEN_SITE_URL = "https://tokenwarden.ai";
|
|
5
|
+
let cachedConfig;
|
|
6
|
+
export function readBuildConfig(configPath = defaultBuildConfigPath()) {
|
|
7
|
+
if (configPath === defaultBuildConfigPath() && cachedConfig)
|
|
8
|
+
return cachedConfig;
|
|
9
|
+
let config = defaultBuildConfig();
|
|
10
|
+
try {
|
|
11
|
+
const parsed = JSON.parse(readFileSync(configPath, "utf8"));
|
|
12
|
+
config = {
|
|
13
|
+
siteURL: validSiteURL(parsed.siteURL) ? parsed.siteURL : config.siteURL,
|
|
14
|
+
generatedAt: typeof parsed.generatedAt === "string" ? parsed.generatedAt : undefined,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
config = defaultBuildConfig();
|
|
19
|
+
}
|
|
20
|
+
if (configPath === defaultBuildConfigPath())
|
|
21
|
+
cachedConfig = config;
|
|
22
|
+
return config;
|
|
23
|
+
}
|
|
24
|
+
export function tokenWardenSiteURL() {
|
|
25
|
+
return readBuildConfig().siteURL;
|
|
26
|
+
}
|
|
27
|
+
function defaultBuildConfig() {
|
|
28
|
+
return {
|
|
29
|
+
siteURL: DEFAULT_TOKENWARDEN_SITE_URL,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function defaultBuildConfigPath() {
|
|
33
|
+
return fileURLToPath(new URL("../../../native/bin/tokenwarden-build-config.json", import.meta.url));
|
|
34
|
+
}
|
|
35
|
+
function validSiteURL(value) {
|
|
36
|
+
if (typeof value !== "string")
|
|
37
|
+
return false;
|
|
38
|
+
try {
|
|
39
|
+
new URL(value);
|
|
40
|
+
return true;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { readdir, readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { countTokens, trimToTokenBudget } from "./tokens.js";
|
|
4
|
+
import { createSavingsEvent } from "./savings.js";
|
|
5
|
+
export async function smartRead(input) {
|
|
6
|
+
const absPath = resolveInsideRoot(input.root, input.path);
|
|
7
|
+
const content = await readFile(absPath, "utf8");
|
|
8
|
+
const mode = input.mode ?? "symbols";
|
|
9
|
+
const budget = input.budget ?? 1600;
|
|
10
|
+
const relPath = relative(input.root, absPath) || input.path;
|
|
11
|
+
const rawTokens = countTokens(content);
|
|
12
|
+
const rendered = trimToTokenBudget(renderSmartRead(relPath, content, mode, input.query), budget);
|
|
13
|
+
const output = countTokens(rendered) > rawTokens ? trimToTokenBudget(content, budget) : rendered;
|
|
14
|
+
const usedTokens = countTokens(output);
|
|
15
|
+
const event = createSavingsEvent({
|
|
16
|
+
sessionID: input.sessionID,
|
|
17
|
+
source: "smart-read",
|
|
18
|
+
label: relPath,
|
|
19
|
+
wouldHaveUsedTokens: rawTokens,
|
|
20
|
+
usedTokens,
|
|
21
|
+
metadata: { mode, query: input.query },
|
|
22
|
+
});
|
|
23
|
+
return { output, rawTokens, usedTokens, event };
|
|
24
|
+
}
|
|
25
|
+
export async function smartPack(input) {
|
|
26
|
+
const budget = input.budget ?? 4000;
|
|
27
|
+
const files = await collectFiles(input.root, input.paths);
|
|
28
|
+
const sections = [];
|
|
29
|
+
let rawTokens = 0;
|
|
30
|
+
for (const file of files.slice(0, 40)) {
|
|
31
|
+
const content = await readFile(file, "utf8");
|
|
32
|
+
rawTokens += countTokens(content);
|
|
33
|
+
const relPath = relative(input.root, file);
|
|
34
|
+
sections.push(renderSmartRead(relPath, content, input.query ? "relevant" : "symbols", input.query));
|
|
35
|
+
if (countTokens(sections.join("\n\n")) >= budget)
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
const output = trimToTokenBudget([`Smart pack for ${input.paths.join(", ")}`, `Files considered: ${files.length}`, "", sections.join("\n\n")].join("\n"), budget);
|
|
39
|
+
const usedTokens = countTokens(output);
|
|
40
|
+
const event = createSavingsEvent({
|
|
41
|
+
sessionID: input.sessionID,
|
|
42
|
+
source: "smart-pack",
|
|
43
|
+
label: input.paths.join(", "),
|
|
44
|
+
wouldHaveUsedTokens: rawTokens,
|
|
45
|
+
usedTokens,
|
|
46
|
+
metadata: { query: input.query, files: files.length },
|
|
47
|
+
});
|
|
48
|
+
return { output, rawTokens, usedTokens, event };
|
|
49
|
+
}
|
|
50
|
+
function renderSmartRead(path, content, mode, query) {
|
|
51
|
+
const lines = content.split(/\r?\n/);
|
|
52
|
+
const imports = collectMatchingLines(lines, /^\s*import\s|^\s*from\s.+import\s|^\s*const\s+.+require\(/);
|
|
53
|
+
const exports = collectMatchingLines(lines, /^\s*export\s/);
|
|
54
|
+
const definitions = collectMatchingLines(lines, /^\s*(export\s+)?(async\s+)?(function|class|interface|type|enum)\s+[A-Za-z0-9_$]+|^\s*(export\s+)?(const|let|var)\s+[A-Za-z0-9_$]+\s*=\s*(async\s*)?(\([^)]*\)|[A-Za-z0-9_$]+)\s*=>/);
|
|
55
|
+
if (mode === "raw-budgeted") {
|
|
56
|
+
return [`File: ${path}`, "Mode: raw-budgeted", "", content].join("\n");
|
|
57
|
+
}
|
|
58
|
+
const sections = [`File: ${path}`, `Estimated raw tokens: ${countTokens(content)}`];
|
|
59
|
+
if (mode === "relevant" && query) {
|
|
60
|
+
const relevant = collectRelevantBlocks(lines, query);
|
|
61
|
+
sections.push("", `Relevant blocks for: ${query}`, ...(relevant.length > 0 ? relevant : ["No direct matches found."]));
|
|
62
|
+
return sections.join("\n");
|
|
63
|
+
}
|
|
64
|
+
if (imports.length > 0)
|
|
65
|
+
sections.push("", "Imports:", ...imports);
|
|
66
|
+
if (exports.length > 0)
|
|
67
|
+
sections.push("", "Exports:", ...exports);
|
|
68
|
+
if (definitions.length > 0)
|
|
69
|
+
sections.push("", "Definitions:", ...definitions);
|
|
70
|
+
if (mode === "summary") {
|
|
71
|
+
sections.push("", "Summary:", summarizeByShape(path, lines, imports.length, definitions.length));
|
|
72
|
+
}
|
|
73
|
+
return sections.join("\n");
|
|
74
|
+
}
|
|
75
|
+
function collectMatchingLines(lines, pattern) {
|
|
76
|
+
return lines
|
|
77
|
+
.map((line, index) => ({ line: line.trim(), index }))
|
|
78
|
+
.filter(({ line }) => pattern.test(line))
|
|
79
|
+
.slice(0, 80)
|
|
80
|
+
.map(({ line, index }) => `- L${index + 1}: ${line}`);
|
|
81
|
+
}
|
|
82
|
+
function collectRelevantBlocks(lines, query) {
|
|
83
|
+
const terms = query
|
|
84
|
+
.toLowerCase()
|
|
85
|
+
.split(/[^a-z0-9_]+/)
|
|
86
|
+
.filter((term) => term.length > 2);
|
|
87
|
+
const keep = new Set();
|
|
88
|
+
lines.forEach((line, index) => {
|
|
89
|
+
const lower = line.toLowerCase();
|
|
90
|
+
if (!terms.some((term) => lower.includes(term)))
|
|
91
|
+
return;
|
|
92
|
+
for (let offset = -3; offset <= 8; offset += 1)
|
|
93
|
+
keep.add(index + offset);
|
|
94
|
+
});
|
|
95
|
+
return [...keep]
|
|
96
|
+
.filter((index) => index >= 0 && index < lines.length)
|
|
97
|
+
.sort((a, b) => a - b)
|
|
98
|
+
.map((index) => `L${index + 1}: ${lines[index]}`)
|
|
99
|
+
.slice(0, 120);
|
|
100
|
+
}
|
|
101
|
+
function summarizeByShape(path, lines, importCount, definitionCount) {
|
|
102
|
+
return `${path} has ${lines.length} lines, ${importCount} import-like lines, and ${definitionCount} top-level definition-like lines. Use relevant mode with a query for focused blocks.`;
|
|
103
|
+
}
|
|
104
|
+
function resolveInsideRoot(root, requestedPath) {
|
|
105
|
+
const absRoot = resolve(root);
|
|
106
|
+
const absPath = isAbsolute(requestedPath) ? resolve(requestedPath) : resolve(absRoot, requestedPath);
|
|
107
|
+
const rel = relative(absRoot, absPath);
|
|
108
|
+
if (rel.startsWith("..") || rel === ".." || rel.includes(`..${sep}`)) {
|
|
109
|
+
throw new Error(`Path is outside project root: ${requestedPath}`);
|
|
110
|
+
}
|
|
111
|
+
return absPath;
|
|
112
|
+
}
|
|
113
|
+
async function collectFiles(root, patterns) {
|
|
114
|
+
const absRoot = resolve(root);
|
|
115
|
+
const allFiles = await walk(absRoot);
|
|
116
|
+
const regexes = patterns.map(globToRegExp);
|
|
117
|
+
return allFiles
|
|
118
|
+
.filter((file) => regexes.some((regex) => regex.test(relative(absRoot, file).replaceAll(sep, "/"))))
|
|
119
|
+
.sort();
|
|
120
|
+
}
|
|
121
|
+
async function walk(dir) {
|
|
122
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
123
|
+
const files = [];
|
|
124
|
+
for (const entry of entries) {
|
|
125
|
+
if (entry.name === ".git" || entry.name === "node_modules" || entry.name === "dist")
|
|
126
|
+
continue;
|
|
127
|
+
const full = join(dir, entry.name);
|
|
128
|
+
if (entry.isDirectory()) {
|
|
129
|
+
files.push(...(await walk(full)));
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (!entry.isFile())
|
|
133
|
+
continue;
|
|
134
|
+
const info = await stat(full);
|
|
135
|
+
if (info.size > 512_000)
|
|
136
|
+
continue;
|
|
137
|
+
files.push(full);
|
|
138
|
+
}
|
|
139
|
+
return files;
|
|
140
|
+
}
|
|
141
|
+
function globToRegExp(pattern) {
|
|
142
|
+
const normalized = pattern.replaceAll("\\", "/");
|
|
143
|
+
let source = "^";
|
|
144
|
+
for (let index = 0; index < normalized.length; index += 1) {
|
|
145
|
+
const char = normalized[index];
|
|
146
|
+
const next = normalized[index + 1];
|
|
147
|
+
if (char === "*" && next === "*") {
|
|
148
|
+
source += ".*";
|
|
149
|
+
index += 1;
|
|
150
|
+
}
|
|
151
|
+
else if (char === "*") {
|
|
152
|
+
source += "[^/]*";
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
source += escapeRegExp(char);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return new RegExp(`${source}$`);
|
|
159
|
+
}
|
|
160
|
+
function escapeRegExp(value) {
|
|
161
|
+
return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
|
|
162
|
+
}
|