code-guardian-ai 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +24 -0
- package/LICENSE +21 -0
- package/README.md +774 -2
- package/bin/cli.js +2 -0
- package/dist/build-manifest.json +136 -0
- package/dist/cli/commands/check.js +56 -0
- package/dist/cli/commands/doctor.js +55 -0
- package/dist/cli/commands/find.js +31 -0
- package/dist/cli/commands/init-files.js +13 -0
- package/dist/cli/commands/init.js +50 -0
- package/dist/cli/commands/optimize.js +53 -0
- package/dist/cli/commands/scan.js +47 -0
- package/dist/cli/index.js +62 -0
- package/dist/cli/output.js +4 -0
- package/dist/context.js +11 -0
- package/dist/core/ast/analysis.js +455 -0
- package/dist/core/ast/cached-info.js +46 -0
- package/dist/core/ast/discovery.js +485 -0
- package/dist/core/ast/extract-js.js +531 -0
- package/dist/core/ast/extract-py.js +298 -0
- package/dist/core/ast/graph.js +92 -0
- package/dist/core/ast/health.js +338 -0
- package/dist/core/ast/indexer.js +286 -0
- package/dist/core/ast/packages.js +343 -0
- package/dist/core/ast/parser.js +91 -0
- package/dist/core/ast/report.js +154 -0
- package/dist/core/ast/resolver.js +392 -0
- package/dist/core/ast/rules.js +37 -0
- package/dist/core/ast/scope-js.js +191 -0
- package/dist/core/ast/shared-index.js +131 -0
- package/dist/core/ast/tsconfig.js +301 -0
- package/dist/core/ast/types.js +1 -0
- package/dist/core/ast/util.js +156 -0
- package/dist/core/git/git.js +430 -0
- package/dist/core/llm/policy.js +35 -0
- package/dist/core/llm/provider.js +199 -0
- package/dist/core/prompt/compressor.js +118 -0
- package/dist/core/prompt/criteria.js +36 -0
- package/dist/core/prompt/find.js +47 -0
- package/dist/core/prompt/guidelines.js +22 -0
- package/dist/core/prompt/model-output.js +178 -0
- package/dist/core/prompt/refiner.js +194 -0
- package/dist/core/prompt/relevance.js +265 -0
- package/dist/core/prompt/untrusted.js +57 -0
- package/dist/core/security/masker.js +330 -0
- package/dist/core/security/outbound.js +40 -0
- package/dist/core/security/safe-file.js +396 -0
- package/dist/core/security/sandbox.js +181 -0
- package/dist/core/validation/baseline.js +121 -0
- package/dist/core/validation/delta.js +243 -0
- package/dist/core/validation/policy.js +123 -0
- package/dist/db/cache-key.js +221 -0
- package/dist/db/client.js +563 -0
- package/dist/db/schema-check.js +149 -0
- package/dist/db/schema.js +36 -0
- package/dist/db/windows-acl.js +150 -0
- package/dist/index.js +55 -0
- package/dist/mcp/server.js +20 -0
- package/dist/mcp/tools/find-reusable.js +22 -0
- package/dist/mcp/tools/get-context.js +39 -0
- package/dist/mcp/tools/optimize-prompt.js +31 -0
- package/dist/mcp/tools/result.js +31 -0
- package/dist/mcp/tools/validate-rules.js +28 -0
- package/dist/utils/config.js +117 -0
- package/dist/utils/errors.js +9 -0
- package/dist/utils/logger.js +19 -0
- package/dist/utils/pace.js +19 -0
- package/dist/utils/text.js +22 -0
- package/dist/utils/tokens.js +14 -0
- package/package.json +69 -4
|
@@ -0,0 +1,563 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import Database from 'better-sqlite3';
|
|
5
|
+
import { UnsafeFileError, createFileExclusive, hardLinkMessage, isHardLinked, regularFileIdentity } from '../core/security/safe-file.js';
|
|
6
|
+
import { resolveInside } from '../core/security/sandbox.js';
|
|
7
|
+
import { errorCode, errorMessage } from '../utils/errors.js';
|
|
8
|
+
import { log } from '../utils/logger.js';
|
|
9
|
+
import { SEALED_OVERHEAD, loadMachineKey, openInfo, sealInfo } from './cache-key.js';
|
|
10
|
+
import { CACHE_LIMITS, CACHE_VERSION, schema } from './schema.js';
|
|
11
|
+
import { currentSchemaProblem, inspectSchema } from './schema-check.js';
|
|
12
|
+
const STATE_DIR = '.guardian';
|
|
13
|
+
const DB_NAME = 'cache.db';
|
|
14
|
+
// SQLite opens these next to the database by path. A planted link here would redirect its writes, so they are checked too.
|
|
15
|
+
const SIDECARS = ['-wal', '-shm', '-journal'];
|
|
16
|
+
// better-sqlite3 reports extended result codes, so a damaged file can arrive as SQLITE_CORRUPT_INDEX, SQLITE_CORRUPT_VTAB, ...
|
|
17
|
+
const isCorruptionCode = (code) => code === 'SQLITE_NOTADB' || code === 'SQLITE_CORRUPT' || code.startsWith('SQLITE_CORRUPT_');
|
|
18
|
+
// Tightens the permissions of the state directory through a descriptor, never through the path. chmod(path) follows symbolic
|
|
19
|
+
// links, so a lstat followed by a chmod lets a .guardian that is swapped for a link in between redirect the change to its
|
|
20
|
+
// target. Here the directory is opened with O_DIRECTORY | O_NOFOLLOW (a link in the last position makes the open fail), the
|
|
21
|
+
// descriptor is checked with fstat to be a directory with the very identity (dev, ino) that the earlier lstat saw, and only then
|
|
22
|
+
// is the mode changed, with fchmod on that descriptor, which cannot reach any other object.
|
|
23
|
+
// The result is verified, not assumed: after the change the mode is read back with fstat on the same descriptor and must be exactly
|
|
24
|
+
// 0700. A failed fchmod is not an error in itself (the directory may already be 0700, or the file system may refuse the call), but
|
|
25
|
+
// then the directory is only used if that read-back shows 0700; otherwise it is rejected. Nothing that cannot be verified is trusted
|
|
26
|
+
// (a failing fstat, no O_DIRECTORY or O_NOFOLLOW on this platform, a link, a replaced or non-directory entry, permission bits other
|
|
27
|
+
// than 0700), and the caller then keeps the cache in memory. Afterwards the path must still name the same directory, and the
|
|
28
|
+
// descriptor is closed on every path out.
|
|
29
|
+
// Windows has no meaningful mode bits (the key and cache protections there are ACL based) and keeps its behaviour: no change.
|
|
30
|
+
function tightenStateDir(dir, hooks) {
|
|
31
|
+
if (process.platform === 'win32')
|
|
32
|
+
return true;
|
|
33
|
+
const { O_RDONLY, O_DIRECTORY, O_NOFOLLOW } = fs.constants;
|
|
34
|
+
if (O_DIRECTORY === undefined || O_NOFOLLOW === undefined || typeof fs.fchmodSync !== 'function') {
|
|
35
|
+
log.warn(`Cannot verify ${dir} safely on this platform`);
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
let fd = null;
|
|
39
|
+
try {
|
|
40
|
+
const seen = fs.lstatSync(dir, { bigint: true });
|
|
41
|
+
if (seen.isSymbolicLink() || !seen.isDirectory())
|
|
42
|
+
throw new UnsafeFileError(`${STATE_DIR} is not a plain directory`);
|
|
43
|
+
hooks.beforePermissionOpen?.();
|
|
44
|
+
fd = fs.openSync(dir, O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
|
|
45
|
+
const held = fs.fstatSync(fd, { bigint: true });
|
|
46
|
+
if (!held.isDirectory() || held.ino !== seen.ino || held.dev !== seen.dev)
|
|
47
|
+
throw new UnsafeFileError(`${STATE_DIR} was replaced while it was being checked`);
|
|
48
|
+
hooks.afterPermissionOpen?.();
|
|
49
|
+
let changeFailure = null;
|
|
50
|
+
try {
|
|
51
|
+
fs.fchmodSync(fd, 0o700);
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
changeFailure = errorCode(err) ?? errorMessage(err);
|
|
55
|
+
}
|
|
56
|
+
hooks.afterPermissionChange?.();
|
|
57
|
+
// What the directory now has, read from the descriptor that fchmod acted on (not from the path). Whether or not the change
|
|
58
|
+
// succeeded, the cache is trusted only if the verified directory really is 0700.
|
|
59
|
+
const applied = fs.fstatSync(fd, { bigint: true });
|
|
60
|
+
if (!applied.isDirectory() || applied.ino !== held.ino || applied.dev !== held.dev)
|
|
61
|
+
throw new UnsafeFileError(`${STATE_DIR} was replaced while its permissions were being set`);
|
|
62
|
+
const mode = applied.mode & 511n;
|
|
63
|
+
if (mode !== 448n) {
|
|
64
|
+
const shown = `0${mode.toString(8)}`;
|
|
65
|
+
throw new UnsafeFileError(changeFailure === null ? `${STATE_DIR} has permissions ${shown} after they were set to 0700, which could not be confirmed` : `${STATE_DIR} has permissions ${shown} and they could not be changed to 0700 (${changeFailure})`);
|
|
66
|
+
}
|
|
67
|
+
if (changeFailure !== null)
|
|
68
|
+
log.debug(`Could not tighten permissions on ${dir} (${changeFailure}), but it already has 0700`);
|
|
69
|
+
const now = fs.lstatSync(dir, { bigint: true });
|
|
70
|
+
if (now.isSymbolicLink() || !now.isDirectory() || now.ino !== held.ino || now.dev !== held.dev)
|
|
71
|
+
throw new UnsafeFileError(`${STATE_DIR} was replaced while its permissions were being set`);
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
log.warn(`Cannot use ${dir} (${err instanceof UnsafeFileError ? err.message : (errorCode(err) ?? errorMessage(err))}), caching in memory for this session`);
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
if (fd !== null) {
|
|
80
|
+
try {
|
|
81
|
+
fs.closeSync(fd);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
/* nothing useful to do */
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
export function prepareStateDir(root, hooks = {}) {
|
|
90
|
+
const dir = path.join(root, STATE_DIR);
|
|
91
|
+
try {
|
|
92
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
93
|
+
resolveInside(root, STATE_DIR);
|
|
94
|
+
if (!fs.statSync(dir).isDirectory())
|
|
95
|
+
throw new Error('not a directory');
|
|
96
|
+
fs.accessSync(dir, fs.constants.R_OK | fs.constants.W_OK);
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
log.warn(`Cannot use ${dir} (${errorCode(err) ?? errorMessage(err)}), caching in memory for this session`);
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
return tightenStateDir(dir, hooks) ? dir : null;
|
|
103
|
+
}
|
|
104
|
+
// A database that opens fine but is not the supported cache: a schema that differs from schema.ts in any way (virtual tables,
|
|
105
|
+
// triggers or views that run inside our own statements, other indexes, other column types or constraints, an unsupported
|
|
106
|
+
// version). Checked with SQLite's own introspection (schema-check.ts). Treated like corruption: replaced, never trusted.
|
|
107
|
+
class PoisonedCacheError extends Error {
|
|
108
|
+
constructor(what) {
|
|
109
|
+
super(`${DB_NAME} is not the supported cache: ${what}`);
|
|
110
|
+
this.name = 'PoisonedCacheError';
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// A cache.db that is larger than the limits allow. Not corruption and not an attack that can be repaired by deleting the file
|
|
114
|
+
// (it may be tracked by the repository), so it is never recovered or deleted: the session uses an in-memory cache.
|
|
115
|
+
class CacheLimitError extends Error {
|
|
116
|
+
constructor(what) {
|
|
117
|
+
super(what);
|
|
118
|
+
this.name = 'CacheLimitError';
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// The size on disk of cache.db and its sidecars, checked before SQLite opens anything: opening and scanning a database costs
|
|
122
|
+
// time proportional to its size, and a repository chooses that size.
|
|
123
|
+
function assertCacheSize(file, limits) {
|
|
124
|
+
let total = 0;
|
|
125
|
+
for (const suffix of ['', ...SIDECARS]) {
|
|
126
|
+
try {
|
|
127
|
+
total += fs.lstatSync(`${file}${suffix}`).size;
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
if (errorCode(err) !== 'ENOENT')
|
|
131
|
+
throw err;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (total > limits.maxDbBytes)
|
|
135
|
+
throw new CacheLimitError(`${DB_NAME} and its journal files take ${Math.ceil(total / 1048576)} MiB, over the ${limits.maxDbBytes / 1048576} MiB limit`);
|
|
136
|
+
}
|
|
137
|
+
// Counting stops one row past the limit, so a table with millions of rows costs no more to refuse than one just over it.
|
|
138
|
+
function assertRowCount(db, limits) {
|
|
139
|
+
const { n } = db.prepare('SELECT COUNT(*) AS n FROM (SELECT 1 FROM files LIMIT ?)').get(limits.maxRows + 1);
|
|
140
|
+
if (n > limits.maxRows)
|
|
141
|
+
throw new CacheLimitError(`the files table has more than ${limits.maxRows} rows`);
|
|
142
|
+
}
|
|
143
|
+
// The usage table is unsigned statistics: only the newest rows are kept, so it cannot grow without bound. This is the trim for a table
|
|
144
|
+
// that arrives with the file (shipped, or from a version that did not trim while running); recordUsage keeps it within the limit after
|
|
145
|
+
// that. Rows that came with the file were never counted by this installation, so they are dropped, not archived.
|
|
146
|
+
function trimUsage(db, limits) {
|
|
147
|
+
db.prepare('DELETE FROM usage WHERE id <= (SELECT MAX(id) FROM usage) - ?').run(limits.maxUsageRows);
|
|
148
|
+
}
|
|
149
|
+
function realDirectory(dir) {
|
|
150
|
+
try {
|
|
151
|
+
return fs.realpathSync.native(dir);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
throw new UnsafeFileError('the cache directory changed while it was being opened');
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function assertSameDirectory(state) {
|
|
158
|
+
const replaced = () => new UnsafeFileError('the cache directory was replaced while it was being used');
|
|
159
|
+
let now;
|
|
160
|
+
try {
|
|
161
|
+
now = fs.lstatSync(state.dir, { bigint: true });
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
throw replaced();
|
|
165
|
+
}
|
|
166
|
+
if (now.isSymbolicLink() || !now.isDirectory() || now.ino !== state.identity.ino || now.dev !== state.identity.dev || realDirectory(state.dir) !== state.realDir)
|
|
167
|
+
throw replaced();
|
|
168
|
+
}
|
|
169
|
+
function bindStateDir(dir) {
|
|
170
|
+
const found = fs.lstatSync(dir, { bigint: true });
|
|
171
|
+
if (found.isSymbolicLink() || !found.isDirectory())
|
|
172
|
+
throw new UnsafeFileError(`${STATE_DIR} is not a plain directory`);
|
|
173
|
+
const state = { dir, realDir: realDirectory(dir), identity: { dev: found.dev, ino: found.ino }, at: (name) => path.join(dir, name), release: () => undefined };
|
|
174
|
+
if (process.platform === 'linux' && fs.constants.O_DIRECTORY !== undefined) {
|
|
175
|
+
try {
|
|
176
|
+
const fd = fs.openSync(dir, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | (fs.constants.O_NOFOLLOW ?? 0));
|
|
177
|
+
const held = fs.fstatSync(fd, { bigint: true });
|
|
178
|
+
if (held.ino === state.identity.ino && held.dev === state.identity.dev && fs.statSync(`/proc/self/fd/${fd}`).isDirectory()) {
|
|
179
|
+
state.at = (name) => `/proc/self/fd/${fd}/${name}`;
|
|
180
|
+
state.release = () => fs.closeSync(fd);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
fs.closeSync(fd);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
/* no /proc or no descriptor: the path-based checks still apply */
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
try {
|
|
191
|
+
assertSameDirectory(state);
|
|
192
|
+
}
|
|
193
|
+
catch (err) {
|
|
194
|
+
state.release();
|
|
195
|
+
throw err;
|
|
196
|
+
}
|
|
197
|
+
return state;
|
|
198
|
+
}
|
|
199
|
+
// Deletes one entry of the bound directory, only after proving that the directory is still the verified one and that
|
|
200
|
+
// the entry is a plain, singly linked file (and, when given, the very file that was checked earlier). Anything else is
|
|
201
|
+
// refused, not deleted.
|
|
202
|
+
function removeVerified(state, name, expected) {
|
|
203
|
+
assertSameDirectory(state);
|
|
204
|
+
const target = state.at(name);
|
|
205
|
+
let found;
|
|
206
|
+
try {
|
|
207
|
+
found = fs.lstatSync(target, { bigint: true });
|
|
208
|
+
}
|
|
209
|
+
catch (err) {
|
|
210
|
+
if (errorCode(err) === 'ENOENT')
|
|
211
|
+
return;
|
|
212
|
+
throw err;
|
|
213
|
+
}
|
|
214
|
+
if (found.isSymbolicLink() || !found.isFile())
|
|
215
|
+
throw new UnsafeFileError(`${name} is not a regular file, it was left in place`);
|
|
216
|
+
if (isHardLinked(found))
|
|
217
|
+
throw new UnsafeFileError(`${hardLinkMessage(name)}, it was left in place`);
|
|
218
|
+
if (expected && (found.ino !== expected.ino || found.dev !== expected.dev))
|
|
219
|
+
throw new UnsafeFileError(`${name} is not the file that was checked, it was left in place`);
|
|
220
|
+
fs.unlinkSync(target);
|
|
221
|
+
}
|
|
222
|
+
// Refuses a symlinked or non-regular cache.db and sidecars, and creates a missing cache.db exclusively. SQLite then
|
|
223
|
+
// only ever opens a plain file that already exists, so it cannot create or follow anything through a planted link.
|
|
224
|
+
function guardDatabaseFile(state) {
|
|
225
|
+
const file = path.join(state.dir, DB_NAME);
|
|
226
|
+
assertSameDirectory(state);
|
|
227
|
+
for (const suffix of SIDECARS)
|
|
228
|
+
regularFileIdentity(`${file}${suffix}`);
|
|
229
|
+
createFileExclusive(state.dir, DB_NAME, '', 0o600);
|
|
230
|
+
assertSameDirectory(state);
|
|
231
|
+
const identity = regularFileIdentity(file);
|
|
232
|
+
if (!identity)
|
|
233
|
+
throw new UnsafeFileError(`${DB_NAME} could not be created`);
|
|
234
|
+
assertSameDirectory(state);
|
|
235
|
+
return { state, file, identity };
|
|
236
|
+
}
|
|
237
|
+
function verifyGuard(guard) {
|
|
238
|
+
assertSameDirectory(guard.state);
|
|
239
|
+
const now = regularFileIdentity(guard.file);
|
|
240
|
+
if (!now || now.ino !== guard.identity.ino || now.dev !== guard.identity.dev)
|
|
241
|
+
throw new UnsafeFileError(`${DB_NAME} was replaced while it was being opened`);
|
|
242
|
+
for (const suffix of SIDECARS)
|
|
243
|
+
regularFileIdentity(`${guard.file}${suffix}`);
|
|
244
|
+
}
|
|
245
|
+
// Every database handle has the key its rows are signed with. A handle opened by openDb gets the machine key; one that
|
|
246
|
+
// was not (an in-memory cache, a handle made directly) gets a random key of its own, which is enough because nothing
|
|
247
|
+
// outside the process can put rows into it.
|
|
248
|
+
const keys = new WeakMap();
|
|
249
|
+
function keyOf(db) {
|
|
250
|
+
let key = keys.get(db);
|
|
251
|
+
if (!key) {
|
|
252
|
+
key = crypto.randomBytes(32);
|
|
253
|
+
keys.set(db, key);
|
|
254
|
+
}
|
|
255
|
+
return key;
|
|
256
|
+
}
|
|
257
|
+
function openMemory() {
|
|
258
|
+
try {
|
|
259
|
+
return open(':memory:');
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
throw new Error(`Cannot create an in-memory SQLite database: ${errorMessage(err)}`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function open(file, guard, limits = CACHE_LIMITS) {
|
|
266
|
+
// Size first, before SQLite reads a byte of it.
|
|
267
|
+
if (guard)
|
|
268
|
+
assertCacheSize(file, limits);
|
|
269
|
+
// A constructor failure leaves no handle to close; it propagates to the caller, which never treats it as a reason to delete.
|
|
270
|
+
const db = new Database(file, guard ? { fileMustExist: true } : undefined);
|
|
271
|
+
try {
|
|
272
|
+
// Nothing below writes until the path is confirmed to still name the file that was checked.
|
|
273
|
+
if (guard)
|
|
274
|
+
verifyGuard(guard);
|
|
275
|
+
db.pragma('trusted_schema = OFF');
|
|
276
|
+
db.pragma('busy_timeout = 5000');
|
|
277
|
+
db.pragma('journal_mode = WAL');
|
|
278
|
+
if (guard)
|
|
279
|
+
verifyGuard(guard);
|
|
280
|
+
db.pragma('synchronous = NORMAL');
|
|
281
|
+
db.transaction(() => {
|
|
282
|
+
// The structure is inspected before anything is created, dropped or migrated: nothing runs against a schema that has not
|
|
283
|
+
// been checked, and a database that is not the supported cache is refused (and then replaced), never repaired in place.
|
|
284
|
+
const inspected = inspectSchema(db, db.pragma('user_version', { simple: true }));
|
|
285
|
+
if (inspected.problem !== null)
|
|
286
|
+
throw new PoisonedCacheError(inspected.problem);
|
|
287
|
+
if (inspected.state === 'migrate') {
|
|
288
|
+
// A supported older version: its file results are rebuilt (the row format changed), its usage statistics are kept.
|
|
289
|
+
db.exec('DROP TABLE IF EXISTS files');
|
|
290
|
+
}
|
|
291
|
+
if (inspected.state !== 'current')
|
|
292
|
+
db.pragma(`user_version = ${CACHE_VERSION}`);
|
|
293
|
+
db.exec(schema);
|
|
294
|
+
// CREATE TABLE IF NOT EXISTS keeps a table that was already there; the finished structure must be exactly the supported one.
|
|
295
|
+
const unexpected = currentSchemaProblem(db);
|
|
296
|
+
if (unexpected !== null)
|
|
297
|
+
throw new PoisonedCacheError(unexpected);
|
|
298
|
+
// A valid schema says nothing about what is in the tables.
|
|
299
|
+
assertRowCount(db, limits);
|
|
300
|
+
trimUsage(db, limits);
|
|
301
|
+
}).immediate();
|
|
302
|
+
// Whatever the file held, it cannot grow past the limit through this handle either.
|
|
303
|
+
const pageSize = db.pragma('page_size', { simple: true });
|
|
304
|
+
if (pageSize > 0)
|
|
305
|
+
db.pragma(`max_page_count = ${Math.max(1, Math.floor(limits.maxDbBytes / pageSize))}`);
|
|
306
|
+
return db;
|
|
307
|
+
}
|
|
308
|
+
catch (err) {
|
|
309
|
+
db.close();
|
|
310
|
+
throw err;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const recoverable = (err) => err instanceof PoisonedCacheError || isCorruptionCode(errorCode(err) ?? '');
|
|
314
|
+
export function openDb(root, hooks = {}, limits = CACHE_LIMITS) {
|
|
315
|
+
const dir = prepareStateDir(root, hooks);
|
|
316
|
+
if (!dir)
|
|
317
|
+
return openMemory();
|
|
318
|
+
const inMemory = (reason) => {
|
|
319
|
+
log.warn(`${reason}, caching in memory`);
|
|
320
|
+
return openMemory();
|
|
321
|
+
};
|
|
322
|
+
const withKey = (db, key) => {
|
|
323
|
+
keys.set(db, key);
|
|
324
|
+
return db;
|
|
325
|
+
};
|
|
326
|
+
// Rows are signed with a key kept outside the workspace; without one the file cache could not be trusted.
|
|
327
|
+
const key = loadMachineKey(root);
|
|
328
|
+
if (!key)
|
|
329
|
+
return openMemory();
|
|
330
|
+
let state;
|
|
331
|
+
try {
|
|
332
|
+
state = bindStateDir(dir);
|
|
333
|
+
}
|
|
334
|
+
catch (err) {
|
|
335
|
+
return inMemory(`Cannot use ${dir} (${errorMessage(err)}), it was left untouched`);
|
|
336
|
+
}
|
|
337
|
+
try {
|
|
338
|
+
hooks.afterStateDir?.();
|
|
339
|
+
let guard;
|
|
340
|
+
try {
|
|
341
|
+
guard = guardDatabaseFile(state);
|
|
342
|
+
}
|
|
343
|
+
catch (err) {
|
|
344
|
+
const refused = err instanceof UnsafeFileError;
|
|
345
|
+
return inMemory(`${refused ? 'Refusing to use' : 'Cannot use'} ${path.join(dir, DB_NAME)} (${errorMessage(err)})${refused ? ', it was left untouched' : ''}`);
|
|
346
|
+
}
|
|
347
|
+
try {
|
|
348
|
+
hooks.afterGuard?.();
|
|
349
|
+
return withKey(open(guard.file, guard, limits), key);
|
|
350
|
+
}
|
|
351
|
+
catch (err) {
|
|
352
|
+
if (!recoverable(err))
|
|
353
|
+
return inMemory(`Could not open ${DB_NAME} (${errorMessage(err)})`);
|
|
354
|
+
log.warn(`${DB_NAME} is corrupted or not a plain cache, rebuilding it`);
|
|
355
|
+
}
|
|
356
|
+
// Recovery deletes files, so every delete is preceded by proof that the directory and the file are the ones that were
|
|
357
|
+
// verified; if anything differs nothing is deleted and the session uses memory.
|
|
358
|
+
try {
|
|
359
|
+
hooks.beforeRecovery?.();
|
|
360
|
+
removeVerified(state, DB_NAME, guard.identity);
|
|
361
|
+
for (const suffix of SIDECARS)
|
|
362
|
+
removeVerified(state, `${DB_NAME}${suffix}`);
|
|
363
|
+
const fresh = guardDatabaseFile(state);
|
|
364
|
+
return withKey(open(fresh.file, fresh, limits), key);
|
|
365
|
+
}
|
|
366
|
+
catch (retryErr) {
|
|
367
|
+
return inMemory(`Could not recreate ${DB_NAME} (${errorMessage(retryErr)})`);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
finally {
|
|
371
|
+
state.release();
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
export function closeDb(db) {
|
|
375
|
+
try {
|
|
376
|
+
db.close();
|
|
377
|
+
}
|
|
378
|
+
catch (err) {
|
|
379
|
+
log.debug(`Closing the database failed: ${errorMessage(err)}`);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
// The path of a row, as a file of the workspace is named in the cache: relative, no NUL, nothing that climbs out.
|
|
383
|
+
function validCachePath(file, limits) {
|
|
384
|
+
return (file.length > 0 &&
|
|
385
|
+
file.length <= limits.maxPathBytes &&
|
|
386
|
+
Buffer.byteLength(file) <= limits.maxPathBytes &&
|
|
387
|
+
!file.includes('\0') &&
|
|
388
|
+
!file.startsWith('/') &&
|
|
389
|
+
!/^[A-Za-z]:/.test(file) &&
|
|
390
|
+
!file.split('/').includes('..'));
|
|
391
|
+
}
|
|
392
|
+
// A content hash is a short run of letters and digits (SHA-256 hex in practice).
|
|
393
|
+
const validCacheHash = (hash) => /^[0-9A-Za-z]{1,128}$/.test(hash);
|
|
394
|
+
// Only rows that can be believed are read, and only as many as the limits allow. The WHERE clause rejects a row of the wrong
|
|
395
|
+
// type or size inside SQLite, so a huge value is never copied into the process; iterate() hands rows over one at a time
|
|
396
|
+
// instead of building the whole result; and the total read is capped before any signature is computed.
|
|
397
|
+
export function loadCache(db, limits = CACHE_LIMITS) {
|
|
398
|
+
try {
|
|
399
|
+
const key = keyOf(db);
|
|
400
|
+
assertRowCount(db, limits);
|
|
401
|
+
const select = db.prepare(`SELECT path, hash, size, mtime_ms, info FROM files
|
|
402
|
+
WHERE typeof(path) = 'text' AND length(CAST(path AS BLOB)) BETWEEN 1 AND @maxPath
|
|
403
|
+
AND typeof(hash) = 'text' AND length(hash) BETWEEN 1 AND 128
|
|
404
|
+
AND typeof(size) IN ('integer', 'real') AND typeof(mtime_ms) IN ('integer', 'real')
|
|
405
|
+
AND typeof(info) = 'text' AND length(CAST(info AS BLOB)) <= @maxInfo
|
|
406
|
+
ORDER BY rowid LIMIT @maxRows`);
|
|
407
|
+
const trusted = new Map();
|
|
408
|
+
let seen = 0;
|
|
409
|
+
let rejected = 0;
|
|
410
|
+
let budget = limits.maxLoadBytes;
|
|
411
|
+
let overBudget = false;
|
|
412
|
+
for (const row of select.iterate({ maxPath: limits.maxPathBytes, maxInfo: limits.maxInfoBytes + SEALED_OVERHEAD, maxRows: limits.maxRows })) {
|
|
413
|
+
seen++;
|
|
414
|
+
if (!validCachePath(row.path, limits) || !validCacheHash(row.hash) || !Number.isFinite(row.size) || !Number.isFinite(row.mtime_ms)) {
|
|
415
|
+
rejected++;
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
// The budget is in bytes of stored data, like every other cache limit: .length counts UTF-16 code units, which is a quarter to a
|
|
419
|
+
// third of the bytes for text outside ASCII. The query already bounds each value, so counting the bytes of one is cheap. The budget
|
|
420
|
+
// is spent before the signature is computed, so the work done on a database has a ceiling however it is filled.
|
|
421
|
+
const bytes = Buffer.byteLength(row.info, 'utf8');
|
|
422
|
+
if (bytes > budget) {
|
|
423
|
+
overBudget = true;
|
|
424
|
+
break;
|
|
425
|
+
}
|
|
426
|
+
budget -= bytes;
|
|
427
|
+
// A row only counts when it carries a signature made with this machine's key for exactly this path and hash.
|
|
428
|
+
const info = openInfo(key, row.path, row.hash, row.info);
|
|
429
|
+
if (info === null)
|
|
430
|
+
rejected++;
|
|
431
|
+
else
|
|
432
|
+
trusted.set(row.path, { ...row, info });
|
|
433
|
+
}
|
|
434
|
+
// Rows the query itself left out (wrong type, too long) were never read; they count as rejected too.
|
|
435
|
+
const total = db.prepare('SELECT COUNT(*) AS n FROM files').get().n;
|
|
436
|
+
if (!overBudget)
|
|
437
|
+
rejected += Math.max(0, total - seen);
|
|
438
|
+
if (rejected > 0)
|
|
439
|
+
log.warn(`${rejected} cached result(s) were not written by this installation, or were malformed or oversized, and will be recomputed`);
|
|
440
|
+
if (overBudget)
|
|
441
|
+
log.warn(`The cache holds more than ${Math.floor(limits.maxLoadBytes / 1048576)} MiB of results; the rest will be recomputed`);
|
|
442
|
+
return trusted;
|
|
443
|
+
}
|
|
444
|
+
catch (err) {
|
|
445
|
+
log.warn(`Cache unreadable, re-parsing everything: ${errorMessage(err)}`);
|
|
446
|
+
return new Map();
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
// `retain` is every path that still exists and is eligible, whether or not this run indexed it.
|
|
450
|
+
// Without it nothing is pruned: used when the scan stopped early and so does not know every path.
|
|
451
|
+
export function applyCacheWrites(db, writes, retain, limits = CACHE_LIMITS) {
|
|
452
|
+
const save = db.prepare(`INSERT INTO files (path, hash, size, mtime_ms, info) VALUES (@path, @hash, @size, @mtime_ms, @info)
|
|
453
|
+
ON CONFLICT(path) DO UPDATE SET hash = @hash, size = @size, mtime_ms = @mtime_ms, info = @info`);
|
|
454
|
+
const touch = db.prepare('UPDATE files SET size = @size, mtime_ms = @mtime_ms WHERE path = @path');
|
|
455
|
+
const remove = db.prepare('DELETE FROM files WHERE path = ?');
|
|
456
|
+
const key = keyOf(db);
|
|
457
|
+
db.transaction(() => {
|
|
458
|
+
for (const write of writes) {
|
|
459
|
+
if (write.kind === 'touch') {
|
|
460
|
+
touch.run(write);
|
|
461
|
+
continue;
|
|
462
|
+
}
|
|
463
|
+
// A result that loadCache would refuse is not stored; the file is simply parsed again next time, and an older row for it is dropped.
|
|
464
|
+
const { row } = write;
|
|
465
|
+
if (!validCachePath(row.path, limits) || !validCacheHash(row.hash) || Buffer.byteLength(row.info) > limits.maxInfoBytes) {
|
|
466
|
+
log.debug(`Not caching ${row.path.slice(0, 200)}: the result is too large or the entry is malformed`);
|
|
467
|
+
remove.run(row.path);
|
|
468
|
+
continue;
|
|
469
|
+
}
|
|
470
|
+
save.run({ ...row, info: sealInfo(key, row.path, row.hash, row.info) });
|
|
471
|
+
}
|
|
472
|
+
if (retain) {
|
|
473
|
+
// Done inside SQLite from a temporary table: a database that holds many or huge rows is never read into the process.
|
|
474
|
+
db.exec('CREATE TEMP TABLE IF NOT EXISTS temp.retain_paths (path TEXT PRIMARY KEY); DELETE FROM temp.retain_paths');
|
|
475
|
+
const keep = db.prepare('INSERT OR IGNORE INTO temp.retain_paths (path) VALUES (?)');
|
|
476
|
+
for (const kept of retain)
|
|
477
|
+
keep.run(kept);
|
|
478
|
+
db.exec('DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM temp.retain_paths r WHERE r.path = files.path); DROP TABLE temp.retain_paths');
|
|
479
|
+
}
|
|
480
|
+
// The table never ends a run above the row limit: the oldest rows (lowest rowid) are the ones to go.
|
|
481
|
+
const { n } = db.prepare('SELECT COUNT(*) AS n FROM files').get();
|
|
482
|
+
if (n > limits.maxRows)
|
|
483
|
+
db.prepare('DELETE FROM files WHERE rowid IN (SELECT rowid FROM files ORDER BY rowid LIMIT ?)').run(n - limits.maxRows);
|
|
484
|
+
}).immediate();
|
|
485
|
+
}
|
|
486
|
+
// The statements of the usage table, in one place so that a test can ask SQLite how each one is executed (EXPLAIN QUERY PLAN): none
|
|
487
|
+
// of them may scan the table. `id` is the rowid, so every range below is a b-tree seek that touches only the rows it names.
|
|
488
|
+
export const USAGE_SQL = {
|
|
489
|
+
insert: 'INSERT INTO usage (at, tool, raw_tokens, sent_tokens) VALUES (?, ?, ?, ?)',
|
|
490
|
+
// What is about to be trimmed away: its row count and the same believable-value sums that lifetimeUsage computes.
|
|
491
|
+
expiring: `SELECT COUNT(*) AS calls, COALESCE(SUM(${believable('raw_tokens')}), 0) AS raw, COALESCE(SUM(${believable('sent_tokens')}), 0) AS sent FROM usage WHERE id <= @cutoff`,
|
|
492
|
+
archive: `INSERT INTO usage_archive (id, calls, raw_tokens, sent_tokens) VALUES (1, @calls, @raw, @sent)
|
|
493
|
+
ON CONFLICT(id) DO UPDATE SET calls = MIN(CASE WHEN typeof(calls) = 'integer' AND calls BETWEEN 0 AND @cap THEN calls ELSE 0 END + @calls, @cap),
|
|
494
|
+
raw_tokens = MIN(CASE WHEN typeof(raw_tokens) = 'integer' AND raw_tokens BETWEEN 0 AND @cap THEN raw_tokens ELSE 0 END + @raw, @cap),
|
|
495
|
+
sent_tokens = MIN(CASE WHEN typeof(sent_tokens) = 'integer' AND sent_tokens BETWEEN 0 AND @cap THEN sent_tokens ELSE 0 END + @sent, @cap)`,
|
|
496
|
+
trim: 'DELETE FROM usage WHERE id <= ?',
|
|
497
|
+
};
|
|
498
|
+
function believable(column) {
|
|
499
|
+
return `CASE WHEN typeof(${column}) = 'integer' AND ${column} BETWEEN 0 AND @maxTokens THEN ${column} ELSE 0 END`;
|
|
500
|
+
}
|
|
501
|
+
// The largest total that is kept: every number handed back is a safe integer, however the tables were filled.
|
|
502
|
+
const TOTAL_CAP = Number.MAX_SAFE_INTEGER;
|
|
503
|
+
// Every call is one transaction: the row is added, and in the same transaction the rows beyond the newest `maxUsageRows` are folded
|
|
504
|
+
// into usage_archive and deleted. So the table is within the limit after every write, not just when the file is opened, and the
|
|
505
|
+
// lifetime totals do not lose what is trimmed.
|
|
506
|
+
// - Bounded work. `id` is an AUTOINCREMENT rowid: the new row has the highest id, the rows to trim are exactly those with
|
|
507
|
+
// id <= newest - limit, and both the count/sum and the delete are range seeks over that rowid range only. In steady state that is
|
|
508
|
+
// one row per call; nothing counts the table or scans it.
|
|
509
|
+
// - Concurrent writers (the server and a CLI run on the same cache.db). BEGIN IMMEDIATE takes the write lock up front, and ids only
|
|
510
|
+
// ever increase, so whichever order the writers run in, each one leaves at most `maxUsageRows` rows with ids above its own cutoff.
|
|
511
|
+
// A writer that cannot get the lock within busy_timeout drops its record.
|
|
512
|
+
// - Failure. Any error (locked, read-only, disk or page limit, a damaged table) rolls the whole transaction back, so the table never
|
|
513
|
+
// holds the row without its trim, and the error is not thrown: statistics must never break a tool call.
|
|
514
|
+
export function recordUsage(db, tool, rawTokens, sentTokens, limits = CACHE_LIMITS) {
|
|
515
|
+
try {
|
|
516
|
+
db.transaction(() => {
|
|
517
|
+
const inserted = db.prepare(USAGE_SQL.insert).run(Date.now(), tool, rawTokens, sentTokens);
|
|
518
|
+
const cutoff = Number(inserted.lastInsertRowid) - limits.maxUsageRows;
|
|
519
|
+
if (!(cutoff > 0))
|
|
520
|
+
return;
|
|
521
|
+
const gone = db.prepare(USAGE_SQL.expiring).get({ cutoff, maxTokens: limits.maxUsageTokens });
|
|
522
|
+
if (gone.calls > 0) {
|
|
523
|
+
db.prepare(USAGE_SQL.archive).run({ calls: gone.calls, raw: gone.raw, sent: gone.sent, cap: TOTAL_CAP });
|
|
524
|
+
db.prepare(USAGE_SQL.trim).run(cutoff);
|
|
525
|
+
}
|
|
526
|
+
}).immediate();
|
|
527
|
+
}
|
|
528
|
+
catch (err) {
|
|
529
|
+
log.debug(`Usage not recorded: ${errorMessage(err)}`);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
// Unsigned statistics. The total is what usage_archive holds (rows that were trimmed away) plus the rows still in the table. Only
|
|
533
|
+
// the newest `maxUsageRows` rows are read, and a value that is not a plausible token count counts as 0, so a crafted table can
|
|
534
|
+
// neither make this scan unbounded nor skew the totals beyond what the limits allow. Archived values are used only when they are
|
|
535
|
+
// non-negative safe integers, and the result is capped at the largest safe integer.
|
|
536
|
+
export function lifetimeUsage(db, limits = CACHE_LIMITS) {
|
|
537
|
+
try {
|
|
538
|
+
const live = db
|
|
539
|
+
.prepare(`SELECT COUNT(*) AS calls, COALESCE(SUM(r), 0) AS raw, COALESCE(SUM(s), 0) AS sent FROM (
|
|
540
|
+
SELECT ${believable('raw_tokens')} AS r, ${believable('sent_tokens')} AS s FROM usage ORDER BY id DESC LIMIT @maxRows
|
|
541
|
+
)`)
|
|
542
|
+
.get({ maxTokens: limits.maxUsageTokens, maxRows: limits.maxUsageRows });
|
|
543
|
+
const archived = archivedUsage(db);
|
|
544
|
+
const total = (a, b) => Math.min(a + b, TOTAL_CAP);
|
|
545
|
+
return { calls: total(live.calls, archived.calls), raw: total(live.raw, archived.raw), sent: total(live.sent, archived.sent) };
|
|
546
|
+
}
|
|
547
|
+
catch {
|
|
548
|
+
return { calls: 0, raw: 0, sent: 0 };
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
function archivedUsage(db) {
|
|
552
|
+
const none = { calls: 0, raw: 0, sent: 0 };
|
|
553
|
+
try {
|
|
554
|
+
const row = db.prepare('SELECT calls, raw_tokens AS raw, sent_tokens AS sent FROM usage_archive WHERE id = 1').get();
|
|
555
|
+
if (!row)
|
|
556
|
+
return none;
|
|
557
|
+
const sane = (v) => (typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 ? v : 0);
|
|
558
|
+
return { calls: sane(row.calls), raw: sane(row.raw), sent: sane(row.sent) };
|
|
559
|
+
}
|
|
560
|
+
catch {
|
|
561
|
+
return none;
|
|
562
|
+
}
|
|
563
|
+
}
|