knodin 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/bin/cli.js +489 -78
- package/dist/src/agent-events.js +128 -0
- package/dist/src/agent-hooks.js +156 -0
- package/dist/src/cli-model.js +42 -1
- package/dist/src/engine/candidate-database.js +167 -0
- package/dist/src/engine/embeddings.js +8 -5
- package/dist/src/engine/index.js +930 -262
- package/dist/src/engine/prune.js +8 -0
- package/dist/src/file-metrics.js +153 -0
- package/dist/src/output-telemetry.js +8 -3
- package/dist/src/repair-lease.js +8 -5
- package/dist/src/response-budget.js +2 -1
- package/dist/src/session-telemetry.js +163 -0
- package/dist/src/shared-index/artifact.js +65 -0
- package/dist/src/shared-index/cache.js +267 -0
- package/dist/src/shared-index/compatibility.js +70 -0
- package/dist/src/shared-index/config.js +224 -0
- package/dist/src/shared-index/contract.js +117 -0
- package/dist/src/shared-index/manifest.js +71 -0
- package/dist/src/shared-index/opportunistic.js +37 -0
- package/dist/src/shared-index/overlay.js +98 -0
- package/dist/src/shared-index/provenance.js +149 -0
- package/dist/src/shared-index/publisher.js +198 -0
- package/dist/src/shared-index/restore.js +209 -0
- package/dist/src/shared-index/s3-client.js +124 -0
- package/dist/src/shared-index/schemas.js +86 -0
- package/dist/src/shared-index/selection.js +111 -0
- package/dist/src/tools/knodin-tools.js +70 -3
- package/docs/CLI.md +56 -0
- package/docs/INSTALLATION.md +11 -0
- package/docs/MCP.md +12 -0
- package/docs/SHARED-INDEX-CONTRACT.md +132 -0
- package/docs/TELEMETRY.md +19 -1
- package/docs/releases/0.8.4.md +22 -0
- package/docs/releases/0.8.5.md +25 -0
- package/package.json +12 -3
- package/roadmap/competitive-roadmap.md +20 -0
- package/schemas/shared-index-branch-pointer-v1.schema.json +66 -0
- package/schemas/shared-index-config-v1.schema.json +97 -0
- package/schemas/shared-index-manifest-v1.schema.json +104 -0
- package/schemas/shared-index-provenance-v1.schema.json +100 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { Database } from "../engine/sqlite.js";
|
|
3
|
+
import { acquireRepairLease } from "../repair-lease.js";
|
|
4
|
+
import { decompressArtifactToCandidate } from "./artifact.js";
|
|
5
|
+
import { SharedIndexCache } from "./cache.js";
|
|
6
|
+
import { canonicalManifestBytes, sha256Hex } from "./contract.js";
|
|
7
|
+
import { deriveSharedOverlay } from "./overlay.js";
|
|
8
|
+
import { readSharedIndexProvenance, writeSharedIndexProvenance, writeSharedRestoreAttempt, } from "./provenance.js";
|
|
9
|
+
import { SharedIndexError, } from "./s3-client.js";
|
|
10
|
+
import { selectSharedSnapshot } from "./selection.js";
|
|
11
|
+
function progress(listener, phase, message, extra = {}) {
|
|
12
|
+
try {
|
|
13
|
+
listener?.({ phase, timestamp: new Date().toISOString(), message, ...extra });
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
// Observability cannot change restore correctness.
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
function boundedFailure(error) {
|
|
20
|
+
if (error instanceof SharedIndexError)
|
|
21
|
+
return { category: error.category, reason: error.message.slice(0, 512) };
|
|
22
|
+
return { category: "unavailable", reason: "shared-index restore failed" };
|
|
23
|
+
}
|
|
24
|
+
function metadata(db, key) {
|
|
25
|
+
return (db.query("SELECT value FROM meta WHERE key = ?").get(key)?.value ??
|
|
26
|
+
null);
|
|
27
|
+
}
|
|
28
|
+
function validateSnapshotDatabase(databasePath, commit, inputFingerprint, databaseSchemaVersion) {
|
|
29
|
+
const db = new Database(databasePath, { readonly: true });
|
|
30
|
+
try {
|
|
31
|
+
const schema = db
|
|
32
|
+
.query("PRAGMA user_version")
|
|
33
|
+
.get()?.user_version;
|
|
34
|
+
if (schema !== databaseSchemaVersion)
|
|
35
|
+
throw new SharedIndexError("integrity", "shared-index database schema does not match signed manifest");
|
|
36
|
+
if (metadata(db, "lastIndexedHead") !== commit)
|
|
37
|
+
throw new SharedIndexError("integrity", "shared-index database commit does not match signed manifest");
|
|
38
|
+
const embeddedFingerprint = metadata(db, "sharedIndexInputFingerprint");
|
|
39
|
+
if (embeddedFingerprint !== inputFingerprint)
|
|
40
|
+
throw new SharedIndexError("integrity", "shared-index database compatibility does not match signed manifest");
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
db.close();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function stampCandidate(databasePath, values) {
|
|
47
|
+
const db = new Database(databasePath);
|
|
48
|
+
try {
|
|
49
|
+
db.run("BEGIN IMMEDIATE");
|
|
50
|
+
for (const [key, value] of Object.entries(values))
|
|
51
|
+
db.run("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", [key, value]);
|
|
52
|
+
db.run("COMMIT");
|
|
53
|
+
db.run("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
try {
|
|
57
|
+
db.run("ROLLBACK");
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// The original database failure is authoritative.
|
|
61
|
+
}
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
db.close();
|
|
66
|
+
}
|
|
67
|
+
for (const suffix of ["-wal", "-shm"])
|
|
68
|
+
fs.rmSync(`${databasePath}${suffix}`, { force: true });
|
|
69
|
+
}
|
|
70
|
+
export async function restoreSharedIndex(options) {
|
|
71
|
+
const startedAt = Date.now();
|
|
72
|
+
let candidate;
|
|
73
|
+
let lease;
|
|
74
|
+
let promoted = false;
|
|
75
|
+
let lifecycleLease;
|
|
76
|
+
try {
|
|
77
|
+
if (!options.config.enabled)
|
|
78
|
+
throw new SharedIndexError("configuration", "shared-index restore is disabled");
|
|
79
|
+
lifecycleLease = acquireRepairLease(options.repo, "shared pull");
|
|
80
|
+
progress(options.onProgress, "discovery", "Discovering shared snapshots");
|
|
81
|
+
const selection = await selectSharedSnapshot({
|
|
82
|
+
repo: options.repo,
|
|
83
|
+
config: options.config,
|
|
84
|
+
store: options.store,
|
|
85
|
+
clientCompatibility: options.clientCompatibility,
|
|
86
|
+
signal: options.signal,
|
|
87
|
+
});
|
|
88
|
+
const selected = selection.selected;
|
|
89
|
+
if (!selected) {
|
|
90
|
+
const first = selection.rejected[0];
|
|
91
|
+
throw new SharedIndexError(first?.category ?? "not-found", first?.reason ?? "no usable shared-index snapshot was found");
|
|
92
|
+
}
|
|
93
|
+
progress(options.onProgress, "verification", "Verified signed shared snapshot");
|
|
94
|
+
const cache = options.cache ?? new SharedIndexCache();
|
|
95
|
+
progress(options.onProgress, "download", "Resolving immutable shared artifact");
|
|
96
|
+
lease = await cache.acquire(options.store, selected.manifest, options.signal);
|
|
97
|
+
progress(options.onProgress, "download", lease.state === "hit" ? "Using verified cache entry" : "Verified shared artifact download", { bytes: selected.manifest.graph.compressedBytes });
|
|
98
|
+
candidate = await options.engine.createCandidate(options.repo, { empty: true });
|
|
99
|
+
progress(options.onProgress, "decompression", "Decompressing into candidate graph");
|
|
100
|
+
await decompressArtifactToCandidate({
|
|
101
|
+
repo: options.repo,
|
|
102
|
+
candidate,
|
|
103
|
+
artifact: lease,
|
|
104
|
+
manifest: selected.manifest,
|
|
105
|
+
signal: options.signal,
|
|
106
|
+
});
|
|
107
|
+
validateSnapshotDatabase(candidate.databasePath, selected.pointer.commit, selected.pointer.inputFingerprint, selected.manifest.compatibility.databaseSchemaVersion);
|
|
108
|
+
const overlay = deriveSharedOverlay(options.repo, selected.pointer.commit);
|
|
109
|
+
const restoredAt = new Date().toISOString();
|
|
110
|
+
stampCandidate(candidate.databasePath, {
|
|
111
|
+
sharedIndexRegistrationId: options.config.registrationId,
|
|
112
|
+
sharedIndexBaseCommit: selected.pointer.commit,
|
|
113
|
+
sharedIndexManifestDigest: sha256Hex(selected.manifestBytes),
|
|
114
|
+
sharedIndexInputFingerprint: selected.pointer.inputFingerprint,
|
|
115
|
+
sharedIndexEffectiveTreeFingerprint: overlay.effectiveTreeFingerprint,
|
|
116
|
+
sharedIndexOverlay: canonicalManifestBytes(overlay.counts).toString("utf8"),
|
|
117
|
+
sharedIndexCache: lease.state,
|
|
118
|
+
sharedIndexRestoredAt: restoredAt,
|
|
119
|
+
});
|
|
120
|
+
progress(options.onProgress, "reconciliation", "Reconciling local repository overlay", {
|
|
121
|
+
completed: 0,
|
|
122
|
+
total: overlay.changedPaths.length,
|
|
123
|
+
});
|
|
124
|
+
await options.engine.reconcileCandidate(candidate, options.repo, overlay.changedPaths, {
|
|
125
|
+
onProgress: (_phase, completed, message, details) => {
|
|
126
|
+
const elapsedMs = Math.max(1, Date.now() - startedAt);
|
|
127
|
+
const total = details?.phaseTotal;
|
|
128
|
+
const ratePerSecond = completed > 0 ? completed / (elapsedMs / 1000) : undefined;
|
|
129
|
+
progress(options.onProgress, "reconciliation", message, {
|
|
130
|
+
completed,
|
|
131
|
+
...(total !== undefined ? { total } : {}),
|
|
132
|
+
...(ratePerSecond !== undefined ? { ratePerSecond } : {}),
|
|
133
|
+
...(ratePerSecond && total !== undefined && total > completed
|
|
134
|
+
? { etaSeconds: (total - completed) / ratePerSecond }
|
|
135
|
+
: {}),
|
|
136
|
+
elapsedMs,
|
|
137
|
+
});
|
|
138
|
+
},
|
|
139
|
+
});
|
|
140
|
+
progress(options.onProgress, "audit", "Deep-auditing reconciled candidate");
|
|
141
|
+
const audit = await options.engine.auditCandidate(candidate, { mode: "deep" });
|
|
142
|
+
if (!audit.healthy)
|
|
143
|
+
throw new SharedIndexError("integrity", `shared-index candidate audit failed (${audit.issues.slice(0, 8).join(",")})`);
|
|
144
|
+
progress(options.onProgress, "promotion", "Promoting verified candidate graph");
|
|
145
|
+
const promotion = await options.engine.promoteCandidate(candidate);
|
|
146
|
+
promoted = true;
|
|
147
|
+
const provenance = {
|
|
148
|
+
schemaVersion: 1,
|
|
149
|
+
enabled: true,
|
|
150
|
+
state: "restored-and-verified",
|
|
151
|
+
registrationId: options.config.registrationId,
|
|
152
|
+
baseCommit: selected.pointer.commit,
|
|
153
|
+
manifestDigest: sha256Hex(selected.manifestBytes),
|
|
154
|
+
source: "shared-snapshot",
|
|
155
|
+
cache: lease.state,
|
|
156
|
+
overlay: overlay.counts,
|
|
157
|
+
effectiveTreeFingerprint: overlay.effectiveTreeFingerprint,
|
|
158
|
+
restoredAt,
|
|
159
|
+
};
|
|
160
|
+
try {
|
|
161
|
+
writeSharedIndexProvenance(options.repo, provenance);
|
|
162
|
+
writeSharedRestoreAttempt(options.repo, {
|
|
163
|
+
schemaVersion: 1,
|
|
164
|
+
attemptedAt: new Date().toISOString(),
|
|
165
|
+
outcome: "restored",
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
// The promoted database contains the same provenance metadata and remains
|
|
170
|
+
// authoritative if the convenience sidecar cannot be written.
|
|
171
|
+
}
|
|
172
|
+
progress(options.onProgress, "completed", "Shared snapshot restored and verified");
|
|
173
|
+
return { restored: true, provenance, backupPath: promotion.backupPath };
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
const failure = boundedFailure(error);
|
|
177
|
+
if (!promoted) {
|
|
178
|
+
try {
|
|
179
|
+
writeSharedRestoreAttempt(options.repo, {
|
|
180
|
+
schemaVersion: 1,
|
|
181
|
+
attemptedAt: new Date().toISOString(),
|
|
182
|
+
outcome: "failed",
|
|
183
|
+
failureCategory: failure.category,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
// A diagnostic write cannot alter active graph preservation.
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
progress(options.onProgress, "failed", failure.reason);
|
|
191
|
+
return {
|
|
192
|
+
restored: false,
|
|
193
|
+
...failure,
|
|
194
|
+
activeProvenance: readSharedIndexProvenance(options.repo),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
198
|
+
lease?.release();
|
|
199
|
+
if (candidate && !promoted) {
|
|
200
|
+
try {
|
|
201
|
+
await options.engine.discardCandidate(candidate);
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
// Candidate cleanup never justifies touching the active graph.
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
lifecycleLease?.release();
|
|
208
|
+
}
|
|
209
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
2
|
+
import { GetObjectCommand, HeadObjectCommand, S3Client, } from "@aws-sdk/client-s3";
|
|
3
|
+
import { NodeHttpHandler } from "@smithy/node-http-handler";
|
|
4
|
+
import { sharedRepositoryPrefix } from "./contract.js";
|
|
5
|
+
export class SharedIndexError extends Error {
|
|
6
|
+
category;
|
|
7
|
+
constructor(category, message) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = "SharedIndexError";
|
|
10
|
+
this.category = category;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function classified(error) {
|
|
14
|
+
if (error instanceof SharedIndexError)
|
|
15
|
+
return error;
|
|
16
|
+
const record = typeof error === "object" && error !== null ? error : {};
|
|
17
|
+
const name = typeof record.name === "string" ? record.name : "";
|
|
18
|
+
const status = Number(record.$metadata?.httpStatusCode);
|
|
19
|
+
if ([
|
|
20
|
+
"CredentialsProviderError",
|
|
21
|
+
"TokenProviderError",
|
|
22
|
+
"ExpiredToken",
|
|
23
|
+
"InvalidAccessKeyId",
|
|
24
|
+
"UnrecognizedClientException",
|
|
25
|
+
].includes(name))
|
|
26
|
+
return new SharedIndexError("authentication", "shared-index AWS authentication failed");
|
|
27
|
+
if (name === "AccessDenied" || name === "AccessDeniedException" || status === 403)
|
|
28
|
+
return new SharedIndexError("authorization", "shared-index object access was denied");
|
|
29
|
+
if (["NoSuchKey", "NotFound", "NoSuchBucket"].includes(name) || status === 404)
|
|
30
|
+
return new SharedIndexError("not-found", "shared-index object was not found");
|
|
31
|
+
if (name === "AbortError" || name === "TimeoutError" || name === "RequestTimeout")
|
|
32
|
+
return new SharedIndexError("timeout", "shared-index object request timed out or was cancelled");
|
|
33
|
+
return new SharedIndexError("unavailable", "shared-index object service is unavailable");
|
|
34
|
+
}
|
|
35
|
+
function requestSignal(signal, timeoutMs) {
|
|
36
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
37
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
38
|
+
}
|
|
39
|
+
function readableBody(body) {
|
|
40
|
+
if (body instanceof Readable)
|
|
41
|
+
return body;
|
|
42
|
+
if (body && typeof body === "object" && "transformToWebStream" in body) {
|
|
43
|
+
const stream = body.transformToWebStream();
|
|
44
|
+
return Readable.fromWeb(stream);
|
|
45
|
+
}
|
|
46
|
+
throw new SharedIndexError("unavailable", "shared-index object response had no readable body");
|
|
47
|
+
}
|
|
48
|
+
export class AwsSharedObjectStore {
|
|
49
|
+
client;
|
|
50
|
+
bucket;
|
|
51
|
+
prefix;
|
|
52
|
+
constructor(config, client) {
|
|
53
|
+
if (!config.enabled)
|
|
54
|
+
throw new SharedIndexError("configuration", "shared-index object access is disabled");
|
|
55
|
+
this.bucket = config.aws.bucket;
|
|
56
|
+
this.prefix = sharedRepositoryPrefix(config.repository);
|
|
57
|
+
const clientConfig = {
|
|
58
|
+
region: config.aws.region,
|
|
59
|
+
maxAttempts: 3,
|
|
60
|
+
requestHandler: new NodeHttpHandler({
|
|
61
|
+
connectionTimeout: 5_000,
|
|
62
|
+
requestTimeout: 30_000,
|
|
63
|
+
}),
|
|
64
|
+
};
|
|
65
|
+
// Omitting `credentials` intentionally selects the SDK's native Node default
|
|
66
|
+
// provider chain (environment, shared config/SSO, web identity, ECS, EC2).
|
|
67
|
+
this.client = client ?? new S3Client(clientConfig);
|
|
68
|
+
}
|
|
69
|
+
key(key) {
|
|
70
|
+
if (!key.startsWith(this.prefix) ||
|
|
71
|
+
key.length <= this.prefix.length ||
|
|
72
|
+
key.includes("\\") ||
|
|
73
|
+
key.split("/").some((segment) => segment === "" || segment === "." || segment === ".."))
|
|
74
|
+
throw new SharedIndexError("configuration", "shared-index object key escaped its repository prefix");
|
|
75
|
+
return key;
|
|
76
|
+
}
|
|
77
|
+
async head(key, options = {}) {
|
|
78
|
+
try {
|
|
79
|
+
const response = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: this.key(key) }), { abortSignal: requestSignal(options.signal, options.timeoutMs ?? 15_000) });
|
|
80
|
+
const bytes = Number(response.ContentLength);
|
|
81
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0)
|
|
82
|
+
throw new SharedIndexError("integrity", "shared-index object size metadata is invalid");
|
|
83
|
+
return {
|
|
84
|
+
bytes,
|
|
85
|
+
...(typeof response.ETag === "string" ? { etag: response.ETag } : {}),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
throw classified(error);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async get(key, limits) {
|
|
93
|
+
if (!Number.isSafeInteger(limits.maxBytes) || limits.maxBytes <= 0)
|
|
94
|
+
throw new SharedIndexError("configuration", "shared-index download limit is invalid");
|
|
95
|
+
try {
|
|
96
|
+
const response = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: this.key(key) }), { abortSignal: requestSignal(limits.signal, limits.timeoutMs ?? 30_000) });
|
|
97
|
+
const declared = Number(response.ContentLength);
|
|
98
|
+
if (Number.isFinite(declared) && declared > limits.maxBytes)
|
|
99
|
+
throw new SharedIndexError("integrity", "shared-index object exceeds the download limit");
|
|
100
|
+
const source = readableBody(response.Body);
|
|
101
|
+
let received = 0;
|
|
102
|
+
return Readable.from((async function* bounded() {
|
|
103
|
+
try {
|
|
104
|
+
for await (const chunk of source) {
|
|
105
|
+
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
106
|
+
received += bytes.length;
|
|
107
|
+
if (received > limits.maxBytes)
|
|
108
|
+
throw new SharedIndexError("integrity", "shared-index object exceeded the download limit");
|
|
109
|
+
yield bytes;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
source.destroy();
|
|
114
|
+
}
|
|
115
|
+
})());
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
throw classified(error);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
export function createSharedObjectStore(config, client) {
|
|
123
|
+
return config.enabled ? new AwsSharedObjectStore(config, client) : null;
|
|
124
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { SHARED_INDEX_LIMITS } from "./contract.js";
|
|
3
|
+
import { SharedIndexError } from "./s3-client.js";
|
|
4
|
+
const sha256 = z.string().regex(/^[a-f0-9]{64}$/);
|
|
5
|
+
const gitObject = z.string().regex(/^[a-f0-9]{40,64}$/);
|
|
6
|
+
const timestamp = z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/);
|
|
7
|
+
const repository = z
|
|
8
|
+
.object({
|
|
9
|
+
host: z
|
|
10
|
+
.string()
|
|
11
|
+
.max(253)
|
|
12
|
+
.regex(/^(?!.*\.\.)(?!.*(?:^|\.)-)(?!.*-(?:\.|$))[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$/),
|
|
13
|
+
owner: z
|
|
14
|
+
.string()
|
|
15
|
+
.max(255)
|
|
16
|
+
.regex(/^[a-z0-9](?:[a-z0-9-]{0,253}[a-z0-9])?$/),
|
|
17
|
+
name: z
|
|
18
|
+
.string()
|
|
19
|
+
.max(255)
|
|
20
|
+
.regex(/^[a-z0-9](?:[a-z0-9._-]{0,253}[a-z0-9])?$/),
|
|
21
|
+
})
|
|
22
|
+
.strict();
|
|
23
|
+
export const branchPointerSchema = z
|
|
24
|
+
.object({
|
|
25
|
+
schemaVersion: z.literal(1),
|
|
26
|
+
registrationId: z.string().regex(/^[a-z0-9][a-z0-9._-]{0,127}$/),
|
|
27
|
+
repository,
|
|
28
|
+
branch: z.string().min(1).max(1024),
|
|
29
|
+
commit: gitObject,
|
|
30
|
+
inputFingerprint: sha256,
|
|
31
|
+
manifestKey: z.string().min(1).max(2048),
|
|
32
|
+
manifestSha256: sha256,
|
|
33
|
+
updatedAt: timestamp,
|
|
34
|
+
})
|
|
35
|
+
.strict();
|
|
36
|
+
export const compatibilitySchema = z
|
|
37
|
+
.object({
|
|
38
|
+
knodinVersion: z.string().regex(/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/),
|
|
39
|
+
databaseSchemaVersion: z.number().int().positive().max(2_147_483_647),
|
|
40
|
+
modelId: z.string().min(1).max(512),
|
|
41
|
+
modelDigest: sha256,
|
|
42
|
+
parserBundleDigest: sha256,
|
|
43
|
+
indexConfigDigest: sha256,
|
|
44
|
+
prunePolicyDigest: sha256,
|
|
45
|
+
})
|
|
46
|
+
.strict();
|
|
47
|
+
export const manifestSchema = z
|
|
48
|
+
.object({
|
|
49
|
+
schemaVersion: z.literal(1),
|
|
50
|
+
registrationId: z.string().regex(/^[a-z0-9][a-z0-9._-]{0,127}$/),
|
|
51
|
+
repository,
|
|
52
|
+
commit: gitObject,
|
|
53
|
+
generatedAt: timestamp,
|
|
54
|
+
compatibility: compatibilitySchema,
|
|
55
|
+
graph: z
|
|
56
|
+
.object({
|
|
57
|
+
key: z.string().min(1).max(2048),
|
|
58
|
+
compressedSha256: sha256,
|
|
59
|
+
compressedBytes: z.number().int().positive().max(SHARED_INDEX_LIMITS.compressedBytes),
|
|
60
|
+
uncompressedSha256: sha256,
|
|
61
|
+
uncompressedBytes: z.number().int().positive().max(SHARED_INDEX_LIMITS.uncompressedBytes),
|
|
62
|
+
})
|
|
63
|
+
.strict(),
|
|
64
|
+
})
|
|
65
|
+
.strict();
|
|
66
|
+
function parse(label, bytes, limit, schema) {
|
|
67
|
+
if (bytes.byteLength > limit)
|
|
68
|
+
throw new SharedIndexError("integrity", `shared-index ${label} exceeds its byte limit`);
|
|
69
|
+
let value;
|
|
70
|
+
try {
|
|
71
|
+
value = JSON.parse(Buffer.from(bytes).toString("utf8"));
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
throw new SharedIndexError("integrity", `shared-index ${label} is not valid JSON`);
|
|
75
|
+
}
|
|
76
|
+
const result = schema.safeParse(value);
|
|
77
|
+
if (!result.success)
|
|
78
|
+
throw new SharedIndexError("integrity", `shared-index ${label} does not match schema v1`);
|
|
79
|
+
return result.data;
|
|
80
|
+
}
|
|
81
|
+
export function parseBranchPointer(bytes) {
|
|
82
|
+
return parse("branch pointer", bytes, SHARED_INDEX_LIMITS.pointerBytes, branchPointerSchema);
|
|
83
|
+
}
|
|
84
|
+
export function parseManifest(bytes) {
|
|
85
|
+
return parse("manifest", bytes, SHARED_INDEX_LIMITS.manifestBytes, manifestSchema);
|
|
86
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import child_process from "node:child_process";
|
|
2
|
+
import { SHARED_INDEX_LIMITS, sharedBranchMatches, } from "./contract.js";
|
|
3
|
+
import { branchPointerKey, signatureKey, validatePointerBinding, verifyManifest, } from "./manifest.js";
|
|
4
|
+
import { SharedIndexError } from "./s3-client.js";
|
|
5
|
+
import { parseBranchPointer, parseManifest, } from "./schemas.js";
|
|
6
|
+
function git(repo, args) {
|
|
7
|
+
const result = child_process.spawnSync("git", args, {
|
|
8
|
+
cwd: repo,
|
|
9
|
+
encoding: "utf8",
|
|
10
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
11
|
+
});
|
|
12
|
+
return { status: result.status, stdout: result.stdout.trim() };
|
|
13
|
+
}
|
|
14
|
+
function localHead(repo) {
|
|
15
|
+
const result = git(repo, ["rev-parse", "--verify", "HEAD"]);
|
|
16
|
+
if (result.status !== 0 || !/^[a-f0-9]{40,64}$/.test(result.stdout))
|
|
17
|
+
throw new SharedIndexError("unavailable", "shared-index could not resolve local HEAD");
|
|
18
|
+
return result.stdout;
|
|
19
|
+
}
|
|
20
|
+
function currentBranch(repo) {
|
|
21
|
+
const result = git(repo, ["symbolic-ref", "--quiet", "--short", "HEAD"]);
|
|
22
|
+
return result.status === 0 && result.stdout ? result.stdout.normalize("NFC") : undefined;
|
|
23
|
+
}
|
|
24
|
+
export function configuredBranchCandidates(config, branch) {
|
|
25
|
+
const candidates = new Set();
|
|
26
|
+
for (const pattern of config.branches) {
|
|
27
|
+
if (pattern.endsWith("/*")) {
|
|
28
|
+
if (branch && sharedBranchMatches(pattern, branch))
|
|
29
|
+
candidates.add(branch);
|
|
30
|
+
}
|
|
31
|
+
else
|
|
32
|
+
candidates.add(pattern.normalize("NFC"));
|
|
33
|
+
}
|
|
34
|
+
return [...candidates];
|
|
35
|
+
}
|
|
36
|
+
async function readBounded(stream, maxBytes) {
|
|
37
|
+
const chunks = [];
|
|
38
|
+
let received = 0;
|
|
39
|
+
for await (const chunk of stream) {
|
|
40
|
+
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
41
|
+
received += bytes.byteLength;
|
|
42
|
+
if (received > maxBytes)
|
|
43
|
+
throw new SharedIndexError("integrity", "shared-index object exceeded its byte limit");
|
|
44
|
+
chunks.push(bytes);
|
|
45
|
+
}
|
|
46
|
+
return Buffer.concat(chunks, received);
|
|
47
|
+
}
|
|
48
|
+
async function objectBytes(store, key, maxBytes, signal) {
|
|
49
|
+
const metadata = await store.head(key, { signal });
|
|
50
|
+
if (metadata.bytes > maxBytes)
|
|
51
|
+
throw new SharedIndexError("integrity", "shared-index object exceeds its byte limit");
|
|
52
|
+
const stream = await store.get(key, { maxBytes, signal });
|
|
53
|
+
const bytes = await readBounded(stream, maxBytes);
|
|
54
|
+
if (bytes.byteLength !== metadata.bytes)
|
|
55
|
+
throw new SharedIndexError("integrity", "shared-index object length changed during download");
|
|
56
|
+
return bytes;
|
|
57
|
+
}
|
|
58
|
+
function commitRelation(repo, head, commit) {
|
|
59
|
+
if (commit === head)
|
|
60
|
+
return { relation: "exact", distance: 0 };
|
|
61
|
+
const ancestor = git(repo, ["merge-base", "--is-ancestor", commit, head]);
|
|
62
|
+
if (ancestor.status !== 0)
|
|
63
|
+
return null;
|
|
64
|
+
const count = git(repo, ["rev-list", "--count", `${commit}..${head}`]);
|
|
65
|
+
const distance = Number(count.stdout);
|
|
66
|
+
if (count.status !== 0 || !Number.isSafeInteger(distance) || distance < 1)
|
|
67
|
+
return null;
|
|
68
|
+
return { relation: "ancestor", distance };
|
|
69
|
+
}
|
|
70
|
+
function rejection(branch, error) {
|
|
71
|
+
if (error instanceof SharedIndexError)
|
|
72
|
+
return { branch, category: error.category, reason: error.message.slice(0, 512) };
|
|
73
|
+
return {
|
|
74
|
+
branch,
|
|
75
|
+
category: "unavailable",
|
|
76
|
+
reason: "shared-index snapshot discovery failed",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export async function selectSharedSnapshot(options) {
|
|
80
|
+
const head = localHead(options.repo);
|
|
81
|
+
const branches = configuredBranchCandidates(options.config, currentBranch(options.repo));
|
|
82
|
+
const settled = await Promise.all(branches.map(async (branch) => {
|
|
83
|
+
try {
|
|
84
|
+
const pointerBytes = await objectBytes(options.store, branchPointerKey(options.config, branch), SHARED_INDEX_LIMITS.pointerBytes, options.signal);
|
|
85
|
+
const pointer = parseBranchPointer(pointerBytes);
|
|
86
|
+
if (pointer.branch.normalize("NFC") !== branch)
|
|
87
|
+
throw new SharedIndexError("integrity", "shared-index pointer branch does not match its discovery key");
|
|
88
|
+
validatePointerBinding(options.config, pointer);
|
|
89
|
+
const relation = commitRelation(options.repo, head, pointer.commit);
|
|
90
|
+
if (!relation)
|
|
91
|
+
throw new SharedIndexError("compatibility", "shared-index snapshot commit is not a proven local ancestor");
|
|
92
|
+
const [manifestBytes, signature] = await Promise.all([
|
|
93
|
+
objectBytes(options.store, pointer.manifestKey, SHARED_INDEX_LIMITS.manifestBytes, options.signal),
|
|
94
|
+
objectBytes(options.store, signatureKey(pointer.manifestKey), SHARED_INDEX_LIMITS.signatureBytes, options.signal),
|
|
95
|
+
]);
|
|
96
|
+
const manifest = parseManifest(manifestBytes);
|
|
97
|
+
verifyManifest(options.config, pointer, manifestBytes, manifest, signature, options.clientCompatibility);
|
|
98
|
+
return { branch, pointer, manifest, manifestBytes, signature, ...relation };
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
return rejection(branch, error);
|
|
102
|
+
}
|
|
103
|
+
}));
|
|
104
|
+
const verified = settled.filter((item) => "manifest" in item);
|
|
105
|
+
verified.sort((left, right) => left.distance - right.distance || left.branch.localeCompare(right.branch));
|
|
106
|
+
return {
|
|
107
|
+
...(verified[0] ? { selected: verified[0] } : {}),
|
|
108
|
+
verified,
|
|
109
|
+
rejected: settled.filter((item) => !("manifest" in item)),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* `server.tool()` / Zod-to-handler inference — that path OOMs `tsc`). One flat
|
|
7
7
|
* gateway keeps the client LLM's context small.
|
|
8
8
|
*/
|
|
9
|
+
import fs from "node:fs";
|
|
9
10
|
import nodePath from "node:path";
|
|
10
11
|
import { fileURLToPath } from "node:url";
|
|
11
12
|
import { resolveCliRuntimeCommand } from "../cli-args.js";
|
|
@@ -17,6 +18,7 @@ import { getDocSection, listDocTopics } from "../docs-sections.js";
|
|
|
17
18
|
import { diagnoseInstallation } from "../doctor.js";
|
|
18
19
|
import { createEngine, REPO_WIDE_QUERY_PATTERNS, } from "../engine/index.js";
|
|
19
20
|
import { measurePerfPhaseSync } from "../engine/perf.js";
|
|
21
|
+
import { resolveDbPath } from "../engine/state-paths.js";
|
|
20
22
|
import { runExecutionProfile } from "../execution-profile.js";
|
|
21
23
|
import { diagnoseFailure, } from "../failure-diagnosis.js";
|
|
22
24
|
import { decorateGraphQueryResult, inspectGraphQueryHealth } from "../graph-query-health.js";
|
|
@@ -31,6 +33,8 @@ import { createRepairPlan } from "../repair-progress.js";
|
|
|
31
33
|
import { runRepositoryInitializationProcess } from "../repository-init-process.js";
|
|
32
34
|
import { discoverRepositories, initializeRepositories, inventoryRepository, searchRepositories, } from "../repository-management.js";
|
|
33
35
|
import { applyResponseBudget } from "../response-budget.js";
|
|
36
|
+
import { loadSharedIndexConfig } from "../shared-index/config.js";
|
|
37
|
+
import { EMPTY_OVERLAY, readSharedIndexProvenance, readSharedRestoreAttempt, } from "../shared-index/provenance.js";
|
|
34
38
|
import { configuredRepositoryInitMemoryLimitBytes, enrichSystemRelationships, incorporateSystemQueryEvidence, indexModeForPath, loadSystemConfiguration, queryConfiguredSystem, systemMembershipsForPath, validateSystemHealth, } from "../system-config.js";
|
|
35
39
|
import { trustedUpdateStatus } from "../update-policy.js";
|
|
36
40
|
import { waitForFresh } from "../wait-for-fresh.js";
|
|
@@ -113,6 +117,7 @@ const QUERY_PATTERNS = [
|
|
|
113
117
|
"structural_implementations_of",
|
|
114
118
|
"tests_for",
|
|
115
119
|
"file_summary",
|
|
120
|
+
"file_metrics",
|
|
116
121
|
"batch_outline",
|
|
117
122
|
"project_overview",
|
|
118
123
|
"shortest_path",
|
|
@@ -150,6 +155,7 @@ const GRAPH_INDEPENDENT_QUERY_PATTERNS = new Set([
|
|
|
150
155
|
"lsp_definitions",
|
|
151
156
|
"lsp_declarations",
|
|
152
157
|
"lsp_implementations",
|
|
158
|
+
"file_metrics",
|
|
153
159
|
]);
|
|
154
160
|
function buildDocumentedKnodinTools() {
|
|
155
161
|
return [
|
|
@@ -302,7 +308,7 @@ function buildDocumentedKnodinTools() {
|
|
|
302
308
|
},
|
|
303
309
|
base: {
|
|
304
310
|
type: "string",
|
|
305
|
-
description: "review/context: compatibility base ref (default HEAD~1). For review this maps to scope=all versus the base; compare uses it as the default older ref. context uses it only for risk.",
|
|
311
|
+
description: "review/context: compatibility base ref (default HEAD~1). query file_metrics: local ref whose merge-base is measured. For review this maps to scope=all versus the base; compare uses it as the default older ref. context uses it only for risk.",
|
|
306
312
|
},
|
|
307
313
|
diffScope: {
|
|
308
314
|
type: "string",
|
|
@@ -379,7 +385,7 @@ function buildDocumentedKnodinTools() {
|
|
|
379
385
|
testScope: {
|
|
380
386
|
type: "string",
|
|
381
387
|
enum: ["all", "test", "production"],
|
|
382
|
-
description: "search: include all, test-only, or production-only
|
|
388
|
+
description: "search/file_metrics: include all, test-only, or production-only files.",
|
|
383
389
|
},
|
|
384
390
|
offset: {
|
|
385
391
|
type: "number",
|
|
@@ -1305,6 +1311,45 @@ async function dispatchKnodinTool(args) {
|
|
|
1305
1311
|
tokens: tokenBudget,
|
|
1306
1312
|
items: itemBudget,
|
|
1307
1313
|
};
|
|
1314
|
+
let sharedEnvelope;
|
|
1315
|
+
const sharedIndexEnvelope = () => {
|
|
1316
|
+
if (sharedEnvelope)
|
|
1317
|
+
return sharedEnvelope;
|
|
1318
|
+
const configuration = loadSharedIndexConfig(repo);
|
|
1319
|
+
const active = readSharedIndexProvenance(repo);
|
|
1320
|
+
const lastRestoreAttempt = readSharedRestoreAttempt(repo);
|
|
1321
|
+
if (active) {
|
|
1322
|
+
sharedEnvelope = {
|
|
1323
|
+
...active,
|
|
1324
|
+
enabled: configuration.state === "configured" ? configuration.config.enabled : false,
|
|
1325
|
+
...(lastRestoreAttempt ? { lastRestoreAttempt } : {}),
|
|
1326
|
+
};
|
|
1327
|
+
return sharedEnvelope;
|
|
1328
|
+
}
|
|
1329
|
+
const state = configuration.state === "not-configured"
|
|
1330
|
+
? "not-configured"
|
|
1331
|
+
: configuration.state === "invalid"
|
|
1332
|
+
? "configuration-invalid"
|
|
1333
|
+
: !configuration.config.enabled
|
|
1334
|
+
? "disabled"
|
|
1335
|
+
: lastRestoreAttempt?.outcome === "failed"
|
|
1336
|
+
? "restore-failed"
|
|
1337
|
+
: "local-only";
|
|
1338
|
+
sharedEnvelope = {
|
|
1339
|
+
schemaVersion: 1,
|
|
1340
|
+
enabled: configuration.state === "configured" && configuration.config.enabled,
|
|
1341
|
+
state,
|
|
1342
|
+
source: "local-index",
|
|
1343
|
+
...(configuration.state === "configured"
|
|
1344
|
+
? { cache: "not-used", overlay: EMPTY_OVERLAY }
|
|
1345
|
+
: {}),
|
|
1346
|
+
...(lastRestoreAttempt ? { lastRestoreAttempt } : {}),
|
|
1347
|
+
...(lastRestoreAttempt?.failureCategory
|
|
1348
|
+
? { failureCategory: lastRestoreAttempt.failureCategory }
|
|
1349
|
+
: {}),
|
|
1350
|
+
};
|
|
1351
|
+
return sharedEnvelope;
|
|
1352
|
+
};
|
|
1308
1353
|
const observe = (output, op, minimal = false) => {
|
|
1309
1354
|
const root = output;
|
|
1310
1355
|
const budgetMeta = root.responseBudget;
|
|
@@ -1333,11 +1378,31 @@ async function dispatchKnodinTool(args) {
|
|
|
1333
1378
|
return root;
|
|
1334
1379
|
};
|
|
1335
1380
|
const bounded = (result, op = String(operation), minimal = false) => {
|
|
1381
|
+
const rootOperation = op.split(":")[0];
|
|
1382
|
+
const includeSharedIndex = [
|
|
1383
|
+
"status",
|
|
1384
|
+
"context",
|
|
1385
|
+
"query",
|
|
1386
|
+
"explain",
|
|
1387
|
+
"review",
|
|
1388
|
+
"map",
|
|
1389
|
+
"search",
|
|
1390
|
+
].includes(rootOperation);
|
|
1391
|
+
const enriched = includeSharedIndex && result && typeof result === "object" && !Array.isArray(result)
|
|
1392
|
+
? { ...result, sharedIndex: sharedIndexEnvelope() }
|
|
1393
|
+
: result;
|
|
1336
1394
|
const key = `${op}${minimal ? ":minimal" : ""}`;
|
|
1337
|
-
const output = applyResponseBudget(
|
|
1395
|
+
const output = applyResponseBudget(enriched, op, budget, RESPONSE_DEFAULTS[key] ?? RESPONSE_DEFAULTS.default);
|
|
1338
1396
|
return observe(output, op, minimal);
|
|
1339
1397
|
};
|
|
1340
1398
|
const graphRead = async (op, run, minimal = false) => {
|
|
1399
|
+
const sharedConfiguration = loadSharedIndexConfig(repo);
|
|
1400
|
+
if (!fs.existsSync(resolveDbPath(repo)) &&
|
|
1401
|
+
sharedConfiguration.state === "configured" &&
|
|
1402
|
+
sharedConfiguration.config.enabled) {
|
|
1403
|
+
const { attemptOpportunisticSharedRestore } = await import("../shared-index/opportunistic.js");
|
|
1404
|
+
await attemptOpportunisticSharedRestore(repo, engine);
|
|
1405
|
+
}
|
|
1341
1406
|
const health = await inspectGatewayGraphHealth(repo);
|
|
1342
1407
|
if (!health.available)
|
|
1343
1408
|
return bounded(health, op, minimal);
|
|
@@ -1597,6 +1662,8 @@ async function dispatchKnodinTool(args) {
|
|
|
1597
1662
|
includeDataFlow,
|
|
1598
1663
|
}
|
|
1599
1664
|
: undefined, {
|
|
1665
|
+
base: pattern === "file_metrics" ? base : undefined,
|
|
1666
|
+
testScope: pattern === "file_metrics" ? testScope : undefined,
|
|
1600
1667
|
minLines,
|
|
1601
1668
|
minComplexity,
|
|
1602
1669
|
kinds,
|