knodin 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/bin/cli.js +489 -78
- package/dist/src/agent-events.js +128 -0
- package/dist/src/agent-hooks.js +156 -0
- package/dist/src/cli-model.js +42 -1
- package/dist/src/engine/candidate-database.js +167 -0
- package/dist/src/engine/embeddings.js +8 -5
- package/dist/src/engine/index.js +930 -262
- package/dist/src/engine/prune.js +8 -0
- package/dist/src/file-metrics.js +153 -0
- package/dist/src/output-telemetry.js +8 -3
- package/dist/src/repair-lease.js +8 -5
- package/dist/src/response-budget.js +2 -1
- package/dist/src/session-telemetry.js +163 -0
- package/dist/src/shared-index/artifact.js +65 -0
- package/dist/src/shared-index/cache.js +267 -0
- package/dist/src/shared-index/compatibility.js +70 -0
- package/dist/src/shared-index/config.js +224 -0
- package/dist/src/shared-index/contract.js +117 -0
- package/dist/src/shared-index/manifest.js +71 -0
- package/dist/src/shared-index/opportunistic.js +37 -0
- package/dist/src/shared-index/overlay.js +98 -0
- package/dist/src/shared-index/provenance.js +149 -0
- package/dist/src/shared-index/publisher.js +198 -0
- package/dist/src/shared-index/restore.js +209 -0
- package/dist/src/shared-index/s3-client.js +124 -0
- package/dist/src/shared-index/schemas.js +86 -0
- package/dist/src/shared-index/selection.js +111 -0
- package/dist/src/tools/knodin-tools.js +70 -3
- package/docs/CLI.md +56 -0
- package/docs/INSTALLATION.md +11 -0
- package/docs/MCP.md +12 -0
- package/docs/SHARED-INDEX-CONTRACT.md +132 -0
- package/docs/TELEMETRY.md +19 -1
- package/docs/releases/0.8.4.md +22 -0
- package/docs/releases/0.8.5.md +25 -0
- package/package.json +12 -3
- package/roadmap/competitive-roadmap.md +20 -0
- package/schemas/shared-index-branch-pointer-v1.schema.json +66 -0
- package/schemas/shared-index-config-v1.schema.json +97 -0
- package/schemas/shared-index-manifest-v1.schema.json +104 -0
- package/schemas/shared-index-provenance-v1.schema.json +100 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { publicKeyFingerprint } from "./config.js";
|
|
4
|
+
import { canonicalManifestBytes, canonicalRepositoryIdentity, encodeSharedIndexComponent, SHARED_INDEX_LIMITS, sha256Hex, sharedIndexCompatibility, sharedRepositoryPrefix, } from "./contract.js";
|
|
5
|
+
import { SharedIndexError } from "./s3-client.js";
|
|
6
|
+
function sameIdentity(left, right) {
|
|
7
|
+
return (JSON.stringify(canonicalRepositoryIdentity(left)) ===
|
|
8
|
+
JSON.stringify(canonicalRepositoryIdentity(right)));
|
|
9
|
+
}
|
|
10
|
+
export function branchPointerKey(config, branch) {
|
|
11
|
+
return `${sharedRepositoryPrefix(config.repository)}branches/${encodeSharedIndexComponent(branch)}.json`;
|
|
12
|
+
}
|
|
13
|
+
export function signatureKey(manifestKey) {
|
|
14
|
+
if (!manifestKey.endsWith("/manifest.json"))
|
|
15
|
+
throw new SharedIndexError("integrity", "shared-index manifest key has an invalid filename");
|
|
16
|
+
return `${manifestKey.slice(0, -"manifest.json".length)}manifest.sig`;
|
|
17
|
+
}
|
|
18
|
+
export function compatibilityFingerprint(compatibility) {
|
|
19
|
+
return sha256Hex(canonicalManifestBytes(compatibility));
|
|
20
|
+
}
|
|
21
|
+
export function validatePointerBinding(config, pointer) {
|
|
22
|
+
if (pointer.registrationId !== config.registrationId)
|
|
23
|
+
throw new SharedIndexError("integrity", "shared-index pointer registration does not match configuration");
|
|
24
|
+
if (!sameIdentity(pointer.repository, config.repository))
|
|
25
|
+
throw new SharedIndexError("integrity", "shared-index pointer repository does not match configuration");
|
|
26
|
+
const directory = `${sharedRepositoryPrefix(config.repository)}snapshots/${pointer.commit}/${pointer.inputFingerprint}/`;
|
|
27
|
+
if (pointer.manifestKey !== `${directory}manifest.json`)
|
|
28
|
+
throw new SharedIndexError("integrity", "shared-index manifest key escaped its immutable snapshot prefix");
|
|
29
|
+
}
|
|
30
|
+
function verifyConfiguredTrustRoot(config) {
|
|
31
|
+
let fingerprint;
|
|
32
|
+
try {
|
|
33
|
+
fingerprint = publicKeyFingerprint(config.signing.publicKeyPem);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
throw new SharedIndexError("integrity", "shared-index configured signing key is invalid");
|
|
37
|
+
}
|
|
38
|
+
const expected = Buffer.from(config.signing.publicKeySha256, "hex");
|
|
39
|
+
const actual = Buffer.from(fingerprint, "hex");
|
|
40
|
+
if (expected.byteLength !== actual.byteLength || !crypto.timingSafeEqual(expected, actual))
|
|
41
|
+
throw new SharedIndexError("integrity", "shared-index configured signing-key fingerprint does not match");
|
|
42
|
+
}
|
|
43
|
+
export function verifyManifest(config, pointer, rawManifest, manifest, signature, clientCompatibility) {
|
|
44
|
+
if (signature.byteLength === 0 || signature.byteLength > SHARED_INDEX_LIMITS.signatureBytes)
|
|
45
|
+
throw new SharedIndexError("integrity", "shared-index signature exceeds its byte limit");
|
|
46
|
+
verifyConfiguredTrustRoot(config);
|
|
47
|
+
if (sha256Hex(rawManifest) !== pointer.manifestSha256)
|
|
48
|
+
throw new SharedIndexError("integrity", "shared-index manifest digest does not match pointer");
|
|
49
|
+
if (!Buffer.from(rawManifest).equals(canonicalManifestBytes(manifest)))
|
|
50
|
+
throw new SharedIndexError("integrity", "shared-index manifest bytes are not canonical JSON v1");
|
|
51
|
+
const verified = crypto.verify("sha256", rawManifest, {
|
|
52
|
+
key: config.signing.publicKeyPem,
|
|
53
|
+
padding: crypto.constants.RSA_PKCS1_PSS_PADDING,
|
|
54
|
+
saltLength: 32,
|
|
55
|
+
}, signature);
|
|
56
|
+
if (!verified)
|
|
57
|
+
throw new SharedIndexError("integrity", "shared-index manifest signature is invalid");
|
|
58
|
+
if (manifest.registrationId !== config.registrationId ||
|
|
59
|
+
!sameIdentity(manifest.repository, config.repository))
|
|
60
|
+
throw new SharedIndexError("integrity", "shared-index signed identity does not match configuration");
|
|
61
|
+
if (manifest.commit !== pointer.commit)
|
|
62
|
+
throw new SharedIndexError("integrity", "shared-index signed commit does not match pointer");
|
|
63
|
+
if (compatibilityFingerprint(manifest.compatibility) !== pointer.inputFingerprint)
|
|
64
|
+
throw new SharedIndexError("integrity", "shared-index compatibility fingerprint does not match pointer");
|
|
65
|
+
const immutableDirectory = path.posix.dirname(pointer.manifestKey);
|
|
66
|
+
if (manifest.graph.key !== `${immutableDirectory}/graph.sqlite.zst`)
|
|
67
|
+
throw new SharedIndexError("integrity", "shared-index graph key escaped its immutable snapshot prefix");
|
|
68
|
+
const compatibility = sharedIndexCompatibility(clientCompatibility, manifest.compatibility);
|
|
69
|
+
if (!compatibility.compatible)
|
|
70
|
+
throw new SharedIndexError("compatibility", `shared-index snapshot is incompatible (${compatibility.reasons.join(",")})`);
|
|
71
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { resolveDbPath } from "../engine/state-paths.js";
|
|
3
|
+
import { currentSharedIndexCompatibility } from "./compatibility.js";
|
|
4
|
+
import { loadSharedIndexConfig } from "./config.js";
|
|
5
|
+
import { readSharedIndexProvenance, readSharedRestoreAttempt } from "./provenance.js";
|
|
6
|
+
import { restoreSharedIndex } from "./restore.js";
|
|
7
|
+
import { createSharedObjectStore } from "./s3-client.js";
|
|
8
|
+
const FAILURE_BACKOFF_MS = 15 * 60 * 1000;
|
|
9
|
+
export function shouldAttemptOpportunisticRestore(repo, now = Date.now()) {
|
|
10
|
+
const configuration = loadSharedIndexConfig(repo);
|
|
11
|
+
if (configuration.state !== "configured" || !configuration.config.enabled)
|
|
12
|
+
return false;
|
|
13
|
+
if (fs.existsSync(resolveDbPath(repo)) && readSharedIndexProvenance(repo))
|
|
14
|
+
return false;
|
|
15
|
+
const last = readSharedRestoreAttempt(repo);
|
|
16
|
+
if (last?.outcome === "failed" && now - Date.parse(last.attemptedAt) < FAILURE_BACKOFF_MS)
|
|
17
|
+
return false;
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
/** Safe cold-start optimization: every failure returns to the caller for local fallback. */
|
|
21
|
+
export async function attemptOpportunisticSharedRestore(repo, engine) {
|
|
22
|
+
if (!shouldAttemptOpportunisticRestore(repo))
|
|
23
|
+
return null;
|
|
24
|
+
const configuration = loadSharedIndexConfig(repo);
|
|
25
|
+
if (configuration.state !== "configured")
|
|
26
|
+
return null;
|
|
27
|
+
const store = createSharedObjectStore(configuration.config);
|
|
28
|
+
if (!store)
|
|
29
|
+
return null;
|
|
30
|
+
return restoreSharedIndex({
|
|
31
|
+
repo,
|
|
32
|
+
config: configuration.config,
|
|
33
|
+
store,
|
|
34
|
+
engine,
|
|
35
|
+
clientCompatibility: currentSharedIndexCompatibility(),
|
|
36
|
+
});
|
|
37
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import child_process from "node:child_process";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { compareBytes } from "../compare.js";
|
|
6
|
+
import { isIndexableSourcePath } from "../engine/source-policy.js";
|
|
7
|
+
import { SharedIndexError } from "./s3-client.js";
|
|
8
|
+
function split(output) {
|
|
9
|
+
return output.split("\0").filter(Boolean);
|
|
10
|
+
}
|
|
11
|
+
function gitPaths(repo, args) {
|
|
12
|
+
try {
|
|
13
|
+
return split(child_process.execFileSync("git", args, {
|
|
14
|
+
cwd: repo,
|
|
15
|
+
encoding: "utf8",
|
|
16
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
17
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
18
|
+
}));
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
throw new SharedIndexError("unavailable", "shared-index could not derive the local overlay");
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function safeRelativePath(repo, value) {
|
|
25
|
+
const normalized = value.replaceAll("\\", "/").normalize("NFC");
|
|
26
|
+
const resolved = path.resolve(repo, normalized);
|
|
27
|
+
const relative = path.relative(path.resolve(repo), resolved);
|
|
28
|
+
if (!relative ||
|
|
29
|
+
relative.startsWith("..") ||
|
|
30
|
+
path.isAbsolute(relative) ||
|
|
31
|
+
normalized.includes("\0"))
|
|
32
|
+
throw new SharedIndexError("integrity", "shared-index Git overlay contains an unsafe path");
|
|
33
|
+
return relative.replaceAll("\\", "/");
|
|
34
|
+
}
|
|
35
|
+
function digestFile(target) {
|
|
36
|
+
const hash = crypto.createHash("sha256");
|
|
37
|
+
const descriptor = fs.openSync(target, "r");
|
|
38
|
+
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
39
|
+
try {
|
|
40
|
+
for (;;) {
|
|
41
|
+
const bytes = fs.readSync(descriptor, buffer, 0, buffer.byteLength, null);
|
|
42
|
+
if (bytes === 0)
|
|
43
|
+
break;
|
|
44
|
+
hash.update(buffer.subarray(0, bytes));
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
fs.closeSync(descriptor);
|
|
49
|
+
}
|
|
50
|
+
return hash.digest("hex");
|
|
51
|
+
}
|
|
52
|
+
export function deriveSharedOverlay(repo, snapshotCommit) {
|
|
53
|
+
const resolved = path.resolve(repo);
|
|
54
|
+
const committed = new Set(gitPaths(resolved, [
|
|
55
|
+
"diff",
|
|
56
|
+
"--name-only",
|
|
57
|
+
"--no-renames",
|
|
58
|
+
"-z",
|
|
59
|
+
snapshotCommit,
|
|
60
|
+
"HEAD",
|
|
61
|
+
"--",
|
|
62
|
+
]).map((file) => safeRelativePath(resolved, file)));
|
|
63
|
+
const staged = new Set(gitPaths(resolved, ["diff", "--cached", "--name-only", "--no-renames", "-z", "HEAD", "--"]).map((file) => safeRelativePath(resolved, file)));
|
|
64
|
+
const working = new Set(gitPaths(resolved, ["diff", "--name-only", "--no-renames", "-z", "--"]).map((file) => safeRelativePath(resolved, file)));
|
|
65
|
+
const untracked = new Set(gitPaths(resolved, ["ls-files", "--others", "--exclude-standard", "-z", "--"])
|
|
66
|
+
.map((file) => safeRelativePath(resolved, file))
|
|
67
|
+
.filter((file) => isIndexableSourcePath(file)));
|
|
68
|
+
const changedPaths = [...new Set([...committed, ...staged, ...working, ...untracked])].sort(compareBytes);
|
|
69
|
+
const deletedPaths = changedPaths.filter((file) => !fs.existsSync(path.join(resolved, file))).length;
|
|
70
|
+
const fingerprint = crypto.createHash("sha256");
|
|
71
|
+
fingerprint.update(`snapshot\0${snapshotCommit}\0`);
|
|
72
|
+
for (const file of changedPaths) {
|
|
73
|
+
const absolute = path.join(resolved, file);
|
|
74
|
+
let digest = "deleted";
|
|
75
|
+
try {
|
|
76
|
+
const stat = fs.lstatSync(absolute);
|
|
77
|
+
if (stat.isFile() && !stat.isSymbolicLink())
|
|
78
|
+
digest = digestFile(absolute);
|
|
79
|
+
else
|
|
80
|
+
digest = "ineligible";
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// Final disk truth is deletion even when a path changed in several layers.
|
|
84
|
+
}
|
|
85
|
+
fingerprint.update(`${Buffer.byteLength(file)}:${file}\0${digest}\0`);
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
changedPaths,
|
|
89
|
+
counts: {
|
|
90
|
+
committedPaths: committed.size,
|
|
91
|
+
stagedPaths: staged.size,
|
|
92
|
+
workingTreePaths: working.size,
|
|
93
|
+
deletedPaths,
|
|
94
|
+
untrackedPaths: untracked.size,
|
|
95
|
+
},
|
|
96
|
+
effectiveTreeFingerprint: fingerprint.digest("hex"),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { Database } from "../engine/sqlite.js";
|
|
6
|
+
import { resolveDbPath, resolveStateDir } from "../engine/state-paths.js";
|
|
7
|
+
const counts = z
|
|
8
|
+
.object({
|
|
9
|
+
committedPaths: z.number().int().nonnegative(),
|
|
10
|
+
stagedPaths: z.number().int().nonnegative(),
|
|
11
|
+
workingTreePaths: z.number().int().nonnegative(),
|
|
12
|
+
deletedPaths: z.number().int().nonnegative(),
|
|
13
|
+
untrackedPaths: z.number().int().nonnegative(),
|
|
14
|
+
})
|
|
15
|
+
.strict();
|
|
16
|
+
export const sharedIndexProvenanceSchema = z
|
|
17
|
+
.object({
|
|
18
|
+
schemaVersion: z.literal(1),
|
|
19
|
+
enabled: z.boolean(),
|
|
20
|
+
state: z.enum([
|
|
21
|
+
"disabled",
|
|
22
|
+
"not-configured",
|
|
23
|
+
"configuration-invalid",
|
|
24
|
+
"local-only",
|
|
25
|
+
"restore-pending",
|
|
26
|
+
"restored-and-verified",
|
|
27
|
+
"restore-failed",
|
|
28
|
+
]),
|
|
29
|
+
registrationId: z.string().optional(),
|
|
30
|
+
baseCommit: z
|
|
31
|
+
.string()
|
|
32
|
+
.regex(/^[a-f0-9]{40,64}$/)
|
|
33
|
+
.optional(),
|
|
34
|
+
manifestDigest: z
|
|
35
|
+
.string()
|
|
36
|
+
.regex(/^[a-f0-9]{64}$/)
|
|
37
|
+
.optional(),
|
|
38
|
+
source: z.enum(["local-index", "shared-snapshot"]),
|
|
39
|
+
cache: z.enum(["not-used", "downloaded", "hit", "unavailable", "corrupt-refetched"]).optional(),
|
|
40
|
+
overlay: counts.optional(),
|
|
41
|
+
failureCategory: z
|
|
42
|
+
.enum([
|
|
43
|
+
"authentication",
|
|
44
|
+
"authorization",
|
|
45
|
+
"not-found",
|
|
46
|
+
"timeout",
|
|
47
|
+
"integrity",
|
|
48
|
+
"compatibility",
|
|
49
|
+
"configuration",
|
|
50
|
+
"unavailable",
|
|
51
|
+
])
|
|
52
|
+
.optional(),
|
|
53
|
+
effectiveTreeFingerprint: z
|
|
54
|
+
.string()
|
|
55
|
+
.regex(/^[a-f0-9]{64}$/)
|
|
56
|
+
.optional(),
|
|
57
|
+
restoredAt: z.iso.datetime().optional(),
|
|
58
|
+
})
|
|
59
|
+
.strict()
|
|
60
|
+
.superRefine((value, context) => {
|
|
61
|
+
if (value.state === "restored-and-verified" &&
|
|
62
|
+
(!value.registrationId ||
|
|
63
|
+
!value.baseCommit ||
|
|
64
|
+
!value.manifestDigest ||
|
|
65
|
+
!value.cache ||
|
|
66
|
+
!value.overlay ||
|
|
67
|
+
!value.effectiveTreeFingerprint ||
|
|
68
|
+
!value.restoredAt))
|
|
69
|
+
context.addIssue({ code: "custom", message: "verified provenance is incomplete" });
|
|
70
|
+
if (value.state === "restore-failed" && !value.failureCategory)
|
|
71
|
+
context.addIssue({ code: "custom", message: "failed provenance has no category" });
|
|
72
|
+
});
|
|
73
|
+
export const EMPTY_OVERLAY = {
|
|
74
|
+
committedPaths: 0,
|
|
75
|
+
stagedPaths: 0,
|
|
76
|
+
workingTreePaths: 0,
|
|
77
|
+
deletedPaths: 0,
|
|
78
|
+
untrackedPaths: 0,
|
|
79
|
+
};
|
|
80
|
+
function atomicJson(target, value) {
|
|
81
|
+
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
82
|
+
const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
|
83
|
+
fs.writeFileSync(temporary, `${JSON.stringify(value)}\n`, { flag: "wx", mode: 0o600 });
|
|
84
|
+
const descriptor = fs.openSync(temporary, "r");
|
|
85
|
+
try {
|
|
86
|
+
fs.fsyncSync(descriptor);
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
fs.closeSync(descriptor);
|
|
90
|
+
}
|
|
91
|
+
fs.renameSync(temporary, target);
|
|
92
|
+
}
|
|
93
|
+
export function readSharedIndexProvenance(repo) {
|
|
94
|
+
// The active database is authoritative. A sidecar from an older restore must
|
|
95
|
+
// never survive a later local rebuild as misleading current provenance.
|
|
96
|
+
const databasePath = resolveDbPath(repo);
|
|
97
|
+
if (!fs.existsSync(databasePath))
|
|
98
|
+
return null;
|
|
99
|
+
try {
|
|
100
|
+
const db = new Database(databasePath, { readonly: true });
|
|
101
|
+
try {
|
|
102
|
+
const get = (key) => db.query("SELECT value FROM meta WHERE key = ?").get(key)
|
|
103
|
+
?.value ?? null;
|
|
104
|
+
const registrationId = get("sharedIndexRegistrationId");
|
|
105
|
+
const baseCommit = get("sharedIndexBaseCommit");
|
|
106
|
+
const manifestDigest = get("sharedIndexManifestDigest");
|
|
107
|
+
if (!registrationId || !baseCommit || !manifestDigest)
|
|
108
|
+
return null;
|
|
109
|
+
return sharedIndexProvenanceSchema.parse({
|
|
110
|
+
schemaVersion: 1,
|
|
111
|
+
enabled: true,
|
|
112
|
+
state: "restored-and-verified",
|
|
113
|
+
registrationId,
|
|
114
|
+
baseCommit,
|
|
115
|
+
manifestDigest,
|
|
116
|
+
source: "shared-snapshot",
|
|
117
|
+
cache: get("sharedIndexCache") ?? "unavailable",
|
|
118
|
+
overlay: JSON.parse(get("sharedIndexOverlay") ?? "null"),
|
|
119
|
+
effectiveTreeFingerprint: get("sharedIndexEffectiveTreeFingerprint") ?? undefined,
|
|
120
|
+
restoredAt: get("sharedIndexRestoredAt") ?? undefined,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
db.close();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
export function writeSharedIndexProvenance(repo, value) {
|
|
132
|
+
atomicJson(path.join(resolveStateDir(repo), "shared-index-provenance.json"), sharedIndexProvenanceSchema.parse(value));
|
|
133
|
+
}
|
|
134
|
+
export function readSharedRestoreAttempt(repo) {
|
|
135
|
+
try {
|
|
136
|
+
const value = JSON.parse(fs.readFileSync(path.join(resolveStateDir(repo), "shared-index-last-attempt.json"), "utf8"));
|
|
137
|
+
if (value.schemaVersion !== 1 ||
|
|
138
|
+
(value.outcome !== "restored" && value.outcome !== "failed") ||
|
|
139
|
+
!Number.isFinite(Date.parse(value.attemptedAt)))
|
|
140
|
+
return null;
|
|
141
|
+
return value;
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
export function writeSharedRestoreAttempt(repo, value) {
|
|
148
|
+
atomicJson(path.join(resolveStateDir(repo), "shared-index-last-attempt.json"), value);
|
|
149
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import child_process from "node:child_process";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { Transform } from "node:stream";
|
|
6
|
+
import { pipeline } from "node:stream/promises";
|
|
7
|
+
import { createZstdCompress } from "node:zlib";
|
|
8
|
+
import { Database } from "../engine/sqlite.js";
|
|
9
|
+
import { resolveDbPath } from "../engine/state-paths.js";
|
|
10
|
+
import { currentSharedIndexCompatibility } from "./compatibility.js";
|
|
11
|
+
import { canonicalManifestBytes, SHARED_INDEX_LIMITS, sha256Hex, sharedBranchMatches, sharedRepositoryPrefix, } from "./contract.js";
|
|
12
|
+
import { branchPointerKey, compatibilityFingerprint, signatureKey, validatePointerBinding, verifyManifest, } from "./manifest.js";
|
|
13
|
+
import { SharedIndexError } from "./s3-client.js";
|
|
14
|
+
import { parseBranchPointer, parseManifest } from "./schemas.js";
|
|
15
|
+
/** Stamp the exact compatibility fingerprint into the checkpointed publisher DB. */
|
|
16
|
+
export function prepareSharedPublisherDatabase(repo) {
|
|
17
|
+
const databasePath = resolveDbPath(repo);
|
|
18
|
+
if (!fs.existsSync(databasePath))
|
|
19
|
+
throw new SharedIndexError("configuration", "shared-index publisher database is missing");
|
|
20
|
+
const compatibility = currentSharedIndexCompatibility();
|
|
21
|
+
const inputFingerprint = compatibilityFingerprint(compatibility);
|
|
22
|
+
const db = new Database(databasePath);
|
|
23
|
+
try {
|
|
24
|
+
const commit = db.query("SELECT value FROM meta WHERE key='lastIndexedHead'").get()
|
|
25
|
+
?.value ?? "";
|
|
26
|
+
if (!/^[a-f0-9]{40,64}$/.test(commit))
|
|
27
|
+
throw new SharedIndexError("integrity", "shared-index publisher database has no valid indexed commit");
|
|
28
|
+
const schema = db
|
|
29
|
+
.query("PRAGMA user_version")
|
|
30
|
+
.get()?.user_version;
|
|
31
|
+
if (schema !== compatibility.databaseSchemaVersion)
|
|
32
|
+
throw new SharedIndexError("compatibility", "shared-index publisher database schema is incompatible");
|
|
33
|
+
db.run("INSERT INTO meta(key, value) VALUES ('sharedIndexInputFingerprint', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", [inputFingerprint]);
|
|
34
|
+
db.run("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
35
|
+
return { schemaVersion: 1, compatibility, inputFingerprint, commit };
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
db.close();
|
|
39
|
+
for (const suffix of ["-wal", "-shm"])
|
|
40
|
+
fs.rmSync(`${databasePath}${suffix}`, { force: true });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function safeOutputDirectory(target) {
|
|
44
|
+
const resolved = path.resolve(target);
|
|
45
|
+
if (fs.existsSync(resolved)) {
|
|
46
|
+
const stat = fs.lstatSync(resolved);
|
|
47
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || fs.readdirSync(resolved).length > 0)
|
|
48
|
+
throw new SharedIndexError("configuration", "shared-index publisher output must be a new or empty non-symlink directory");
|
|
49
|
+
}
|
|
50
|
+
else
|
|
51
|
+
fs.mkdirSync(resolved, { recursive: true, mode: 0o700 });
|
|
52
|
+
fs.chmodSync(resolved, 0o700);
|
|
53
|
+
return resolved;
|
|
54
|
+
}
|
|
55
|
+
function countingHash(hash, count) {
|
|
56
|
+
return new Transform({
|
|
57
|
+
transform(chunk, _encoding, callback) {
|
|
58
|
+
count.bytes += chunk.byteLength;
|
|
59
|
+
hash.update(chunk);
|
|
60
|
+
callback(null, chunk);
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
export async function createSharedPublisherBundle(options) {
|
|
65
|
+
if (!options.config.enabled)
|
|
66
|
+
throw new SharedIndexError("configuration", "shared-index publishing is disabled");
|
|
67
|
+
if (!options.config.branches.some((pattern) => sharedBranchMatches(pattern, options.branch)))
|
|
68
|
+
throw new SharedIndexError("configuration", "shared-index publishing branch is not configured");
|
|
69
|
+
const metadata = prepareSharedPublisherDatabase(options.repo);
|
|
70
|
+
const head = child_process
|
|
71
|
+
.execFileSync("git", ["rev-parse", "--verify", "HEAD"], {
|
|
72
|
+
cwd: options.repo,
|
|
73
|
+
encoding: "utf8",
|
|
74
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
75
|
+
})
|
|
76
|
+
.trim();
|
|
77
|
+
if (head !== metadata.commit)
|
|
78
|
+
throw new SharedIndexError("integrity", "shared-index publisher graph does not match repository HEAD");
|
|
79
|
+
const output = safeOutputDirectory(options.outputDirectory);
|
|
80
|
+
const graphFile = path.join(output, "graph.sqlite.zst");
|
|
81
|
+
const manifestFile = path.join(output, "manifest.json");
|
|
82
|
+
const signatureFile = path.join(output, "manifest.sig");
|
|
83
|
+
const pointerFile = path.join(output, "branch-pointer.json");
|
|
84
|
+
const databasePath = resolveDbPath(options.repo);
|
|
85
|
+
const uncompressedHash = crypto.createHash("sha256");
|
|
86
|
+
const compressedHash = crypto.createHash("sha256");
|
|
87
|
+
const uncompressed = { bytes: 0 };
|
|
88
|
+
const compressed = { bytes: 0 };
|
|
89
|
+
await pipeline(fs.createReadStream(databasePath), countingHash(uncompressedHash, uncompressed), createZstdCompress(), countingHash(compressedHash, compressed), fs.createWriteStream(graphFile, { flags: "wx", mode: 0o600 }));
|
|
90
|
+
if (uncompressed.bytes <= 0 ||
|
|
91
|
+
uncompressed.bytes > SHARED_INDEX_LIMITS.uncompressedBytes ||
|
|
92
|
+
compressed.bytes <= 0 ||
|
|
93
|
+
compressed.bytes > SHARED_INDEX_LIMITS.compressedBytes)
|
|
94
|
+
throw new SharedIndexError("integrity", "shared-index publisher graph exceeds contract bounds");
|
|
95
|
+
const immutable = `${sharedRepositoryPrefix(options.config.repository)}snapshots/${metadata.commit}/${metadata.inputFingerprint}`;
|
|
96
|
+
const graphKey = `${immutable}/graph.sqlite.zst`;
|
|
97
|
+
const manifestKey = `${immutable}/manifest.json`;
|
|
98
|
+
const manifest = {
|
|
99
|
+
schemaVersion: 1,
|
|
100
|
+
registrationId: options.config.registrationId,
|
|
101
|
+
repository: options.config.repository,
|
|
102
|
+
commit: metadata.commit,
|
|
103
|
+
generatedAt: new Date().toISOString(),
|
|
104
|
+
compatibility: metadata.compatibility,
|
|
105
|
+
graph: {
|
|
106
|
+
key: graphKey,
|
|
107
|
+
compressedSha256: compressedHash.digest("hex"),
|
|
108
|
+
compressedBytes: compressed.bytes,
|
|
109
|
+
uncompressedSha256: uncompressedHash.digest("hex"),
|
|
110
|
+
uncompressedBytes: uncompressed.bytes,
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
const manifestBytes = canonicalManifestBytes(manifest);
|
|
114
|
+
fs.writeFileSync(manifestFile, manifestBytes, { flag: "wx", mode: 0o600 });
|
|
115
|
+
const bundle = {
|
|
116
|
+
schemaVersion: 1,
|
|
117
|
+
branch: options.branch,
|
|
118
|
+
commit: metadata.commit,
|
|
119
|
+
inputFingerprint: metadata.inputFingerprint,
|
|
120
|
+
manifestKey,
|
|
121
|
+
manifestSha256: sha256Hex(manifestBytes),
|
|
122
|
+
signatureKey: signatureKey(manifestKey),
|
|
123
|
+
pointerKey: branchPointerKey(options.config, options.branch),
|
|
124
|
+
graphKey,
|
|
125
|
+
region: options.config.aws.region,
|
|
126
|
+
bucket: options.config.aws.bucket,
|
|
127
|
+
keyId: options.config.signing.keyId,
|
|
128
|
+
files: {
|
|
129
|
+
graph: graphFile,
|
|
130
|
+
manifest: manifestFile,
|
|
131
|
+
signature: signatureFile,
|
|
132
|
+
pointer: pointerFile,
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
fs.writeFileSync(path.join(output, "bundle.json"), `${JSON.stringify(bundle)}\n`, {
|
|
136
|
+
flag: "wx",
|
|
137
|
+
mode: 0o600,
|
|
138
|
+
});
|
|
139
|
+
return bundle;
|
|
140
|
+
}
|
|
141
|
+
export function finalizeSharedPublisherBundle(options) {
|
|
142
|
+
const bundle = options.bundle;
|
|
143
|
+
const directory = path.dirname(path.resolve(bundle.files?.manifest ?? ""));
|
|
144
|
+
const expectedFiles = {
|
|
145
|
+
graph: "graph.sqlite.zst",
|
|
146
|
+
manifest: "manifest.json",
|
|
147
|
+
signature: "manifest.sig",
|
|
148
|
+
pointer: "branch-pointer.json",
|
|
149
|
+
};
|
|
150
|
+
if (bundle.schemaVersion !== 1 ||
|
|
151
|
+
!/^[a-f0-9]{40,64}$/.test(bundle.commit) ||
|
|
152
|
+
!/^[a-f0-9]{64}$/.test(bundle.inputFingerprint) ||
|
|
153
|
+
!/^[a-f0-9]{64}$/.test(bundle.manifestSha256) ||
|
|
154
|
+
bundle.region !== options.config.aws.region ||
|
|
155
|
+
bundle.bucket !== options.config.aws.bucket ||
|
|
156
|
+
bundle.keyId !== options.config.signing.keyId ||
|
|
157
|
+
Object.entries(expectedFiles).some(([name, filename]) => path.resolve(bundle.files[name] ?? "") !==
|
|
158
|
+
path.join(directory, filename)))
|
|
159
|
+
throw new SharedIndexError("integrity", "shared-index publisher bundle descriptor is invalid");
|
|
160
|
+
const manifestBytes = fs.readFileSync(options.bundle.files.manifest);
|
|
161
|
+
const signature = fs.readFileSync(options.bundle.files.signature);
|
|
162
|
+
const manifest = parseManifest(manifestBytes);
|
|
163
|
+
if (!options.config.branches.some((pattern) => sharedBranchMatches(pattern, bundle.branch)) ||
|
|
164
|
+
bundle.graphKey !== manifest.graph.key ||
|
|
165
|
+
bundle.signatureKey !== signatureKey(bundle.manifestKey) ||
|
|
166
|
+
sha256Hex(manifestBytes) !== bundle.manifestSha256)
|
|
167
|
+
throw new SharedIndexError("integrity", "shared-index publisher bundle bindings are invalid");
|
|
168
|
+
const pointer = parseBranchPointer(canonicalManifestBytes({
|
|
169
|
+
schemaVersion: 1,
|
|
170
|
+
registrationId: options.config.registrationId,
|
|
171
|
+
repository: options.config.repository,
|
|
172
|
+
branch: options.bundle.branch,
|
|
173
|
+
commit: options.bundle.commit,
|
|
174
|
+
inputFingerprint: options.bundle.inputFingerprint,
|
|
175
|
+
manifestKey: options.bundle.manifestKey,
|
|
176
|
+
manifestSha256: options.bundle.manifestSha256,
|
|
177
|
+
updatedAt: new Date().toISOString(),
|
|
178
|
+
}));
|
|
179
|
+
validatePointerBinding(options.config, pointer);
|
|
180
|
+
verifyManifest(options.config, pointer, manifestBytes, manifest, signature, manifest.compatibility);
|
|
181
|
+
const pointerBytes = canonicalManifestBytes(pointer);
|
|
182
|
+
fs.writeFileSync(options.bundle.files.pointer, pointerBytes, { flag: "wx", mode: 0o600 });
|
|
183
|
+
return pointerBytes;
|
|
184
|
+
}
|
|
185
|
+
export function guardPublisherPointer(options) {
|
|
186
|
+
const pointer = parseBranchPointer(options.existingPointerBytes);
|
|
187
|
+
if (pointer.branch !== options.branch)
|
|
188
|
+
throw new SharedIndexError("integrity", "shared-index existing pointer branch is invalid");
|
|
189
|
+
validatePointerBinding(options.config, pointer);
|
|
190
|
+
if (pointer.commit === options.newCommit)
|
|
191
|
+
return;
|
|
192
|
+
const relation = child_process.spawnSync("git", ["merge-base", "--is-ancestor", pointer.commit, options.newCommit], { cwd: options.repo, stdio: "ignore" });
|
|
193
|
+
if (relation.status !== 0)
|
|
194
|
+
throw new SharedIndexError("compatibility", "shared-index publisher refuses a backward or diverged pointer update");
|
|
195
|
+
}
|
|
196
|
+
export function publisherPointerKey(config, branch) {
|
|
197
|
+
return branchPointerKey(config, branch);
|
|
198
|
+
}
|