@intx/hub-sessions 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +82 -1
- package/dist/agent-repo.d.ts +93 -0
- package/dist/agent-repo.js +109 -0
- package/dist/agent-state-kind.d.ts +12 -0
- package/dist/agent-state-kind.js +189 -0
- package/dist/asset-service.d.ts +104 -0
- package/dist/asset-service.js +267 -0
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +33 -0
- package/dist/credential-push.js +109 -0
- package/dist/event-collector-registry.d.ts +20 -0
- package/dist/event-collector-registry.js +115 -0
- package/dist/event-collector.d.ts +39 -0
- package/dist/event-collector.js +365 -0
- package/dist/hub-session-lookups.d.ts +135 -0
- package/dist/hub-session-lookups.js +663 -0
- package/dist/hub-session-orchestrator.d.ts +25 -0
- package/dist/hub-session-orchestrator.js +87 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.js +24 -0
- package/dist/package-registry-kind.d.ts +70 -0
- package/dist/package-registry-kind.js +260 -0
- package/dist/repo-store/index.d.ts +4 -0
- package/dist/repo-store/index.js +3 -0
- package/dist/repo-store/store.d.ts +41 -0
- package/dist/repo-store/store.js +1829 -0
- package/dist/repo-store/subscribe-kind.d.ts +56 -0
- package/dist/repo-store/subscribe-kind.js +144 -0
- package/dist/repo-store/types.d.ts +571 -0
- package/dist/repo-store/types.js +42 -0
- package/dist/session-service.d.ts +416 -0
- package/dist/session-service.js +1191 -0
- package/dist/sidecar-allocation/contracts.d.ts +78 -0
- package/dist/sidecar-allocation/contracts.js +21 -0
- package/dist/sidecar-allocation/index.d.ts +4 -0
- package/dist/sidecar-allocation/index.js +3 -0
- package/dist/sidecar-allocation/placement-policy.d.ts +11 -0
- package/dist/sidecar-allocation/placement-policy.js +21 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +11 -0
- package/dist/sidecar-allocation/plugin-registry.js +37 -0
- package/dist/sidecar-allocation/reconciler.d.ts +42 -0
- package/dist/sidecar-allocation/reconciler.js +431 -0
- package/dist/skill-kind.d.ts +41 -0
- package/dist/skill-kind.js +292 -0
- package/dist/substrate.d.ts +8 -0
- package/dist/substrate.js +21 -0
- package/dist/workflow-allocation-service.d.ts +58 -0
- package/dist/workflow-allocation-service.js +239 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +40 -0
- package/dist/workflow-dispatch-service.js +146 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +37 -0
- package/dist/workflow-kind.js +310 -0
- package/dist/workflow-probe-gate.d.ts +214 -0
- package/dist/workflow-probe-gate.js +207 -0
- package/dist/workflow-run-event-log.d.ts +21 -0
- package/dist/workflow-run-event-log.js +51 -0
- package/dist/workflow-run-kind.d.ts +440 -0
- package/dist/workflow-run-kind.js +2916 -0
- package/dist/workflow-run-reader.d.ts +47 -0
- package/dist/workflow-run-reader.js +153 -0
- package/dist/workflow-run-restore.d.ts +15 -0
- package/dist/workflow-run-restore.js +26 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/ws/index.d.ts +3 -0
- package/dist/ws/index.js +3 -0
- package/dist/ws/sidecar-events.d.ts +222 -0
- package/dist/ws/sidecar-events.js +72 -0
- package/dist/ws/sidecar-handler.d.ts +305 -0
- package/dist/ws/sidecar-handler.js +2537 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +17 -0
- package/dist/ws/sidecar-token-authenticator.js +81 -0
- package/package.json +35 -12
- package/src/agent-repo.test.ts +0 -310
- package/src/agent-repo.ts +0 -165
- package/src/agent-state-kind.test.ts +0 -247
- package/src/agent-state-kind.ts +0 -204
- package/src/asset-service.test.ts +0 -540
- package/src/asset-service.ts +0 -378
- package/src/available-skills-stanza.test.ts +0 -87
- package/src/available-skills-stanza.ts +0 -47
- package/src/credential-push.ts +0 -65
- package/src/event-collector-registry.test.ts +0 -73
- package/src/event-collector-registry.ts +0 -171
- package/src/event-collector.test.ts +0 -1387
- package/src/event-collector.ts +0 -424
- package/src/hub-session-lookups.ts +0 -206
- package/src/hub-session-orchestrator.test.ts +0 -510
- package/src/hub-session-orchestrator.ts +0 -213
- package/src/index.ts +0 -78
- package/src/repo-store/index.ts +0 -15
- package/src/repo-store/store.test.ts +0 -1169
- package/src/repo-store/store.ts +0 -428
- package/src/repo-store/types.ts +0 -253
- package/src/session-service.test.ts +0 -895
- package/src/session-service.ts +0 -464
- package/src/skill-kind.test.ts +0 -599
- package/src/skill-kind.ts +0 -350
- package/src/ws/index.ts +0 -18
- package/src/ws/sidecar-events.test.ts +0 -96
- package/src/ws/sidecar-events.ts +0 -231
- package/src/ws/sidecar-handler.test.ts +0 -2217
- package/src/ws/sidecar-handler.ts +0 -1574
- package/tsconfig.json +0 -4
- package/tsconfig.tsbuildinfo +0 -1
|
@@ -0,0 +1,1829 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import git, {} from "isomorphic-git";
|
|
4
|
+
import { createSSHSignature } from "@intx/crypto";
|
|
5
|
+
import { initRepo as storageInitRepo, createDeployPack, receivePackObjects, collectReachableObjects, maybeGC, } from "@intx/storage-isogit/node";
|
|
6
|
+
import { hasCode } from "@intx/types";
|
|
7
|
+
import { getLogger } from "@intx/log";
|
|
8
|
+
import { SAFE_REPO_ID } from "./types.js";
|
|
9
|
+
const DEFAULT_SUBSCRIBE_BUFFER_LIMIT = 1024;
|
|
10
|
+
const AUTHOR = {
|
|
11
|
+
name: "interchange-hub",
|
|
12
|
+
email: "hub@interchange.local",
|
|
13
|
+
};
|
|
14
|
+
const logger = getLogger(["hub", "repo-store"]);
|
|
15
|
+
/**
|
|
16
|
+
* In-process push-serialization lock. Keyed by `${kind}/${id}`. Each
|
|
17
|
+
* entry holds the tail of the chain of in-flight critical sections for
|
|
18
|
+
* that repo; the next acquirer awaits the current tail and replaces it
|
|
19
|
+
* with its own pending completion. On release the tail-check
|
|
20
|
+
* (`if (locks.get(key) === myTail) locks.delete(key)`) prevents the map
|
|
21
|
+
* from leaking entries once the chain drains.
|
|
22
|
+
*
|
|
23
|
+
* Single-process assumption: this protects against concurrent operations
|
|
24
|
+
* inside a single hub instance. Cross-process serialization (e.g. a
|
|
25
|
+
* second hub replica or an external git client touching the same
|
|
26
|
+
* on-disk repo) would need a filesystem-backed lock; the migration path
|
|
27
|
+
* is to swap the body of `withRepoLock` for an FS-lock acquire/release
|
|
28
|
+
* around the same critical section.
|
|
29
|
+
*/
|
|
30
|
+
const locks = new Map();
|
|
31
|
+
async function withRepoLock(repoId, fn) {
|
|
32
|
+
const key = `${repoId.kind}/${repoId.id}`;
|
|
33
|
+
const previous = locks.get(key) ?? Promise.resolve();
|
|
34
|
+
let releaseFn = () => undefined;
|
|
35
|
+
const tail = new Promise((resolve) => {
|
|
36
|
+
releaseFn = resolve;
|
|
37
|
+
});
|
|
38
|
+
locks.set(key, tail);
|
|
39
|
+
try {
|
|
40
|
+
await previous;
|
|
41
|
+
return await fn();
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
if (locks.get(key) === tail) {
|
|
45
|
+
locks.delete(key);
|
|
46
|
+
}
|
|
47
|
+
releaseFn();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export function createRepoStore(config) {
|
|
51
|
+
const { dataDir, signingKey, handlers, authorize, signingCallback, gc } = config;
|
|
52
|
+
// Per-repo-directory isomorphic-git memoization cache. The store
|
|
53
|
+
// threads that dir's cache through every index-touching and
|
|
54
|
+
// object-reading git.* call, so isomorphic-git reuses the parsed
|
|
55
|
+
// on-disk index across the repo's serialized write sequence instead
|
|
56
|
+
// of re-reading and re-parsing it on every git.add / remove /
|
|
57
|
+
// updateIndex / commit / listFiles, and reuses parsed packfile
|
|
58
|
+
// indexes across object reads. The index-free tree assembly
|
|
59
|
+
// (git.writeTree / git.writeBlob) uses no index and threads no cache.
|
|
60
|
+
// The store is the single writer under withRepoLock, so a long-lived
|
|
61
|
+
// per-repo cache never races a concurrent mutator.
|
|
62
|
+
//
|
|
63
|
+
// The cache stays a pure accelerator, never a second source of truth,
|
|
64
|
+
// because the on-disk repo remains authoritative on every axis:
|
|
65
|
+
// isomorphic-git persists the index to disk on each dirty mutation and
|
|
66
|
+
// re-reads it on a stat mismatch; the object cache is OID-keyed
|
|
67
|
+
// (content-addressed, so it is never stale for a committed object);
|
|
68
|
+
// object reads enumerate pack files from disk on every call (so a GC
|
|
69
|
+
// repack's now-pruned packs are simply never consulted again); and
|
|
70
|
+
// refs are not cached at all — resolveRef and friends take no cache —
|
|
71
|
+
// so the cache can never serve a stale ref tip. Those same properties
|
|
72
|
+
// make it safe to drop a dir's cache at any instant: the next call
|
|
73
|
+
// just re-reads from disk.
|
|
74
|
+
//
|
|
75
|
+
// Two bounds keep a long-lived store from retaining parsed packfiles
|
|
76
|
+
// (and their pack bytes) without limit: a dir's cache is rebuilt once
|
|
77
|
+
// it has served GIT_CACHE_MAX_OPS calls, and the store holds at most
|
|
78
|
+
// GIT_CACHE_MAX_REPOS dir caches, evicting the least-recently-used.
|
|
79
|
+
// `invalidateGitCache` additionally drops a dir's cache after an
|
|
80
|
+
// out-of-band mutation that bypasses it — a received pack writes
|
|
81
|
+
// objects and advances a ref without threading this cache — so the
|
|
82
|
+
// next read rebuilds from the mutated repo.
|
|
83
|
+
const GIT_CACHE_MAX_OPS = 8192;
|
|
84
|
+
const GIT_CACHE_MAX_REPOS = 256;
|
|
85
|
+
const gitCaches = new Map();
|
|
86
|
+
function cacheFor(dir) {
|
|
87
|
+
let entry = gitCaches.get(dir);
|
|
88
|
+
if (entry === undefined) {
|
|
89
|
+
entry = { cache: {}, ops: 0 };
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
// Re-insert so this dir ranks most-recently-used for the LRU
|
|
93
|
+
// eviction below; rebuild once it has spent its op budget.
|
|
94
|
+
gitCaches.delete(dir);
|
|
95
|
+
if (entry.ops >= GIT_CACHE_MAX_OPS)
|
|
96
|
+
entry = { cache: {}, ops: 0 };
|
|
97
|
+
}
|
|
98
|
+
entry.ops += 1;
|
|
99
|
+
gitCaches.set(dir, entry);
|
|
100
|
+
while (gitCaches.size > GIT_CACHE_MAX_REPOS) {
|
|
101
|
+
const lru = gitCaches.keys().next().value;
|
|
102
|
+
if (lru === undefined)
|
|
103
|
+
break;
|
|
104
|
+
gitCaches.delete(lru);
|
|
105
|
+
}
|
|
106
|
+
return entry.cache;
|
|
107
|
+
}
|
|
108
|
+
function invalidateGitCache(dir) {
|
|
109
|
+
gitCaches.delete(dir);
|
|
110
|
+
}
|
|
111
|
+
// Per-(repoId, ref) seq cache. Value is the seq of the ref's
|
|
112
|
+
// current tip; the next commit on the ref gets `cached + 1`. The
|
|
113
|
+
// cache is populated lazily — on each ref update we either bump
|
|
114
|
+
// the cached value or recompute it from `git.log` walk. Cleared
|
|
115
|
+
// on any failure inside the update path so a half-applied state
|
|
116
|
+
// never poisons future reads.
|
|
117
|
+
const seqCache = new Map();
|
|
118
|
+
// Per-(repoId, ref) subscriber set. Each subscriber holds its own
|
|
119
|
+
// buffer, kind filter, and waiter callback so concurrent
|
|
120
|
+
// subscribers do not interfere with each other.
|
|
121
|
+
const subscribers = new Map();
|
|
122
|
+
// Repo-scoped cache key shared by the per-repoId caches below
|
|
123
|
+
// (existing-commit set, and any future per-repo bookkeeping).
|
|
124
|
+
function indexCacheKey(repoId) {
|
|
125
|
+
return `${repoId.kind}/${repoId.id}`;
|
|
126
|
+
}
|
|
127
|
+
// Per-commit reachable-object cache. createPack for a workflow-run
|
|
128
|
+
// ref walks the first-parent chain and unions every commit's
|
|
129
|
+
// reachable objects so the receiver gets the full history needed to
|
|
130
|
+
// validate per-commit transitions. Without this cache the walk
|
|
131
|
+
// recomputes the reachable set for every ancestor on every push, so
|
|
132
|
+
// a long-running deployment's createPack cost scales as O(N^2) in
|
|
133
|
+
// the number of commits.
|
|
134
|
+
const chainReachabilityCache = new Map();
|
|
135
|
+
// Per-repoId cache of "every commit OID currently reachable from
|
|
136
|
+
// any branch or tag in the repo". The substrate uses this in
|
|
137
|
+
// `receivePack` to skip per-commit validation for commits the
|
|
138
|
+
// receiver already had before the pack arrived. Computing the set
|
|
139
|
+
// fresh on every receive scans every ref's parent chain — for a
|
|
140
|
+
// long-running workflow-run repo this scales with history depth.
|
|
141
|
+
// Caching it as a flat set and incrementally extending it on each
|
|
142
|
+
// ref update lets receivePack pay O(new commits) instead of
|
|
143
|
+
// O(history) per call. The cache is initialised lazily on first
|
|
144
|
+
// receivePack to avoid the cold-start scan for repos that only
|
|
145
|
+
// ever write via writeTree.
|
|
146
|
+
const existingCommitsCache = new Map();
|
|
147
|
+
// Per-(repoId, ref) cache of "the commit OID the receiver has acked
|
|
148
|
+
// for a workflow-run pack". After the first push lands a ref's tip on
|
|
149
|
+
// the receiver, every subsequent push only needs to carry the commits
|
|
150
|
+
// added since then — the parent chain older than this tip is already
|
|
151
|
+
// on the receiver, so re-shipping it wastes pack bytes and inflates
|
|
152
|
+
// `receivePack` time per push. The cursor advances in
|
|
153
|
+
// `commitPackedTip`, which the push pipeline calls once the receiver
|
|
154
|
+
// acks the transfer — NOT when `createPack` builds the pack. Advancing
|
|
155
|
+
// on build would strand the receiver whenever a transfer is cancelled
|
|
156
|
+
// before its ack (a reconnect drops the in-flight push): the next
|
|
157
|
+
// rebuild would walk back only to the un-acked commit and omit it, so
|
|
158
|
+
// the receiver would reject the pack with a dangling parent forever.
|
|
159
|
+
// Gating the advance on the ack keeps a cancelled transfer cleanly
|
|
160
|
+
// re-shippable. The cache key is `${repoId.kind}/${repoId.id}/${ref}`
|
|
161
|
+
// so writes against different refs on the same repo (events vs the
|
|
162
|
+
// workflow-run ref) do not interfere.
|
|
163
|
+
const lastPackedTip = new Map();
|
|
164
|
+
function lastPackedTipKey(repoId, ref) {
|
|
165
|
+
return `${repoId.kind}/${repoId.id}/${ref}`;
|
|
166
|
+
}
|
|
167
|
+
function refKey(repoId, ref) {
|
|
168
|
+
return `${repoId.kind}/${repoId.id}/${ref}`;
|
|
169
|
+
}
|
|
170
|
+
// Count commits reachable from `ref`. Returns 0 when the ref does
|
|
171
|
+
// not yet exist. `git.log` returns newest-first; the count gives
|
|
172
|
+
// the seq the next commit would land at if added now (the current
|
|
173
|
+
// tip's seq is `count - 1`).
|
|
174
|
+
async function countCommits(dir, ref) {
|
|
175
|
+
try {
|
|
176
|
+
const entries = await git.log({ fs, dir, cache: cacheFor(dir), ref });
|
|
177
|
+
return entries.length;
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
181
|
+
return 0;
|
|
182
|
+
throw err;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Walk the ref's commit history oldest-first, assigning seq 0 to
|
|
186
|
+
// the root commit and incrementing toward HEAD. Each entry's
|
|
187
|
+
// `event` is the substrate-level commit descriptor — same shape
|
|
188
|
+
// the live path emits, so replay and live are vocabulary-identical.
|
|
189
|
+
async function replayHistory(dir, ref) {
|
|
190
|
+
let entries;
|
|
191
|
+
try {
|
|
192
|
+
entries = await git.log({ fs, dir, cache: cacheFor(dir), ref });
|
|
193
|
+
}
|
|
194
|
+
catch (err) {
|
|
195
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
196
|
+
return [];
|
|
197
|
+
throw err;
|
|
198
|
+
}
|
|
199
|
+
const reversed = [...entries].reverse();
|
|
200
|
+
const out = [];
|
|
201
|
+
let prev = null;
|
|
202
|
+
for (let i = 0; i < reversed.length; i++) {
|
|
203
|
+
const entry = reversed[i];
|
|
204
|
+
if (entry === undefined)
|
|
205
|
+
throw new Error("unreachable");
|
|
206
|
+
const event = {
|
|
207
|
+
type: "ref.updated",
|
|
208
|
+
ref,
|
|
209
|
+
oldSha: prev,
|
|
210
|
+
newSha: entry.oid,
|
|
211
|
+
};
|
|
212
|
+
out.push({ seq: i, event });
|
|
213
|
+
prev = entry.oid;
|
|
214
|
+
}
|
|
215
|
+
return out;
|
|
216
|
+
}
|
|
217
|
+
function deliverToSubscriber(sub, entry) {
|
|
218
|
+
if (sub.closed)
|
|
219
|
+
return;
|
|
220
|
+
if (sub.waiter !== null) {
|
|
221
|
+
const w = sub.waiter;
|
|
222
|
+
sub.waiter = null;
|
|
223
|
+
w({ value: entry, done: false });
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
if (sub.buffer.length >= sub.bufferLimit) {
|
|
227
|
+
sub.error = new Error(`subscribe_buffer_overrun: subscriber exceeded bufferLimit=${String(sub.bufferLimit)}`);
|
|
228
|
+
sub.closed = true;
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
sub.buffer.push(entry);
|
|
232
|
+
}
|
|
233
|
+
// Called from inside the per-repo lock immediately after a
|
|
234
|
+
// successful ref update. Computes the new tip's seq (either by
|
|
235
|
+
// bumping the cached value or by walking the log), then fans the
|
|
236
|
+
// event out to every subscriber registered for this (repoId, ref).
|
|
237
|
+
// Errors raised by individual subscriber delivery (e.g. buffer
|
|
238
|
+
// overrun) are captured on the subscriber's state so the
|
|
239
|
+
// ref-update path itself is never destabilised by a slow consumer.
|
|
240
|
+
async function emitRefUpdate(repoId, ref, oldSha, newSha) {
|
|
241
|
+
const key = refKey(repoId, ref);
|
|
242
|
+
let seq;
|
|
243
|
+
const cached = seqCache.get(key);
|
|
244
|
+
if (cached !== undefined) {
|
|
245
|
+
seq = cached + 1;
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
const dir = repoDir(repoId);
|
|
249
|
+
const count = await countCommits(dir, ref);
|
|
250
|
+
// `count` is the number of commits including the one we just
|
|
251
|
+
// produced. The tip's seq is `count - 1`.
|
|
252
|
+
seq = Math.max(0, count - 1);
|
|
253
|
+
}
|
|
254
|
+
seqCache.set(key, seq);
|
|
255
|
+
const event = {
|
|
256
|
+
type: "ref.updated",
|
|
257
|
+
ref,
|
|
258
|
+
oldSha,
|
|
259
|
+
newSha,
|
|
260
|
+
};
|
|
261
|
+
const entry = { seq, event };
|
|
262
|
+
const set = subscribers.get(key);
|
|
263
|
+
if (set === undefined)
|
|
264
|
+
return;
|
|
265
|
+
for (const sub of set)
|
|
266
|
+
deliverToSubscriber(sub, entry);
|
|
267
|
+
}
|
|
268
|
+
function handlerFor(repoId) {
|
|
269
|
+
const handler = handlers[repoId.kind];
|
|
270
|
+
if (handler === undefined) {
|
|
271
|
+
throw new Error(`no handler registered for kind: ${repoId.kind}`);
|
|
272
|
+
}
|
|
273
|
+
return handler;
|
|
274
|
+
}
|
|
275
|
+
function signerFor(repoId) {
|
|
276
|
+
return signingCallback === undefined ? undefined : signingCallback(repoId);
|
|
277
|
+
}
|
|
278
|
+
function repoDir(repoId) {
|
|
279
|
+
if (!SAFE_REPO_ID.test(repoId.id)) {
|
|
280
|
+
throw new Error(`repo_id_invalid: ${repoId.id}`);
|
|
281
|
+
}
|
|
282
|
+
const handler = handlerFor(repoId);
|
|
283
|
+
return path.join(dataDir, handler.directoryPrefix, repoId.id);
|
|
284
|
+
}
|
|
285
|
+
// Write-path reclaim. Called at the end of every successful write for an
|
|
286
|
+
// allowlisted kind, from inside the substrate's `withRepoLock`. The shared
|
|
287
|
+
// evaluator acquires the storage per-directory lock, applies the policy
|
|
288
|
+
// (reclaim over threshold, warn over byte budget), and logs rather than
|
|
289
|
+
// propagates a reclaim failure — the triggering write has already
|
|
290
|
+
// committed durably, so failing the caller would misreport it.
|
|
291
|
+
async function maybeRunGC(repoId) {
|
|
292
|
+
if (gc === undefined || !gc.kinds.includes(repoId.kind))
|
|
293
|
+
return;
|
|
294
|
+
await maybeGC(repoDir(repoId), gc);
|
|
295
|
+
}
|
|
296
|
+
function gateAccess(principal, repoId, ref, action) {
|
|
297
|
+
const verdict = authorize(principal, repoId, ref, action);
|
|
298
|
+
if (!verdict.allowed) {
|
|
299
|
+
throw new Error(`authorize_denied: ${verdict.reason}`);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
function validateClearPrefix(clearPrefix) {
|
|
303
|
+
const malformed = clearPrefix.length === 0 ||
|
|
304
|
+
!clearPrefix.endsWith("/") ||
|
|
305
|
+
clearPrefix.startsWith("/") ||
|
|
306
|
+
clearPrefix.split("/").includes("..");
|
|
307
|
+
if (malformed) {
|
|
308
|
+
throw new Error(`clear_prefix_invalid: ${clearPrefix}`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
// Validate a delta write path. A put is always a file (no trailing
|
|
312
|
+
// slash); a delete is either an exact file or a subtree prefix
|
|
313
|
+
// (trailing slash allowed). Both reject empties, absolute paths, and
|
|
314
|
+
// any `..` traversal segment.
|
|
315
|
+
function validateDeltaPath(p, isDelete) {
|
|
316
|
+
const malformed = p.length === 0 ||
|
|
317
|
+
p.startsWith("/") ||
|
|
318
|
+
p.split("/").includes("..") ||
|
|
319
|
+
(!isDelete && p.endsWith("/"));
|
|
320
|
+
if (malformed) {
|
|
321
|
+
throw new Error(`delta_path_invalid: ${JSON.stringify(p)}`);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
// Reject an ambiguous delta rather than silently pick a winner: a path
|
|
325
|
+
// that is both put and deleted, or a put that lands under a
|
|
326
|
+
// subtree-prefix delete. assembleTree would let the put win in both
|
|
327
|
+
// cases; making the contradiction loud keeps a caller from committing a
|
|
328
|
+
// tree that does not match its stated intent.
|
|
329
|
+
function assertDeltaUnambiguous(puts, deletes) {
|
|
330
|
+
const deleteSet = new Set(deletes);
|
|
331
|
+
const deletePrefixes = deletes.filter((d) => d.endsWith("/"));
|
|
332
|
+
for (const p of Object.keys(puts)) {
|
|
333
|
+
if (deleteSet.has(p)) {
|
|
334
|
+
throw new Error(`delta_ambiguous: path ${JSON.stringify(p)} is in both puts and deletes`);
|
|
335
|
+
}
|
|
336
|
+
for (const dp of deletePrefixes) {
|
|
337
|
+
if (p.startsWith(dp)) {
|
|
338
|
+
throw new Error(`delta_ambiguous: put ${JSON.stringify(p)} lands under deleted subtree ${JSON.stringify(dp)}`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
// When a delta declares a change scope, every put and delete must fall
|
|
344
|
+
// under it — otherwise a handler that scopes validation to
|
|
345
|
+
// `changedPathPrefixes` would skip a region the write actually mutated,
|
|
346
|
+
// an exactly-once hole. An undefined scope means validate-all, so there
|
|
347
|
+
// is nothing to under-cover.
|
|
348
|
+
function assertDeltaScoped(puts, deletes, changedPathPrefixes) {
|
|
349
|
+
if (changedPathPrefixes === undefined)
|
|
350
|
+
return;
|
|
351
|
+
const covered = (p) => {
|
|
352
|
+
for (const prefix of changedPathPrefixes) {
|
|
353
|
+
if (p === prefix || p.startsWith(prefix))
|
|
354
|
+
return true;
|
|
355
|
+
}
|
|
356
|
+
return false;
|
|
357
|
+
};
|
|
358
|
+
for (const p of Object.keys(puts)) {
|
|
359
|
+
if (!covered(p)) {
|
|
360
|
+
throw new Error(`delta_out_of_scope: put ${JSON.stringify(p)} is not under any changedPathPrefix`);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
for (const d of deletes) {
|
|
364
|
+
if (!covered(d)) {
|
|
365
|
+
throw new Error(`delta_out_of_scope: delete ${JSON.stringify(d)} is not under any changedPathPrefix`);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
function storageOptsFor(repoId, opts) {
|
|
370
|
+
const out = {};
|
|
371
|
+
const signer = signerFor(repoId);
|
|
372
|
+
if (signer !== undefined)
|
|
373
|
+
out.signer = signer;
|
|
374
|
+
if (opts?.gitignore !== undefined)
|
|
375
|
+
out.gitignore = opts.gitignore;
|
|
376
|
+
return out;
|
|
377
|
+
}
|
|
378
|
+
async function initRepo(repoId, opts) {
|
|
379
|
+
await storageInitRepo(repoDir(repoId), storageOptsFor(repoId, opts));
|
|
380
|
+
}
|
|
381
|
+
function getRepoDir(repoId) {
|
|
382
|
+
return repoDir(repoId);
|
|
383
|
+
}
|
|
384
|
+
async function listRefs(principal, repoId) {
|
|
385
|
+
gateAccess(principal, repoId, "*", "resolveRef");
|
|
386
|
+
const dir = repoDir(repoId);
|
|
387
|
+
const repoExists = await fs.promises
|
|
388
|
+
.stat(path.join(dir, ".git"))
|
|
389
|
+
.then(() => true)
|
|
390
|
+
.catch(() => false);
|
|
391
|
+
if (!repoExists)
|
|
392
|
+
return [];
|
|
393
|
+
const [branches, tags] = await Promise.all([
|
|
394
|
+
git.listBranches({ fs, dir }),
|
|
395
|
+
git.listTags({ fs, dir }),
|
|
396
|
+
]);
|
|
397
|
+
const names = [];
|
|
398
|
+
for (const b of branches)
|
|
399
|
+
names.push(`refs/heads/${b}`);
|
|
400
|
+
for (const t of tags)
|
|
401
|
+
names.push(`refs/tags/${t}`);
|
|
402
|
+
names.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
|
403
|
+
const entries = [];
|
|
404
|
+
for (const name of names) {
|
|
405
|
+
try {
|
|
406
|
+
const sha = await git.resolveRef({ fs, dir, ref: name });
|
|
407
|
+
entries.push({ name, sha });
|
|
408
|
+
}
|
|
409
|
+
catch (err) {
|
|
410
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
411
|
+
continue;
|
|
412
|
+
throw err;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
return entries;
|
|
416
|
+
}
|
|
417
|
+
async function resolveHead(principal, repoId) {
|
|
418
|
+
gateAccess(principal, repoId, "*", "resolveRef");
|
|
419
|
+
const dir = repoDir(repoId);
|
|
420
|
+
const repoExists = await fs.promises
|
|
421
|
+
.stat(path.join(dir, ".git"))
|
|
422
|
+
.then(() => true)
|
|
423
|
+
.catch(() => false);
|
|
424
|
+
if (!repoExists)
|
|
425
|
+
return null;
|
|
426
|
+
const symbolicTarget = await git.currentBranch({ fs, dir, fullname: true });
|
|
427
|
+
if (symbolicTarget === undefined)
|
|
428
|
+
return null;
|
|
429
|
+
const sha = await resolveRefSha(dir, symbolicTarget);
|
|
430
|
+
if (sha === null)
|
|
431
|
+
return null;
|
|
432
|
+
return { symbolicTarget, sha };
|
|
433
|
+
}
|
|
434
|
+
async function resolveRefSha(dir, ref) {
|
|
435
|
+
try {
|
|
436
|
+
return await git.resolveRef({ fs, dir, ref });
|
|
437
|
+
}
|
|
438
|
+
catch (err) {
|
|
439
|
+
if (hasCode(err) && err.code === "NotFoundError") {
|
|
440
|
+
return null;
|
|
441
|
+
}
|
|
442
|
+
throw err;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
// Walk from a tree object's root to the tree-or-blob entry at
|
|
446
|
+
// `relPath`. Returns `null` when any path segment is missing, or when
|
|
447
|
+
// the final entry does not match `expectedType`. `relPath === ""`
|
|
448
|
+
// resolves to the root tree itself, and only when a tree is expected.
|
|
449
|
+
async function resolveTreeOid(dir, rootTreeOid, relPath, expectedType) {
|
|
450
|
+
if (relPath === "") {
|
|
451
|
+
return expectedType === "tree" ? rootTreeOid : null;
|
|
452
|
+
}
|
|
453
|
+
const segments = relPath.split("/").filter((s) => s !== "");
|
|
454
|
+
let currentOid = rootTreeOid;
|
|
455
|
+
for (let i = 0; i < segments.length; i++) {
|
|
456
|
+
const segment = segments[i];
|
|
457
|
+
if (segment === undefined)
|
|
458
|
+
throw new Error("unreachable");
|
|
459
|
+
const isLast = i === segments.length - 1;
|
|
460
|
+
const { tree } = await git.readTree({
|
|
461
|
+
fs,
|
|
462
|
+
dir,
|
|
463
|
+
cache: cacheFor(dir),
|
|
464
|
+
oid: currentOid,
|
|
465
|
+
});
|
|
466
|
+
const entry = tree.find((e) => e.path === segment);
|
|
467
|
+
if (entry === undefined)
|
|
468
|
+
return null;
|
|
469
|
+
if (isLast) {
|
|
470
|
+
if (entry.type !== expectedType)
|
|
471
|
+
return null;
|
|
472
|
+
return entry.oid;
|
|
473
|
+
}
|
|
474
|
+
if (entry.type !== "tree")
|
|
475
|
+
return null;
|
|
476
|
+
currentOid = entry.oid;
|
|
477
|
+
}
|
|
478
|
+
return currentOid;
|
|
479
|
+
}
|
|
480
|
+
// Walk from a commit's root tree to the tree-or-blob entry at
|
|
481
|
+
// `relPath`. Used to back both `priorReadBlob` and `priorListDir` so a
|
|
482
|
+
// handler can inspect the parent commit's tree from inside validatePush
|
|
483
|
+
// without each call duplicating the tree-walk.
|
|
484
|
+
async function resolveTreeEntry(dir, commitSha, relPath, expectedType) {
|
|
485
|
+
const { commit } = await git.readCommit({
|
|
486
|
+
fs,
|
|
487
|
+
dir,
|
|
488
|
+
cache: cacheFor(dir),
|
|
489
|
+
oid: commitSha,
|
|
490
|
+
});
|
|
491
|
+
return resolveTreeOid(dir, commit.tree, relPath, expectedType);
|
|
492
|
+
}
|
|
493
|
+
// Build the `(priorReadBlob, priorListDir)` pair fed into a kind
|
|
494
|
+
// handler's validatePush. When `commitSha` is `null`, both closures
|
|
495
|
+
// surface the "ref had no prior commit" state — readBlob returns
|
|
496
|
+
// null, listDir returns an empty array. When non-null, they read
|
|
497
|
+
// against that commit's tree. Read failures on a path that does
|
|
498
|
+
// exist (an EIO mid-walk) surface as a thrown error so a handler's
|
|
499
|
+
// append-only check cannot silently degrade into an accept.
|
|
500
|
+
function buildPriorTreeClosures(dir, commitSha) {
|
|
501
|
+
// Read any blob by its object id, cache-backed. Object ids are
|
|
502
|
+
// content addresses, so this is independent of `commitSha` and is
|
|
503
|
+
// available even on the no-prior-commit branch (where callers have
|
|
504
|
+
// no ids to read).
|
|
505
|
+
const readBlobByOid = async (oid) => {
|
|
506
|
+
const { blob } = await git.readBlob({
|
|
507
|
+
fs,
|
|
508
|
+
dir,
|
|
509
|
+
cache: cacheFor(dir),
|
|
510
|
+
oid,
|
|
511
|
+
});
|
|
512
|
+
return blob;
|
|
513
|
+
};
|
|
514
|
+
if (commitSha === null) {
|
|
515
|
+
return {
|
|
516
|
+
priorReadBlob: async () => null,
|
|
517
|
+
priorListDir: async () => [],
|
|
518
|
+
priorListDirOids: async () => [],
|
|
519
|
+
readBlobByOid,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
const priorReadBlob = async (relPath) => {
|
|
523
|
+
const oid = await resolveTreeEntry(dir, commitSha, relPath, "blob");
|
|
524
|
+
if (oid === null)
|
|
525
|
+
return null;
|
|
526
|
+
const { blob } = await git.readBlob({
|
|
527
|
+
fs,
|
|
528
|
+
dir,
|
|
529
|
+
cache: cacheFor(dir),
|
|
530
|
+
oid,
|
|
531
|
+
});
|
|
532
|
+
return blob;
|
|
533
|
+
};
|
|
534
|
+
const priorListDir = async (relPath) => {
|
|
535
|
+
const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree");
|
|
536
|
+
if (oid === null)
|
|
537
|
+
return [];
|
|
538
|
+
const { tree } = await git.readTree({
|
|
539
|
+
fs,
|
|
540
|
+
dir,
|
|
541
|
+
cache: cacheFor(dir),
|
|
542
|
+
oid,
|
|
543
|
+
});
|
|
544
|
+
return tree.map((e) => e.path);
|
|
545
|
+
};
|
|
546
|
+
// Same walk as `priorListDir` but carries each child's git object id
|
|
547
|
+
// out of the tree listing. A kind handler validating a large subtree
|
|
548
|
+
// by its per-commit delta uses the OID to prove a retained entry is
|
|
549
|
+
// byte-unchanged without re-reading the blob.
|
|
550
|
+
const priorListDirOids = async (relPath) => {
|
|
551
|
+
const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree");
|
|
552
|
+
if (oid === null)
|
|
553
|
+
return [];
|
|
554
|
+
const { tree } = await git.readTree({
|
|
555
|
+
fs,
|
|
556
|
+
dir,
|
|
557
|
+
cache: cacheFor(dir),
|
|
558
|
+
oid,
|
|
559
|
+
});
|
|
560
|
+
return tree.map((e) => ({ name: e.path, oid: e.oid }));
|
|
561
|
+
};
|
|
562
|
+
return { priorReadBlob, priorListDir, priorListDirOids, readBlobByOid };
|
|
563
|
+
}
|
|
564
|
+
// Build the `(readBlob, listDir, topLevelTreePaths)` triple a kind
|
|
565
|
+
// handler's validatePush sees for the prospective tree of a given
|
|
566
|
+
// commit. Mirrors the tip-only closures the storage layer builds for
|
|
567
|
+
// its validateTree callback, except parameterised on the commit OID
|
|
568
|
+
// so the substrate can walk per-commit during a multi-commit pack.
|
|
569
|
+
async function buildCommitTreeClosures(dir, commitSha) {
|
|
570
|
+
const { commit } = await git.readCommit({
|
|
571
|
+
fs,
|
|
572
|
+
dir,
|
|
573
|
+
cache: cacheFor(dir),
|
|
574
|
+
oid: commitSha,
|
|
575
|
+
});
|
|
576
|
+
const { tree: rootTree } = await git.readTree({
|
|
577
|
+
fs,
|
|
578
|
+
dir,
|
|
579
|
+
cache: cacheFor(dir),
|
|
580
|
+
oid: commit.tree,
|
|
581
|
+
});
|
|
582
|
+
const topLevelTreePaths = rootTree.map((e) => e.path);
|
|
583
|
+
const readBlob = async (relPath) => {
|
|
584
|
+
const oid = await resolveTreeEntry(dir, commitSha, relPath, "blob");
|
|
585
|
+
if (oid === null) {
|
|
586
|
+
throw new Error(`readBlob: path ${relPath} not found in commit ${commitSha} tree`);
|
|
587
|
+
}
|
|
588
|
+
const { blob } = await git.readBlob({
|
|
589
|
+
fs,
|
|
590
|
+
dir,
|
|
591
|
+
cache: cacheFor(dir),
|
|
592
|
+
oid,
|
|
593
|
+
});
|
|
594
|
+
return blob;
|
|
595
|
+
};
|
|
596
|
+
const listDir = async (relPath) => {
|
|
597
|
+
if (relPath === "")
|
|
598
|
+
return rootTree.map((e) => e.path);
|
|
599
|
+
const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree");
|
|
600
|
+
// An absent directory lists as empty, matching the writeTree-path
|
|
601
|
+
// `listDir` and the prior-tree closures: every `listDir` the
|
|
602
|
+
// substrate hands a kind handler returns `[]` for a missing path,
|
|
603
|
+
// so a handler that walks an optional or scoped-but-absent subtree
|
|
604
|
+
// (e.g. workflow-run's per-run scoped walk over a run the commit
|
|
605
|
+
// dropped) hits its own empty-directory guard and produces a clean
|
|
606
|
+
// `path_violation` reason instead of a raw substrate throw. A real
|
|
607
|
+
// read fault inside `git.readTree` below still bubbles.
|
|
608
|
+
if (oid === null)
|
|
609
|
+
return [];
|
|
610
|
+
const { tree } = await git.readTree({
|
|
611
|
+
fs,
|
|
612
|
+
dir,
|
|
613
|
+
cache: cacheFor(dir),
|
|
614
|
+
oid,
|
|
615
|
+
});
|
|
616
|
+
return tree.map((e) => e.path);
|
|
617
|
+
};
|
|
618
|
+
return { topLevelTreePaths, readBlob, listDir };
|
|
619
|
+
}
|
|
620
|
+
// Read a tree's child entries as a name->oid map. Returns null when
|
|
621
|
+
// the path is absent or is not a tree at the given commit, so the
|
|
622
|
+
// diff below treats "subtree gained/lost" uniformly with "subtree
|
|
623
|
+
// changed".
|
|
624
|
+
async function readTreeEntryMap(dir, commitSha, relPath) {
|
|
625
|
+
const oid = relPath === ""
|
|
626
|
+
? (await git.readCommit({
|
|
627
|
+
fs,
|
|
628
|
+
dir,
|
|
629
|
+
cache: cacheFor(dir),
|
|
630
|
+
oid: commitSha,
|
|
631
|
+
})).commit.tree
|
|
632
|
+
: await resolveTreeEntry(dir, commitSha, relPath, "tree");
|
|
633
|
+
if (oid === null)
|
|
634
|
+
return null;
|
|
635
|
+
const { tree } = await git.readTree({ fs, dir, cache: cacheFor(dir), oid });
|
|
636
|
+
const out = new Map();
|
|
637
|
+
for (const e of tree)
|
|
638
|
+
out.set(e.path, e.oid);
|
|
639
|
+
return out;
|
|
640
|
+
}
|
|
641
|
+
// git tree-entry modes an admitted asset tree must not carry: a symlink can
|
|
642
|
+
// escape its directory when the tree is checked out, and a submodule gitlink
|
|
643
|
+
// is always a dangling reference (a pushed pack ships no submodule object).
|
|
644
|
+
const GIT_MODE_SYMLINK = "120000";
|
|
645
|
+
const GIT_MODE_SUBMODULE = "160000";
|
|
646
|
+
// Walk a commit's tree and return the first entry whose mode is disallowed
|
|
647
|
+
// (a symlink or a submodule), or null when the tree is clean. This mirrors
|
|
648
|
+
// the materialization backstop in `writeTreeToDisk`, but at the PUSH boundary
|
|
649
|
+
// so a directory-escaping symlink (or a dangling gitlink) is refused at
|
|
650
|
+
// admission rather than only caught later at checkout. It reads trees only,
|
|
651
|
+
// never blobs; `seen` deduplicates shared subtree oids across the commits of
|
|
652
|
+
// one pack so a subtree is walked once.
|
|
653
|
+
async function findDisallowedTreeMode(dir, treeOid, seen, prefix = "") {
|
|
654
|
+
if (seen.has(treeOid))
|
|
655
|
+
return null;
|
|
656
|
+
seen.add(treeOid);
|
|
657
|
+
const { tree } = await git.readTree({
|
|
658
|
+
fs,
|
|
659
|
+
dir,
|
|
660
|
+
cache: cacheFor(dir),
|
|
661
|
+
oid: treeOid,
|
|
662
|
+
});
|
|
663
|
+
for (const entry of tree) {
|
|
664
|
+
const entryPath = prefix === "" ? entry.path : `${prefix}/${entry.path}`;
|
|
665
|
+
if (entry.mode === GIT_MODE_SYMLINK) {
|
|
666
|
+
return { path: entryPath, kind: "symlink" };
|
|
667
|
+
}
|
|
668
|
+
if (entry.type === "commit" || entry.mode === GIT_MODE_SUBMODULE) {
|
|
669
|
+
return { path: entryPath, kind: "submodule" };
|
|
670
|
+
}
|
|
671
|
+
if (entry.type === "tree") {
|
|
672
|
+
const nested = await findDisallowedTreeMode(dir, entry.oid, seen, entryPath);
|
|
673
|
+
if (nested !== null)
|
|
674
|
+
return nested;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
return null;
|
|
678
|
+
}
|
|
679
|
+
// Bound the set of paths a received-pack commit may have changed
|
|
680
|
+
// relative to its parent, as repo-root-relative POSIX prefixes ending
|
|
681
|
+
// in `/`. Git is content-addressed: a subtree whose object id is
|
|
682
|
+
// unchanged between the two commits is byte-identical, so the diff
|
|
683
|
+
// only descends into entries whose oid differs. Top-level entries that
|
|
684
|
+
// differ are emitted as `<name>/`; the `runs/` subtree is descended
|
|
685
|
+
// one level further so a per-run handler can scope to the exact
|
|
686
|
+
// `runs/<runId>/` directories that changed rather than re-validating
|
|
687
|
+
// every run. `parentSha === null` (no readable parent) means the
|
|
688
|
+
// substrate cannot bound the change set, so it returns `undefined`
|
|
689
|
+
// and the handler validates the whole tree.
|
|
690
|
+
async function computeChangedPathPrefixes(dir, commitSha, parentSha) {
|
|
691
|
+
if (parentSha === null)
|
|
692
|
+
return undefined;
|
|
693
|
+
const prefixes = new Set();
|
|
694
|
+
const newTop = (await readTreeEntryMap(dir, commitSha, "")) ?? new Map();
|
|
695
|
+
const oldTop = (await readTreeEntryMap(dir, parentSha, "")) ?? new Map();
|
|
696
|
+
const topNames = new Set([...newTop.keys(), ...oldTop.keys()]);
|
|
697
|
+
for (const name of topNames) {
|
|
698
|
+
if (newTop.get(name) === oldTop.get(name))
|
|
699
|
+
continue;
|
|
700
|
+
if (name === "runs") {
|
|
701
|
+
const newRuns = (await readTreeEntryMap(dir, commitSha, "runs")) ?? new Map();
|
|
702
|
+
const oldRuns = (await readTreeEntryMap(dir, parentSha, "runs")) ?? new Map();
|
|
703
|
+
const runNames = new Set([
|
|
704
|
+
...newRuns.keys(),
|
|
705
|
+
...oldRuns.keys(),
|
|
706
|
+
]);
|
|
707
|
+
for (const runId of runNames) {
|
|
708
|
+
if (newRuns.get(runId) === oldRuns.get(runId))
|
|
709
|
+
continue;
|
|
710
|
+
prefixes.add(`runs/${runId}/`);
|
|
711
|
+
}
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
prefixes.add(`${name}/`);
|
|
715
|
+
}
|
|
716
|
+
return prefixes;
|
|
717
|
+
}
|
|
718
|
+
// Enumerate every commit OID reachable from any branch or tag in
|
|
719
|
+
// the repo. The substrate uses this to define the set of "old"
|
|
720
|
+
// commits — anything reachable via an existing ref existed before
|
|
721
|
+
// the in-flight pack landed, so a pack-walk that reaches one of
|
|
722
|
+
// these has crossed the boundary between new and prior history.
|
|
723
|
+
// Returns an empty set when the repo's `.git` directory has no refs
|
|
724
|
+
// yet (a freshly-initialised repo with HEAD pointing at an unborn
|
|
725
|
+
// branch). Read failures bubble; a silent empty would let the walk
|
|
726
|
+
// re-validate commits the kind handler already accepted.
|
|
727
|
+
async function snapshotExistingCommits(dir) {
|
|
728
|
+
const out = new Set();
|
|
729
|
+
const visit = async (start) => {
|
|
730
|
+
const stack = [start];
|
|
731
|
+
while (stack.length > 0) {
|
|
732
|
+
const oid = stack.pop();
|
|
733
|
+
if (oid === undefined)
|
|
734
|
+
break;
|
|
735
|
+
if (out.has(oid))
|
|
736
|
+
continue;
|
|
737
|
+
let parsed;
|
|
738
|
+
try {
|
|
739
|
+
parsed = await git.readCommit({ fs, dir, cache: cacheFor(dir), oid });
|
|
740
|
+
}
|
|
741
|
+
catch (err) {
|
|
742
|
+
// A previously-received single-commit pack may leave the
|
|
743
|
+
// tip's `parent` field pointing at a SHA the receiver has
|
|
744
|
+
// never seen (the producer ships only the tip's tree, not
|
|
745
|
+
// its ancestor commits). That dangling parent is structural
|
|
746
|
+
// for this repo kind, not a corruption, so the walk stops
|
|
747
|
+
// at the missing node instead of erroring.
|
|
748
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
749
|
+
continue;
|
|
750
|
+
throw err;
|
|
751
|
+
}
|
|
752
|
+
out.add(oid);
|
|
753
|
+
for (const parent of parsed.commit.parent) {
|
|
754
|
+
if (!out.has(parent))
|
|
755
|
+
stack.push(parent);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
};
|
|
759
|
+
const branches = await git.listBranches({ fs, dir });
|
|
760
|
+
for (const b of branches) {
|
|
761
|
+
const sha = await resolveRefSha(dir, `refs/heads/${b}`);
|
|
762
|
+
if (sha !== null)
|
|
763
|
+
await visit(sha);
|
|
764
|
+
}
|
|
765
|
+
const tags = await git.listTags({ fs, dir });
|
|
766
|
+
for (const t of tags) {
|
|
767
|
+
const sha = await resolveRefSha(dir, `refs/tags/${t}`);
|
|
768
|
+
if (sha !== null)
|
|
769
|
+
await visit(sha);
|
|
770
|
+
}
|
|
771
|
+
return out;
|
|
772
|
+
}
|
|
773
|
+
// Walk parent links from `tipSha` back to identify the new commits
|
|
774
|
+
// a pack just published, returning the chain in topological order —
|
|
775
|
+
// oldest new commit first, tip last. A commit is considered "new"
|
|
776
|
+
// when it is readable from the object store, was not present before
|
|
777
|
+
// the pack arrived (not in `existingCommits`), and does not match
|
|
778
|
+
// the CAS-pinned `expectedOldSha`. When the parent's commit object
|
|
779
|
+
// is not in the store at all — the common case for the deploy-pack
|
|
780
|
+
// shape `createDeployPack` produces, which packs only the tip's
|
|
781
|
+
// tree and not its ancestor commits — that absence is the
|
|
782
|
+
// boundary: history older than the readable commit is opaque from
|
|
783
|
+
// this pack's perspective, so the walker stops without pushing the
|
|
784
|
+
// unreadable parent.
|
|
785
|
+
//
|
|
786
|
+
// The walk chases the first parent only; multi-parent merge commits
|
|
787
|
+
// are not produced by any current writer of repo-store-managed
|
|
788
|
+
// kinds, so a multi-parent commit in a received pack is a
|
|
789
|
+
// structural defect the substrate refuses outright. Each returned
|
|
790
|
+
// commit is one the substrate must validate against its predecessor
|
|
791
|
+
// before the ref advances.
|
|
792
|
+
async function collectNewCommits(dir, tipSha, expectedOldSha, existingCommits) {
|
|
793
|
+
const chain = [];
|
|
794
|
+
let current = tipSha;
|
|
795
|
+
while (true) {
|
|
796
|
+
if (current === expectedOldSha)
|
|
797
|
+
break;
|
|
798
|
+
if (existingCommits.has(current))
|
|
799
|
+
break;
|
|
800
|
+
let parsed;
|
|
801
|
+
try {
|
|
802
|
+
parsed = await git.readCommit({
|
|
803
|
+
fs,
|
|
804
|
+
dir,
|
|
805
|
+
cache: cacheFor(dir),
|
|
806
|
+
oid: current,
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
catch (err) {
|
|
810
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
811
|
+
break;
|
|
812
|
+
throw err;
|
|
813
|
+
}
|
|
814
|
+
chain.push(current);
|
|
815
|
+
const parents = parsed.commit.parent;
|
|
816
|
+
if (parents.length === 0)
|
|
817
|
+
break;
|
|
818
|
+
if (parents.length > 1) {
|
|
819
|
+
throw new Error(`pack_walk_multi_parent: commit ${current} has ${String(parents.length)} parents; merge commits are not supported in repo-store packs`);
|
|
820
|
+
}
|
|
821
|
+
const next = parents[0];
|
|
822
|
+
if (next === undefined)
|
|
823
|
+
throw new Error("unreachable");
|
|
824
|
+
current = next;
|
|
825
|
+
}
|
|
826
|
+
return chain.reverse();
|
|
827
|
+
}
|
|
828
|
+
// Assemble a new root tree by splicing `puts` (repo-root-relative path
|
|
829
|
+
// -> blob oid) and `deletes` onto the parent's root tree, reusing every
|
|
830
|
+
// unchanged entry by oid. The result is committed directly via
|
|
831
|
+
// `git.commit({ tree })`, so the on-disk index is never touched.
|
|
832
|
+
//
|
|
833
|
+
// `deletes` is a set of repo-root-relative paths, and removal must
|
|
834
|
+
// match the base entry's type. A no-slash entry names an exact path
|
|
835
|
+
// and clears the blob there; deleting a path that is a directory in
|
|
836
|
+
// the base is rejected with `delete_type_mismatch` (there is no
|
|
837
|
+
// git-rm-style recursive drop). A trailing-slash entry names a
|
|
838
|
+
// subtree prefix and clears it (the clear-prefix shape); a
|
|
839
|
+
// trailing-slash delete whose leading segment is a base blob is
|
|
840
|
+
// rejected with `delete_type_mismatch`, unless a `put` drives the
|
|
841
|
+
// same descent -- that is a legitimate file-to-directory replacement
|
|
842
|
+
// and is accepted (the put wins). A `put` at a path overrides a
|
|
843
|
+
// delete of the same path. A delete of a path absent from the parent
|
|
844
|
+
// tree is an idempotent no-op (the entry is simply never emitted),
|
|
845
|
+
// matching `git rm --ignore-unmatch` and the working-tree `rm` with
|
|
846
|
+
// `force: true`.
|
|
847
|
+
//
|
|
848
|
+
// Recursion is scoped to the touched subtrees: a level is read and
|
|
849
|
+
// rewritten only when a put lands under it or a delete removes within
|
|
850
|
+
// it. Every entry the write does not touch — sibling subtrees AND
|
|
851
|
+
// sibling blobs inside a subtree being touched — is carried forward by
|
|
852
|
+
// its existing oid without a re-hash or a walk, so the per-commit cost
|
|
853
|
+
// tracks the size of the change rather than the repo. Returns the new
|
|
854
|
+
// tree oid, or `null` when the subtree ends up empty (the caller writes
|
|
855
|
+
// an empty root tree when the whole repo empties).
|
|
856
|
+
async function assembleTree(dir, baseTreeOid, prefix, puts, deletes) {
|
|
857
|
+
// A subtree-delete naming exactly this node drops the base wholesale;
|
|
858
|
+
// only puts under it survive.
|
|
859
|
+
const cleared = prefix !== "" && deletes.has(prefix);
|
|
860
|
+
const baseEntries = new Map();
|
|
861
|
+
if (baseTreeOid !== null && !cleared) {
|
|
862
|
+
const { tree } = await git.readTree({
|
|
863
|
+
fs,
|
|
864
|
+
dir,
|
|
865
|
+
cache: cacheFor(dir),
|
|
866
|
+
oid: baseTreeOid,
|
|
867
|
+
});
|
|
868
|
+
for (const e of tree) {
|
|
869
|
+
baseEntries.set(e.path, { mode: e.mode, oid: e.oid, type: e.type });
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
// Classify what changes at this level: direct blob puts, exact-file
|
|
873
|
+
// deletes here, and subtrees a put or a delete descends into.
|
|
874
|
+
const blobPutsHere = new Set();
|
|
875
|
+
const subtreeNames = new Set();
|
|
876
|
+
const subtreePutNames = new Set();
|
|
877
|
+
const fileDeletesHere = new Set();
|
|
878
|
+
for (const full of puts.keys()) {
|
|
879
|
+
if (prefix !== "" && !full.startsWith(prefix))
|
|
880
|
+
continue;
|
|
881
|
+
const rest = full.slice(prefix.length);
|
|
882
|
+
if (rest.length === 0)
|
|
883
|
+
continue;
|
|
884
|
+
const slash = rest.indexOf("/");
|
|
885
|
+
if (slash === -1)
|
|
886
|
+
blobPutsHere.add(rest);
|
|
887
|
+
else {
|
|
888
|
+
const child = rest.slice(0, slash);
|
|
889
|
+
subtreeNames.add(child);
|
|
890
|
+
subtreePutNames.add(child);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
for (const del of deletes) {
|
|
894
|
+
if (prefix !== "" && !del.startsWith(prefix))
|
|
895
|
+
continue;
|
|
896
|
+
const rest = del.slice(prefix.length);
|
|
897
|
+
if (rest.length === 0)
|
|
898
|
+
continue; // del === prefix, handled by `cleared`
|
|
899
|
+
const slash = rest.indexOf("/");
|
|
900
|
+
if (slash === -1)
|
|
901
|
+
fileDeletesHere.add(rest);
|
|
902
|
+
else
|
|
903
|
+
subtreeNames.add(rest.slice(0, slash));
|
|
904
|
+
}
|
|
905
|
+
// A name written both as a direct blob and as a directory (a put/base
|
|
906
|
+
// `foo` plus a put or delete under `foo/`) is contradictory. Left
|
|
907
|
+
// implicit, the blob-put branch below wins and the subtree side is
|
|
908
|
+
// silently dropped; reject it loudly instead. This covers every
|
|
909
|
+
// caller — writeTree's `content.files`, writeTreePreservingPrefix's
|
|
910
|
+
// merge output, and writeTreeDelta's puts/deletes — since all funnel
|
|
911
|
+
// into this same classification.
|
|
912
|
+
for (const name of blobPutsHere) {
|
|
913
|
+
if (subtreeNames.has(name)) {
|
|
914
|
+
throw new Error(`tree_name_collision: ${JSON.stringify(prefix + name)} is written both as a file and as a directory`);
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
const names = new Set([
|
|
918
|
+
...baseEntries.keys(),
|
|
919
|
+
...blobPutsHere,
|
|
920
|
+
...subtreeNames,
|
|
921
|
+
]);
|
|
922
|
+
const entries = [];
|
|
923
|
+
for (const name of names) {
|
|
924
|
+
const full = prefix + name;
|
|
925
|
+
const putOid = puts.get(full);
|
|
926
|
+
if (putOid !== undefined) {
|
|
927
|
+
// A put overrides whatever the base held and any delete of the
|
|
928
|
+
// same path. Puts are always regular files (mode 100644). If a
|
|
929
|
+
// put ever overwrites a base entry with a different mode (an
|
|
930
|
+
// executable 100755 blob, a symlink 120000, or a submodule), this
|
|
931
|
+
// downgrades it to 100644 — the store only ever writes 100644
|
|
932
|
+
// content blobs today, so no caller hits it, but a future caller
|
|
933
|
+
// that needs to preserve an executable bit would have to carry
|
|
934
|
+
// the mode through `puts` rather than assume 100644.
|
|
935
|
+
entries.push({
|
|
936
|
+
mode: "100644",
|
|
937
|
+
path: name,
|
|
938
|
+
oid: putOid,
|
|
939
|
+
type: "blob",
|
|
940
|
+
});
|
|
941
|
+
continue;
|
|
942
|
+
}
|
|
943
|
+
if (fileDeletesHere.has(name)) {
|
|
944
|
+
const base = baseEntries.get(name);
|
|
945
|
+
if (base !== undefined && base.type === "tree") {
|
|
946
|
+
throw new Error(`delete_type_mismatch: ${JSON.stringify(prefix + name)} is deleted as a file but is a directory in the base tree`);
|
|
947
|
+
}
|
|
948
|
+
continue; // file removed
|
|
949
|
+
}
|
|
950
|
+
if (subtreeNames.has(name)) {
|
|
951
|
+
const base = baseEntries.get(name);
|
|
952
|
+
// A trailing-slash delete descending into a base blob
|
|
953
|
+
// contradicts the base type -- reject it. The
|
|
954
|
+
// `!subtreePutNames` carve-out lets a put-driven descent
|
|
955
|
+
// through: a `put` under `name/` is a file-to-directory
|
|
956
|
+
// replacement, not a delete mismatch, so it must not raise
|
|
957
|
+
// delete_type_mismatch. That replacement still dead-ends at
|
|
958
|
+
// working-tree materialization in writeTreeUnderLock (mkdir
|
|
959
|
+
// over the base file EEXISTs) -- a separate limitation the
|
|
960
|
+
// claim-check callers never reach, which must not be masked by
|
|
961
|
+
// mislabeling a put as a delete mismatch.
|
|
962
|
+
if (base !== undefined &&
|
|
963
|
+
base.type === "blob" &&
|
|
964
|
+
!subtreePutNames.has(name)) {
|
|
965
|
+
throw new Error(`delete_type_mismatch: ${JSON.stringify(prefix + name)} is deleted as a subtree but is a file in the base tree`);
|
|
966
|
+
}
|
|
967
|
+
const baseChildOid = base !== undefined && base.type === "tree" ? base.oid : null;
|
|
968
|
+
const childOid = await assembleTree(dir, baseChildOid, `${full}/`, puts, deletes);
|
|
969
|
+
if (childOid !== null) {
|
|
970
|
+
entries.push({
|
|
971
|
+
mode: "040000",
|
|
972
|
+
path: name,
|
|
973
|
+
oid: childOid,
|
|
974
|
+
type: "tree",
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
continue;
|
|
978
|
+
}
|
|
979
|
+
const base = baseEntries.get(name);
|
|
980
|
+
if (base === undefined)
|
|
981
|
+
continue;
|
|
982
|
+
entries.push({
|
|
983
|
+
mode: base.mode,
|
|
984
|
+
path: name,
|
|
985
|
+
oid: base.oid,
|
|
986
|
+
type: base.type,
|
|
987
|
+
});
|
|
988
|
+
}
|
|
989
|
+
if (entries.length === 0)
|
|
990
|
+
return null;
|
|
991
|
+
return await git.writeTree({ fs, dir, tree: entries });
|
|
992
|
+
}
|
|
993
|
+
// Prospective-side validatePush closures sourced from the assembled
|
|
994
|
+
// tree oid, so a handler sees exactly the tree the commit will carry.
|
|
995
|
+
// The blobs and trees the assembly wrote are unreferenced objects
|
|
996
|
+
// until the commit lands, so reading them here advances nothing.
|
|
997
|
+
function buildTreeReadClosures(dir, rootTreeOid) {
|
|
998
|
+
const topLevelTreePaths = async () => {
|
|
999
|
+
const { tree } = await git.readTree({
|
|
1000
|
+
fs,
|
|
1001
|
+
dir,
|
|
1002
|
+
cache: cacheFor(dir),
|
|
1003
|
+
oid: rootTreeOid,
|
|
1004
|
+
});
|
|
1005
|
+
return tree.map((e) => e.path);
|
|
1006
|
+
};
|
|
1007
|
+
const readBlob = async (relPath) => {
|
|
1008
|
+
const oid = await resolveTreeOid(dir, rootTreeOid, relPath, "blob");
|
|
1009
|
+
if (oid === null) {
|
|
1010
|
+
throw new Error(`readBlob: path ${relPath} not present in prospective tree`);
|
|
1011
|
+
}
|
|
1012
|
+
const { blob } = await git.readBlob({
|
|
1013
|
+
fs,
|
|
1014
|
+
dir,
|
|
1015
|
+
cache: cacheFor(dir),
|
|
1016
|
+
oid,
|
|
1017
|
+
});
|
|
1018
|
+
return blob;
|
|
1019
|
+
};
|
|
1020
|
+
const listDir = async (relPath) => {
|
|
1021
|
+
const oid = relPath === ""
|
|
1022
|
+
? rootTreeOid
|
|
1023
|
+
: await resolveTreeOid(dir, rootTreeOid, relPath, "tree");
|
|
1024
|
+
if (oid === null)
|
|
1025
|
+
return [];
|
|
1026
|
+
const { tree } = await git.readTree({
|
|
1027
|
+
fs,
|
|
1028
|
+
dir,
|
|
1029
|
+
cache: cacheFor(dir),
|
|
1030
|
+
oid,
|
|
1031
|
+
});
|
|
1032
|
+
return tree.map((e) => e.path);
|
|
1033
|
+
};
|
|
1034
|
+
// Same walk as `listDir` but carries each child's git object id out of
|
|
1035
|
+
// the assembled tree's listing, mirroring the prior side's
|
|
1036
|
+
// `priorListDirOids`. A handler validating a large retained subtree by
|
|
1037
|
+
// its per-commit delta (workflow-run's consumed dedup index) reads the
|
|
1038
|
+
// prospective OID straight from here instead of re-reading and hashing
|
|
1039
|
+
// every retained blob.
|
|
1040
|
+
const listDirOids = async (relPath) => {
|
|
1041
|
+
const oid = relPath === ""
|
|
1042
|
+
? rootTreeOid
|
|
1043
|
+
: await resolveTreeOid(dir, rootTreeOid, relPath, "tree");
|
|
1044
|
+
if (oid === null)
|
|
1045
|
+
return [];
|
|
1046
|
+
const { tree } = await git.readTree({
|
|
1047
|
+
fs,
|
|
1048
|
+
dir,
|
|
1049
|
+
cache: cacheFor(dir),
|
|
1050
|
+
oid,
|
|
1051
|
+
});
|
|
1052
|
+
return tree.map((e) => ({ name: e.path, oid: e.oid }));
|
|
1053
|
+
};
|
|
1054
|
+
return { topLevelTreePaths, readBlob, listDir, listDirOids };
|
|
1055
|
+
}
|
|
1056
|
+
// Unlocked body of writeTree. The caller is responsible for
|
|
1057
|
+
// acquiring the per-repo lock before invoking this and for not
|
|
1058
|
+
// releasing it until the returned promise settles. Extracted so
|
|
1059
|
+
// writeTreePreservingPrefix can run a read-then-merge step under the
|
|
1060
|
+
// same lock without holding two nested acquisitions.
|
|
1061
|
+
async function writeTreeUnderLock(principal, repoId, ref, w,
|
|
1062
|
+
// When present, the parent tip already resolved under this lock. The
|
|
1063
|
+
// delta path pins it once and hands the SAME oid to both computeDelta
|
|
1064
|
+
// (its dedup reads) and this assembly, so the dedup snapshot, the
|
|
1065
|
+
// committed tree, and validation are provably one pre-image. Absent,
|
|
1066
|
+
// the write resolves its own (the writeTree / preserving-prefix path,
|
|
1067
|
+
// which has no prior read to keep consistent with). `{ sha }` wraps
|
|
1068
|
+
// the value so a genuinely-null pin is distinct from "not provided".
|
|
1069
|
+
pinnedParent) {
|
|
1070
|
+
const dir = repoDir(repoId);
|
|
1071
|
+
await storageInitRepo(dir, storageOptsFor(repoId, undefined));
|
|
1072
|
+
const handler = handlerFor(repoId);
|
|
1073
|
+
// Assemble the commit's tree directly and commit that tree oid,
|
|
1074
|
+
// never staging into the on-disk index. The index is a single
|
|
1075
|
+
// repo-global structure shared across refs; routing writes through
|
|
1076
|
+
// it forced a full index rebuild on every events<->workflow-run ref
|
|
1077
|
+
// flip and re-serialized the whole index once per staged blob.
|
|
1078
|
+
// Splicing the tree from the parent's root tree — applying `w.files`
|
|
1079
|
+
// as puts and `w.deletes` as removals, reusing every untouched entry
|
|
1080
|
+
// by oid — keeps the per-commit cost tracking the change rather than
|
|
1081
|
+
// the accumulated history.
|
|
1082
|
+
// Pin the parent under the lock: the new commit's parent is the
|
|
1083
|
+
// ref's tip, and the splice runs against THAT commit's root tree,
|
|
1084
|
+
// read under the same lock, so the pre-image is race-free. A ref
|
|
1085
|
+
// that does not yet exist has no base tree (the splice starts from
|
|
1086
|
+
// empty) and parents on HEAD, matching the prior index-reset path
|
|
1087
|
+
// which seeded an empty index for a missing ref.
|
|
1088
|
+
const parentCommitSha = pinnedParent !== undefined
|
|
1089
|
+
? pinnedParent.sha
|
|
1090
|
+
: await resolveRefSha(dir, ref);
|
|
1091
|
+
let baseRootTreeOid = null;
|
|
1092
|
+
if (parentCommitSha !== null) {
|
|
1093
|
+
const { commit } = await git.readCommit({
|
|
1094
|
+
fs,
|
|
1095
|
+
dir,
|
|
1096
|
+
cache: cacheFor(dir),
|
|
1097
|
+
oid: parentCommitSha,
|
|
1098
|
+
});
|
|
1099
|
+
baseRootTreeOid = commit.tree;
|
|
1100
|
+
}
|
|
1101
|
+
// Write the put blobs, then splice them and `w.deletes` onto the
|
|
1102
|
+
// parent root tree. writeBlob and writeTree emit unreferenced
|
|
1103
|
+
// objects; nothing the ref can reach moves until the commit lands.
|
|
1104
|
+
const puts = new Map();
|
|
1105
|
+
for (const [relPath, contents] of Object.entries(w.files)) {
|
|
1106
|
+
const bytes = typeof contents === "string"
|
|
1107
|
+
? new TextEncoder().encode(contents)
|
|
1108
|
+
: contents;
|
|
1109
|
+
const oid = await git.writeBlob({ fs, dir, blob: bytes });
|
|
1110
|
+
puts.set(relPath, oid);
|
|
1111
|
+
}
|
|
1112
|
+
const assembled = await assembleTree(dir, baseRootTreeOid, "", puts, w.deletes);
|
|
1113
|
+
const newRootTreeOid = assembled ?? (await git.writeTree({ fs, dir, tree: [] }));
|
|
1114
|
+
// validatePush sees the full prospective tree via closures over the
|
|
1115
|
+
// assembled tree oid; the prior-side closures read the parent commit
|
|
1116
|
+
// (`parentCommitSha`) exactly as before. `w.changedPathPrefixes` is
|
|
1117
|
+
// the handler's scoping hint — the prefixes this write may have
|
|
1118
|
+
// touched — and stays undefined for an unbounded (validate-all) write.
|
|
1119
|
+
const { priorReadBlob, priorListDir, priorListDirOids } = buildPriorTreeClosures(dir, parentCommitSha);
|
|
1120
|
+
const prospective = buildTreeReadClosures(dir, newRootTreeOid);
|
|
1121
|
+
const changedPathPrefixes = w.changedPathPrefixes;
|
|
1122
|
+
// No disallowed-mode (symlink/submodule) gate here on purpose: the
|
|
1123
|
+
// substrate mode gate lives on the pack-receive path. This write path
|
|
1124
|
+
// cannot introduce a `120000`/`160000` entry -- `assembleTree` writes puts
|
|
1125
|
+
// as `100644` and passes base-tree entries through unchanged, and no
|
|
1126
|
+
// ingress ever admits such a mode into a base tree (the pack path is now
|
|
1127
|
+
// gated; this path only writes `100644`). Adding a walk here would gate a
|
|
1128
|
+
// tree that cannot carry those modes.
|
|
1129
|
+
const validation = await handler.validatePush({
|
|
1130
|
+
repoId,
|
|
1131
|
+
ref,
|
|
1132
|
+
principal,
|
|
1133
|
+
topLevelTreePaths: await prospective.topLevelTreePaths(),
|
|
1134
|
+
readBlob: prospective.readBlob,
|
|
1135
|
+
listDir: prospective.listDir,
|
|
1136
|
+
listDirOids: prospective.listDirOids,
|
|
1137
|
+
priorReadBlob,
|
|
1138
|
+
priorListDir,
|
|
1139
|
+
priorListDirOids,
|
|
1140
|
+
changedPathPrefixes,
|
|
1141
|
+
});
|
|
1142
|
+
if (!validation.ok) {
|
|
1143
|
+
// Nothing was staged and no ref advanced: the assembly wrote only
|
|
1144
|
+
// unreferenced blob and tree objects, which the next GC reclaims.
|
|
1145
|
+
// There is no half-applied index or working-tree state to roll
|
|
1146
|
+
// back, so the refusal just surfaces.
|
|
1147
|
+
throw new Error(`path_violation: ${validation.reason}`);
|
|
1148
|
+
}
|
|
1149
|
+
// Materialize the working tree for the paths this write touched:
|
|
1150
|
+
// remove each deleted path, then write each put file. The store's own
|
|
1151
|
+
// reads resolve through the object store, but some consumers (the
|
|
1152
|
+
// workflow-run claim-check processing scan) read these files straight
|
|
1153
|
+
// from disk, so the working tree must mirror the committed change.
|
|
1154
|
+
// Only the deleted paths and the put paths are touched, so this is
|
|
1155
|
+
// O(change), not O(repo). It runs only after validation passes, so a
|
|
1156
|
+
// rejected push leaves the working tree untouched — the failure
|
|
1157
|
+
// atomicity the old index rollback provided, now without any
|
|
1158
|
+
// rollback. `rm` with `force` no-ops a missing path and `recursive`
|
|
1159
|
+
// covers both a file delete and a subtree-prefix delete.
|
|
1160
|
+
for (const del of w.deletes) {
|
|
1161
|
+
await fs.promises.rm(path.join(dir, del), {
|
|
1162
|
+
recursive: true,
|
|
1163
|
+
force: true,
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
for (const [relPath, contents] of Object.entries(w.files)) {
|
|
1167
|
+
const fullPath = path.join(dir, relPath);
|
|
1168
|
+
await fs.promises.mkdir(path.dirname(fullPath), { recursive: true });
|
|
1169
|
+
await fs.promises.writeFile(fullPath, contents);
|
|
1170
|
+
}
|
|
1171
|
+
// The parent is the pinned tip, or HEAD for a never-written ref so a
|
|
1172
|
+
// first write parents on the repo's initial commit. `oldSha` is
|
|
1173
|
+
// precise: null only when the ref truly does not exist.
|
|
1174
|
+
const oldSha = parentCommitSha;
|
|
1175
|
+
const parentSha = parentCommitSha ?? (await git.resolveRef({ fs, dir, ref: "HEAD" }));
|
|
1176
|
+
const commitSha = await git.commit({
|
|
1177
|
+
fs,
|
|
1178
|
+
dir,
|
|
1179
|
+
cache: cacheFor(dir),
|
|
1180
|
+
tree: newRootTreeOid,
|
|
1181
|
+
message: w.message,
|
|
1182
|
+
author: AUTHOR,
|
|
1183
|
+
parent: [parentSha],
|
|
1184
|
+
ref,
|
|
1185
|
+
signingKey: "sshsig",
|
|
1186
|
+
onSign: async ({ payload }) => ({
|
|
1187
|
+
signature: await createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey),
|
|
1188
|
+
}),
|
|
1189
|
+
});
|
|
1190
|
+
const cachedExisting = existingCommitsCache.get(indexCacheKey(repoId));
|
|
1191
|
+
if (cachedExisting !== undefined)
|
|
1192
|
+
cachedExisting.add(commitSha);
|
|
1193
|
+
await handler.onRefUpdated({ repoId, ref, oldSha, newSha: commitSha });
|
|
1194
|
+
await emitRefUpdate(repoId, ref, oldSha, commitSha);
|
|
1195
|
+
await maybeRunGC(repoId);
|
|
1196
|
+
return { commitSha, newlyTerminalRuns: validation.newlyTerminalRuns ?? [] };
|
|
1197
|
+
}
|
|
1198
|
+
// Normalize a `TreeContent` (files + optional clearPrefix) into the
|
|
1199
|
+
// puts/deletes/scope shape writeTreeUnderLock consumes. A clearPrefix
|
|
1200
|
+
// becomes a single subtree-delete and the handler's change scope; its
|
|
1201
|
+
// absence is a purely-additive write validated in full.
|
|
1202
|
+
function normalizeTreeContent(content) {
|
|
1203
|
+
if (content.clearPrefix !== undefined) {
|
|
1204
|
+
validateClearPrefix(content.clearPrefix);
|
|
1205
|
+
return {
|
|
1206
|
+
files: content.files,
|
|
1207
|
+
deletes: new Set([content.clearPrefix]),
|
|
1208
|
+
changedPathPrefixes: new Set([content.clearPrefix]),
|
|
1209
|
+
message: content.message,
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
return {
|
|
1213
|
+
files: content.files,
|
|
1214
|
+
deletes: new Set(),
|
|
1215
|
+
changedPathPrefixes: undefined,
|
|
1216
|
+
message: content.message,
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
async function writeTree(principal, repoId, ref, content) {
|
|
1220
|
+
gateAccess(principal, repoId, ref, "writeTree");
|
|
1221
|
+
// The lock spans the entire substrate body of writeTree: tree
|
|
1222
|
+
// assembly, validatePush, commit, and the onRefUpdated hook. Holding
|
|
1223
|
+
// the lock through onRefUpdated keeps post-update consumers
|
|
1224
|
+
// serialized against the same ref's next writer.
|
|
1225
|
+
return withRepoLock(repoId, () => writeTreeUnderLock(principal, repoId, ref, normalizeTreeContent(content)));
|
|
1226
|
+
}
|
|
1227
|
+
// Enumerate every blob directly under `prefix` in the tree at
|
|
1228
|
+
// `ref`, returning a map from repo-root-relative path (including the
|
|
1229
|
+
// prefix) to bytes. The empty map covers the ref-missing /
|
|
1230
|
+
// prefix-missing cases — both legitimate first-write states for the
|
|
1231
|
+
// prefix-preserving primitive.
|
|
1232
|
+
async function readPrefixBlobs(repoId, ref, prefix) {
|
|
1233
|
+
const dir = repoDir(repoId);
|
|
1234
|
+
const out = new Map();
|
|
1235
|
+
const repoExists = await fs.promises
|
|
1236
|
+
.stat(path.join(dir, ".git"))
|
|
1237
|
+
.then(() => true)
|
|
1238
|
+
.catch(() => false);
|
|
1239
|
+
if (!repoExists)
|
|
1240
|
+
return out;
|
|
1241
|
+
const commitSha = await resolveRefSha(dir, ref);
|
|
1242
|
+
if (commitSha === null)
|
|
1243
|
+
return out;
|
|
1244
|
+
const { commit } = await git.readCommit({
|
|
1245
|
+
fs,
|
|
1246
|
+
dir,
|
|
1247
|
+
cache: cacheFor(dir),
|
|
1248
|
+
oid: commitSha,
|
|
1249
|
+
});
|
|
1250
|
+
let currentOid = commit.tree;
|
|
1251
|
+
const segments = prefix
|
|
1252
|
+
.replace(/\/$/, "")
|
|
1253
|
+
.split("/")
|
|
1254
|
+
.filter((s) => s !== "");
|
|
1255
|
+
for (const segment of segments) {
|
|
1256
|
+
const { tree } = await git.readTree({
|
|
1257
|
+
fs,
|
|
1258
|
+
dir,
|
|
1259
|
+
cache: cacheFor(dir),
|
|
1260
|
+
oid: currentOid,
|
|
1261
|
+
});
|
|
1262
|
+
const entry = tree.find((e) => e.path === segment);
|
|
1263
|
+
if (entry === undefined || entry.type !== "tree") {
|
|
1264
|
+
return out;
|
|
1265
|
+
}
|
|
1266
|
+
currentOid = entry.oid;
|
|
1267
|
+
}
|
|
1268
|
+
const { tree } = await git.readTree({
|
|
1269
|
+
fs,
|
|
1270
|
+
dir,
|
|
1271
|
+
cache: cacheFor(dir),
|
|
1272
|
+
oid: currentOid,
|
|
1273
|
+
});
|
|
1274
|
+
// N+1 isomorphic-git round-trips: one tree read plus one
|
|
1275
|
+
// readBlob per blob entry. Acceptable at the current scale
|
|
1276
|
+
// (single-digit tarballs per registry); when a registry grows
|
|
1277
|
+
// to hundreds of entries this becomes the obvious optimization
|
|
1278
|
+
// target — readBlob can take the entry oid directly, avoiding
|
|
1279
|
+
// the per-call path resolution.
|
|
1280
|
+
for (const entry of tree) {
|
|
1281
|
+
if (entry.type !== "blob")
|
|
1282
|
+
continue;
|
|
1283
|
+
const { blob } = await git.readBlob({
|
|
1284
|
+
fs,
|
|
1285
|
+
dir,
|
|
1286
|
+
cache: cacheFor(dir),
|
|
1287
|
+
oid: commitSha,
|
|
1288
|
+
filepath: `${prefix}${entry.path}`,
|
|
1289
|
+
});
|
|
1290
|
+
out.set(`${prefix}${entry.path}`, blob);
|
|
1291
|
+
}
|
|
1292
|
+
return out;
|
|
1293
|
+
}
|
|
1294
|
+
async function writeTreePreservingPrefix(principal, repoId, ref, args) {
|
|
1295
|
+
gateAccess(principal, repoId, ref, "writeTree");
|
|
1296
|
+
validateClearPrefix(args.preservePrefix);
|
|
1297
|
+
return withRepoLock(repoId, async () => {
|
|
1298
|
+
// Reading and merging both happen inside the lock so two
|
|
1299
|
+
// concurrent callers targeting the same prefix observe each
|
|
1300
|
+
// other's commits in serial order — no lost-update window
|
|
1301
|
+
// between the read and the writeTree.
|
|
1302
|
+
await storageInitRepo(repoDir(repoId), storageOptsFor(repoId, undefined));
|
|
1303
|
+
const existing = await readPrefixBlobs(repoId, ref, args.preservePrefix);
|
|
1304
|
+
const files = await args.merge(existing);
|
|
1305
|
+
return writeTreeUnderLock(principal, repoId, ref, {
|
|
1306
|
+
files,
|
|
1307
|
+
deletes: new Set([args.preservePrefix]),
|
|
1308
|
+
changedPathPrefixes: new Set([args.preservePrefix]),
|
|
1309
|
+
message: args.message,
|
|
1310
|
+
});
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
// Commit a targeted delta: `computeDelta` runs under the per-repo lock
|
|
1314
|
+
// against the pinned parent tip and returns the exact files to put and
|
|
1315
|
+
// paths to delete; everything else is carried forward by oid. Unlike
|
|
1316
|
+
// writeTreePreservingPrefix — which clears and rebuilds a whole prefix
|
|
1317
|
+
// from a full merge output — a delta touches only the entries it names,
|
|
1318
|
+
// so a caller that mutates one file in a large directory (a claim-check
|
|
1319
|
+
// move that adds one entry and deletes another) does not re-hash or
|
|
1320
|
+
// re-materialize the untouched siblings. `changedPathPrefixes` is the
|
|
1321
|
+
// handler's scoping hint for the touched region; the caller supplies it
|
|
1322
|
+
// because the delta has no single clear-prefix to derive it from.
|
|
1323
|
+
async function writeTreeDelta(principal, repoId, ref, args) {
|
|
1324
|
+
gateAccess(principal, repoId, ref, "writeTree");
|
|
1325
|
+
return withRepoLock(repoId, async () => {
|
|
1326
|
+
const dir = repoDir(repoId);
|
|
1327
|
+
await storageInitRepo(dir, storageOptsFor(repoId, undefined));
|
|
1328
|
+
// Pin the parent tip ONCE under the lock and hand the same oid to
|
|
1329
|
+
// computeDelta's dedup reads and to the assembly below, so the
|
|
1330
|
+
// pre-image the delta is computed against is exactly the pre-image
|
|
1331
|
+
// it is committed against — no lost-update window, no second
|
|
1332
|
+
// resolve that could (in principle) observe a different tip.
|
|
1333
|
+
const parentCommitSha = await resolveRefSha(dir, ref);
|
|
1334
|
+
const { priorListDirOids, readBlobByOid } = buildPriorTreeClosures(dir, parentCommitSha);
|
|
1335
|
+
const delta = await args.computeDelta(parentCommitSha, {
|
|
1336
|
+
readBlobByOid,
|
|
1337
|
+
listDirOids: priorListDirOids,
|
|
1338
|
+
});
|
|
1339
|
+
for (const p of Object.keys(delta.puts))
|
|
1340
|
+
validateDeltaPath(p, false);
|
|
1341
|
+
for (const d of delta.deletes)
|
|
1342
|
+
validateDeltaPath(d, true);
|
|
1343
|
+
assertDeltaUnambiguous(delta.puts, delta.deletes);
|
|
1344
|
+
assertDeltaScoped(delta.puts, delta.deletes, args.changedPathPrefixes);
|
|
1345
|
+
return writeTreeUnderLock(principal, repoId, ref, {
|
|
1346
|
+
files: delta.puts,
|
|
1347
|
+
deletes: new Set(delta.deletes),
|
|
1348
|
+
changedPathPrefixes: args.changedPathPrefixes,
|
|
1349
|
+
message: args.message,
|
|
1350
|
+
}, { sha: parentCommitSha });
|
|
1351
|
+
});
|
|
1352
|
+
}
|
|
1353
|
+
async function receivePack(principal, repoId, ref, pack, commitSha, expectedOldSha) {
|
|
1354
|
+
gateAccess(principal, repoId, ref, "receivePack");
|
|
1355
|
+
// The lock spans the entire substrate body of receivePack: the
|
|
1356
|
+
// packfile index, the CAS check against `expectedOldSha`, the
|
|
1357
|
+
// validateTree hook, the ref write, and the onRefUpdated hook.
|
|
1358
|
+
// `oldSha` for onRefUpdated is taken from the receivePackObjects
|
|
1359
|
+
// return value so the post-update hook sees the same pre-image the
|
|
1360
|
+
// CAS read observed, without a second resolveRef.
|
|
1361
|
+
return withRepoLock(repoId, async () => {
|
|
1362
|
+
const dir = repoDir(repoId);
|
|
1363
|
+
await storageInitRepo(dir, storageOptsFor(repoId, undefined));
|
|
1364
|
+
const handler = handlerFor(repoId);
|
|
1365
|
+
const transferId = crypto.randomUUID().replace(/-/g, "");
|
|
1366
|
+
const existingKey = indexCacheKey(repoId);
|
|
1367
|
+
let existingCommits = existingCommitsCache.get(existingKey);
|
|
1368
|
+
if (existingCommits === undefined) {
|
|
1369
|
+
existingCommits = await snapshotExistingCommits(dir);
|
|
1370
|
+
existingCommitsCache.set(existingKey, existingCommits);
|
|
1371
|
+
}
|
|
1372
|
+
const newCommitsFromPack = [];
|
|
1373
|
+
const newlyTerminalRuns = [];
|
|
1374
|
+
const oldSha = await receivePackObjects(dir, pack, ref, commitSha, transferId, expectedOldSha,
|
|
1375
|
+
// A pack may carry more than one new commit (e.g. supervisor
|
|
1376
|
+
// bootstrap that batches enqueue + dequeue before the hub has
|
|
1377
|
+
// the workflow-run repo bootstrapped). The kind handler's
|
|
1378
|
+
// prior-tree closures must point at THAT commit's parent — not
|
|
1379
|
+
// the ref's tip before the pack arrived — so an intra-pack
|
|
1380
|
+
// transition (inbox in commit N, processing in commit N+1) is
|
|
1381
|
+
// validated against the right pre-image. We walk the parent
|
|
1382
|
+
// chain from tip back to the first commit already present in
|
|
1383
|
+
// the pre-pack history, then call validatePush once per new
|
|
1384
|
+
// commit in topological (oldest-first) order. A single-commit
|
|
1385
|
+
// pack collapses to the same behaviour as the tip-only path.
|
|
1386
|
+
async () => {
|
|
1387
|
+
const newCommits = await collectNewCommits(dir, commitSha, expectedOldSha, existingCommits);
|
|
1388
|
+
newCommitsFromPack.push(...newCommits);
|
|
1389
|
+
// Shared across the pack's commits so a subtree common to several is
|
|
1390
|
+
// walked once by the disallowed-mode gate below.
|
|
1391
|
+
const seenTreeOids = new Set();
|
|
1392
|
+
for (const newCommit of newCommits) {
|
|
1393
|
+
const { commit: parsed } = await git.readCommit({
|
|
1394
|
+
fs,
|
|
1395
|
+
dir,
|
|
1396
|
+
cache: cacheFor(dir),
|
|
1397
|
+
oid: newCommit,
|
|
1398
|
+
});
|
|
1399
|
+
const parents = parsed.parent;
|
|
1400
|
+
const declaredParent = parents.length === 0 ? null : (parents[0] ?? null);
|
|
1401
|
+
// When the commit declares a parent the receiver does not
|
|
1402
|
+
// have in its object store, the substrate cannot
|
|
1403
|
+
// reconstruct the prior tree the kind handler would
|
|
1404
|
+
// validate against. For kinds whose handler reads prior
|
|
1405
|
+
// bytes to enforce append-only invariants (workflow-run),
|
|
1406
|
+
// silently degrading to "empty prior" lets a path in the
|
|
1407
|
+
// new commit that overwrites an immutable prior-tree
|
|
1408
|
+
// entry slip through unchecked — append-only enforcement
|
|
1409
|
+
// accepts a brand-new path, not a path that contradicts
|
|
1410
|
+
// an entry the handler cannot read. The workflow-run
|
|
1411
|
+
// `createPack` above ships the full parent chain, so this
|
|
1412
|
+
// branch is unreachable on the production pack-push path
|
|
1413
|
+
// for that kind; a producer that does ship an incomplete
|
|
1414
|
+
// chain is rejected outright.
|
|
1415
|
+
//
|
|
1416
|
+
// Other kinds ship deploy-shape packs (tip commit + tree
|
|
1417
|
+
// only) as their normal flow and their handlers do not
|
|
1418
|
+
// read prior bytes, so a dangling parent there is
|
|
1419
|
+
// structurally normal and the substrate continues to
|
|
1420
|
+
// collapse to the no-prior path.
|
|
1421
|
+
let parentSha = null;
|
|
1422
|
+
if (declaredParent !== null) {
|
|
1423
|
+
try {
|
|
1424
|
+
await git.readCommit({
|
|
1425
|
+
fs,
|
|
1426
|
+
dir,
|
|
1427
|
+
cache: cacheFor(dir),
|
|
1428
|
+
oid: declaredParent,
|
|
1429
|
+
});
|
|
1430
|
+
parentSha = declaredParent;
|
|
1431
|
+
}
|
|
1432
|
+
catch (err) {
|
|
1433
|
+
if (!hasCode(err) || err.code !== "NotFoundError")
|
|
1434
|
+
throw err;
|
|
1435
|
+
if (repoId.kind === "workflow-run") {
|
|
1436
|
+
throw new Error(`pack_walk_dangling_parent: commit ${newCommit} declares parent ${declaredParent} which is neither in the receiver's store nor in the pack`);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
const { priorReadBlob, priorListDir, priorListDirOids } = buildPriorTreeClosures(dir, parentSha);
|
|
1441
|
+
const { topLevelTreePaths, readBlob, listDir } = await buildCommitTreeClosures(dir, newCommit);
|
|
1442
|
+
const changedPathPrefixes = await computeChangedPathPrefixes(dir, newCommit, parentSha);
|
|
1443
|
+
// Admission gate for EVERY kind, before the per-kind validatePush:
|
|
1444
|
+
// an asset tree carrying a symlink (which can escape its directory
|
|
1445
|
+
// at checkout) or a submodule (a dangling gitlink the pack cannot
|
|
1446
|
+
// reproduce) is refused here, so the ref never advances. No kind
|
|
1447
|
+
// legitimately pushes either; `writeTreeToDisk` rejects both at
|
|
1448
|
+
// materialization as the defense-in-depth backstop. Per-commit, so
|
|
1449
|
+
// an intermediate commit that adds a symlink a later commit deletes
|
|
1450
|
+
// is still caught.
|
|
1451
|
+
const disallowed = await findDisallowedTreeMode(dir, parsed.tree, seenTreeOids);
|
|
1452
|
+
if (disallowed !== null) {
|
|
1453
|
+
// The substrate owns the `path_violation:` prefix on the thrown
|
|
1454
|
+
// message, so the reason omits it (matching every handler reason).
|
|
1455
|
+
const reason = `${disallowed.kind} at ${disallowed.path} is not allowed in a pushed asset tree (commit ${newCommit})`;
|
|
1456
|
+
logger.debug `validatePush rejected ${repoId.kind}/${repoId.id} on ${ref} at commit ${newCommit}: ${reason}`;
|
|
1457
|
+
return { ok: false, reason };
|
|
1458
|
+
}
|
|
1459
|
+
const result = await handler.validatePush({
|
|
1460
|
+
repoId,
|
|
1461
|
+
ref,
|
|
1462
|
+
principal,
|
|
1463
|
+
topLevelTreePaths,
|
|
1464
|
+
readBlob,
|
|
1465
|
+
listDir,
|
|
1466
|
+
priorReadBlob,
|
|
1467
|
+
priorListDir,
|
|
1468
|
+
priorListDirOids,
|
|
1469
|
+
changedPathPrefixes,
|
|
1470
|
+
});
|
|
1471
|
+
if (!result.ok) {
|
|
1472
|
+
logger.debug `validatePush rejected ${repoId.kind}/${repoId.id} on ${ref} at commit ${newCommit}: ${result.reason}`;
|
|
1473
|
+
return { ok: false, reason: result.reason };
|
|
1474
|
+
}
|
|
1475
|
+
if (result.newlyTerminalRuns !== undefined) {
|
|
1476
|
+
newlyTerminalRuns.push(...result.newlyTerminalRuns);
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
return true;
|
|
1480
|
+
});
|
|
1481
|
+
// receivePackObjects wrote new objects and advanced the ref
|
|
1482
|
+
// straight to disk without threading the memoization cache, so
|
|
1483
|
+
// drop the dir's cache; the next read rebuilds against the packed
|
|
1484
|
+
// objects and the new tip.
|
|
1485
|
+
invalidateGitCache(dir);
|
|
1486
|
+
for (const sha of newCommitsFromPack)
|
|
1487
|
+
existingCommits.add(sha);
|
|
1488
|
+
await handler.onRefUpdated({ repoId, ref, oldSha, newSha: commitSha });
|
|
1489
|
+
await emitRefUpdate(repoId, ref, oldSha, commitSha);
|
|
1490
|
+
await maybeRunGC(repoId);
|
|
1491
|
+
return newlyTerminalRuns;
|
|
1492
|
+
});
|
|
1493
|
+
}
|
|
1494
|
+
async function createPack(principal, repoId, ref) {
|
|
1495
|
+
gateAccess(principal, repoId, ref, "createPack");
|
|
1496
|
+
// `createDeployPack` packs only the tip commit + its tree, which
|
|
1497
|
+
// covers the deploy-pack shape every other kind ships (the
|
|
1498
|
+
// receiver starts from genesis and applies the tree wholesale).
|
|
1499
|
+
// The workflow-run kind ships incrementally and the receiver
|
|
1500
|
+
// (the hub) needs to validate per-commit transitions against the
|
|
1501
|
+
// sender's prior tree, so the pack must carry the full parent
|
|
1502
|
+
// chain from the supplied ref's tip. Walking the chain here keeps
|
|
1503
|
+
// the kind-specific branching at the substrate boundary so the
|
|
1504
|
+
// kind handler stays receive-side only.
|
|
1505
|
+
if (repoId.kind === "workflow-run") {
|
|
1506
|
+
const dir = repoDir(repoId);
|
|
1507
|
+
const commitSha = await git.resolveRef({ fs, dir, ref });
|
|
1508
|
+
const tipKey = lastPackedTipKey(repoId, ref);
|
|
1509
|
+
const stopAt = lastPackedTip.get(tipKey) ?? null;
|
|
1510
|
+
const oids = await collectChainReachableObjects(dir, commitSha, stopAt);
|
|
1511
|
+
const result = await git.packObjects({
|
|
1512
|
+
fs,
|
|
1513
|
+
dir,
|
|
1514
|
+
cache: cacheFor(dir),
|
|
1515
|
+
oids,
|
|
1516
|
+
write: false,
|
|
1517
|
+
});
|
|
1518
|
+
if (result.packfile === undefined) {
|
|
1519
|
+
throw new Error(`packObjects returned no packfile for ref "${ref}" (${commitSha})`);
|
|
1520
|
+
}
|
|
1521
|
+
// The cursor is NOT advanced here. Building a pack only produces
|
|
1522
|
+
// bytes; the shipped tip advances in `commitPackedTip` once the
|
|
1523
|
+
// receiver acks the transfer that carried these commits. A
|
|
1524
|
+
// transfer a reconnect cancels before its ack thus leaves the
|
|
1525
|
+
// cursor where it was, so the next `createPack` re-includes the
|
|
1526
|
+
// un-acked commits and the receiver still gets a self-consistent
|
|
1527
|
+
// chain instead of a pack whose base commit it never received.
|
|
1528
|
+
return { pack: result.packfile, commitSha, ref };
|
|
1529
|
+
}
|
|
1530
|
+
const { pack, commitSha } = await createDeployPack(repoDir(repoId), ref);
|
|
1531
|
+
return { pack, commitSha, ref };
|
|
1532
|
+
}
|
|
1533
|
+
// Advance the incremental-pack cursor for `(repoId, ref)` to
|
|
1534
|
+
// `commitSha`. Called by the pack-push pipeline once the receiver has
|
|
1535
|
+
// acked the transfer that shipped `commitSha`. Gating the advance on
|
|
1536
|
+
// the ack — rather than advancing inside `createPack` when the pack
|
|
1537
|
+
// bytes are produced — is what lets a cancelled-before-ack transfer
|
|
1538
|
+
// (a reconnect drops the in-flight push) be re-shipped cleanly: the
|
|
1539
|
+
// cursor stays put, so the next `createPack` re-includes the un-acked
|
|
1540
|
+
// commits and the receiver gets a self-consistent chain. Only the
|
|
1541
|
+
// `workflow-run` kind ships incremental packs; for every other kind
|
|
1542
|
+
// `createPack` produces a self-contained deploy pack and the cursor
|
|
1543
|
+
// is unused, so this is a no-op there.
|
|
1544
|
+
function commitPackedTip(repoId, ref, commitSha) {
|
|
1545
|
+
if (repoId.kind !== "workflow-run")
|
|
1546
|
+
return;
|
|
1547
|
+
lastPackedTip.set(lastPackedTipKey(repoId, ref), commitSha);
|
|
1548
|
+
}
|
|
1549
|
+
// Collect every object OID reachable from `tipSha` and from every
|
|
1550
|
+
// ancestor commit along its first-parent chain. Mirrors what the
|
|
1551
|
+
// upload-pack layer's negotiated walker produces when the requester
|
|
1552
|
+
// advertises no `haves` but is sized for the substrate's own use:
|
|
1553
|
+
// a workflow-run pack push from the supervisor needs to carry every
|
|
1554
|
+
// commit the hub does not yet have, and the substrate has no
|
|
1555
|
+
// negotiation channel to ask the hub what it already has. The walk
|
|
1556
|
+
// stops at the first parent whose commit object is not in the local
|
|
1557
|
+
// store — that commit is by definition not part of the local
|
|
1558
|
+
// supervisor's history and so cannot be a relevant ancestor.
|
|
1559
|
+
async function collectChainReachableObjects(dir, tipSha, stopAt) {
|
|
1560
|
+
const seen = new Set();
|
|
1561
|
+
let current = tipSha;
|
|
1562
|
+
while (current !== null) {
|
|
1563
|
+
if (current === stopAt)
|
|
1564
|
+
break;
|
|
1565
|
+
let perCommit = chainReachabilityCache.get(current);
|
|
1566
|
+
if (perCommit === undefined) {
|
|
1567
|
+
perCommit = await collectReachableObjects(dir, current);
|
|
1568
|
+
chainReachabilityCache.set(current, perCommit);
|
|
1569
|
+
}
|
|
1570
|
+
for (const o of perCommit)
|
|
1571
|
+
seen.add(o);
|
|
1572
|
+
let parsed;
|
|
1573
|
+
try {
|
|
1574
|
+
parsed = await git.readCommit({
|
|
1575
|
+
fs,
|
|
1576
|
+
dir,
|
|
1577
|
+
cache: cacheFor(dir),
|
|
1578
|
+
oid: current,
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
catch (err) {
|
|
1582
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
1583
|
+
break;
|
|
1584
|
+
throw err;
|
|
1585
|
+
}
|
|
1586
|
+
const parents = parsed.commit.parent;
|
|
1587
|
+
if (parents.length === 0)
|
|
1588
|
+
break;
|
|
1589
|
+
const next = parents[0];
|
|
1590
|
+
if (next === undefined)
|
|
1591
|
+
throw new Error("unreachable");
|
|
1592
|
+
current = next;
|
|
1593
|
+
}
|
|
1594
|
+
return Array.from(seen);
|
|
1595
|
+
}
|
|
1596
|
+
async function resolveRef(principal, repoId, ref) {
|
|
1597
|
+
gateAccess(principal, repoId, ref, "resolveRef");
|
|
1598
|
+
return resolveRefSha(repoDir(repoId), ref);
|
|
1599
|
+
}
|
|
1600
|
+
// Build the committed-read closures pinned to a commit. Shared by the
|
|
1601
|
+
// by-ref (`openCommittedReads`) and by-commit
|
|
1602
|
+
// (`openCommittedReadsAtCommit`) entrypoints, which differ only in how
|
|
1603
|
+
// they obtain and validate the commit SHA.
|
|
1604
|
+
function committedReadsAt(dir, commitSha) {
|
|
1605
|
+
const { readBlobByOid } = buildPriorTreeClosures(dir, commitSha);
|
|
1606
|
+
const treeOid = async (relPath) => {
|
|
1607
|
+
// "." and "" both name the root tree; resolveTreeEntry treats the
|
|
1608
|
+
// empty string as the root, so normalize "." to it.
|
|
1609
|
+
const normalized = relPath === "." ? "" : relPath;
|
|
1610
|
+
return resolveTreeEntry(dir, commitSha, normalized, "tree");
|
|
1611
|
+
};
|
|
1612
|
+
const listDir = async (relPath) => {
|
|
1613
|
+
const oid = await treeOid(relPath);
|
|
1614
|
+
if (oid === null)
|
|
1615
|
+
return [];
|
|
1616
|
+
const { tree } = await git.readTree({
|
|
1617
|
+
fs,
|
|
1618
|
+
dir,
|
|
1619
|
+
cache: cacheFor(dir),
|
|
1620
|
+
oid,
|
|
1621
|
+
});
|
|
1622
|
+
return tree.map((e) => ({ name: e.path, oid: e.oid, type: e.type }));
|
|
1623
|
+
};
|
|
1624
|
+
return { listDir, readBlobByOid, treeOid };
|
|
1625
|
+
}
|
|
1626
|
+
async function repoHasGitDir(dir) {
|
|
1627
|
+
return fs.promises
|
|
1628
|
+
.stat(path.join(dir, ".git"))
|
|
1629
|
+
.then(() => true)
|
|
1630
|
+
.catch(() => false);
|
|
1631
|
+
}
|
|
1632
|
+
async function openCommittedReads(principal, repoId, ref) {
|
|
1633
|
+
gateAccess(principal, repoId, ref, "resolveRef");
|
|
1634
|
+
const dir = repoDir(repoId);
|
|
1635
|
+
if (!(await repoHasGitDir(dir)))
|
|
1636
|
+
return null;
|
|
1637
|
+
const commitSha = await resolveRefSha(dir, ref);
|
|
1638
|
+
if (commitSha === null)
|
|
1639
|
+
return null;
|
|
1640
|
+
return committedReadsAt(dir, commitSha);
|
|
1641
|
+
}
|
|
1642
|
+
async function openCommittedReadsAtCommit(principal, repoId, commitSha) {
|
|
1643
|
+
// No single ref to gate on; use the same `"*"`/`resolveRef` bulk-read
|
|
1644
|
+
// gate that `listRefs` and `resolveHead` use.
|
|
1645
|
+
gateAccess(principal, repoId, "*", "resolveRef");
|
|
1646
|
+
if (!/^[0-9a-f]{40}$/.test(commitSha)) {
|
|
1647
|
+
throw new Error(`commit_sha_invalid: ${commitSha}`);
|
|
1648
|
+
}
|
|
1649
|
+
const dir = repoDir(repoId);
|
|
1650
|
+
if (!(await repoHasGitDir(dir)))
|
|
1651
|
+
return null;
|
|
1652
|
+
// Confirm the commit object is present so a pruned commit reads as a
|
|
1653
|
+
// clean `null` rather than a lazy throw on the first tree walk.
|
|
1654
|
+
const present = await git
|
|
1655
|
+
.readCommit({ fs, dir, cache: cacheFor(dir), oid: commitSha })
|
|
1656
|
+
.then(() => true)
|
|
1657
|
+
.catch((err) => {
|
|
1658
|
+
if (hasCode(err) && err.code === "NotFoundError")
|
|
1659
|
+
return false;
|
|
1660
|
+
throw err;
|
|
1661
|
+
});
|
|
1662
|
+
if (!present)
|
|
1663
|
+
return null;
|
|
1664
|
+
return committedReadsAt(dir, commitSha);
|
|
1665
|
+
}
|
|
1666
|
+
function subscribe(principal, repoId, ref, opts) {
|
|
1667
|
+
gateAccess(principal, repoId, ref, "resolveRef");
|
|
1668
|
+
const bufferLimit = opts.bufferLimit ?? DEFAULT_SUBSCRIBE_BUFFER_LIMIT;
|
|
1669
|
+
if (!Number.isInteger(bufferLimit) || bufferLimit <= 0) {
|
|
1670
|
+
throw new Error(`subscribe_buffer_limit_invalid: ${String(opts.bufferLimit)}`);
|
|
1671
|
+
}
|
|
1672
|
+
const sub = {
|
|
1673
|
+
bufferLimit,
|
|
1674
|
+
buffer: [],
|
|
1675
|
+
closed: false,
|
|
1676
|
+
error: null,
|
|
1677
|
+
waiter: null,
|
|
1678
|
+
};
|
|
1679
|
+
const key = refKey(repoId, ref);
|
|
1680
|
+
let set = subscribers.get(key);
|
|
1681
|
+
if (set === undefined) {
|
|
1682
|
+
set = new Set();
|
|
1683
|
+
subscribers.set(key, set);
|
|
1684
|
+
}
|
|
1685
|
+
set.add(sub);
|
|
1686
|
+
const removeSubscriber = () => {
|
|
1687
|
+
const current = subscribers.get(key);
|
|
1688
|
+
if (current === undefined)
|
|
1689
|
+
return;
|
|
1690
|
+
current.delete(sub);
|
|
1691
|
+
if (current.size === 0)
|
|
1692
|
+
subscribers.delete(key);
|
|
1693
|
+
};
|
|
1694
|
+
const finish = () => {
|
|
1695
|
+
if (sub.closed) {
|
|
1696
|
+
// Already closed by abort or error. Still flush any waiter
|
|
1697
|
+
// so the consumer's pending `next()` resolves promptly.
|
|
1698
|
+
}
|
|
1699
|
+
sub.closed = true;
|
|
1700
|
+
removeSubscriber();
|
|
1701
|
+
if (sub.waiter !== null) {
|
|
1702
|
+
const w = sub.waiter;
|
|
1703
|
+
sub.waiter = null;
|
|
1704
|
+
w({ value: undefined, done: true });
|
|
1705
|
+
}
|
|
1706
|
+
};
|
|
1707
|
+
const onAbort = () => {
|
|
1708
|
+
sub.closed = true;
|
|
1709
|
+
removeSubscriber();
|
|
1710
|
+
if (sub.waiter !== null) {
|
|
1711
|
+
const w = sub.waiter;
|
|
1712
|
+
sub.waiter = null;
|
|
1713
|
+
w({ value: undefined, done: true });
|
|
1714
|
+
}
|
|
1715
|
+
};
|
|
1716
|
+
if (opts.signal.aborted) {
|
|
1717
|
+
// Aborted before any work — return an iterator that yields
|
|
1718
|
+
// {done: true} immediately. The subscriber is registered then
|
|
1719
|
+
// removed for symmetry with the live path's cleanup.
|
|
1720
|
+
onAbort();
|
|
1721
|
+
}
|
|
1722
|
+
else {
|
|
1723
|
+
opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
1724
|
+
}
|
|
1725
|
+
// Replay queue: events sourced from history that the iterator
|
|
1726
|
+
// surfaces before falling through to the live buffer. Filled
|
|
1727
|
+
// synchronously by the first `next()` call (replayHistory does
|
|
1728
|
+
// the git.log walk), then drained one entry per next().
|
|
1729
|
+
let replayQueue = null;
|
|
1730
|
+
let replayPrimed = false;
|
|
1731
|
+
async function primeReplay() {
|
|
1732
|
+
replayPrimed = true;
|
|
1733
|
+
const dir = repoDir(repoId);
|
|
1734
|
+
if (opts.from === "head") {
|
|
1735
|
+
// Seed the seq cache so the next commit on the ref carries
|
|
1736
|
+
// the correct seq even if no prior commit had ever populated
|
|
1737
|
+
// it. Subscribers that come in with `from: "head"` see only
|
|
1738
|
+
// new commits — there is no history to replay.
|
|
1739
|
+
const cached = seqCache.get(key);
|
|
1740
|
+
if (cached === undefined) {
|
|
1741
|
+
const count = await countCommits(dir, ref);
|
|
1742
|
+
if (count > 0)
|
|
1743
|
+
seqCache.set(key, count - 1);
|
|
1744
|
+
}
|
|
1745
|
+
replayQueue = [];
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
const all = await replayHistory(dir, ref);
|
|
1749
|
+
// Seed the seq cache from the replay so live deliveries
|
|
1750
|
+
// continue the seq sequence correctly.
|
|
1751
|
+
if (all.length > 0) {
|
|
1752
|
+
const last = all[all.length - 1];
|
|
1753
|
+
if (last === undefined)
|
|
1754
|
+
throw new Error("unreachable");
|
|
1755
|
+
seqCache.set(key, last.seq);
|
|
1756
|
+
}
|
|
1757
|
+
const fromSeq = opts.from.seq;
|
|
1758
|
+
replayQueue = all.filter((e) => e.seq >= fromSeq);
|
|
1759
|
+
}
|
|
1760
|
+
const iterator = {
|
|
1761
|
+
[Symbol.asyncIterator]() {
|
|
1762
|
+
return iterator;
|
|
1763
|
+
},
|
|
1764
|
+
async next() {
|
|
1765
|
+
if (!replayPrimed) {
|
|
1766
|
+
try {
|
|
1767
|
+
await primeReplay();
|
|
1768
|
+
}
|
|
1769
|
+
catch (err) {
|
|
1770
|
+
finish();
|
|
1771
|
+
throw err;
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
if (replayQueue !== null && replayQueue.length > 0) {
|
|
1775
|
+
const entry = replayQueue.shift();
|
|
1776
|
+
if (entry === undefined)
|
|
1777
|
+
throw new Error("unreachable");
|
|
1778
|
+
return { value: entry, done: false };
|
|
1779
|
+
}
|
|
1780
|
+
if (sub.buffer.length > 0) {
|
|
1781
|
+
const entry = sub.buffer.shift();
|
|
1782
|
+
if (entry === undefined)
|
|
1783
|
+
throw new Error("unreachable");
|
|
1784
|
+
return { value: entry, done: false };
|
|
1785
|
+
}
|
|
1786
|
+
if (sub.error !== null) {
|
|
1787
|
+
const err = sub.error;
|
|
1788
|
+
sub.error = null;
|
|
1789
|
+
finish();
|
|
1790
|
+
throw err;
|
|
1791
|
+
}
|
|
1792
|
+
if (sub.closed) {
|
|
1793
|
+
finish();
|
|
1794
|
+
return { value: undefined, done: true };
|
|
1795
|
+
}
|
|
1796
|
+
return new Promise((resolve) => {
|
|
1797
|
+
sub.waiter = resolve;
|
|
1798
|
+
});
|
|
1799
|
+
},
|
|
1800
|
+
async return() {
|
|
1801
|
+
opts.signal.removeEventListener("abort", onAbort);
|
|
1802
|
+
finish();
|
|
1803
|
+
return { value: undefined, done: true };
|
|
1804
|
+
},
|
|
1805
|
+
async throw(err) {
|
|
1806
|
+
opts.signal.removeEventListener("abort", onAbort);
|
|
1807
|
+
finish();
|
|
1808
|
+
throw err;
|
|
1809
|
+
},
|
|
1810
|
+
};
|
|
1811
|
+
return iterator;
|
|
1812
|
+
}
|
|
1813
|
+
return {
|
|
1814
|
+
initRepo,
|
|
1815
|
+
writeTree,
|
|
1816
|
+
writeTreePreservingPrefix,
|
|
1817
|
+
writeTreeDelta,
|
|
1818
|
+
receivePack,
|
|
1819
|
+
createPack,
|
|
1820
|
+
commitPackedTip,
|
|
1821
|
+
resolveRef,
|
|
1822
|
+
listRefs,
|
|
1823
|
+
resolveHead,
|
|
1824
|
+
getRepoDir,
|
|
1825
|
+
openCommittedReads,
|
|
1826
|
+
openCommittedReadsAtCommit,
|
|
1827
|
+
subscribe,
|
|
1828
|
+
};
|
|
1829
|
+
}
|