@vzn/vx-dte 0.0.652 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +135 -62
- package/package.json +3 -3
- package/src/dispatch.ts +39 -4
- package/src/github.ts +0 -3
- package/src/index.ts +1 -0
- package/src/init.ts +9 -10
- package/src/local.ts +112 -28
- package/src/messages.ts +14 -3
- package/src/orchestrator.ts +265 -33
- package/src/planner.ts +183 -55
- package/src/plugin.ts +209 -30
- package/src/seal.ts +159 -43
- package/src/server-fit.ts +80 -0
- package/src/tree.ts +215 -7
- package/src/usage.ts +65 -3
- package/src/warm-ups.ts +72 -6
- package/src/wire.ts +84 -42
- package/src/worker.ts +148 -15
package/src/seal.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// opens no past run.
|
|
8
8
|
|
|
9
9
|
import { createHash } from 'node:crypto'
|
|
10
|
+
import { zstdCompressSync, zstdDecompressSync } from 'node:zlib'
|
|
10
11
|
import { type Bytes, type CipherState, Handshake, MAX_PLAINTEXT } from './noise.js'
|
|
11
12
|
import { RelayError, unsent as unsentBy, type Participant, type RelayMessage } from './wire.js'
|
|
12
13
|
import { ui } from '@vzn/vx'
|
|
@@ -192,12 +193,16 @@ const REPLAY = Symbol('replay')
|
|
|
192
193
|
|
|
193
194
|
/**
|
|
194
195
|
* The size a body is padded to, so the relay learns its size class, not
|
|
195
|
-
* its size: a power of two from
|
|
196
|
+
* its size: a power of two from 64 bytes, then whole transport messages.
|
|
197
|
+
* From 256, each `ran`, `end` and `report`, mostly some 100 bytes, sealed
|
|
198
|
+
* in two to eight times its size, and their order and timing already
|
|
199
|
+
* say which is which (2026-11-06).
|
|
196
200
|
*/
|
|
197
201
|
function padded(n: number): number {
|
|
198
202
|
if (n >= MAX_PLAINTEXT) return Math.ceil(n / MAX_PLAINTEXT) * MAX_PLAINTEXT
|
|
199
|
-
return Math.min(2 ** Math.ceil(Math.log2(Math.max(n,
|
|
203
|
+
return Math.min(2 ** Math.ceil(Math.log2(Math.max(n, MIN_CLASS))), MAX_PLAINTEXT)
|
|
200
204
|
}
|
|
205
|
+
const MIN_CLASS = 64
|
|
201
206
|
|
|
202
207
|
/**
|
|
203
208
|
* One worker and the orchestrator after their handshake. Each body is
|
|
@@ -208,37 +213,68 @@ function padded(n: number): number {
|
|
|
208
213
|
*/
|
|
209
214
|
class Session {
|
|
210
215
|
private sent = 0
|
|
216
|
+
/** What this side sealed and what it opened: the orchestrator's bodies' zstd dictionary. */
|
|
217
|
+
private readonly sealed: History
|
|
218
|
+
private readonly opened: History
|
|
211
219
|
private readonly seen = new Set<number>()
|
|
212
220
|
/** The last nonce of the newest body opened. */
|
|
213
221
|
private newest = -1
|
|
222
|
+
/** A body's parts from its sends so far, until the send that ends it. */
|
|
223
|
+
private held: { n: number; c: string }[] = []
|
|
214
224
|
/** The last send handed to the wire, or still sealing: the next waits on it. */
|
|
215
225
|
private turn: Promise<unknown> = Promise.resolve()
|
|
226
|
+
/** Bodies asked for in this turn of the event loop, or while the send before seals: sealed as one. */
|
|
227
|
+
private group: { bodies: string[]; bytes: number; sent: Promise<void> } | undefined
|
|
216
228
|
|
|
217
229
|
constructor(
|
|
218
230
|
private readonly tx: CipherState,
|
|
219
231
|
private readonly rx: CipherState,
|
|
220
|
-
|
|
232
|
+
orchestrator: boolean,
|
|
233
|
+
) {
|
|
234
|
+
this.sealed = new History(orchestrator ? HISTORY_BYTES : 0)
|
|
235
|
+
this.opened = new History(orchestrator ? 0 : HISTORY_BYTES)
|
|
236
|
+
}
|
|
221
237
|
|
|
222
238
|
/**
|
|
223
239
|
* `body` sealed and handed to `wire`, in the order sends were asked for:
|
|
224
240
|
* sealing is async, and a big body sealed first reached the wire after a
|
|
225
241
|
* small one sealed next (a worker's `end` before its `ran`, 2026-10-26).
|
|
226
242
|
* The wire numbers a send as it is handed over, so only the handing waits.
|
|
243
|
+
* Bodies that wait together go as one: each was a send, a relay frame
|
|
244
|
+
* and its answer of its own. A body of more parts than a frame holds
|
|
245
|
+
* goes over several sends, each but the last led by `+`.
|
|
227
246
|
*/
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
const
|
|
231
|
-
|
|
247
|
+
send(body: unknown, wire: (sealed: string[]) => Promise<void>): Promise<void> {
|
|
248
|
+
const json = JSON.stringify(body)
|
|
249
|
+
const g = this.group
|
|
250
|
+
if (g !== undefined && g.bytes + json.length <= GROUP_BYTES) {
|
|
251
|
+
g.bodies.push(json)
|
|
252
|
+
g.bytes += json.length
|
|
253
|
+
return g.sent
|
|
254
|
+
}
|
|
255
|
+
const mine = { bodies: [json], bytes: json.length, sent: Promise.resolve() }
|
|
256
|
+
this.group = mine
|
|
257
|
+
let handed: Promise<void> | undefined
|
|
258
|
+
const sealed = this.turn.then(async () => {
|
|
259
|
+
if (this.group === mine) this.group = undefined
|
|
260
|
+
const parts = await this.seal(`[${mine.bodies.join(',')}]`)
|
|
261
|
+
const sends: Promise<void>[] = []
|
|
262
|
+
for (let at = 0; at < parts.length; at += SEND_PARTS) {
|
|
263
|
+
const some = parts.slice(at, at + SEND_PARTS)
|
|
264
|
+
sends.push(wire(at + SEND_PARTS < parts.length ? [MORE, ...some] : some))
|
|
265
|
+
}
|
|
266
|
+
handed = Promise.all(sends).then(() => undefined)
|
|
232
267
|
})
|
|
233
|
-
// quiet: `
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
await sent
|
|
268
|
+
// quiet: `sent` hands each caller this error; the next turn must not inherit it.
|
|
269
|
+
this.turn = sealed.catch(() => undefined)
|
|
270
|
+
mine.sent = sealed.then(() => handed)
|
|
271
|
+
return mine.sent
|
|
238
272
|
}
|
|
239
273
|
|
|
240
|
-
private async seal(
|
|
241
|
-
const
|
|
274
|
+
private async seal(bodies: string): Promise<string[]> {
|
|
275
|
+
const json = text.encode(bodies)
|
|
276
|
+
const plain = squeeze(json, this.sealed)
|
|
277
|
+
this.sealed.add(json)
|
|
242
278
|
// Trailing spaces: JSON.parse reads past them, and `unsqueeze` stops before them.
|
|
243
279
|
const bytes = new Uint8Array(padded(plain.length)).fill(0x20)
|
|
244
280
|
bytes.set(plain)
|
|
@@ -249,15 +285,17 @@ class Session {
|
|
|
249
285
|
const c = await this.tx.encrypt(n, ad, bytes.subarray(at, at + MAX_PLAINTEXT))
|
|
250
286
|
out.push(`${n}.${b64(c)}`)
|
|
251
287
|
}
|
|
252
|
-
return
|
|
288
|
+
return out
|
|
253
289
|
}
|
|
254
290
|
|
|
255
|
-
/**
|
|
256
|
-
async open(
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
291
|
+
/** Its bodies; `REPLAY` for one delivered before; undefined for one that does not open. */
|
|
292
|
+
async open(s: unknown): Promise<unknown[] | typeof REPLAY | undefined> {
|
|
293
|
+
const parts = this.held
|
|
294
|
+
this.held = []
|
|
295
|
+
if (!Array.isArray(s)) return undefined
|
|
296
|
+
const more = s[0] === MORE
|
|
297
|
+
if (s.length === (more ? 1 : 0)) return undefined
|
|
298
|
+
for (const p of more ? s.slice(1) : s) {
|
|
261
299
|
if (typeof p !== 'string') return undefined
|
|
262
300
|
const [n, c] = p.split('.')
|
|
263
301
|
const seq = Number(n)
|
|
@@ -271,16 +309,23 @@ class Session {
|
|
|
271
309
|
// The wire numbers sends outside the seal, in the order they were
|
|
272
310
|
// sealed: one older than a body already opened was held back by the relay.
|
|
273
311
|
if (parts[0]!.n <= this.newest) return undefined
|
|
312
|
+
if (more) {
|
|
313
|
+
this.held = parts
|
|
314
|
+
return []
|
|
315
|
+
}
|
|
274
316
|
// Each part is sealed to its body's first nonce and part count: unbound,
|
|
275
317
|
// a relay spliced a fresh first part onto a delivered body's second.
|
|
276
318
|
const ad = bodyAd(parts[0]!.n, parts.length)
|
|
277
319
|
try {
|
|
278
320
|
const plain: Uint8Array[] = []
|
|
279
321
|
for (const p of parts) plain.push(await this.rx.decrypt(p.n, ad, unb64(p.c)))
|
|
280
|
-
const
|
|
322
|
+
const json = unsqueeze(Buffer.concat(plain), this.opened)
|
|
323
|
+
const bodies: unknown = JSON.parse(Buffer.from(json).toString())
|
|
324
|
+
if (!Array.isArray(bodies)) return undefined
|
|
325
|
+
this.opened.add(json)
|
|
281
326
|
for (const p of parts) this.seen.add(p.n)
|
|
282
327
|
this.newest = parts.at(-1)!.n
|
|
283
|
-
return
|
|
328
|
+
return bodies
|
|
284
329
|
// quiet: a body that does not open is the relay's tampering, which the caller says
|
|
285
330
|
} catch {
|
|
286
331
|
return undefined
|
|
@@ -288,29 +333,100 @@ class Session {
|
|
|
288
333
|
}
|
|
289
334
|
}
|
|
290
335
|
|
|
291
|
-
/**
|
|
336
|
+
/**
|
|
337
|
+
* Transport messages per send: six of 64 KiB, in base64, pass the wire's
|
|
338
|
+
* 512 KiB frame, and a task's env of 1.3 MB sealed as one send passed the
|
|
339
|
+
* relay's 1 MiB (2026-11-08).
|
|
340
|
+
*/
|
|
341
|
+
const SEND_PARTS = 5
|
|
342
|
+
/** Leads each send of a body but its last. */
|
|
343
|
+
const MORE = '+'
|
|
344
|
+
/** Leads a body zstd-compressed, and one as its JSON. */
|
|
292
345
|
const ZSTD = 0
|
|
346
|
+
const RAW = 1
|
|
347
|
+
/**
|
|
348
|
+
* Bodies of up to this much JSON go sealed as one; each body of more goes
|
|
349
|
+
* alone, over as many sends as its parts take (`SEND_PARTS`).
|
|
350
|
+
*/
|
|
351
|
+
const GROUP_BYTES = 64 * 1024
|
|
352
|
+
/** The lead byte and its sender's history length; a compressed one's frame length next. */
|
|
353
|
+
const HEADER = 5
|
|
354
|
+
/**
|
|
355
|
+
* The last this many bytes the orchestrator sealed to a worker are its
|
|
356
|
+
* next body's zstd dictionary: each batch sent its tasks' env and paths
|
|
357
|
+
* again. 32 KiB sealed no less for 1.7 times the CPU (1000 tasks,
|
|
358
|
+
* 2026-11-06). A worker's bodies go without one: a dictionary costs some
|
|
359
|
+
* 100 µs a body (node:zlib's per-call context), and the orchestrator
|
|
360
|
+
* opening each in turn ran a 1000-task run 0.3 s longer.
|
|
361
|
+
*/
|
|
362
|
+
const HISTORY_BYTES = 16 * 1024
|
|
293
363
|
|
|
294
364
|
/**
|
|
295
|
-
*
|
|
296
|
-
*
|
|
297
|
-
* The padding still hides its size within a class.
|
|
365
|
+
* One direction's bodies, as JSON: both ends add each, in the order sealed,
|
|
366
|
+
* as the wire delivers each once and in order.
|
|
298
367
|
*/
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
368
|
+
class History {
|
|
369
|
+
constructor(private readonly bytes: number) {}
|
|
370
|
+
window = new Uint8Array(0)
|
|
371
|
+
/**
|
|
372
|
+
* Every body's bytes, before the window too: each body names its
|
|
373
|
+
* sender's, so one sealed after a body the relay kept back or refused
|
|
374
|
+
* does not open.
|
|
375
|
+
*/
|
|
376
|
+
total = 0
|
|
377
|
+
add(json: Uint8Array): void {
|
|
378
|
+
this.total = (this.total + json.length) >>> 0
|
|
379
|
+
if (this.bytes === 0) return
|
|
380
|
+
const all = new Uint8Array(this.window.length + json.length)
|
|
381
|
+
all.set(this.window)
|
|
382
|
+
all.set(json, this.window.length)
|
|
383
|
+
this.window = all.subarray(Math.max(0, all.length - this.bytes))
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* A body zstd-compressed against what this side sealed before, before it
|
|
389
|
+
* is sealed, when smaller so: sealed, it is random bytes no relay or
|
|
390
|
+
* socket compression shrinks. The padding still hides its size within a
|
|
391
|
+
* class.
|
|
392
|
+
*/
|
|
393
|
+
function squeeze(json: Uint8Array, h: History): Uint8Array {
|
|
394
|
+
const z =
|
|
395
|
+
h.window.length === 0
|
|
396
|
+
? Bun.zstdCompressSync(json)
|
|
397
|
+
: zstdCompressSync(json, { dictionary: h.window })
|
|
398
|
+
const raw = z.length + 4 >= json.length
|
|
399
|
+
const out = new Uint8Array(HEADER + (raw ? json.length : 4 + z.length))
|
|
400
|
+
const v = new DataView(out.buffer)
|
|
401
|
+
out[0] = raw ? RAW : ZSTD
|
|
402
|
+
v.setUint32(1, h.total)
|
|
403
|
+
if (raw) out.set(json, HEADER)
|
|
404
|
+
else {
|
|
405
|
+
v.setUint32(HEADER, z.length)
|
|
406
|
+
out.set(z, HEADER + 4)
|
|
407
|
+
}
|
|
307
408
|
return out
|
|
308
409
|
}
|
|
309
410
|
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
411
|
+
/**
|
|
412
|
+
* Its JSON, unpadded. Throws for one sealed after a body this side never
|
|
413
|
+
* opened: compressed, it decoded against another dictionary to other
|
|
414
|
+
* bytes, and either way the relay kept that body back.
|
|
415
|
+
*/
|
|
416
|
+
function unsqueeze(plain: Uint8Array, h: History): Uint8Array {
|
|
417
|
+
const v = new DataView(plain.buffer, plain.byteOffset)
|
|
418
|
+
if (v.getUint32(1) !== h.total) throw new Error('sealed after a body never opened')
|
|
419
|
+
if (plain[0] === RAW) {
|
|
420
|
+
// JSON.stringify ends no body with a space: what trails it is padding.
|
|
421
|
+
let end = plain.length
|
|
422
|
+
while (end > HEADER && plain[end - 1] === 0x20) end--
|
|
423
|
+
return plain.subarray(HEADER, end)
|
|
424
|
+
}
|
|
425
|
+
if (plain[0] !== ZSTD) throw new Error('no body')
|
|
426
|
+
const z = plain.subarray(HEADER + 4, HEADER + 4 + v.getUint32(HEADER))
|
|
427
|
+
return h.window.length === 0
|
|
428
|
+
? Bun.zstdDecompressSync(z)
|
|
429
|
+
: zstdDecompressSync(z, { dictionary: h.window })
|
|
314
430
|
}
|
|
315
431
|
|
|
316
432
|
/** What each part of one body is sealed to: its first nonce and how many parts. */
|
|
@@ -367,7 +483,7 @@ export function sealOrchestrator(
|
|
|
367
483
|
proof.psk === undefined
|
|
368
484
|
? { type: 'hello', noise: answer, token: await proof.sign!(['hello', run, b64(hs.hash)]) }
|
|
369
485
|
: { type: 'hello', noise: answer }
|
|
370
|
-
return { info: parsed, hello, session: new Session(send, receive) }
|
|
486
|
+
return { info: parsed, hello, session: new Session(send, receive, true) }
|
|
371
487
|
}
|
|
372
488
|
return {
|
|
373
489
|
id: me.id,
|
|
@@ -424,7 +540,7 @@ export function sealOrchestrator(
|
|
|
424
540
|
out.push({ from: 'relay', body: { type: 'lost', id: m.from } })
|
|
425
541
|
continue
|
|
426
542
|
}
|
|
427
|
-
out.push({ from: m.from, body })
|
|
543
|
+
for (const one of body) out.push({ from: m.from, body: one })
|
|
428
544
|
}
|
|
429
545
|
return out
|
|
430
546
|
},
|
|
@@ -464,7 +580,7 @@ export async function sealWorker(
|
|
|
464
580
|
)
|
|
465
581
|
return false
|
|
466
582
|
const { send, receive } = await hs.split()
|
|
467
|
-
session = new Session(send, receive)
|
|
583
|
+
session = new Session(send, receive, false)
|
|
468
584
|
return true
|
|
469
585
|
}
|
|
470
586
|
return {
|
|
@@ -501,7 +617,7 @@ export async function sealWorker(
|
|
|
501
617
|
throw new SealError(
|
|
502
618
|
'a message from the orchestrator did not open: the relay changed or replayed it',
|
|
503
619
|
)
|
|
504
|
-
out.push({ from: m.from, body })
|
|
620
|
+
for (const one of body) out.push({ from: m.from, body: one })
|
|
505
621
|
}
|
|
506
622
|
return out
|
|
507
623
|
},
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// What a worker's servers use, charged to the tasks that make them work.
|
|
2
|
+
// A server is no task of the run's: its CPU was no one's, so history
|
|
3
|
+
// booked its dependants by their own trees and a server they kept at 100%
|
|
4
|
+
// went unbooked. Each look gives one sample per server: the cores its
|
|
5
|
+
// tree used and exactly which dependants ran then. The model is
|
|
6
|
+
// cores = base + the sum of each running dependant's own cost, fitted
|
|
7
|
+
// online (projected gradient: never below 0). Each sample's cores are
|
|
8
|
+
// split by it: each dependant is charged its own cost's part, added to its
|
|
9
|
+
// CPU, so history books it; the base is the server's own, booked once on
|
|
10
|
+
// each worker that runs it (`Need.servers`), and the planner still sees
|
|
11
|
+
// only tasks.
|
|
12
|
+
|
|
13
|
+
/** The step of each update, as a share of the error: small enough not to swing on one noisy look. */
|
|
14
|
+
const RATE = 0.3
|
|
15
|
+
|
|
16
|
+
interface Model {
|
|
17
|
+
base: number
|
|
18
|
+
readonly cost: Map<string, number>
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface ServerFit {
|
|
22
|
+
/**
|
|
23
|
+
* One look at `server`: `cores` its tree used over the last `ms` while
|
|
24
|
+
* `users` ran. `capped`: the machine was full, so `cores` is only what
|
|
25
|
+
* it got, a floor of what it asked for.
|
|
26
|
+
*/
|
|
27
|
+
sample(server: string, users: readonly string[], cores: number, ms: number, capped: boolean): void
|
|
28
|
+
/** The server CPU charged to `task` since it began, in core-ms; taking it clears it. */
|
|
29
|
+
take(task: string): number
|
|
30
|
+
/** `server`'s model now: its base and each dependant's cost, cores. */
|
|
31
|
+
model(server: string): { base: number; cost: ReadonlyMap<string, number> } | undefined
|
|
32
|
+
/** Each server's base now, cores: what a worker reports, rounded to a hundredth. */
|
|
33
|
+
bases(): Record<string, number>
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function serverFit(): ServerFit {
|
|
37
|
+
const models = new Map<string, Model>()
|
|
38
|
+
const owed = new Map<string, number>()
|
|
39
|
+
return {
|
|
40
|
+
sample(server, users, cores, ms, capped) {
|
|
41
|
+
let m = models.get(server)
|
|
42
|
+
if (m === undefined) models.set(server, (m = { base: 0, cost: new Map() }))
|
|
43
|
+
const cost = (u: string): number => m.cost.get(u) ?? 0
|
|
44
|
+
let predicted = m.base
|
|
45
|
+
for (const u of users) predicted += cost(u)
|
|
46
|
+
const error = cores - predicted
|
|
47
|
+
// A capped look says only "at least": it may raise the model, never
|
|
48
|
+
// lower it. The base learns only from looks with no dependant, each
|
|
49
|
+
// cost only from looks with: from looks always with one, both moved
|
|
50
|
+
// alike and the base took half of what the dependant made it do.
|
|
51
|
+
// Unlearned, the base is 0: a dependant is charged too much, never
|
|
52
|
+
// too little.
|
|
53
|
+
if (!(capped && error < 0)) {
|
|
54
|
+
if (users.length === 0) m.base = Math.max(0, m.base + RATE * error)
|
|
55
|
+
else {
|
|
56
|
+
const step = (RATE * error) / users.length
|
|
57
|
+
for (const u of users) m.cost.set(u, Math.max(0, cost(u) + step))
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (users.length === 0 || !(cores > 0) || !(ms > 0)) return
|
|
61
|
+
let total = m.base
|
|
62
|
+
for (const u of users) total += cost(u)
|
|
63
|
+
if (!(total > 0)) return
|
|
64
|
+
for (const u of users) owed.set(u, (owed.get(u) ?? 0) + (cores * ms * cost(u)) / total)
|
|
65
|
+
},
|
|
66
|
+
take(task) {
|
|
67
|
+
const ms = owed.get(task) ?? 0
|
|
68
|
+
owed.delete(task)
|
|
69
|
+
return ms
|
|
70
|
+
},
|
|
71
|
+
model(server) {
|
|
72
|
+
return models.get(server)
|
|
73
|
+
},
|
|
74
|
+
bases() {
|
|
75
|
+
const out: Record<string, number> = {}
|
|
76
|
+
for (const [server, m] of models) out[server] = Math.round(m.base * 100) / 100
|
|
77
|
+
return out
|
|
78
|
+
},
|
|
79
|
+
}
|
|
80
|
+
}
|
package/src/tree.ts
CHANGED
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
// them. Workers compare trees, not commits, so a run on uncommitted changes
|
|
3
3
|
// still matches the worktrees `local()` makes from them.
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { existsSync, lstatSync } from 'node:fs'
|
|
6
|
+
import { copyFile, mkdtemp, rm, stat, utimes, writeFile } from 'node:fs/promises'
|
|
6
7
|
import os from 'node:os'
|
|
7
8
|
import path from 'node:path'
|
|
9
|
+
import { PROJECT_CONFIG_FILENAMES, UserError } from '@vzn/vx'
|
|
8
10
|
|
|
9
11
|
/**
|
|
10
12
|
* A commit of the tracked files as they are now (`HEAD`'s when none changed),
|
|
@@ -18,10 +20,14 @@ export async function snapshot(root: string): Promise<{ commit: string; tree: st
|
|
|
18
20
|
// leaves the index and the working tree as they are. `git stash create`
|
|
19
21
|
// did the same, but refused an intent-to-add entry (`git add -N`), and
|
|
20
22
|
// HEAD stood in for the changes (2026-10-28).
|
|
21
|
-
const index =
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
const [top, index] =
|
|
24
|
+
(
|
|
25
|
+
await git(
|
|
26
|
+
['rev-parse', '--show-toplevel', '--path-format=absolute', '--git-path', 'index'],
|
|
27
|
+
root,
|
|
28
|
+
)
|
|
29
|
+
)?.split('\n') ?? []
|
|
30
|
+
if (top === undefined || index === undefined) return none
|
|
25
31
|
const dir = await mkdtemp(path.join(os.tmpdir(), 'vx-dte-index-'))
|
|
26
32
|
try {
|
|
27
33
|
const env = { GIT_INDEX_FILE: path.join(dir, 'index') }
|
|
@@ -37,7 +43,16 @@ export async function snapshot(root: string): Promise<{ commit: string; tree: st
|
|
|
37
43
|
// quiet: no index yet, so HEAD's tree is its tracked set.
|
|
38
44
|
if ((await git(['read-tree', head], root, env)) === undefined) return none
|
|
39
45
|
}
|
|
40
|
-
|
|
46
|
+
await unflag(top, env)
|
|
47
|
+
// The executable bit as on disk, which core keys from an lstat under
|
|
48
|
+
// `core.fileMode=false` too: there `add -u` staged no `chmod +x`,
|
|
49
|
+
// workers checked the file out 644, and saved what it built under the
|
|
50
|
+
// key of its 755 (2026-11-08).
|
|
51
|
+
if (
|
|
52
|
+
(await git([...UNSPARSE, '-c', 'core.fileMode=true', 'add', '-u', '--', ':/'], root, env)) ===
|
|
53
|
+
undefined
|
|
54
|
+
)
|
|
55
|
+
return none
|
|
41
56
|
const tree = (await git(['write-tree'], root, env))?.trim()
|
|
42
57
|
if (tree === undefined || tree === '') return none
|
|
43
58
|
const was = (await git(['rev-parse', `${head}^{tree}`], root))?.trim()
|
|
@@ -53,6 +68,198 @@ export async function snapshot(root: string): Promise<{ commit: string; tree: st
|
|
|
53
68
|
}
|
|
54
69
|
}
|
|
55
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Each tracked file a git filter of this machine checked out other than
|
|
73
|
+
* the commit holds it (`text`, `eol`, `ident`, `filter`,
|
|
74
|
+
* `working-tree-encoding`, `core.autocrlf`), written back as the commit's
|
|
75
|
+
* bytes; the paths written. The run keys its disk's bytes and refuses a
|
|
76
|
+
* task that keys a file the commit holds otherwise, so these are the bytes
|
|
77
|
+
* its key read: a worktree that smudged LF to CRLF built CRLF output under
|
|
78
|
+
* the key of the LF file, a stale hit (2026-11-08).
|
|
79
|
+
*/
|
|
80
|
+
export async function unfilter(root: string): Promise<string[]> {
|
|
81
|
+
const top = (await git(['rev-parse', '--show-toplevel'], root))?.trim()
|
|
82
|
+
if (top === undefined || top === '') return []
|
|
83
|
+
const vars = await git(['var', '-l'], top)
|
|
84
|
+
if (vars === undefined) return []
|
|
85
|
+
const config = varList(vars)
|
|
86
|
+
const every = autocrlf(config)
|
|
87
|
+
// The common repository names no filter: no index read.
|
|
88
|
+
if (!every && !(await attributed(top, config))) return []
|
|
89
|
+
const staged = await git([...UNSPARSE, 'ls-files', '-s', '-z'], top)
|
|
90
|
+
if (staged === undefined) return []
|
|
91
|
+
const blobs = new Map<string, string>()
|
|
92
|
+
for (const e of staged.split('\0')) {
|
|
93
|
+
const m = /^(?:100644|100755) ([0-9a-f]+) 0\t(.*)$/s.exec(e)
|
|
94
|
+
if (m !== null) blobs.set(m[2]!, m[1]!)
|
|
95
|
+
}
|
|
96
|
+
let paths = [...blobs.keys()]
|
|
97
|
+
if (!every) paths = await filtered(top, paths)
|
|
98
|
+
// `--stdin-paths` reads lines; a file off disk (a sparse checkout's) is no checkout.
|
|
99
|
+
paths = paths.filter((p) => !p.includes('\n') && isFile(path.join(top, p)))
|
|
100
|
+
if (paths.length === 0) return []
|
|
101
|
+
const raw = await git(
|
|
102
|
+
['hash-object', '--no-filters', '--stdin-paths'],
|
|
103
|
+
top,
|
|
104
|
+
undefined,
|
|
105
|
+
paths.join('\n') + '\n',
|
|
106
|
+
)
|
|
107
|
+
if (raw === undefined) return []
|
|
108
|
+
const oids = raw.split('\n')
|
|
109
|
+
const off = paths.filter((p, i) => oids[i] !== blobs.get(p))
|
|
110
|
+
if (off.length === 0) return []
|
|
111
|
+
const p = Bun.spawn(['git', 'cat-file', '--batch'], {
|
|
112
|
+
cwd: top,
|
|
113
|
+
stdin: Buffer.from(off.map((f) => blobs.get(f)).join('\n') + '\n'),
|
|
114
|
+
stdout: 'pipe',
|
|
115
|
+
stderr: 'ignore',
|
|
116
|
+
})
|
|
117
|
+
const [out, code] = await Promise.all([new Response(p.stdout).bytes(), p.exited])
|
|
118
|
+
if (code !== 0) throw new UserError(`vx dte worker: git could not read ${off[0]}'s blob`)
|
|
119
|
+
// Each `<oid> blob <size>\n<bytes>\n`.
|
|
120
|
+
let at = 0
|
|
121
|
+
for (const f of off) {
|
|
122
|
+
const nl = out.indexOf(10, at)
|
|
123
|
+
const size = Number(new TextDecoder().decode(out.subarray(at, nl)).split(' ')[2])
|
|
124
|
+
await writeFile(path.join(top, f), out.subarray(nl + 1, nl + 1 + size))
|
|
125
|
+
at = nl + 1 + size + 1
|
|
126
|
+
}
|
|
127
|
+
return off
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Whether `file` is a regular file on disk. */
|
|
131
|
+
function isFile(file: string): boolean {
|
|
132
|
+
try {
|
|
133
|
+
return lstatSync(file).isFile()
|
|
134
|
+
} catch {
|
|
135
|
+
// quiet: off disk is no file
|
|
136
|
+
return false
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** `git var -l`'s last value of each name; a bare key (true) has none. */
|
|
141
|
+
function varList(vars: string): Map<string, string | undefined> {
|
|
142
|
+
const out = new Map<string, string | undefined>()
|
|
143
|
+
for (const line of vars.split('\n')) {
|
|
144
|
+
const eq = line.indexOf('=')
|
|
145
|
+
if (eq === -1) out.set(line.toLowerCase(), undefined)
|
|
146
|
+
else out.set(line.slice(0, eq).toLowerCase(), line.slice(eq + 1))
|
|
147
|
+
}
|
|
148
|
+
return out
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Whether `core.autocrlf`, as git reads it, converts line ends. */
|
|
152
|
+
function autocrlf(config: Map<string, string | undefined>): boolean {
|
|
153
|
+
if (!config.has('core.autocrlf')) return false
|
|
154
|
+
const v = config.get('core.autocrlf')
|
|
155
|
+
if (v === undefined) return true
|
|
156
|
+
const s = v.trim().toLowerCase()
|
|
157
|
+
if (s === 'input' || s === 'true' || s === 'yes' || s === 'on') return true
|
|
158
|
+
return /^-?\d+$/.test(s) && Number(s) !== 0
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Whether any attributes file may name a filter: a `.gitattributes` in the
|
|
163
|
+
* tree, `$GIT_DIR/info/attributes`, or the global or system one git names
|
|
164
|
+
* (`git var`, from git 2.42; an older git names neither, so any may).
|
|
165
|
+
*/
|
|
166
|
+
async function attributed(top: string, config: Map<string, string | undefined>): Promise<boolean> {
|
|
167
|
+
if (existsSync(path.join(top, '.gitattributes'))) return true
|
|
168
|
+
const [tracked, info] = await Promise.all([
|
|
169
|
+
git([...UNSPARSE, 'ls-files', '-z', '--', ':(glob)**/.gitattributes'], top),
|
|
170
|
+
git(['rev-parse', '--path-format=absolute', '--git-path', 'info/attributes'], top),
|
|
171
|
+
])
|
|
172
|
+
if (tracked !== '' || info === undefined || existsSync(info.trim())) return true
|
|
173
|
+
if (!config.has('git_attr_global')) return true
|
|
174
|
+
for (const name of ['git_attr_global', 'git_attr_system']) {
|
|
175
|
+
const f = config.get(name)
|
|
176
|
+
if (f !== undefined && f !== '' && existsSync(f)) return true
|
|
177
|
+
}
|
|
178
|
+
return false
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Of `paths`, those a filter attribute names (set, or given a value). */
|
|
182
|
+
async function filtered(top: string, paths: string[]): Promise<string[]> {
|
|
183
|
+
const out = await git(
|
|
184
|
+
['check-attr', '-z', '--stdin', 'text', 'eol', 'ident', 'filter', 'working-tree-encoding'],
|
|
185
|
+
top,
|
|
186
|
+
undefined,
|
|
187
|
+
paths.join('\0') + '\0',
|
|
188
|
+
)
|
|
189
|
+
if (out === undefined) return paths
|
|
190
|
+
const named = new Set<string>()
|
|
191
|
+
const f = out.split('\0')
|
|
192
|
+
for (let i = 0; i + 2 < f.length; i += 3)
|
|
193
|
+
if (f[i + 2] !== 'unspecified' && f[i + 2] !== 'unset') named.add(f[i]!)
|
|
194
|
+
return paths.filter((p) => named.has(p))
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* The workspace config core loads (the first `.vx/workspace.*` under
|
|
199
|
+
* `root`), when `tree` lacks it: every worker started without one failed
|
|
200
|
+
* `unknown command: dte`, and started again three times, before a task
|
|
201
|
+
* said why (2026-11-07).
|
|
202
|
+
*/
|
|
203
|
+
export async function untrackedWorkspace(root: string, tree: string): Promise<string | undefined> {
|
|
204
|
+
for (const name of WORKSPACE_FILES) {
|
|
205
|
+
if (!(await Bun.file(path.join(root, name)).exists())) continue
|
|
206
|
+
return (await git(['cat-file', '-e', `${tree}:./${name}`], root)) === undefined
|
|
207
|
+
? name
|
|
208
|
+
: undefined
|
|
209
|
+
}
|
|
210
|
+
return undefined
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Core's names in its order: a project config's extensions. */
|
|
214
|
+
const WORKSPACE_FILES = PROJECT_CONFIG_FILENAMES.map((n) => `.vx/workspace${path.extname(n)}`)
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* The copy's assume-unchanged and skip-worktree flags cleared, so `add -u`
|
|
218
|
+
* takes those files from disk: it passed over them, and the tree kept
|
|
219
|
+
* HEAD's blob of a file core's key hashed as edited, a stale hit
|
|
220
|
+
* (2026-11-07). A skip-worktree file off disk (a sparse checkout's) stays
|
|
221
|
+
* flagged: its bytes are the index's, not a deletion.
|
|
222
|
+
*/
|
|
223
|
+
async function unflag(top: string, env: Record<string, string>): Promise<void> {
|
|
224
|
+
const listed = await git([...UNSPARSE, 'ls-files', '-v', '-z'], top, env)
|
|
225
|
+
if (listed === undefined) return
|
|
226
|
+
const assumed: string[] = []
|
|
227
|
+
const skipped: string[] = []
|
|
228
|
+
for (const entry of listed.split('\0')) {
|
|
229
|
+
const tag = entry[0]
|
|
230
|
+
if (tag === undefined) continue
|
|
231
|
+
const file = entry.slice(2)
|
|
232
|
+
// Lower case is assume-unchanged; S (s: both flags) skip-worktree.
|
|
233
|
+
if (tag !== tag.toUpperCase()) assumed.push(file)
|
|
234
|
+
if ((tag === 'S' || tag === 's') && existsSync(path.join(top, file))) skipped.push(file)
|
|
235
|
+
}
|
|
236
|
+
// One flag a call: given both, update-index clears the first and skips
|
|
237
|
+
// the second.
|
|
238
|
+
for (const [flag, files] of [
|
|
239
|
+
['--no-assume-unchanged', assumed],
|
|
240
|
+
['--no-skip-worktree', skipped],
|
|
241
|
+
] as const) {
|
|
242
|
+
if (files.length === 0) continue
|
|
243
|
+
const p = Bun.spawn(['git', ...UNSPARSE, 'update-index', flag, '-z', '--stdin'], {
|
|
244
|
+
cwd: top,
|
|
245
|
+
stdin: Buffer.from(files.join('\0') + '\0'),
|
|
246
|
+
stdout: 'ignore',
|
|
247
|
+
stderr: 'pipe',
|
|
248
|
+
env: { ...process.env, ...env },
|
|
249
|
+
})
|
|
250
|
+
const [err, code] = await Promise.all([new Response(p.stderr).text(), p.exited])
|
|
251
|
+
if (code !== 0)
|
|
252
|
+
throw new UserError(
|
|
253
|
+
`dte(): git could not take ${files[0]} (flagged assume-unchanged or skip-worktree) from disk, so workers would run its committed bytes: ${err.trim()}`,
|
|
254
|
+
)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// A sparse checkout's patterns kept `add` off a file outside them that is
|
|
259
|
+
// on disk all the same, and its index reads clear the flag of such a file
|
|
260
|
+
// in memory only, so `ls-files` named it unflagged.
|
|
261
|
+
const UNSPARSE = ['-c', 'core.sparseCheckout=false']
|
|
262
|
+
|
|
56
263
|
const IDENTITY = {
|
|
57
264
|
GIT_AUTHOR_NAME: 'vx',
|
|
58
265
|
GIT_AUTHOR_EMAIL: 'vx@localhost',
|
|
@@ -65,10 +272,11 @@ async function git(
|
|
|
65
272
|
args: string[],
|
|
66
273
|
cwd: string,
|
|
67
274
|
env?: Record<string, string>,
|
|
275
|
+
input?: string,
|
|
68
276
|
): Promise<string | undefined> {
|
|
69
277
|
const p = Bun.spawn(['git', ...args], {
|
|
70
278
|
cwd,
|
|
71
|
-
stdin: 'ignore',
|
|
279
|
+
stdin: input === undefined ? 'ignore' : Buffer.from(input),
|
|
72
280
|
stdout: 'pipe',
|
|
73
281
|
stderr: 'ignore',
|
|
74
282
|
...(env === undefined ? {} : { env: { ...process.env, ...env } }),
|