@awebai/oats 0.69.12 → 0.69.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +216 -178
- package/bin/oats.mjs +254 -201
- package/docs/capabilities.md +58 -13
- package/docs/configuration.md +63 -5
- package/docs/container.md +291 -0
- package/docs/design/2026-09-23-workspace-module-contracts.md +2 -0
- package/docs/desktop-cli-api.md +590 -55
- package/docs/desktop-instance-start.md +4 -1
- package/docs/desktop.md +58 -12
- package/docs/execution-targets.md +84 -9
- package/docs/first-team.md +4 -3
- package/docs/implementation.md +99 -32
- package/docs/integrations.md +3 -2
- package/docs/knowledge-and-learning.md +72 -0
- package/docs/oats-local.schema.json +11 -2
- package/docs/official-catalog.md +39 -5
- package/docs/packages.md +93 -14
- package/docs/release-lane.md +2 -1
- package/docs/release-notes/v0.69.13.md +767 -0
- package/docs/release-notes/v0.69.14.md +83 -0
- package/docs/release-notes/v0.69.15.md +444 -0
- package/docs/schedules.md +159 -5
- package/docs/servers.md +241 -7
- package/docs/souls-and-instances.md +223 -14
- package/docs/tui.md +1 -1
- package/docs/upgrading.md +36 -5
- package/docs/workspaces.md +31 -5
- package/lib/claim.mjs +20 -5
- package/lib/command-flags.mjs +240 -0
- package/lib/core.mjs +1004 -633
- package/lib/deployments.mjs +4 -3
- package/lib/dir-lock.mjs +145 -33
- package/lib/errors.mjs +3 -2
- package/lib/harness-token.mjs +110 -0
- package/lib/host-run.mjs +218 -0
- package/lib/host-stop-child.mjs +21 -0
- package/lib/hot-reads.mjs +4 -3
- package/lib/instance-events.mjs +28 -6
- package/lib/instance-git.mjs +165 -14
- package/lib/instance-inspect.mjs +6 -1
- package/lib/instance-lifecycle.mjs +4 -3
- package/lib/instance-listing.mjs +114 -43
- package/lib/instance-resolution.mjs +56 -3
- package/lib/instance-upgrade.mjs +16 -20
- package/lib/kernel-features.mjs +4 -1
- package/lib/launch-env-exec.sh +7 -0
- package/lib/launch-env.mjs +74 -0
- package/lib/launch-preference.mjs +19 -12
- package/lib/member-clone.mjs +7 -1
- package/lib/packages.mjs +74 -2
- package/lib/read-call.mjs +4 -1
- package/lib/reader.mjs +3 -2
- package/lib/refused-text.mjs +3 -2
- package/lib/remote.mjs +126 -31
- package/lib/retained-after-retire.mjs +2 -7
- package/lib/retire-git-forms.mjs +41 -0
- package/lib/retire-incomplete.mjs +96 -0
- package/lib/schedule.mjs +239 -21
- package/lib/servers.mjs +161 -28
- package/lib/session-end.mjs +28 -8
- package/lib/session-resume.mjs +63 -56
- package/lib/session-seen.mjs +3 -14
- package/lib/start-env.mjs +13 -7
- package/lib/status-words.mjs +11 -4
- package/lib/system-tools.mjs +1 -36
- package/lib/teams.mjs +11 -4
- package/lib/upgrade-state.mjs +13 -2
- package/lib/worktree-hooks.mjs +6 -175
- package/lib/worktree.mjs +68 -10
- package/package-catalog.json +1 -1
- package/package.json +5 -1
- package/packages/client/instance-lifecycle.mjs +1 -1
- package/packages/client/lifecycle-contract.mjs +3 -1
- package/packages/client/session-resume-cli.mjs +2 -1
- package/packages/client/session-start.mjs +10 -7
- package/packages/client/workspace-cli.mjs +5 -1
- package/packages/record/README.md +56 -21
- package/packages/record/docs/capture-service.md +8 -5
- package/packages/record/docs/turn-record-sot.md +9 -0
- package/packages/record/lib/capture-health.mjs +6 -4
- package/packages/record/lib/capture-lock.mjs +31 -27
- package/packages/record/lib/capture-retained-scratch.mjs +4 -3
- package/packages/record/lib/capture-scratch.mjs +2 -1
- package/packages/record/lib/derived-state.mjs +4 -2
- package/packages/record/lib/index-db.mjs +4 -2
- package/packages/record/lib/lock-holder.mjs +21 -0
- package/packages/record/lib/private-files.mjs +18 -0
- package/packages/record/lib/process-identity.mjs +310 -0
- package/packages/record/lib/store.mjs +31 -25
- package/share/oats-login-receive +204 -0
- package/skills/oats-getting-started/SKILL.md +93 -26
- package/lib/tmux-config.mjs +0 -35
package/lib/remote.mjs
CHANGED
|
@@ -36,10 +36,12 @@
|
|
|
36
36
|
* every budget is applied to real sizes. A server that cannot serve partial fetches
|
|
37
37
|
* (no filter support; or blob wants refused, so the commit is fetched again with
|
|
38
38
|
* `--refetch`) gets whole trees from then on: the cache records `oats.fetch = full`,
|
|
39
|
-
* and the session's `notices` say so once. So does a git older than
|
|
40
|
-
* (2.45, which brought GIT_NO_LAZY_FETCH): an older git would fetch a
|
|
41
|
-
* its own or die, so every cache it touches fetches whole trees, and a
|
|
42
|
-
* finds is deleted and fetched again whole (rebuildWhole).
|
|
39
|
+
* and the session's `notices` say so once for that repository. So does a git older than
|
|
40
|
+
* PARTIAL_FETCH_GIT (2.45, which brought GIT_NO_LAZY_FETCH): an older git would fetch a
|
|
41
|
+
* missing blob on its own or die, so every cache it touches fetches whole trees, and a
|
|
42
|
+
* partial cache it finds is deleted and fetched again whole (rebuildWhole). Each such
|
|
43
|
+
* cache records `oats.fetch = full`, but the notice names no repository and is said once
|
|
44
|
+
* per session (once per command, or per reader call), however many caches it starts. A lost `.lock` race on the
|
|
43
45
|
* cache repo's config or objects (another process starting the same cache) is retried.
|
|
44
46
|
*
|
|
45
47
|
* The cache is invisible plumbing: it may be wiped at any time (a wiped cache
|
|
@@ -88,9 +90,10 @@
|
|
|
88
90
|
* CONCURRENCY (awebai/oats#386): per-key operations on one cache repo are serialized
|
|
89
91
|
* in-process (withCacheLock), and every WRITE to a cache repo (init, config, fetch, pin)
|
|
90
92
|
* holds its cross-process write lock `<cacheRoot>/.locks/<repo>.lock` (withCacheWriteLock):
|
|
91
|
-
* a live holder is waited for (bounded by a whole fetch) and never stolen from, a
|
|
92
|
-
* one
|
|
93
|
-
*
|
|
93
|
+
* a live holder is waited for (bounded by a whole fetch) and never stolen from, a gone
|
|
94
|
+
* one (dead, its pid another process's, or of another boot) is reclaimed; reads take no
|
|
95
|
+
* lock. A cache repo is created whole (init into a private directory, then rename).
|
|
96
|
+
* A git `.lock` met under the write lock belongs to an older
|
|
94
97
|
* kernel's live write (retried briefly) or to a git that was killed: one older than the
|
|
95
98
|
* longest fetch, inside the cache and a regular file, is removed (a session notice says
|
|
96
99
|
* so) and the write made once more; any other is E_REMOTE_UNREADABLE { reason: "cache" }
|
|
@@ -160,6 +163,16 @@
|
|
|
160
163
|
* keep a record each. A record is reused only
|
|
161
164
|
* under `maxAge`, only when key, ref args and urlDigest all match, and never in place of
|
|
162
165
|
* a live observation that failed.
|
|
166
|
+
*
|
|
167
|
+
* VERIFIED STORE (`treeVerification`; kernel-private, not a contract):
|
|
168
|
+
* `<cacheRoot>/.verified/<sha256([key, commit, dir, digest])>.json` records that the tree `dir` of
|
|
169
|
+
* (repo key, commit), read from this cache, had the content digest `digest`:
|
|
170
|
+
* { v, key, commit, dir, digest, entry }. `entry` is the state of the cache repo the tree was read from
|
|
171
|
+
* (cacheEntryState: every file under it by path, size, mode, inode, and modification and status-change times),
|
|
172
|
+
* so a record is believed only while every recorded field is the asked one AND the cache repo is file for
|
|
173
|
+
* file what it was: git reads an object without checking it against its id, so an object changed in the
|
|
174
|
+
* cache is answered as it is. A cache repo that reads objects from outside itself (a symlink, an alternates
|
|
175
|
+
* file) has no state and no record. Written atomically, last writer wins; a corrupt record is a miss.
|
|
163
176
|
*/
|
|
164
177
|
import { execFileSync, spawn } from "node:child_process";
|
|
165
178
|
import { createHash, randomBytes } from "node:crypto";
|
|
@@ -173,7 +186,9 @@ import { fileURLToPath } from "node:url";
|
|
|
173
186
|
import { oatsError } from "./errors.mjs";
|
|
174
187
|
import { REFUSED_TEXT } from "./refused-text.mjs";
|
|
175
188
|
import { killGroup, signalGroup, terminateGroup, watchGroup } from "./process-group.mjs";
|
|
189
|
+
import { identityHolderState, thisBoot, validPid } from "./session-seen.mjs";
|
|
176
190
|
import { writeThroughPrivateFile } from "./write-through-private-file.mjs";
|
|
191
|
+
import { processStartToken } from "./worktree-hooks.mjs";
|
|
177
192
|
|
|
178
193
|
export const FILE_BUDGET = 4 * 1024 * 1024; // readRemoteFile: 4 MiB per file
|
|
179
194
|
export const TREE_BUDGET = 64 * 1024 * 1024; // fetchRemoteTree: 64 MiB per subtree
|
|
@@ -639,8 +654,6 @@ const UNREADABLE_LOCK_STALE_MS = 30_000;
|
|
|
639
654
|
* longest fetch any oats allows, so an older kernel's live fetch (it takes no write lock) is never broken. */
|
|
640
655
|
const GIT_LOCK_STALE_MS = GIT_FETCH_TIMEOUT_MS + 60_000;
|
|
641
656
|
|
|
642
|
-
function pidAlive(pid) { try { process.kill(pid, 0); return true; } catch (e) { return e.code === "EPERM"; } }
|
|
643
|
-
|
|
644
657
|
/** The write lock of the cache repo `dir`: `<cacheRoot>/.locks/<repo>.lock`, outside git's own namespace. */
|
|
645
658
|
const cacheWriteLockOf = (dir) => join(dirname(dir), ".locks", `${basename(dir)}.lock`);
|
|
646
659
|
|
|
@@ -662,17 +675,22 @@ function readCacheWriteLock(path) {
|
|
|
662
675
|
* Run `fn` holding the cache repo's cross-process write lock (docs: oats-kernel-expert decision
|
|
663
676
|
* record-lock-liveness-tradeoff): every oats write to a cache repo — init, config, fetch, pin — happens under
|
|
664
677
|
* it, so two processes never write one cache at once; reads take no lock. The lock is an exclusive-create file
|
|
665
|
-
* holding { pid, token, startedAt }, linked into place whole
|
|
666
|
-
*
|
|
667
|
-
*
|
|
668
|
-
*
|
|
678
|
+
* holding { pid, token, startedAt, bootId, processStart }, linked into place whole: its holder named by pid,
|
|
679
|
+
* boot and start, as a claim's is (lib/claim.mjs). A live holder is waited for, never stolen from, whatever
|
|
680
|
+
* its age; a gone holder's lock (isStaleLock: one taken in another boot, or whose pid is gone or runs with
|
|
681
|
+
* another start; or an unreadable one past UNREADABLE_LOCK_STALE_MS) is removed after checking it is still the
|
|
682
|
+
* same file. A record that names no boot or start is its holder's by pid alone. An owner whose liveness
|
|
683
|
+
* cannot be judged (identityHolderState "unknown": a pid or a start that cannot be read, a pid that is no
|
|
684
|
+
* process ID) is waited for like a live one, never reclaimed; the refusal for a pid that is no process ID
|
|
685
|
+
* says so, with no holderPid. Every pass that does not
|
|
686
|
+
* acquire falls through one deadline check and one sleep. At the deadline: E_REMOTE_UNREADABLE { reason: "cache", stage, cacheDir, lock, holderPid }; at once
|
|
669
687
|
* when a reclaimer died holding the reclaim guard (reclaimCacheWriteLock), with details.guard.
|
|
670
688
|
* `fn({ waited })`: whether another process held the lock first (what it wrote is worth checking again).
|
|
671
689
|
*/
|
|
672
690
|
async function withCacheWriteLock(ref, dir, stage, session, fn) {
|
|
673
691
|
const signal = session?.signal;
|
|
674
692
|
const lock = cacheWriteLockOf(dir);
|
|
675
|
-
const owner = { pid: process.pid, token: randomBytes(12).toString("hex"), startedAt: new Date().toISOString() };
|
|
693
|
+
const owner = { pid: process.pid, token: randomBytes(12).toString("hex"), startedAt: new Date().toISOString(), bootId: thisBoot(), processStart: processStartToken(process.pid) };
|
|
676
694
|
const url = displayRepoRef(ref.url);
|
|
677
695
|
const refuse = (message, extra = {}) => fail("E_REMOTE_UNREADABLE", `cannot write the cache of ${url} at ${dir} (cache, ${stage}): ${message}`,
|
|
678
696
|
{ url: ref.url, key: ref.key, reason: "cache", stage, cacheDir: dir, lock, ...extra });
|
|
@@ -684,9 +702,10 @@ async function withCacheWriteLock(ref, dir, stage, session, fn) {
|
|
|
684
702
|
const budgetEnded = () => {
|
|
685
703
|
let now = held;
|
|
686
704
|
try { now = readCacheWriteLock(lock); } catch { /* as last seen */ }
|
|
687
|
-
const
|
|
705
|
+
const pid = now?.owner && validPid(now.owner.pid) ? now.owner.pid : null;
|
|
706
|
+
const holder = pid ? `oats process ${pid}` : "another process";
|
|
688
707
|
return fail("E_REMOTE_UNREADABLE", `cannot write the cache of ${url} at ${dir} (timeout, ${stage}): the read budget ended ${now ? `while ${holder} held its write lock ${lock}` : `before it took its write lock ${lock}`}`,
|
|
689
|
-
{ url: ref.url, key: ref.key, reason: "timeout", stage, cacheDir: dir, lock, ...(
|
|
708
|
+
{ url: ref.url, key: ref.key, reason: "timeout", stage, cacheDir: dir, lock, ...(pid ? { holderPid: pid } : {}) });
|
|
690
709
|
};
|
|
691
710
|
for (let attempt = 0; ; attempt++) {
|
|
692
711
|
if (signal?.aborted) throw unreadable(ref, abortError(signal), { cacheDir: dir, stage });
|
|
@@ -706,7 +725,9 @@ async function withCacheWriteLock(ref, dir, stage, session, fn) {
|
|
|
706
725
|
}
|
|
707
726
|
}
|
|
708
727
|
if (Date.now() >= deadline) {
|
|
709
|
-
|
|
728
|
+
// A record whose pid is no process ID names no process to check: the refusal says what it names.
|
|
729
|
+
if (held?.owner && !validPid(held.owner.pid)) throw refuse(`its write lock ${lock} names ${held.owner.pid}, which is no process ID, as its holder; it is safe to remove once no oats process is running`);
|
|
730
|
+
throw refuse(held?.owner ? `process ${held.owner.pid} has been writing it since ${held.owner.startedAt} (lock ${lock}); try again once it finishes; check whether pid ${held.owner.pid} is an oats process: once no oats process holds it, the lock ${lock} is safe to remove`
|
|
710
731
|
: `its write lock ${lock} is held; it is safe to remove once no oats process is running`, held?.owner ? { holderPid: held.owner.pid } : {});
|
|
711
732
|
}
|
|
712
733
|
if (session?.expired()) throw budgetEnded();
|
|
@@ -721,9 +742,16 @@ async function withCacheWriteLock(ref, dir, stage, session, fn) {
|
|
|
721
742
|
}
|
|
722
743
|
}
|
|
723
744
|
|
|
724
|
-
/** Whether a lock record (readCacheWriteLock) is provably abandoned: its owner
|
|
725
|
-
* readable owner and is older than UNREADABLE_LOCK_STALE_MS. A live owner is never stale, whatever its age. */
|
|
726
|
-
const isStaleLock = (held) => (held.owner ?
|
|
745
|
+
/** Whether a lock record (readCacheWriteLock) is provably abandoned: its owner is gone (ownerGone), or it has
|
|
746
|
+
* no readable owner and is older than UNREADABLE_LOCK_STALE_MS. A live owner is never stale, whatever its age. */
|
|
747
|
+
const isStaleLock = (held) => (held.owner ? ownerGone(held.owner) : Date.now() - held.mtimeMs > UNREADABLE_LOCK_STALE_MS);
|
|
748
|
+
/** Whether the owner a lock record names is gone, as identityHolderState says it
|
|
749
|
+
* (packages/record/lib/process-identity.mjs, where the rule is composed). One that names its boot or start
|
|
750
|
+
* (`bootId`, `processStart`) is gone when taken in another boot, or when its pid is gone, runs with another
|
|
751
|
+
* start or is a zombie. One that names neither is gone when its pid is. An owner whose liveness cannot be
|
|
752
|
+
* judged (a start or a pid that cannot be read, a pid that is no process ID) is never gone: its lock is
|
|
753
|
+
* waited for and refused, as a live holder's is. */
|
|
754
|
+
const ownerGone = (owner) => identityHolderState(owner) === "gone";
|
|
727
755
|
/** The same record still at `path`: the same owner token, or (no readable owner) the same file. */
|
|
728
756
|
const sameLock = (now, held) => (held.owner ? now?.owner?.token === held.owner.token : now && !now.owner && now.ino === held.ino && now.mtimeMs === held.mtimeMs);
|
|
729
757
|
|
|
@@ -1373,8 +1401,8 @@ function touchParsed(file, dir, session) {
|
|
|
1373
1401
|
/**
|
|
1374
1402
|
* Bound the stores under `root` (one scan): the current fingerprint's entries, least recently used first,
|
|
1375
1403
|
* down to keepEntries AND keepBytes once over maxEntries OR maxBytes; other fingerprint directories unused
|
|
1376
|
-
* for staleFingerprintMs; observation records older than staleObservationMs; temp
|
|
1377
|
-
* left. Never throws. → { removed, entries, bytes } for the current fingerprint.
|
|
1404
|
+
* for staleFingerprintMs; observation records and verified-tree records older than staleObservationMs; temp
|
|
1405
|
+
* files a crashed writer left. Never throws. → { removed, entries, bytes } for the current fingerprint.
|
|
1378
1406
|
*/
|
|
1379
1407
|
export function pruneStores(root, { fingerprint = kernelFingerprint(), limits = PARSED_LIMITS, now = Date.now() } = {}) {
|
|
1380
1408
|
const L = { ...PARSED_LIMITS, ...limits };
|
|
@@ -1406,13 +1434,69 @@ export function pruneStores(root, { fingerprint = kernelFingerprint(), limits =
|
|
|
1406
1434
|
if (st && now - st.mtimeMs > L.staleFingerprintMs && remove(dir, { recursive: true })) removed++;
|
|
1407
1435
|
}
|
|
1408
1436
|
}
|
|
1409
|
-
for (const observed of [join(root, ".observed"), join(root, ".observed-facts")]) for (const f of list(observed)) {
|
|
1437
|
+
for (const observed of [join(root, ".observed"), join(root, ".observed-facts"), join(root, ".verified")]) for (const f of list(observed)) {
|
|
1410
1438
|
const p = join(observed, f), st = stat(p);
|
|
1411
1439
|
if (st && now - st.mtimeMs > (f.endsWith(".tmp") ? L.staleTempMs : L.staleObservationMs) && remove(p)) removed++;
|
|
1412
1440
|
}
|
|
1413
1441
|
return { removed, entries, bytes };
|
|
1414
1442
|
}
|
|
1415
1443
|
|
|
1444
|
+
// ---------------------------------------------------------------------------
|
|
1445
|
+
// the verified store: trees whose content digest was checked, bound to the cache repo they were read from
|
|
1446
|
+
// ---------------------------------------------------------------------------
|
|
1447
|
+
|
|
1448
|
+
/** What the cache repo at `dir` is, without reading its content: sha256 over every file under it, in path
|
|
1449
|
+
* order, as (path, size, mode, inode, mtime ns, ctime ns). Adding, removing, replacing or rewriting a file
|
|
1450
|
+
* changes it; so does a rewrite that restores the size and the modification time (the status-change time
|
|
1451
|
+
* moves). → "sha256-<hex>", or null when the repo has no such state: it cannot be listed (absent, or a file
|
|
1452
|
+
* went while it was read: a write in progress), or what git reads there is not all under it (an entry that is
|
|
1453
|
+
* neither a directory nor a regular file, such as a symlink; an alternates file). */
|
|
1454
|
+
function cacheEntryState(dir) {
|
|
1455
|
+
const hash = createHash("sha256");
|
|
1456
|
+
const walk = (abs, rel) => {
|
|
1457
|
+
for (const e of readdirSync(abs, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
|
|
1458
|
+
const p = join(abs, e.name), r = rel ? `${rel}/${e.name}` : e.name;
|
|
1459
|
+
if (e.isDirectory()) { if (!walk(p, r)) return false; continue; }
|
|
1460
|
+
if (!e.isFile() || r === "objects/info/alternates" || r === "objects/info/http-alternates") return false;
|
|
1461
|
+
const st = lstatSync(p, { bigint: true });
|
|
1462
|
+
hash.update(`${r}\0${st.size}\0${st.mode}\0${st.ino}\0${st.mtimeNs}\0${st.ctimeNs}\0`);
|
|
1463
|
+
}
|
|
1464
|
+
return true;
|
|
1465
|
+
};
|
|
1466
|
+
try { return lstatSync(dir).isDirectory() && walk(dir, "") ? `sha256-${hash.digest("hex")}` : null; }
|
|
1467
|
+
catch { return null; }
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
/**
|
|
1471
|
+
* The verified store (see the module header) for the tree `dir` of (refText's key, commit) and the content
|
|
1472
|
+
* digest `digest` it is expected to have. The cache repo's state is taken when this is called, before the
|
|
1473
|
+
* caller reads the tree. → {
|
|
1474
|
+
* holds() → true when a record says this tree had this digest in the cache repo as it is: every
|
|
1475
|
+
* recorded field is the asked one and the recorded state is the state taken;
|
|
1476
|
+
* record() → write the record, after the caller read the tree from the cache and found its digest to be
|
|
1477
|
+
* `digest`. Only when the cache repo is still in the state taken (what was read is then what
|
|
1478
|
+
* that state holds). → whether it was written.
|
|
1479
|
+
* } Starts no git process.
|
|
1480
|
+
*/
|
|
1481
|
+
export function treeVerification(refText, commit, dir, digest, options = {}) {
|
|
1482
|
+
const ref = parseRepoRef(refText, options);
|
|
1483
|
+
const rel = normalizeTreePath(dir, { allowRoot: true });
|
|
1484
|
+
const root = cacheRootOf(options);
|
|
1485
|
+
const file = join(root, ".verified", `${sha256(JSON.stringify([ref.key, commit, rel, digest]))}.json`);
|
|
1486
|
+
const state = () => cacheEntryState(cacheDirOf(root, ref));
|
|
1487
|
+
const entry = state();
|
|
1488
|
+
return {
|
|
1489
|
+
holds() {
|
|
1490
|
+
if (entry === null) return false;
|
|
1491
|
+
const rec = readStoreFile(file);
|
|
1492
|
+
return Boolean(rec) && typeof rec === "object" && rec.v === 1 && rec.key === ref.key && rec.commit === commit && rec.dir === rel && rec.digest === digest && rec.entry === entry;
|
|
1493
|
+
},
|
|
1494
|
+
record() {
|
|
1495
|
+
return entry !== null && state() === entry && writeAtomicQuiet(file, JSON.stringify({ v: 1, key: ref.key, commit, dir: rel, digest, entry }));
|
|
1496
|
+
},
|
|
1497
|
+
};
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1416
1500
|
// ---------------------------------------------------------------------------
|
|
1417
1501
|
// the observation store: successful live head observations, for --max-age
|
|
1418
1502
|
// ---------------------------------------------------------------------------
|
|
@@ -1526,7 +1610,7 @@ async function ensureCommit(ref, oid, options) {
|
|
|
1526
1610
|
const cached = peeled && !waited ? null : await asReadTimeout(repo, peelCommit(repo, oid));
|
|
1527
1611
|
if (cached) return { repo, commit: remember(cached) }; // pinned AND present AND (peels to) a commit
|
|
1528
1612
|
const mode = await fetchMode(repo)
|
|
1529
|
-
?? (keepsPartialCache(version) ? await startPartialFetches(repo, ref) : await
|
|
1613
|
+
?? (keepsPartialCache(version) ? await startPartialFetches(repo, ref) : await recordOlderGitFetches(repo, ref, version));
|
|
1530
1614
|
const filter = mode === "partial" ? [`--filter=${PARTIAL_FILTER}`] : ["--no-filter"];
|
|
1531
1615
|
const { stderr } = await fetchCommit(repo, ref, ["fetch", "-q", "--depth", "1", "--no-tags", "--no-recurse-submodules", ...filter, "origin", oid], oid);
|
|
1532
1616
|
// A server without filters says so and sends the whole tree: nothing is lost, the cache remembers.
|
|
@@ -1606,16 +1690,27 @@ async function writeConfig(repo, ref, key, value) {
|
|
|
1606
1690
|
catch (error) { throw unreadable(ref, error, { cacheDir: repo.dir, stage: "config" }); }
|
|
1607
1691
|
}
|
|
1608
1692
|
|
|
1609
|
-
/** This cache fetches whole trees from now on
|
|
1610
|
-
*
|
|
1611
|
-
async function recordFullFetches(repo, ref
|
|
1693
|
+
/** This cache fetches whole trees from now on, because its server cannot serve partial fetches: recorded, and
|
|
1694
|
+
* said once for this repository (the session's notices; the CLI prints them when the command ends). */
|
|
1695
|
+
async function recordFullFetches(repo, ref) {
|
|
1696
|
+
await writeConfig(repo, ref, "oats.fetch", "full");
|
|
1697
|
+
repo.session?.notices.push(displayDiagnosticText(`${displayRepoRef(ref.url)} does not serve partial fetches; OATS fetches whole trees from it`, ref.url));
|
|
1698
|
+
return "full";
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
/** This cache fetches whole trees from now on, because this git cannot keep a partial cache: recorded per cache,
|
|
1702
|
+
* every time, but said once per session (once per command, or per reader call), however many caches it starts
|
|
1703
|
+
* or rebuilds. The notice names the git, not a repository. */
|
|
1704
|
+
async function recordOlderGitFetches(repo, ref, version) {
|
|
1612
1705
|
await writeConfig(repo, ref, "oats.fetch", "full");
|
|
1613
|
-
repo.session?.notices
|
|
1706
|
+
const notices = repo.session?.notices;
|
|
1707
|
+
const notice = displayDiagnosticText(olderGitNotice(version), null);
|
|
1708
|
+
if (notices && !notices.includes(notice)) notices.push(notice);
|
|
1614
1709
|
return "full";
|
|
1615
1710
|
}
|
|
1616
1711
|
|
|
1617
|
-
const olderGitNotice = (version
|
|
1618
|
-
`git ${version?.text ?? "(unknown version)"} cannot keep a partial cache (it needs ${PARTIAL_FETCH_GIT.join(".")})
|
|
1712
|
+
const olderGitNotice = (version) =>
|
|
1713
|
+
`git ${version?.text ?? "(unknown version)"} cannot keep a partial cache (it needs ${PARTIAL_FETCH_GIT.join(".")}), so OATS fetches whole trees instead; a git of ${PARTIAL_FETCH_GIT.join(".")} or later removes this notice`;
|
|
1619
1714
|
|
|
1620
1715
|
/** One strict probe per exec: share in-flight work and retain only parsed success. A failed or abandoned
|
|
1621
1716
|
* probe cannot authorize rebuilding a cache, nor erase a newer probe installed by a retry. */
|
|
@@ -1680,7 +1775,7 @@ export function keepsPartialCache(version) {
|
|
|
1680
1775
|
* it again recording whole-tree fetches. The cache is disposable: everything in it is fetched again. */
|
|
1681
1776
|
async function rebuildWhole(repo, ref, options, version) {
|
|
1682
1777
|
const fresh = await freshCache(repo, ref, options);
|
|
1683
|
-
await
|
|
1778
|
+
await recordOlderGitFetches(fresh, ref, version);
|
|
1684
1779
|
return fresh;
|
|
1685
1780
|
}
|
|
1686
1781
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
import { closeSync, constants as fsConstants, fstatSync, lstatSync, openSync, readdirSync, readFileSync, realpathSync } from "node:fs";
|
|
18
18
|
import { isAbsolute, join } from "node:path";
|
|
19
19
|
import { gitOperationInProgress, instancesDirs, retainedWorktreesRoot, retirementRecoveryRoot } from "./core.mjs";
|
|
20
|
-
import { gitRead, gitRepoRead } from "./instance-git.mjs";
|
|
20
|
+
import { filterDriverKeys, gitRead, gitRepoRead } from "./instance-git.mjs";
|
|
21
21
|
import { formatBytes } from "./retire-output.mjs";
|
|
22
22
|
import { joinBytes } from "./tree-copy.mjs";
|
|
23
23
|
|
|
@@ -167,12 +167,7 @@ function headOf(read) {
|
|
|
167
167
|
function contentFilters(path) {
|
|
168
168
|
const config = gitRead(path, ["config", "--list", "-z"]);
|
|
169
169
|
if (!config.ok) return { error: config.err };
|
|
170
|
-
const names = new Set();
|
|
171
|
-
for (const entry of config.out.split("\0")) {
|
|
172
|
-
const nl = entry.indexOf("\n");
|
|
173
|
-
const m = /^filter\.(.*)\.(clean|process)$/s.exec(nl < 0 ? entry : entry.slice(0, nl));
|
|
174
|
-
if (m) names.add(m[1]);
|
|
175
|
-
}
|
|
170
|
+
const names = new Set(filterDriverKeys(config.out).filter((key) => key.variable === "clean" || key.variable === "process").map((key) => key.name));
|
|
176
171
|
return { names: [...names].sort(byCodeUnit) };
|
|
177
172
|
}
|
|
178
173
|
/** Whether the index of the tree at `path` holds a submodule (a gitlink,
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which of a retire's Git commands read no file of a work tree. lib/core.mjs
|
|
3
|
+
* (retireGitArgv) runs those without the overrides of the repository's filter
|
|
4
|
+
* drivers, and every other one with them. Kernel-private, dependency-free.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** A form whose first argument is one of `verbs`. */
|
|
8
|
+
const firstIs = (...verbs) => (rest) => verbs.includes(rest[0]);
|
|
9
|
+
/** The forms of Git command a retire runs that read refs, objects, the index or
|
|
10
|
+
* the configuration, and no file of a work tree: no filter driver can run in
|
|
11
|
+
* them, so they are not preceded by the read of the repository's drivers.
|
|
12
|
+
* A subcommand is here with the forms of it that are so, never as a whole:
|
|
13
|
+
* `true` for one that has no form that touches a work tree, else what its
|
|
14
|
+
* arguments must be (`stash store` is here, `stash push` is not). A form that
|
|
15
|
+
* is not here gets the overrides, so one added to the retire is covered until
|
|
16
|
+
* it is listed. */
|
|
17
|
+
const RETIRE_READS_NO_WORK_TREE_FILE = new Map([
|
|
18
|
+
["rev-parse", true], ["symbolic-ref", true], ["show-ref", true], ["for-each-ref", true], ["update-ref", true],
|
|
19
|
+
["cat-file", true], ["ls-tree", true], ["fetch", true], ["config", true],
|
|
20
|
+
// The index's entries, as recorded. Other forms compare the work tree's files with it (--modified, --deleted) or list them (--others).
|
|
21
|
+
["ls-files", (rest) => rest.length > 0 && rest.every((arg) => ["--stage", "-s", "-v", "-z", "--resolve-undo"].includes(arg)) && rest.some((arg) => arg !== "-z" && arg !== "-v")],
|
|
22
|
+
// Content from standard input, hashed as it is. With a file, or with `--path` (which gives the
|
|
23
|
+
// content read from standard input the filters of that path), the content goes through filters.
|
|
24
|
+
["hash-object", (rest) => rest.includes("--stdin") && !rest.some((arg) => arg === "--path" || arg.startsWith("--path="))],
|
|
25
|
+
["reflog", firstIs("show")],
|
|
26
|
+
["remote", firstIs("remove")],
|
|
27
|
+
["stash", firstIs("store")],
|
|
28
|
+
["worktree", firstIs("list")],
|
|
29
|
+
]);
|
|
30
|
+
/** Where the subcommand is in `args`, a retire Git command: after any `-c <setting>` of its own. */
|
|
31
|
+
export function retireSubcommandAt(args) {
|
|
32
|
+
let at = 0;
|
|
33
|
+
while (args[at] === "-c") at += 2;
|
|
34
|
+
return at;
|
|
35
|
+
}
|
|
36
|
+
/** Whether `args` (a retire Git command) is a form in RETIRE_READS_NO_WORK_TREE_FILE. */
|
|
37
|
+
export function retireGitReadsNoWorkTreeFile(args) {
|
|
38
|
+
const at = retireSubcommandAt(args);
|
|
39
|
+
const form = RETIRE_READS_NO_WORK_TREE_FILE.get(args[at]);
|
|
40
|
+
return form === true || (typeof form === "function" && form(args.slice(at + 1)));
|
|
41
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/** `oats doctor`'s `retire-incomplete` problems (oats-project/oats#1747): one per instance home whose
|
|
2
|
+
* retirement was left open, with the command that finishes it and the capability whose retire hook
|
|
3
|
+
* failed last, so the operator knows whose service to check.
|
|
4
|
+
*
|
|
5
|
+
* A retirement is open when the home's status row reads a quarantine (`rollbackIncomplete`, never
|
|
6
|
+
* a spawn still in progress), when its `retire.state` is `failed` or `interrupted`, or when a
|
|
7
|
+
* failed self-retire left its outcome beside it. A home whose retire is `running` or `unknown` is
|
|
8
|
+
* not listed: a retire may be under way, or cannot be read. An outcome whose home is gone is not
|
|
9
|
+
* an open retirement either: a home is removed only once its hooks proved clean or `--force` took
|
|
10
|
+
* over the debt, and `oats status` still lists the outcome.
|
|
11
|
+
*
|
|
12
|
+
* Read with the readers `oats status` uses (openRetirements, lib/instance-listing.mjs): read-only
|
|
13
|
+
* and offline, no claim taken, no file removed, nothing signalled. */
|
|
14
|
+
import { openRetirements } from "./instance-listing.mjs";
|
|
15
|
+
import { shellWord } from "./shell-word.mjs";
|
|
16
|
+
|
|
17
|
+
export const RETIRE_INCOMPLETE = "retire-incomplete";
|
|
18
|
+
|
|
19
|
+
/** The capabilities a failed outcome's items name as `retire hook <capability>: …`, in order. */
|
|
20
|
+
function hooksNamedIn(items) {
|
|
21
|
+
const caps = [];
|
|
22
|
+
for (const item of Array.isArray(items) ? items : []) {
|
|
23
|
+
const m = typeof item === "string" && /^retire hook ([^\s:]+): /.exec(item);
|
|
24
|
+
if (m && !caps.includes(m[1])) caps.push(m[1]);
|
|
25
|
+
}
|
|
26
|
+
return caps;
|
|
27
|
+
}
|
|
28
|
+
const list = (items) => items.join(", ");
|
|
29
|
+
|
|
30
|
+
/** What a quarantine marker owes, as the clauses of a message; the hooks it owes; and the hooks its
|
|
31
|
+
* latest attempt left incomplete (`latest`). */
|
|
32
|
+
function quarantineOwes(marker) {
|
|
33
|
+
const owed = marker?.cleanup?.outstanding;
|
|
34
|
+
if (!owed || typeof owed !== "object") return { said: ["it is quarantined, and its quarantine marker cannot be read"], hooks: [], latest: hooksNamedIn(marker?.incomplete) };
|
|
35
|
+
const hooks = Array.isArray(owed.hooks) ? owed.hooks.filter((h) => typeof h === "string" && h) : [];
|
|
36
|
+
const git = Array.isArray(owed.git) ? owed.git.filter((g) => typeof g === "string" && g) : [];
|
|
37
|
+
const debts = [
|
|
38
|
+
...(hooks.length ? [`retire hooks of ${list(hooks)}`] : []),
|
|
39
|
+
...(git.length ? [`Git steps (${list(git)})`] : []),
|
|
40
|
+
...(owed.directory === true ? ["the preservation of its directory"] : []),
|
|
41
|
+
...(owed.orphanedWork === true ? ["its orphaned work directory"] : []),
|
|
42
|
+
];
|
|
43
|
+
const at = [marker.lastRetryAt, marker.createdAt].find((t) => typeof t === "string" && t);
|
|
44
|
+
// `outstanding` is what the quarantine first owed, and a retry does not rewrite it; `incomplete`
|
|
45
|
+
// is what its latest attempt left, which names the hook that failed last.
|
|
46
|
+
return { said: [`it is quarantined${debts.length ? `, owing ${debts.join(" and ")}` : ""}${at ? ` (since ${at})` : ""}`], hooks, latest: hooksNamedIn(marker.incomplete) };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The `retire-incomplete` problem of one home, or null when its retirement is not open. */
|
|
50
|
+
function problemOf(row, failures) {
|
|
51
|
+
const state = row.retire?.state;
|
|
52
|
+
if (state === "running" || state === "unknown") return null;
|
|
53
|
+
if (!row.rollbackIncomplete && state !== "failed" && state !== "interrupted" && !failures.length) return null;
|
|
54
|
+
const said = [];
|
|
55
|
+
let owed = [], latest = [];
|
|
56
|
+
if (row.rollbackIncomplete) {
|
|
57
|
+
const owes = quarantineOwes(row.rollbackIncomplete);
|
|
58
|
+
said.push(...owes.said);
|
|
59
|
+
owed = owes.hooks;
|
|
60
|
+
latest = owes.latest;
|
|
61
|
+
}
|
|
62
|
+
if (state === "failed") {
|
|
63
|
+
const why = row.retire.message ?? (failures.some((f) => f.incomplete) ? "its cleanup did not finish" : undefined);
|
|
64
|
+
said.push(`its self-retire failed at ${row.retire.since}${row.retire.code ? ` (${row.retire.code})` : ""}${why ? `: ${why}` : ""}`);
|
|
65
|
+
}
|
|
66
|
+
else if (state === "interrupted") said.push(`its self-retire, requested at ${row.retire.since}, was interrupted: its completion ended without an outcome`);
|
|
67
|
+
for (const f of failures) {
|
|
68
|
+
if (f.error === "unreadable result file") said.push(`its result file ${f.resultPath} cannot be read`);
|
|
69
|
+
// A failure the row's `failed` state already said is said once.
|
|
70
|
+
else if (state !== "failed") said.push(`its self-retire failed at ${f.completedAt ?? "an unknown time"}${f.error ? `: ${f.error}` : f.incomplete ? ": its cleanup did not finish" : ""}`);
|
|
71
|
+
}
|
|
72
|
+
if (row.record === "unreadable") said.push(`its record ${row.recordSaid}`);
|
|
73
|
+
// The hook that failed last: as the quarantine's latest attempt names it, else as the outcome's
|
|
74
|
+
// items do, else the hooks the quarantine owes.
|
|
75
|
+
const named = failures.flatMap((f) => hooksNamedIn(f.incomplete)).filter((cap, i, all) => all.indexOf(cap) === i);
|
|
76
|
+
const hooks = latest.length ? latest : named.length ? named : owed;
|
|
77
|
+
const command = `oats retire ${shellWord(row.instance)} --home ${shellWord(row.home)}`;
|
|
78
|
+
const remedy = hooks.length
|
|
79
|
+
? `check the service of ${list(hooks)}, whose retire hook failed last, then run: ${command}`
|
|
80
|
+
: `no failed retire hook is recorded; run: ${command}`;
|
|
81
|
+
return { code: RETIRE_INCOMPLETE, message: `the retirement of ${row.instance} (${row.home}) is not finished: ${said.join("; ")}`, remedy };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Every `retire-incomplete` problem of the deployment whose agents root is `root`, ordered by soul
|
|
85
|
+
* and home. [] when no retirement is open. */
|
|
86
|
+
export function retireIncompleteProblems(root) {
|
|
87
|
+
const problems = [];
|
|
88
|
+
for (const agent of openRetirements(root)) {
|
|
89
|
+
const homes = [...agent.homes].sort((a, b) => (a.home < b.home ? -1 : a.home > b.home ? 1 : 0));
|
|
90
|
+
for (const row of homes) {
|
|
91
|
+
const problem = problemOf(row, agent.retireFailures.filter((f) => f.instance === row.instance));
|
|
92
|
+
if (problem) problems.push(problem);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return problems;
|
|
96
|
+
}
|