@futurelastic/colab-handbook 1.20.4-rc.2 → 1.20.4-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/package.json +1 -1
  2. package/tools/lib/state.js +150 -8
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.20.4-rc.2"
36
+ "version": "1.20.4-rc.3"
37
37
  }
@@ -187,27 +187,168 @@ function sleep(ms) {
187
187
  while (Date.now() < end) { /* spin */ }
188
188
  }
189
189
 
190
- /** Acquire the mkdir lock, run fn, always release. Returns fn's result. */
191
- function withLock(fn, { tries = 100, waitMs = 100 } = {}) {
190
+ /*
191
+ * The state lock (#712). `mkdir LOCK_DIR` is still the acquire — atomic, and the same primitive an
192
+ * older colab on this machine uses, so the two still exclude each other. What changed is that the
193
+ * holder now writes LOCK_OWNER inside it, and a waiter uses that record to tell a dead lock from a
194
+ * live one. Before, the directory recorded nothing: a holder that died inside the critical section
195
+ * left it forever, every state write on the machine (claims, worktrees, ports, places, ship) failed
196
+ * with "Lock stuck", and nobody could tell whether removing it by hand was safe.
197
+ *
198
+ * A waiter breaks the lock itself, logging one line, only when the holder is provably gone:
199
+ * - its pid is dead (same host), or
200
+ * - its pid is alive but started at a different time than the record says (pid reused), or
201
+ * - liveness cannot be proven at all — no readable owner (an older colab, or a holder that died
202
+ * between mkdir and writing the record), another host, or a start time `ps` cannot give — AND
203
+ * the lock is older than LOCK_STALE_MS, far above any real critical section (they are
204
+ * synchronous file edits: milliseconds).
205
+ * A live, matching holder is never broken, however old: the waiter gives up with the holder named.
206
+ *
207
+ * Breaking is race-safe in two layers. Breakers serialize on LOCK_BREAK (a second mkdir lock, held
208
+ * for microseconds), and inside it the breaker renames LOCK_DIR to a private name and re-reads the
209
+ * owner from THAT copy: only the exact stale owner it judged (same token) is removed. Anything else
210
+ * — a waiter that acquired a fresh lock in between — is renamed straight back.
211
+ */
212
+ const LOCK_OWNER = 'owner.json';
213
+ const LOCK_BREAK = LOCK_DIR + '.break';
214
+ const LOCK_STALE_MS = 60 * 1000;
215
+ const PROC_START_MS = Date.now() - Math.round(process.uptime() * 1000);
216
+ const START_TOLERANCE_MS = 3000; // `ps -o lstart` truncates to whole seconds; uptime starts after exec
217
+
218
+ function lockOwnerRecord() {
219
+ return {
220
+ pid: process.pid, host: os.hostname(), start: PROC_START_MS, at: Date.now(),
221
+ token: `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`,
222
+ };
223
+ }
224
+
225
+ /** The owner record in `dir`, or null when absent/unreadable (an older colab, or mid-write). */
226
+ function readLockOwner(dir = LOCK_DIR) {
227
+ try {
228
+ const o = JSON.parse(fs.readFileSync(path.join(dir, LOCK_OWNER), 'utf8'));
229
+ return o && Number.isInteger(o.pid) && typeof o.token === 'string' ? o : null;
230
+ } catch (_) { return null; }
231
+ }
232
+
233
+ /** true = alive, false = no such process, null = cannot tell. */
234
+ function pidAlive(pid) {
235
+ try { process.kill(pid, 0); return true; } catch (e) {
236
+ if (e.code === 'ESRCH') return false;
237
+ if (e.code === 'EPERM') return true; // exists, owned by someone else
238
+ return null;
239
+ }
240
+ }
241
+
242
+ /** Start time (ms) of a live pid per `ps`, or null when ps cannot say. */
243
+ function pidStartMs(pid) {
244
+ try {
245
+ const out = require('child_process').execFileSync('ps', ['-o', 'lstart=', '-p', String(pid)],
246
+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 2000 }).trim();
247
+ const t = Date.parse(out);
248
+ return Number.isFinite(t) ? t : null;
249
+ } catch (_) { return null; }
250
+ }
251
+
252
+ /**
253
+ * Judge the lock currently at LOCK_DIR. Returns { stale, owner, ageMs, reason } or null when there
254
+ * is no lock to judge. `startCache` (token -> ps start) keeps `ps` to once per holder per wait.
255
+ */
256
+ function judgeLock(startCache = new Map(), now = Date.now()) {
257
+ let st;
258
+ try { st = fs.statSync(LOCK_DIR); } catch (_) { return null; }
259
+ const owner = readLockOwner();
260
+ const ageMs = Math.max(0, now - (owner && Number.isFinite(owner.at) ? owner.at : st.mtimeMs));
261
+ const old = ageMs > LOCK_STALE_MS;
262
+ if (!owner) return { stale: old, owner, ageMs, reason: 'no owner record' };
263
+ if (owner.host && owner.host !== os.hostname()) return { stale: old, owner, ageMs, reason: `held from host ${owner.host}` };
264
+ const alive = pidAlive(owner.pid);
265
+ if (alive === false) return { stale: true, owner, ageMs, reason: `pid ${owner.pid} is gone` };
266
+ if (alive === null) return { stale: old, owner, ageMs, reason: `pid ${owner.pid} liveness unknown` };
267
+ if (!startCache.has(owner.token)) startCache.set(owner.token, pidStartMs(owner.pid));
268
+ const psStart = startCache.get(owner.token);
269
+ if (psStart == null || !Number.isFinite(owner.start)) {
270
+ return { stale: old, owner, ageMs, reason: `pid ${owner.pid} alive, start time unverifiable` };
271
+ }
272
+ if (Math.abs(psStart - owner.start) > START_TOLERANCE_MS) {
273
+ return { stale: true, owner, ageMs, reason: `pid ${owner.pid} was reused by another process` };
274
+ }
275
+ return { stale: false, owner, ageMs, reason: `pid ${owner.pid} is alive` };
276
+ }
277
+
278
+ function describeHolder(j) {
279
+ if (!j) return 'holder unknown (lock vanished)';
280
+ const age = `${Math.round(j.ageMs / 1000)}s old`;
281
+ if (!j.owner) return `no owner recorded, ${age}`;
282
+ return `held by pid ${j.owner.pid} on ${j.owner.host || '?'}, ${age} — ${j.reason}`;
283
+ }
284
+
285
+ function rmLockDir(dir) {
286
+ try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_) { /* best effort */ }
287
+ }
288
+
289
+ /**
290
+ * Remove the lock `judged` found stale — but only if it is still that same lock. Returns true when
291
+ * it removed it. Never throws: a failed break just means the waiter keeps waiting.
292
+ */
293
+ function breakStaleLock(judged, log = (m) => process.stderr.write(m + '\n')) {
294
+ try { fs.mkdirSync(LOCK_BREAK); } catch (e) {
295
+ // Another breaker is at it. A breaker that died mid-break leaves an empty LOCK_BREAK; clear it
296
+ // once it is far older than the microseconds a break takes, and let the next round retry.
297
+ try { if (Date.now() - fs.statSync(LOCK_BREAK).mtimeMs > 10 * 1000) fs.rmdirSync(LOCK_BREAK); } catch (_) { /* gone */ }
298
+ return false;
299
+ }
300
+ try {
301
+ const grave = `${LOCK_DIR}.stale-${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
302
+ try { fs.renameSync(LOCK_DIR, grave); } catch (_) { return false; } // released meanwhile
303
+ const now = readLockOwner(grave);
304
+ const same = judged.owner ? (now && now.token === judged.owner.token) : !now;
305
+ if (!same) {
306
+ // A fresh lock replaced the stale one between judging and renaming — put it back untouched.
307
+ try { fs.renameSync(grave, LOCK_DIR); } catch (_) { /* LOCK_DIR re-taken: nothing safe to do */ }
308
+ return false;
309
+ }
310
+ rmLockDir(grave);
311
+ log(`colab: broke a stale state lock ${LOCK_DIR} (${describeHolder(judged)})`);
312
+ return true;
313
+ } finally {
314
+ try { fs.rmdirSync(LOCK_BREAK); } catch (_) { /* gone */ }
315
+ }
316
+ }
317
+
318
+ /** Acquire the state lock, run fn, always release. Returns fn's result. */
319
+ function withLock(fn, { tries = 100, waitMs = 100, log } = {}) {
192
320
  ensureDir();
193
- let acquired = false;
321
+ const startCache = new Map();
322
+ let me = null;
323
+ let written = false;
324
+ let last = null;
194
325
  for (let i = 0; i < tries; i++) {
195
326
  try {
196
327
  fs.mkdirSync(LOCK_DIR);
197
- acquired = true;
328
+ me = lockOwnerRecord();
329
+ try { fs.writeFileSync(path.join(LOCK_DIR, LOCK_OWNER), JSON.stringify(me) + '\n'); written = true; } catch (_) {
330
+ // Unrecorded is still a held lock — it just ages out like an older colab's would.
331
+ }
198
332
  break;
199
333
  } catch (e) {
200
334
  if (e.code !== 'EEXIST') throw e;
335
+ last = judgeLock(startCache);
336
+ if (last && last.stale && breakStaleLock(last, log)) continue; // retry the mkdir at once
201
337
  sleep(waitMs);
202
338
  }
203
339
  }
204
- if (!acquired) {
205
- throw new Error(`Lock stuck: ${LOCK_DIR} (remove it by hand if you are sure no session holds it)`);
340
+ if (!me) {
341
+ const holder = describeHolder(judgeLock(startCache) || last);
342
+ throw new Error(`Lock stuck: ${LOCK_DIR} (${holder}). `
343
+ + 'A live holder is never broken automatically; remove the directory by hand only if that process is not a colab writer.');
206
344
  }
207
345
  try {
208
346
  return fn();
209
347
  } finally {
210
- try { fs.rmdirSync(LOCK_DIR); } catch (_) { /* already gone */ }
348
+ // Release only our own lock: if a waiter judged us dead and broke it, LOCK_DIR is now theirs —
349
+ // including an ownerless one, when we did write our record and it is no longer there.
350
+ const cur = readLockOwner();
351
+ if (cur ? cur.token === me.token : !written) rmLockDir(LOCK_DIR);
211
352
  }
212
353
  }
213
354
 
@@ -257,6 +398,7 @@ function mutate(fn) {
257
398
  }
258
399
 
259
400
  module.exports = {
260
- COLAB_DIR, STATE_FILE, CONFIG_FILE, LOCK_DIR, STATE_VERSION, DEFAULT_CONFIG,
401
+ COLAB_DIR, STATE_FILE, CONFIG_FILE, LOCK_DIR, LOCK_OWNER, LOCK_STALE_MS, STATE_VERSION, DEFAULT_CONFIG,
261
402
  loadConfig, saveConfig, loadState, saveState, withLock, mutate, emptyState, atomicWrite,
403
+ _lock: { judgeLock, breakStaleLock, readLockOwner }, // for tools/lib/state-lock.test.js
262
404
  };