@prjct.app/pi-team 0.6.0 → 0.7.0

package/src/supervisor/runtime-store.ts

@@ -0,0 +1,231 @@
+import { createHash } from 'node:crypto';
+import { Type } from 'typebox';
+import { Value } from 'typebox/value';
+import { EntityIdSchema, TeamIdSchema, TimestampSchema } from '../domain/team.ts';
+import {
+  createAtomicJson, ensurePrivateDirectory, ensurePrivateTree, jsonFileNames, readJson, removeAtomic, replaceAtomicJson,
+  withStorageLock,
+} from '../storage/atomic.ts';
+import { TeamPaths } from '../storage/paths.ts';
+
+export type OwnerIdentity = {
+  readonly ownerSessionId: string;
+  readonly ownerInstanceId: string;
+  readonly ownerProcessNonce: string;
+  readonly ownerEpoch: number;
+};
+
+export type OwnedRuntimeState = 'starting' | 'ready' | 'busy' | 'stopping' | 'terminated' | 'lost';
+
+export type OwnedRuntime = {
+  readonly schemaVersion: 2;
+  readonly runtimeId: string;
+  readonly teamId: string;
+  readonly memberId: string;
+  readonly owner: OwnerIdentity;
+  readonly processPid: number;
+  readonly processStartToken: string;
+  readonly processGroupId?: number;
+  readonly tmuxSession?: string;
+  readonly tmuxOwnershipTokenHash?: string;
+  readonly cwd: string;
+  readonly state: OwnedRuntimeState;
+  readonly activeRequestId?: string;
+  readonly createdAt: string;
+  readonly updatedAt: string;
+};
+
+const OwnerIdentitySchema = Type.Object({
+  ownerSessionId: EntityIdSchema,
+  ownerInstanceId: EntityIdSchema,
+  ownerProcessNonce: Type.String({ pattern: '^[a-f0-9]{64}$' }),
+  ownerEpoch: Type.Integer({ minimum: 1, maximum: Number.MAX_SAFE_INTEGER }),
+}, { additionalProperties: false });
+
+export const OwnedRuntimeSchema = Type.Object({
+  schemaVersion: Type.Literal(2),
+  runtimeId: EntityIdSchema,
+  teamId: TeamIdSchema,
+  memberId: EntityIdSchema,
+  owner: OwnerIdentitySchema,
+  processPid: Type.Integer({ minimum: 2, maximum: Number.MAX_SAFE_INTEGER }),
+  processStartToken: Type.String({ minLength: 1, maxLength: 256 }),
+  processGroupId: Type.Optional(Type.Integer({ minimum: 2, maximum: Number.MAX_SAFE_INTEGER })),
+  tmuxSession: Type.Optional(Type.String({ minLength: 1, maxLength: 128 })),
+  tmuxOwnershipTokenHash: Type.Optional(Type.String({ pattern: '^[a-f0-9]{64}$' })),
+  cwd: Type.String({ minLength: 1, maxLength: 4096 }),
+  state: Type.Union([
+    Type.Literal('starting'), Type.Literal('ready'), Type.Literal('busy'), Type.Literal('stopping'),
+    Type.Literal('terminated'), Type.Literal('lost'),
+  ]),
+  activeRequestId: Type.Optional(EntityIdSchema),
+  createdAt: TimestampSchema,
+  updatedAt: TimestampSchema,
+}, { additionalProperties: false });
+
+const transitions: Readonly<Record<OwnedRuntimeState, readonly OwnedRuntimeState[]>> = {
+  starting: ['ready', 'busy', 'stopping', 'lost'],
+  ready: ['busy', 'stopping', 'lost'],
+  busy: ['ready', 'stopping', 'lost'],
+  stopping: ['terminated', 'lost'],
+  terminated: [],
+  lost: ['stopping', 'terminated'],
+};
+
+export function assertOwnedRuntime(value: unknown): asserts value is OwnedRuntime {
+  if (!Value.Check(OwnedRuntimeSchema, value)) throw new Error('Invalid Team v2 owned runtime record.');
+  const runtime = value as OwnedRuntime;
+  if (!runtime.cwd.startsWith('/')) throw new Error('Owned runtime cwd must be absolute.');
+  if (Date.parse(runtime.updatedAt) < Date.parse(runtime.createdAt)) throw new Error('Owned runtime timestamp moved backwards.');
+  if ((runtime.tmuxSession === undefined) !== (runtime.tmuxOwnershipTokenHash === undefined)) {
+    throw new Error('Tmux runtime identity is incomplete.');
+  }
+  if (runtime.state !== 'busy' && runtime.activeRequestId !== undefined) {
+    throw new Error('Only a busy runtime may hold an active request.');
+  }
+}
+
+export function sameOwner(left: OwnerIdentity, right: OwnerIdentity): boolean {
+  return left.ownerSessionId === right.ownerSessionId && left.ownerInstanceId === right.ownerInstanceId &&
+    left.ownerProcessNonce === right.ownerProcessNonce && left.ownerEpoch === right.ownerEpoch;
+}
+
+export function canHandoff(from: OwnerIdentity, to: OwnerIdentity): boolean {
+  return from.ownerSessionId === to.ownerSessionId && from.ownerProcessNonce === to.ownerProcessNonce &&
+    from.ownerInstanceId !== to.ownerInstanceId && to.ownerEpoch > from.ownerEpoch;
+}
+
+const RUNTIME_MAX_BYTES = 32 * 1024;
+
+export class RuntimeStore {
+  constructor(readonly paths: TeamPaths, private readonly maxRuntimes = 32) {
+    if (!Number.isSafeInteger(maxRuntimes) || maxRuntimes < 1) throw new Error('Invalid runtime quota.');
+  }
+
+  private lockPath(teamId: string, runtimeId: string): string {
+    const hash = createHash('sha256').update(`${teamId}\0${runtimeId}`).digest('hex');
+    return this.paths.lock(`runtime-${hash}`);
+  }
+
+  private rosterLockPath(teamId: string): string {
+    return this.paths.lock(`runtimes-${createHash('sha256').update(teamId).digest('hex')}`);
+  }
+
+  private async prepare(teamId: string): Promise<void> {
+    await ensurePrivateTree(this.paths.root, 'teams');
+    await ensurePrivateTree(this.paths.root, 'control');
+    await ensurePrivateDirectory(this.paths.team(teamId), false);
+    await ensurePrivateDirectory(this.paths.runtimes(teamId), false);
+  }
+
+  async create(runtime: OwnedRuntime): Promise<void> {
+    assertOwnedRuntime(runtime);
+    await this.prepare(runtime.teamId);
+    await withStorageLock(this.rosterLockPath(runtime.teamId), async () => {
+      const ids = await jsonFileNames(this.paths.runtimes(runtime.teamId), false);
+      if (ids.length >= this.maxRuntimes) throw Object.assign(new Error('Team runtime quota reached.'), { code: 'QUOTA_EXCEEDED' });
+      const records = await Promise.all(ids.map(id => this.read(runtime.teamId, id)));
+      if (records.some(existing => existing?.memberId === runtime.memberId && existing.state !== 'terminated')) {
+        throw Object.assign(new Error('Supervised member already has a live runtime.'), { code: 'ALREADY_EXISTS' });
+      }
+      await createAtomicJson(this.paths.runtime(runtime.teamId, runtime.runtimeId), runtime, RUNTIME_MAX_BYTES);
+    });
+  }
+
+  async read(teamId: string, runtimeId: string): Promise<OwnedRuntime | undefined> {
+    await this.prepare(teamId);
+    const runtime = await readJson(this.paths.runtime(teamId, runtimeId), assertOwnedRuntime, RUNTIME_MAX_BYTES);
+    if (runtime && (runtime.teamId !== teamId || runtime.runtimeId !== runtimeId)) {
+      throw Object.assign(new Error('Runtime identity does not match its storage path.'), { code: 'CORRUPT_RECORD' });
+    }
+    return runtime;
+  }
+
+  async list(teamId: string): Promise<readonly OwnedRuntime[]> {
+    await this.prepare(teamId);
+    const ids = await jsonFileNames(this.paths.runtimes(teamId), false);
+    const records = await Promise.all(ids.map(id => this.read(teamId, id)));
+    return records.filter((runtime): runtime is OwnedRuntime => runtime !== undefined);
+  }
+
+  async update(
+    teamId: string,
+    runtimeId: string,
+    owner: OwnerIdentity,
+    change: (runtime: OwnedRuntime) => OwnedRuntime,
+  ): Promise<OwnedRuntime> {
+    await this.prepare(teamId);
+    return withStorageLock(this.lockPath(teamId, runtimeId), async () => {
+      const path = this.paths.runtime(teamId, runtimeId);
+      const current = await readJson(path, assertOwnedRuntime, RUNTIME_MAX_BYTES);
+      if (!current) throw Object.assign(new Error(`Unknown runtime "${runtimeId}".`), { code: 'NOT_FOUND' });
+      if (!sameOwner(current.owner, owner)) throw Object.assign(new Error('Runtime ownership has been fenced.'), { code: 'FENCED' });
+      const next = change(current);
+      assertOwnedRuntime(next);
+      if (next.runtimeId !== current.runtimeId || next.teamId !== current.teamId || next.memberId !== current.memberId ||
+          next.processPid !== current.processPid || next.processStartToken !== current.processStartToken ||
+          next.processGroupId !== current.processGroupId || next.tmuxSession !== current.tmuxSession ||
+          next.tmuxOwnershipTokenHash !== current.tmuxOwnershipTokenHash || next.cwd !== current.cwd ||
+          next.createdAt !== current.createdAt || !sameOwner(next.owner, current.owner)) {
+        throw new Error('Owned runtime identity is immutable outside handoff.');
+      }
+      if (next.state !== current.state && !transitions[current.state].includes(next.state)) {
+        throw Object.assign(new Error(`Invalid runtime transition: ${current.state} → ${next.state}.`), { code: 'INVALID_TRANSITION' });
+      }
+      if (Date.parse(next.updatedAt) < Date.parse(current.updatedAt)) throw new Error('Runtime updatedAt moved backwards.');
+      await replaceAtomicJson(path, next, { maxBytes: RUNTIME_MAX_BYTES, previous: true });
+      return next;
+    });
+  }
+
+  async removeTerminated(teamId: string, runtimeId: string, owner: OwnerIdentity): Promise<boolean> {
+    await this.prepare(teamId);
+    return withStorageLock(this.rosterLockPath(teamId), async () =>
+      withStorageLock(this.lockPath(teamId, runtimeId), async () => {
+        const path = this.paths.runtime(teamId, runtimeId);
+        const current = await readJson(path, assertOwnedRuntime, RUNTIME_MAX_BYTES);
+        if (!current) return false;
+        if (!sameOwner(current.owner, owner)) {
+          throw Object.assign(new Error('Runtime ownership has been fenced.'), { code: 'FENCED' });
+        }
+        if (current.state !== 'terminated') {
+          throw Object.assign(new Error('Only a terminated runtime can be removed.'), { code: 'INVALID_STATE' });
+        }
+        return removeAtomic(path);
+      }));
+  }
+
+  async advanceOwner(teamId: string, runtimeId: string, from: OwnerIdentity, to: OwnerIdentity, at: string): Promise<OwnedRuntime> {
+    await this.prepare(teamId);
+    return withStorageLock(this.lockPath(teamId, runtimeId), async () => {
+      const path = this.paths.runtime(teamId, runtimeId);
+      const current = await readJson(path, assertOwnedRuntime, RUNTIME_MAX_BYTES);
+      if (!current) throw Object.assign(new Error(`Unknown runtime "${runtimeId}".`), { code: 'NOT_FOUND' });
+      const sameIdentity = from.ownerSessionId === to.ownerSessionId && from.ownerInstanceId === to.ownerInstanceId &&
+        from.ownerProcessNonce === to.ownerProcessNonce && to.ownerEpoch > from.ownerEpoch;
+      if (!sameOwner(current.owner, from) || !sameIdentity) {
+        throw Object.assign(new Error('Runtime epoch advancement does not match ownership.'), { code: 'FENCED' });
+      }
+      const next = { ...current, owner: to, updatedAt: at };
+      assertOwnedRuntime(next);
+      await replaceAtomicJson(path, next, { maxBytes: RUNTIME_MAX_BYTES, previous: true });
+      return next;
+    });
+  }
+
+  async handoff(teamId: string, runtimeId: string, from: OwnerIdentity, to: OwnerIdentity, at: string): Promise<OwnedRuntime> {
+    await this.prepare(teamId);
+    return withStorageLock(this.lockPath(teamId, runtimeId), async () => {
+      const path = this.paths.runtime(teamId, runtimeId);
+      const current = await readJson(path, assertOwnedRuntime, RUNTIME_MAX_BYTES);
+      if (!current) throw Object.assign(new Error(`Unknown runtime "${runtimeId}".`), { code: 'NOT_FOUND' });
+      if (!sameOwner(current.owner, from) || !canHandoff(from, to)) {
+        throw Object.assign(new Error('Runtime handoff ownership does not match.'), { code: 'FENCED' });
+      }
+      const next = { ...current, owner: to, updatedAt: at };
+      assertOwnedRuntime(next);
+      await replaceAtomicJson(path, next, { maxBytes: RUNTIME_MAX_BYTES, previous: true });
+      return next;
+    });
+  }
+}

package/src/supervisor/shutdown.ts

@@ -0,0 +1,141 @@
+import { defaultProcessController, sameProcess, type ProcessController, type ProcessIdentity } from '../process-identity.ts';
+import type { OwnedRuntime } from './runtime-store.ts';
+import type { TmuxAdapter } from './tmux-adapter.ts';
+
+export type ShutdownReason = 'stop' | 'close' | 'reload_failed' | 'owner_lost';
+export type ShutdownPhase = 'prepare' | 'graceful' | 'term' | 'kill' | 'tmux' | 'terminated' | 'blocked';
+
+export type ShutdownTimings = {
+  readonly gracefulMs: number;
+  readonly termMs: number;
+  readonly killMs: number;
+  readonly pollMs: number;
+};
+
+export type RuntimeControl = {
+  prepareShutdown(runtimeId: string, reason: ShutdownReason, deadlineAt: string): Promise<boolean>;
+};
+
+export type ShutdownResult = {
+  readonly runtimeId: string;
+  readonly status: 'terminated' | 'blocked';
+  readonly phase: ShutdownPhase;
+  readonly detail?: string;
+};
+
+const DEFAULT_TIMINGS: ShutdownTimings = { gracefulMs: 5_000, termMs: 2_000, killMs: 1_000, pollMs: 50 };
+
+export function shutdownTimings(input: Partial<ShutdownTimings> = {}): ShutdownTimings {
+  const value = { ...DEFAULT_TIMINGS, ...input };
+  if ([value.gracefulMs, value.termMs, value.killMs].some(duration => !Number.isFinite(duration) || duration < 0) ||
+      !Number.isFinite(value.pollMs) || value.pollMs <= 0) {
+    throw new Error('Shutdown timings require non-negative durations and a positive poll interval.');
+  }
+  return value;
+}
+
+function identity(runtime: OwnedRuntime): ProcessIdentity {
+  return {
+    processPid: runtime.processPid,
+    processStartToken: runtime.processStartToken,
+    processGroupId: runtime.processGroupId ?? runtime.processPid,
+  };
+}
+
+export class RuntimeShutdown {
+  private readonly timings: ShutdownTimings;
+
+  constructor(
+    private readonly control: RuntimeControl,
+    private readonly tmux: Pick<TmuxAdapter, 'metadataMatches' | 'killSession'>,
+    private readonly processes: ProcessController = defaultProcessController,
+    options: { readonly timings?: Partial<ShutdownTimings>; readonly phase?: (runtimeId: string, phase: ShutdownPhase) => void } = {},
+  ) {
+    this.timings = shutdownTimings(options.timings);
+    this.phase = options.phase;
+  }
+
+  private readonly phase?: (runtimeId: string, phase: ShutdownPhase) => void;
+
+  private emit(runtimeId: string, phase: ShutdownPhase): void { this.phase?.(runtimeId, phase); }
+
+  private async alive(expected: ProcessIdentity): Promise<boolean> {
+    return sameProcess(expected, await this.processes.inspect(expected.processPid));
+  }
+
+  private async waitForExit(expected: ProcessIdentity, remainingMs: number): Promise<boolean> {
+    if (!await this.alive(expected)) return true;
+    if (remainingMs <= 0) return false;
+    const pause = Math.min(this.timings.pollMs, remainingMs);
+    await this.processes.delay(pause);
+    return this.waitForExit(expected, remainingMs - pause);
+  }
+
+  private async boundedPrepare(runtime: OwnedRuntime, reason: ShutdownReason): Promise<void> {
+    const deadlineAt = new Date(Date.now() + this.timings.gracefulMs).toISOString();
+    const timeout = this.processes.delay(Math.min(this.timings.pollMs, this.timings.gracefulMs));
+    await Promise.race([
+      this.control.prepareShutdown(runtime.runtimeId, reason, deadlineAt).catch(() => false),
+      timeout,
+    ]);
+  }
+
+  private async ownershipSafe(runtime: OwnedRuntime, expected: ProcessIdentity): Promise<'owned' | 'exited' | 'blocked'> {
+    if (!await this.alive(expected)) return 'exited';
+    return await this.tmux.metadataMatches(runtime) ? 'owned' : 'blocked';
+  }
+
+  private async cleanTmux(runtime: OwnedRuntime): Promise<void> {
+    this.emit(runtime.runtimeId, 'tmux');
+    await this.tmux.killSession(runtime).catch(() => false);
+  }
+
+  async stop(runtime: OwnedRuntime, reason: ShutdownReason = 'stop'): Promise<ShutdownResult> {
+    if (runtime.state === 'terminated') return { runtimeId: runtime.runtimeId, status: 'terminated', phase: 'terminated' };
+    const expected = identity(runtime);
+    this.emit(runtime.runtimeId, 'prepare');
+    await this.boundedPrepare(runtime, reason);
+    this.emit(runtime.runtimeId, 'graceful');
+    if (await this.waitForExit(expected, this.timings.gracefulMs)) {
+      await this.cleanTmux(runtime);
+      this.emit(runtime.runtimeId, 'terminated');
+      return { runtimeId: runtime.runtimeId, status: 'terminated', phase: 'terminated' };
+    }
+
+    const termOwnership = await this.ownershipSafe(runtime, expected);
+    if (termOwnership === 'exited') {
+      await this.cleanTmux(runtime);
+      return { runtimeId: runtime.runtimeId, status: 'terminated', phase: 'terminated' };
+    }
+    if (termOwnership === 'blocked') {
+      this.emit(runtime.runtimeId, 'blocked');
+      return { runtimeId: runtime.runtimeId, status: 'blocked', phase: 'blocked', detail: 'Ownership metadata changed before SIGTERM.' };
+    }
+    this.emit(runtime.runtimeId, 'term');
+    await this.processes.signal(expected, 'SIGTERM');
+    if (await this.waitForExit(expected, this.timings.termMs)) {
+      await this.cleanTmux(runtime);
+      this.emit(runtime.runtimeId, 'terminated');
+      return { runtimeId: runtime.runtimeId, status: 'terminated', phase: 'terminated' };
+    }
+
+    const killOwnership = await this.ownershipSafe(runtime, expected);
+    if (killOwnership === 'exited') {
+      await this.cleanTmux(runtime);
+      return { runtimeId: runtime.runtimeId, status: 'terminated', phase: 'terminated' };
+    }
+    if (killOwnership === 'blocked') {
+      this.emit(runtime.runtimeId, 'blocked');
+      return { runtimeId: runtime.runtimeId, status: 'blocked', phase: 'blocked', detail: 'Ownership metadata changed before SIGKILL.' };
+    }
+    this.emit(runtime.runtimeId, 'kill');
+    await this.processes.signal(expected, 'SIGKILL');
+    if (!await this.waitForExit(expected, this.timings.killMs)) {
+      this.emit(runtime.runtimeId, 'blocked');
+      return { runtimeId: runtime.runtimeId, status: 'blocked', phase: 'blocked', detail: 'Verified process survived SIGKILL deadline.' };
+    }
+    await this.cleanTmux(runtime);
+    this.emit(runtime.runtimeId, 'terminated');
+    return { runtimeId: runtime.runtimeId, status: 'terminated', phase: 'terminated' };
+  }
+}