@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/infrastructure/state/state-migration.ts

@@ -0,0 +1,103 @@
+/**
+ * State Migration
+ *
+ * Forward migration of persisted state across schema versions, plus detection
+ * of state written by a NEWER build than the current process understands.
+ *
+ * The state file is shared by every co-installed front end (pi extension, SDK,
+ * CLI / agent skill). When those builds differ in schema version, an older
+ * reader must NOT treat a newer file as corruption and overwrite it — that would
+ * destroy the newer build's live sessions, browser-server lease, and authSecret.
+ * Instead:
+ *   - version < CURRENT  → migrate forward, stepwise, to CURRENT.
+ *   - version === CURRENT → pass through unchanged.
+ *   - version > CURRENT  → throw StateVersionTooNewError (caller quarantines and
+ *     runs degraded rather than clobbering the file).
+ */
+
+import { CURRENT_STATE_VERSION } from '../types/state-types.ts';
+
+/**
+ * Thrown when the on-disk state was written by a build with a higher schema
+ * version than this process supports. Carries the versions for logging.
+ */
+export class StateVersionTooNewError extends Error {
+  readonly fileVersion: number;
+  readonly currentVersion: number;
+  constructor(fileVersion: number, currentVersion: number) {
+    super(
+      `State file version ${fileVersion} is newer than the supported version ${currentVersion}. ` +
+      `It was written by a newer build; this process will not modify it.`,
+    );
+    this.name = 'StateVersionTooNewError';
+    this.fileVersion = fileVersion;
+    this.currentVersion = currentVersion;
+  }
+}
+
+/**
+ * A single forward migration step. Receives a plain parsed object at version
+ * `from` and must return an object at version `from + 1`. Steps run in order
+ * until the object reaches CURRENT_STATE_VERSION.
+ *
+ * Keyed by the source version. When CURRENT_STATE_VERSION is bumped to N, add a
+ * MIGRATIONS[N - 1] entry that upgrades an (N-1)-shaped object to N.
+ */
+type MigrationStep = (raw: Record<string, unknown>) => Record<string, unknown>;
+
+const MIGRATIONS: Record<number, MigrationStep> = {
+  // No migrations yet — CURRENT_STATE_VERSION is 1, the initial schema.
+  // Example for a future bump to 2:
+  // 1: (raw) => ({ ...raw, version: 2, /* fill new required fields */ }),
+};
+
+/**
+ * Read a numeric `version` from a parsed object, or null if it is absent or not
+ * a finite number (in which case the object is malformed and the validator will
+ * reject it as corruption).
+ */
+function readVersion(raw: unknown): number | null {
+  if (!raw || typeof raw !== 'object') return null;
+  const v = (raw as Record<string, unknown>)['version'];
+  return typeof v === 'number' && Number.isFinite(v) ? v : null;
+}
+
+/**
+ * Bring a parsed state object up to CURRENT_STATE_VERSION.
+ *
+ * - Objects without a usable numeric version are returned unchanged so the
+ *   schema validator can reject them as corruption (the existing recovery path).
+ * - Future versions throw StateVersionTooNewError.
+ * - Older versions are migrated forward step by step; a missing step for some
+ *   intermediate version is an unrecoverable gap and throws.
+ */
+export function migrateRawState(raw: unknown): unknown {
+  const version = readVersion(raw);
+  if (version === null) return raw;
+
+  if (version > CURRENT_STATE_VERSION) {
+    throw new StateVersionTooNewError(version, CURRENT_STATE_VERSION);
+  }
+
+  if (version === CURRENT_STATE_VERSION) return raw;
+
+  let current = raw as Record<string, unknown>;
+  let v = version;
+  while (v < CURRENT_STATE_VERSION) {
+    const step = MIGRATIONS[v];
+    if (!step) {
+      throw new Error(
+        `No migration registered from state version ${v} to ${v + 1}; cannot upgrade.`,
+      );
+    }
+    current = step(current);
+    const next = readVersion(current);
+    if (next !== v + 1) {
+      throw new Error(
+        `Migration from state version ${v} produced version ${String(next)}, expected ${v + 1}.`,
+      );
+    }
+    v = next;
+  }
+  return current;
+}

package/src/infrastructure/state/state-path-configuration.ts

@@ -0,0 +1,119 @@
+/**
+ * State Path Configuration Service
+ *
+ * Computes and provides file paths for state management components.
+ * This service is registered first so that FileLockService and StateBackupManager
+ * can depend on it for their path configuration.
+ */
+
+import * as path from 'node:path';
+import { getGlobalConfigDir } from '../../config.ts';
+import type { IService } from '../../core/service-registry.ts';
+import { ServiceLifecycle } from '../../core/service-registry.ts';
+import { ServiceNames } from '../../core/interfaces/service-names.ts';
+
+export interface StatePaths {
+  stateFilePath: string;
+  lockDirPath: string;
+  backupDirPath: string;
+  lockFilePath: string;
+  projectSettingsPath: string;
+}
+
+/**
+ * State Path Configuration Service
+ *
+ * Provides centralized path configuration for state management.
+ */
+export class StatePathConfiguration implements IService {
+  readonly name = ServiceNames.STATE_PATH_CONFIGURATION;
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+  private _initialized = false;
+
+  private readonly stateDir: string;
+  private readonly stateFilePath: string;
+  private readonly lockDirPath: string;
+  private readonly backupDirPath: string;
+  private readonly lockFilePath: string;
+  private readonly projectSettingsPath: string;
+
+  constructor(stateDir?: string) {
+    // pi-research's own namespace (~/.pi/research/state), beside config.env and
+    // knowledge_db — not the host pi config root (~/.pi/state).
+    const resolvedStateDir = stateDir || path.join(getGlobalConfigDir(), 'state');
+    this.stateDir = resolvedStateDir;
+    this.stateFilePath = path.join(resolvedStateDir, 'research-state.json');
+    this.lockDirPath = path.join(resolvedStateDir, '.locks');
+    this.backupDirPath = path.join(resolvedStateDir, 'backups');
+    this.lockFilePath = path.join(this.lockDirPath, 'research-state.lock');
+    this.projectSettingsPath = path.join(resolvedStateDir, 'project-settings.json');
+  }
+
+  async initialize(): Promise<void> {
+    if (this._initialized) {
+      return;
+    }
+    this._initialized = true;
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+  }
+
+  async dispose(): Promise<void> {
+    // Nothing to dispose
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+  }
+
+  /**
+   * Get all state paths
+   */
+  getPaths(): StatePaths {
+    return {
+      stateFilePath: this.stateFilePath,
+      lockDirPath: this.lockDirPath,
+      backupDirPath: this.backupDirPath,
+      lockFilePath: this.lockFilePath,
+      projectSettingsPath: this.projectSettingsPath,
+    };
+  }
+
+  /**
+   * Get the state directory
+   */
+  getStateDir(): string {
+    return this.stateDir;
+  }
+
+  /**
+   * Get the state file path
+   */
+  getStateFilePath(): string {
+    return this.stateFilePath;
+  }
+
+  /**
+   * Get the lock directory path
+   */
+  getLockDirPath(): string {
+    return this.lockDirPath;
+  }
+
+  /**
+   * Get the backup directory path
+   */
+  getBackupDirPath(): string {
+    return this.backupDirPath;
+  }
+
+  /**
+   * Get the lock file path
+   */
+  getLockFilePath(): string {
+    return this.lockFilePath;
+  }
+
+  /**
+   * Get the project settings file path
+   */
+  getProjectSettingsPath(): string {
+    return this.projectSettingsPath;
+  }
+  }

package/src/infrastructure/state/state-session-api.ts

@@ -0,0 +1,94 @@
+/**
+ * State Session API
+ *
+ * Session management API that delegates to StateSessionManager.
+ * This separates the public API from the internal session manager.
+ */
+
+import type { SingletonState } from '../types/state-types.ts';
+import type { StateSessionManager } from './state-session-manager.ts';
+
+/**
+ * Provides session management operations on top of StateSessionManager
+ */
+export class StateSessionApi {
+  constructor(private readonly sessionManager: StateSessionManager) {}
+
+  /**
+   * Add a new session to the state
+   * @param sessionId The session ID to add
+   * @param param PID (number) or container name (string)
+   * @param updateState Function to update state atomically
+   * @param processPid PID of the current process
+   * @param getStartTime Function to get process start time
+   * @throws Error if unable to update state
+   */
+  async addSession(
+    sessionId: string,
+    param: number | string,
+    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>,
+    processPid: number,
+    getStartTime: (pid: number) => Promise<number | null>
+  ): Promise<void> {
+    const pid = typeof param === 'number' ? param : processPid;
+    const startTime = await getStartTime(pid);
+    await updateState(async (state): Promise<SingletonState> => {
+      return this.sessionManager.addSession(state, sessionId, pid, startTime);
+    });
+  }
+
+  /**
+   * Remove a session from the state
+   * @param sessionId The session ID to remove
+   * @param updateState Function to update state atomically
+   */
+  async removeSession(
+    sessionId: string,
+    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>
+  ): Promise<void> {
+    await updateState((state): SingletonState => {
+      return this.sessionManager.removeSession(state, sessionId);
+    });
+  }
+
+  /**
+   * Update the heartbeat timestamp for a session
+   * @param sessionId The session ID to update
+   * @param updateState Function to update state atomically
+   */
+  async updateHeartbeat(
+    sessionId: string,
+    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>
+  ): Promise<void> {
+    await updateState((state): SingletonState => {
+      return this.sessionManager.updateHeartbeat(state, sessionId);
+    });
+  }
+
+  /**
+   * Clean up stale sessions based on timeout and process liveness
+   * @param timeoutMs Timeout in milliseconds for session staleness
+   * @param readState Function to read state
+   * @param updateState Function to update state atomically
+   * @returns Number of sessions removed
+   */
+  async cleanupStaleSessions(
+    timeoutMs: number,
+    readState: () => Promise<SingletonState>,
+    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>
+  ): Promise<number> {
+    const state = await readState();
+    const sessionsToRemove = await this.sessionManager.classifyStaleSessions(state, timeoutMs);
+
+    // Remove stale sessions under a fresh lock. Re-validate lastSeen so a
+    // session that received a heartbeat between the two lock acquisitions is
+    // not accidentally deleted (TOCTOU guard).
+    if (sessionsToRemove.size > 0) {
+      await updateState((state): SingletonState => {
+        return this.sessionManager.removeStaleSessions(state, sessionsToRemove);
+      });
+    }
+
+    return sessionsToRemove.size;
+  }
+}

package/src/infrastructure/state/state-session-manager.ts

@@ -0,0 +1,127 @@
+/**
+ * State Session Manager
+ *
+ * Handles session lifecycle and cleanup operations for the StateManager.
+ */
+
+import type { ProcessLifecycleService } from '../process-lifecycle-service.ts';
+import type { SingletonState } from '../types/state-types.ts';
+import { ServiceLifecycle, type IService } from '../../core/service-registry.ts';
+import { ServiceNames } from '../../core/interfaces/service-names.ts';
+
+/**
+ * Manages research sessions within the state
+ */
+export class StateSessionManager implements IService {
+  readonly name = ServiceNames.STATE_SESSION_MANAGER;
+
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+  constructor(private readonly processLifecycle: ProcessLifecycleService) {}
+
+  /**
+   * Add a new session to the state
+   * @param state The current state
+   * @param sessionId The session ID to add
+   * @param pid The process ID
+   * @param startTime Optional process start time (Linux)
+   * @returns Updated state with new session
+   */
+  addSession(state: SingletonState, sessionId: string, pid: number, startTime?: number | null): SingletonState {
+    state.sessions[sessionId] = {
+      pid,
+      startTime: startTime ?? undefined,
+      lastSeen: Date.now(),
+      connectedAt: Date.now(),
+    };
+
+    return state;
+  }
+
+  /**
+   * Remove a session from the state
+   * @param state The current state
+   * @param sessionId The session ID to remove
+   * @returns Updated state without the session
+   */
+  removeSession(state: SingletonState, sessionId: string): SingletonState {
+    if (state.sessions[sessionId] !== undefined) {
+      delete state.sessions[sessionId];
+    }
+    return state;
+  }
+
+  /**
+   * Update the heartbeat timestamp for a session
+   * @param state The current state
+   * @param sessionId The session ID to update
+   * @returns Updated state with updated heartbeat
+   */
+  updateHeartbeat(state: SingletonState, sessionId: string): SingletonState {
+    const session = state.sessions[sessionId];
+    if (session !== undefined) {
+      session.lastSeen = Date.now();
+    }
+    return state;
+  }
+
+  /**
+   * Clean up stale sessions based on timeout and process liveness
+   * @param state The current state
+   * @param timeoutMs Timeout in milliseconds for session staleness
+   * @returns Number of sessions to remove and their lastSeen timestamps
+   */
+  async classifyStaleSessions(state: SingletonState, timeoutMs: number): Promise<Map<string, number>> {
+    const now = Date.now();
+    const sessionsToRemove = new Map<string, number>();
+
+    for (const [sessionId, sessionInfo] of Object.entries(state.sessions)) {
+      // Primary check: lastSeen timeout
+      const lastSeenAge = now - sessionInfo.lastSeen;
+
+      if (lastSeenAge > timeoutMs) {
+        sessionsToRemove.set(sessionId, sessionInfo.lastSeen);
+        continue;
+      }
+
+      // Secondary check: is process still alive?
+      // Passing startTime prevents PID reuse races on Linux.
+      const isAlive = await this.processLifecycle.isProcessAlive(sessionInfo.pid, sessionInfo.startTime);
+
+      if (!isAlive) {
+        sessionsToRemove.set(sessionId, sessionInfo.lastSeen);
+      }
+    }
+
+    return sessionsToRemove;
+  }
+
+  /**
+   * Remove stale sessions from state
+   * @param state The current state
+   * @param sessionsToRemove Map of session IDs to their lastSeen timestamps
+   * @returns Updated state with stale sessions removed
+   */
+  removeStaleSessions(state: SingletonState, sessionsToRemove: Map<string, number>): SingletonState {
+    for (const [sessionId, lastSeenAtClassify] of sessionsToRemove) {
+      const current = state.sessions[sessionId];
+      if (current && current.lastSeen === lastSeenAtClassify) {
+        delete state.sessions[sessionId];
+      }
+    }
+    return state;
+  }
+
+  async initialize(): Promise<void> {
+    if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
+      return;
+    }
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+  }
+
+  async dispose(): Promise<void> {
+    if (this.lifecycle === ServiceLifecycle.DISPOSED) {
+      return;
+    }
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+  }
+}

package/src/infrastructure/state/state-validator.ts

@@ -0,0 +1,93 @@
+/**
+ * State Validator
+ *
+ * Validates state structure and values.
+ */
+
+import type { SingletonState } from '../types/state-types.ts';
+import { SingletonStateSchema } from '../types/state-types.ts';
+import { migrateRawState } from './state-migration.ts';
+import { Value } from 'typebox/value';
+import { ServiceLifecycle, type IService } from '../../core/service-registry.ts';
+import { ServiceNames } from '../../core/interfaces/service-names.ts';
+
+/**
+ * Validates state objects using TypeBox schemas
+ */
+export class StateValidator implements IService {
+  readonly name = ServiceNames.STATE_VALIDATOR;
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+
+  /**
+   * Validate the structure and version of a state object
+   * @param state The state object to validate
+   * @returns The validated and potentially coerced SingletonState
+   * @throws Error if state structure or version is invalid
+   */
+  validateState(state: unknown): SingletonState {
+    if (!state || typeof state !== 'object') {
+      throw new Error('Invalid state: not an object');
+    }
+
+    // Coerce values if possible
+    const coerced = Value.Convert(SingletonStateSchema, state);
+
+    // Validate against schema
+    if (!Value.Check(SingletonStateSchema, coerced)) {
+      // Use coerced state for error reporting to get better paths
+      const errors = [...Value.Errors(SingletonStateSchema, coerced)];
+      const errorMsg = errors.map((e: any) => {
+        const path = String(e.path || 'root');
+        return `${path}: ${e.message} (${JSON.stringify(e.value)})`;
+      }).join(', ');
+      throw new Error(`Invalid state: ${errorMsg}`);
+    }
+
+    // Additional value range checks not easily covered by basic schemas
+    if (coerced.port < 0 || coerced.port > 65535) {
+      throw new Error(`Invalid state: port must be 0-65535, got ${coerced.port}`);
+    }
+
+    if (coerced.browserServer && (coerced.browserServer.port < 0 || coerced.browserServer.port > 65535)) {
+      throw new Error(`Invalid state: browserServer.port must be 0-65535, got ${coerced.browserServer.port}`);
+    }
+
+    // embeddingServer.port allows the sentinel -1 ("a process is initializing"),
+    // otherwise must be a valid TCP port. This rejects garbage (e.g. -5, 99999)
+    // that would otherwise produce confusing socket errors in EmbeddingClient.
+    if (coerced.embeddingServer) {
+      const ep = coerced.embeddingServer.port;
+      if (ep < -1 || ep > 65535) {
+        throw new Error(`Invalid state: embeddingServer.port must be -1 or 0-65535, got ${ep}`);
+      }
+    }
+
+    return coerced;
+  }
+
+  /**
+   * Migrate a freshly-parsed state object up to the current schema version, then
+   * validate it. Use this for state read from disk (where the version may be
+   * older, or newer than this process understands). Throws
+   * StateVersionTooNewError for future versions so the caller can quarantine
+   * rather than overwrite; throws "Invalid state: ..." for everything the schema
+   * rejects, matching the corruption-recovery contract in _readState.
+   */
+  migrateAndValidate(raw: unknown): SingletonState {
+    return this.validateState(migrateRawState(raw));
+  }
+
+  async initialize(): Promise<void> {
+    if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
+      return;
+    }
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+  }
+
+  async dispose(): Promise<void> {
+    if (this.lifecycle === ServiceLifecycle.DISPOSED) {
+      return;
+    }
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+  }
+}

package/src/infrastructure/types/state-types.ts

@@ -0,0 +1,85 @@
+/**
+ * State Types
+ *
+ * Shared types for state management.
+ * Extracted to separate file to avoid circular dependencies.
+ */
+
+import { Type, type Static } from 'typebox';
+
+/**
+ * Current on-disk state schema version. Bump this (and add a migration step in
+ * state-migration.ts) whenever the persisted shape changes incompatibly. The
+ * schema's `version` literal is pinned to this value so a reader only accepts a
+ * state it actually understands; older/newer files are routed through migration
+ * or future-version quarantine rather than being silently overwritten.
+ */
+export const CURRENT_STATE_VERSION = 1 as const;
+
+/**
+ * State metrics interface
+ */
+export interface StateMetrics {
+  totalSessions: number;
+  activeSessions: number;
+  oldestSession: number | null;
+  newestSession: number | null;
+  containerUptime: number | null;
+  lastHeartbeatAge: number | null;
+}
+
+/**
+ * Session information interface
+ */
+export const SessionInfoSchema = Type.Object({
+  pid: Type.Number(),
+  startTime: Type.Optional(Type.Number()),
+  lastSeen: Type.Number(),
+  connectedAt: Type.Number(),
+});
+
+/**
+ * Main state structure interface
+ */
+export const SingletonStateSchema = Type.Object({
+  version: Type.Literal(CURRENT_STATE_VERSION),
+  containerId: Type.String(),
+  containerName: Type.String(),
+  port: Type.Number(),
+  sessions: Type.Record(Type.String(), SessionInfoSchema),
+  lastUpdated: Type.Number(),
+  browserServer: Type.Optional(Type.Object({
+    port: Type.Number(),
+    pid: Type.Number(),
+    startTime: Type.Optional(Type.Number()),
+    schedulerId: Type.Optional(Type.String()),
+    authSecret: Type.Optional(Type.String()),
+  })),
+  schedulerVersion: Type.Optional(Type.String()),
+  gpuOwner: Type.Optional(Type.Object({
+    pid: Type.Number(),
+    startTime: Type.Optional(Type.Number()),
+    startedAt: Type.Number(),
+    sessionId: Type.Optional(Type.String()),
+    // Re-entrant hold count for the owning process. The lock is shared by N
+    // concurrent in-process embedding calls (activeEmbeddings is designed > 1);
+    // each re-entrant acquire increments and each release decrements, so the
+    // cross-process owner record is only cleared when the last in-process hold
+    // is released. Absent on state written before refcounting — treated as 1.
+    holds: Type.Optional(Type.Number()),
+  })),
+  embeddingServer: Type.Optional(Type.Object({
+    port: Type.Number(),
+    pid: Type.Number(),
+    startTime: Type.Optional(Type.Number()),
+    serverId: Type.String(),
+    // EMBEDDING_MODEL the leader serves. A follower configured with a DIFFERENT
+    // model must not adopt this leader (same-dimension models would silently
+    // cross-embed and corrupt the vector space) — it falls back to a local
+    // in-process embedder instead. Optional for entries written by older
+    // processes, which are tolerated as matching.
+    model: Type.Optional(Type.String()),
+  })),
+});
+
+export type SingletonState = Static<typeof SingletonStateSchema>;