@modelprofile.com/flexharness 10.17.1 → 10.18.0

package/ts_tools_node/classes.workspacereversion.ts

@@ -0,0 +1,255 @@
+import * as plugins from './plugins.js';
+import type { IToolFileChange, IToolFileChangeObserver } from './tool.context.js';
+import {
+  fileImageBytes,
+  fileImagesEqual,
+  missingAncestors,
+  readFileImage,
+  removeEmptyDirectories,
+  writeFileImage,
+  type TFileImage,
+} from './functions.fileimages.js';
+
+export interface IInMemoryWorkspaceReversionOptions {
+  /** The workspace a revertible turn names in the session's reversion info. */
+  workspace: plugins.IFlexAffectedWorkspace;
+  /**
+   * The bytes of file images one turn may hold. A turn that changes more is recorded as
+   * nonrevertible (`capture-too-large`) and its images are dropped. Default: 64 MiB.
+   */
+  maxCaptureBytes?: number;
+}
+
+/** One file a turn changed: what it held before the turn's first change and after the turn. */
+interface IFileRecord {
+  before: TFileImage;
+  after?: TFileImage;
+  /** directories the turn's first write created, deepest first */
+  createdDirectories: string[];
+}
+
+interface ICapture {
+  captureId: string;
+  runId: string;
+  state: 'prepared' | 'finalized';
+  files: Map<string, IFileRecord>;
+  bytes: number;
+  nonrevertibleReason?: string;
+  outcome?: plugins.TFlexTurnReversionFinalizedOutcome;
+  appliedOperations: Set<string>;
+  /** apply operations that failed after a partial change could not be rolled back */
+  uncertainOperations: Set<string>;
+}
+
+interface ICaptureReference {
+  captureId: string;
+}
+
+const DEFAULT_MAX_CAPTURE_BYTES = 64 * 1024 * 1024;
+
+const isCaptureReference = (valueArg: plugins.TJsonValue): valueArg is plugins.TJsonValue & ICaptureReference =>
+  !!valueArg
+  && typeof valueArg === 'object'
+  && !Array.isArray(valueArg)
+  && typeof valueArg.captureId === 'string';
+
+/**
+ * Turn reversion for the local filesystem tools, held in memory: FlexHarness `/undo` and `/redo`
+ * move a turn's file changes together with its transcript.
+ *
+ * Give each run's `createLocalToolExecutionContext` the observer `observerFor(context.runId)`. The
+ * first change of a file in a turn records what the file held before; `finalize` records what it
+ * holds after. Undo puts the before images back and redo the after images, each only while every
+ * file still holds what the other direction left, so a later edit of the user is never overwritten.
+ * A turn that ran a shell command is nonrevertible, since the command may have changed files no
+ * observer saw.
+ *
+ * Captures live as long as the process. A capture FlexHarness asks about after a restart is
+ * `missing`, and undoing past it fails without changing anything. Only root-session runs are
+ * captured; a change of a run without a prepared capture is refused.
+ */
+export class InMemoryWorkspaceReversion<TScope = unknown> implements plugins.IFlexTurnReversionProviderV2<TScope> {
+  public readonly protocolVersion = 2;
+
+  private readonly workspace: plugins.IFlexAffectedWorkspace;
+  private readonly maxCaptureBytes: number;
+  private readonly captures = new Map<string, ICapture>();
+  private readonly captureIdsByRun = new Map<string, string>();
+
+  constructor(optionsArg: IInMemoryWorkspaceReversionOptions) {
+    this.workspace = { id: optionsArg.workspace.id, label: optionsArg.workspace.label };
+    this.maxCaptureBytes = optionsArg.maxCaptureBytes ?? DEFAULT_MAX_CAPTURE_BYTES;
+    if (!Number.isSafeInteger(this.maxCaptureBytes) || this.maxCaptureBytes <= 0) {
+      throw new Error('maxCaptureBytes must be a positive integer.');
+    }
+  }
+
+  /** The observer of one run's file changes and shell commands. */
+  public observerFor(runIdArg: string): IToolFileChangeObserver {
+    return {
+      beforeFileChange: (changeArg) => this.recordChange(runIdArg, changeArg),
+      beforeShellCommand: () => {
+        this.markNonrevertible(this.requirePreparedCapture(runIdArg), 'shell-command');
+      },
+    };
+  }
+
+  public prepare(contextArg: plugins.IFlexTurnReversionBaseContext<TScope>): void {
+    contextArg.signal.throwIfAborted();
+    if (this.captures.has(contextArg.captureId)) return;
+    this.captures.set(contextArg.captureId, {
+      captureId: contextArg.captureId,
+      runId: contextArg.runId,
+      state: 'prepared',
+      files: new Map(),
+      bytes: 0,
+      appliedOperations: new Set(),
+      uncertainOperations: new Set(),
+    });
+    this.captureIdsByRun.set(contextArg.runId, contextArg.captureId);
+  }
+
+  public inspectCapture(
+    contextArg: plugins.IFlexTurnReversionBaseContext<TScope>,
+  ): plugins.TFlexTurnReversionCaptureInspectionV2 {
+    const capture = this.captures.get(contextArg.captureId);
+    if (!capture) return { status: 'missing' };
+    return capture.outcome ? { status: 'finalized', outcome: capture.outcome } : { status: 'prepared' };
+  }
+
+  public async finalize(
+    contextArg: plugins.IFlexTurnReversionBaseContext<TScope>,
+  ): Promise<plugins.TFlexTurnReversionFinalizedOutcome> {
+    contextArg.signal.throwIfAborted();
+    const capture = this.requireCapture(contextArg.captureId);
+    if (capture.outcome) return capture.outcome;
+    const reference: ICaptureReference = { captureId: capture.captureId };
+    if (!capture.nonrevertibleReason) {
+      for (const [filePath, record] of capture.files) {
+        record.after = await readFileImage(filePath);
+        if (fileImagesEqual(record.before, record.after)) capture.files.delete(filePath);
+      }
+    }
+    capture.state = 'finalized';
+    if (capture.nonrevertibleReason) {
+      capture.outcome = {
+        disposition: 'nonrevertible',
+        reference: { ...reference },
+        reasonCode: capture.nonrevertibleReason,
+        affectedWorkspaces: [this.workspace],
+      };
+    } else if (capture.files.size === 0) {
+      capture.outcome = { disposition: 'no-change', reference: { ...reference } };
+    } else {
+      capture.outcome = { disposition: 'revertible', reference: { ...reference }, affectedWorkspaces: [this.workspace] };
+    }
+    this.captureIdsByRun.delete(capture.runId);
+    return capture.outcome;
+  }
+
+  public async apply(contextArg: plugins.IFlexTurnReversionApplyContext<TScope>): Promise<void> {
+    contextArg.signal.throwIfAborted();
+    const capture = this.captureOf(contextArg.reference);
+    if (capture.outcome?.disposition !== 'revertible') {
+      throw new Error('This turn has no revertible workspace changes.');
+    }
+    const records = [...capture.files.entries()];
+    const expected = (recordArg: IFileRecord) => (contextArg.direction === 'undo' ? recordArg.after! : recordArg.before);
+    const target = (recordArg: IFileRecord) => (contextArg.direction === 'undo' ? recordArg.before : recordArg.after!);
+    for (const [filePath, record] of records) {
+      if (!fileImagesEqual(await readFileImage(filePath), expected(record))) {
+        throw new Error(`"${filePath}" changed after the turn; its changes were not ${contextArg.direction === 'undo' ? 'undone' : 'redone'}.`);
+      }
+    }
+    const written: Array<[string, IFileRecord]> = [];
+    try {
+      for (const entry of records) {
+        await writeFileImage(entry[0], target(entry[1]));
+        written.push(entry);
+      }
+    } catch (error) {
+      await this.rollBack(written, expected, contextArg.operationId, capture);
+      throw error;
+    }
+    if (contextArg.direction === 'undo') {
+      for (const [, record] of records) await removeEmptyDirectories(record.createdDirectories);
+    }
+    capture.appliedOperations.add(contextArg.operationId);
+  }
+
+  public inspectApply(
+    contextArg: plugins.IFlexTurnReversionApplyContext<TScope>,
+  ): plugins.IFlexTurnReversionApplyInspection {
+    const capture = this.captures.get(isCaptureReference(contextArg.reference) ? contextArg.reference.captureId : '');
+    if (!capture) return { status: 'not-applied' };
+    if (capture.appliedOperations.has(contextArg.operationId)) return { status: 'applied' };
+    if (capture.uncertainOperations.has(contextArg.operationId)) return { status: 'unknown' };
+    return { status: 'not-applied' };
+  }
+
+  public release(contextArg: plugins.IFlexTurnReversionReleaseContext<TScope>): void {
+    const captureId = isCaptureReference(contextArg.reference) ? contextArg.reference.captureId : contextArg.captureId;
+    const capture = this.captures.get(captureId);
+    if (!capture) return;
+    this.captures.delete(captureId);
+    if (this.captureIdsByRun.get(capture.runId) === captureId) this.captureIdsByRun.delete(capture.runId);
+  }
+
+  private async recordChange(runIdArg: string, changeArg: IToolFileChange): Promise<void> {
+    const capture = this.requirePreparedCapture(runIdArg);
+    if (capture.nonrevertibleReason || capture.files.has(changeArg.path)) return;
+    const before = await readFileImage(changeArg.path);
+    const createdDirectories = before.kind === 'absent' && changeArg.kind === 'write'
+      ? await missingAncestors(changeArg.path)
+      : [];
+    if (capture.files.has(changeArg.path)) return;
+    capture.bytes += fileImageBytes(before);
+    if (capture.bytes > this.maxCaptureBytes) {
+      this.markNonrevertible(capture, 'capture-too-large');
+      return;
+    }
+    capture.files.set(changeArg.path, { before, createdDirectories });
+  }
+
+  private markNonrevertible(captureArg: ICapture, reasonArg: string): void {
+    captureArg.nonrevertibleReason ??= reasonArg;
+    captureArg.files.clear();
+    captureArg.bytes = 0;
+  }
+
+  private async rollBack(
+    writtenArg: Array<[string, IFileRecord]>,
+    expectedArg: (recordArg: IFileRecord) => TFileImage,
+    operationIdArg: string,
+    captureArg: ICapture,
+  ): Promise<void> {
+    try {
+      for (const [filePath, record] of writtenArg.reverse()) await writeFileImage(filePath, expectedArg(record));
+    } catch {
+      // the workspace holds part of the change; FlexHarness learns that the outcome is unknown
+      captureArg.uncertainOperations.add(operationIdArg);
+    }
+  }
+
+  private requirePreparedCapture(runIdArg: string): ICapture {
+    const captureId = this.captureIdsByRun.get(runIdArg);
+    const capture = captureId ? this.captures.get(captureId) : undefined;
+    if (!capture || capture.state !== 'prepared') {
+      throw new Error(`No workspace capture is open for run "${runIdArg}"; the change was refused.`);
+    }
+    return capture;
+  }
+
+  private requireCapture(captureIdArg: string): ICapture {
+    const capture = this.captures.get(captureIdArg);
+    if (!capture) throw new Error(`The workspace capture "${captureIdArg}" is not held.`);
+    return capture;
+  }
+
+  private captureOf(referenceArg: plugins.TJsonValue): ICapture {
+    if (!isCaptureReference(referenceArg)) throw new Error('The workspace capture reference is malformed.');
+    const capture = this.captures.get(referenceArg.captureId);
+    if (!capture) throw new Error('The workspace changes of this turn are no longer held in memory.');
+    return capture;
+  }
+}

package/ts_tools_node/functions.fileimages.ts

@@ -0,0 +1,89 @@
+import * as plugins from './plugins.js';
+
+/** What one path holds at a moment: nothing, a regular file, or a symbolic link. */
+export type TFileImage =
+  | { kind: 'absent' }
+  | { kind: 'file'; bytes: Buffer; mode: number }
+  | { kind: 'symlink'; target: string };
+
+const isMissing = (error: unknown): boolean => {
+  const code = (error as NodeJS.ErrnoException).code;
+  return code === 'ENOENT' || code === 'ENOTDIR';
+};
+
+/** Reads what the path holds, without following a final symbolic link. */
+export const readFileImage = async (pathArg: string): Promise<TFileImage> => {
+  let stats: plugins.fs.Stats;
+  try {
+    stats = await plugins.fs.promises.lstat(pathArg);
+  } catch (error) {
+    if (isMissing(error)) return { kind: 'absent' };
+    throw error;
+  }
+  if (stats.isSymbolicLink()) return { kind: 'symlink', target: await plugins.fs.promises.readlink(pathArg) };
+  if (stats.isFile()) return { kind: 'file', bytes: await plugins.fs.promises.readFile(pathArg), mode: stats.mode & 0o7777 };
+  throw new Error(`"${pathArg}" is neither a file nor a symbolic link.`);
+};
+
+/** The bytes an image keeps in memory. */
+export const fileImageBytes = (imageArg: TFileImage): number =>
+  imageArg.kind === 'file'
+    ? imageArg.bytes.byteLength
+    : imageArg.kind === 'symlink'
+      ? Buffer.byteLength(imageArg.target)
+      : 0;
+
+export const fileImagesEqual = (leftArg: TFileImage, rightArg: TFileImage): boolean => {
+  if (leftArg.kind === 'absent' || rightArg.kind === 'absent') return leftArg.kind === rightArg.kind;
+  if (leftArg.kind === 'symlink' || rightArg.kind === 'symlink') {
+    return leftArg.kind === 'symlink' && rightArg.kind === 'symlink' && leftArg.target === rightArg.target;
+  }
+  return leftArg.bytes.equals(rightArg.bytes);
+};
+
+/** Puts the image at the path, creating parent directories as needed. */
+export const writeFileImage = async (pathArg: string, imageArg: TFileImage): Promise<void> => {
+  if (imageArg.kind === 'absent') {
+    await plugins.fs.promises.rm(pathArg, { force: true });
+    return;
+  }
+  await plugins.fs.promises.mkdir(plugins.path.dirname(pathArg), { recursive: true });
+  if (imageArg.kind === 'symlink') {
+    await plugins.fs.promises.rm(pathArg, { force: true });
+    await plugins.fs.promises.symlink(imageArg.target, pathArg);
+    return;
+  }
+  if ((await readFileImage(pathArg)).kind === 'symlink') await plugins.fs.promises.rm(pathArg);
+  await plugins.fs.promises.writeFile(pathArg, imageArg.bytes);
+  await plugins.fs.promises.chmod(pathArg, imageArg.mode);
+};
+
+/** The ancestors of the path that do not exist yet, deepest first: a write would create them. */
+export const missingAncestors = async (pathArg: string): Promise<string[]> => {
+  const missing: string[] = [];
+  let current = plugins.path.dirname(pathArg);
+  while (current !== plugins.path.dirname(current)) {
+    try {
+      await plugins.fs.promises.lstat(current);
+      break;
+    } catch (error) {
+      if (!isMissing(error)) throw error;
+      missing.push(current);
+      current = plugins.path.dirname(current);
+    }
+  }
+  return missing;
+};
+
+/** Removes the directories, deepest first, as far as they are empty. */
+export const removeEmptyDirectories = async (directoriesArg: readonly string[]): Promise<void> => {
+  for (const directory of directoriesArg) {
+    try {
+      await plugins.fs.promises.rmdir(directory);
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code;
+      if (code === 'ENOTEMPTY' || code === 'EEXIST') return;
+      if (!isMissing(error)) throw error;
+    }
+  }
+};

package/ts_tools_node/index.ts

@@ -1,5 +1,11 @@
 export { createLocalToolExecutionContext } from './tool.context.js';
-export type { ILocalToolExecutionContextOptions } from './tool.context.js';
+export type {
+  ILocalToolExecutionContextOptions,
+  IToolFileChange,
+  IToolFileChangeObserver,
+} from './tool.context.js';
+export { InMemoryWorkspaceReversion } from './classes.workspacereversion.js';
+export type { IInMemoryWorkspaceReversionOptions } from './classes.workspacereversion.js';
 export { FileToolJobStore } from './tool.persistence.js';
 export type { IFileToolJobStoreOptions } from './tool.persistence.js';
 export * from './tools.local.js';

package/ts_tools_node/plugins.ts

@@ -41,3 +41,14 @@ export type {
   TToolJobEvent,
   TToolJobEventListener,
 } from '@modelprofile.com/flexharness-agent';
+export type {
+  IFlexAffectedWorkspace,
+  IFlexTurnReversionApplyContext,
+  IFlexTurnReversionApplyInspection,
+  IFlexTurnReversionBaseContext,
+  IFlexTurnReversionProviderV2,
+  IFlexTurnReversionReleaseContext,
+  TFlexTurnReversionCaptureInspectionV2,
+  TFlexTurnReversionFinalizedOutcome,
+  TJsonValue,
+} from '@modelprofile.com/flexharness';

package/ts_tools_node/readme.md

@@ -11,6 +11,33 @@ pnpm add @modelprofile.com/flexharness
 See the [toolbox usage and migration guide](https://code.foss.global/modelprofile.com/flexharness#readme)
 for composition examples and the SmartAI/SmartAgent migration map.
 
+## Turn reversion for local files
+
+`InMemoryWorkspaceReversion` is a FlexHarness `turnReversionProvider` that undoes and redoes a turn's changes of the local filesystem tools together with its transcript, held in memory for the process's lifetime.
+
+```typescript
+const reversion = new InMemoryWorkspaceReversion({ workspace: { id: rootDir, label: 'project' } });
+const harness = new FlexHarness({
+  scopeResolver,
+  modelResolver,
+  turnReversionProvider: reversion,
+  reversionPolicy: 'workspace-required',
+  toolProvider: {
+    provideTools: (context) => {
+      const execution = createLocalToolExecutionContext({
+        rootDir,
+        cwd: rootDir,
+        fileChangeObserver: reversion.observerFor(context.runId),
+        requestPermission: createFlexToolPermissionRequester(context, { policy: { write: 'allow' } }),
+      });
+      return { tools: { ...createFilesystemTools(execution, { includeEdit: true }), ...createShellTools(execution) } };
+    },
+  },
+});
+```
+
+The `fileChangeObserver` of `createLocalToolExecutionContext` sees every write and delete by its canonical path (a write follows a symbolic link to its target, a delete removes the link itself) after the root checks and before the change, and every shell command before it starts; a rejection refuses the operation. The first change of a file in a turn records its bytes, mode or link target, and the directories a new file's write creates. Undo puts the before images back and removes those directories while empty; redo puts the after images back. Either direction applies only while every file still holds what the other direction left, so a later edit is never overwritten: the undo fails without a change and can be retried. A turn that ran a shell command is nonrevertible (`shell-command`), as is one whose images exceed `maxCaptureBytes` (64 MiB by default; `capture-too-large`). Only root-session runs are captured: a change of a run without a capture is refused. Captures do not survive a restart; FlexHarness then finds them `missing`.
+
 ## Issue Reporting and Security
 
 Report issues and security concerns through [community.foss.global](https://community.foss.global).

package/ts_tools_node/tool.context.ts

@@ -1,6 +1,26 @@
 import * as plugins from './plugins.js';
 import type { IToolJobStore, IToolPermissionRequest, IToolInvocationOptions, IToolRunOptions, IToolShellResult, IShellCommand, IToolShellContext, IToolJobRequest, IShellStartedResult, IShellFinishedResult, IToolJobState, IToolJobHandle, IToolJobEvent, IToolJobOutputEvent, TToolJobEvent, TToolJobEventListener, IToolJobContext, IToolFilesystemReadOptions, IToolFilesystemListOptions, IToolFilesystemContext, IToolBrowserRunOptions, TBrowserToolAction, IBrowserToolInput, IToolBrowserContext, IToolExecutionContext } from './plugins.js';
 
+/** A file a tool is about to change, named by its canonical path. */
+export interface IToolFileChange {
+  kind: 'write' | 'delete';
+  /**
+   * The canonical absolute path the operation changes. A write follows a symlink to its target; a
+   * delete removes the link itself, so its path is the link's.
+   */
+  path: string;
+}
+
+/**
+ * Sees every change the local filesystem tools make, after the path checks and before the change,
+ * and every shell command before it starts. A rejection refuses the operation.
+ */
+export interface IToolFileChangeObserver {
+  beforeFileChange(change: IToolFileChange): Promise<void> | void;
+  /** A shell command or background job may change files the observer does not see. */
+  beforeShellCommand(): Promise<void> | void;
+}
+
 export interface ILocalToolExecutionContextOptions {
   cwd?: string;
   rootDir?: string;
@@ -9,6 +29,8 @@ export interface ILocalToolExecutionContextOptions {
   abortSignal?: AbortSignal;
   /** Guards each local filesystem operation or shell side effect. */
   beforeOperation?: () => Promise<void> | void;
+  /** Observes every file change and shell command, for example to capture a turn for undo. */
+  fileChangeObserver?: IToolFileChangeObserver;
   /** Hard combined stdout/stderr capture limit for each local shell command. Default: 1 MiB. */
   maxShellOutputBytes?: number;
   /** Number of terminal background job snapshots retained for get/list. Default: 100. */
@@ -58,6 +80,13 @@ export const createLocalToolExecutionContext = (options: ILocalToolExecutionCont
   if (!Number.isSafeInteger(jobStoreOperationTimeoutMs) || jobStoreOperationTimeoutMs <= 0) {
     throw new Error('jobStoreOperationTimeoutMs must be a positive integer.');
   }
+  const fileChangeObserver = options.fileChangeObserver;
+  const beforeShellOperation: ILocalToolExecutionContextOptions['beforeOperation'] = fileChangeObserver
+    ? async () => {
+        await options.beforeOperation?.();
+        await fileChangeObserver.beforeShellCommand();
+      }
+    : options.beforeOperation;
   const jobs = createLocalToolJobContext({
     cwd,
     rootDir,
@@ -67,7 +96,7 @@ export const createLocalToolExecutionContext = (options: ILocalToolExecutionCont
     outputListenerTimeoutMs,
     jobStoreOperationTimeoutMs,
     settleOrphanedJob: options.settleOrphanedJob,
-    beforeOperation: options.beforeOperation,
+    beforeOperation: beforeShellOperation,
     abortSignal: options.abortSignal,
   });
   return {
@@ -84,7 +113,7 @@ export const createLocalToolExecutionContext = (options: ILocalToolExecutionCont
         onOutput: runOptions?.onOutput,
         outputListenerTimeoutMs,
         maxOutputBytes: maxShellOutputBytes,
-        beforeOperation: options.beforeOperation,
+        beforeOperation: beforeShellOperation,
       }),
       runStructured: (command, runOptions) => runLocalStructuredCommand(command, {
         cwd: resolveLocalPath(runOptions?.cwd ?? cwd, rootDir),
@@ -93,7 +122,7 @@ export const createLocalToolExecutionContext = (options: ILocalToolExecutionCont
         onOutput: runOptions?.onOutput,
         outputListenerTimeoutMs,
         maxOutputBytes: maxShellOutputBytes,
-        beforeOperation: options.beforeOperation,
+        beforeOperation: beforeShellOperation,
       }),
     },
     jobs,
@@ -147,6 +176,13 @@ export const createLocalToolExecutionContext = (options: ILocalToolExecutionCont
             canonicalBoundary.rootDir,
           );
         }
+        if (fileChangeObserver) {
+          await fileChangeObserver.beforeFileChange({
+            kind: 'write',
+            path: await resolveCanonicalProspectivePath(resolved),
+          });
+          throwIfAborted(abortSignal);
+        }
         await runBeforeLocalOperation(options.beforeOperation, abortSignal);
         await plugins.fs.promises.mkdir(parentPath, { recursive: true });
         throwIfAborted(abortSignal);
@@ -193,6 +229,16 @@ export const createLocalToolExecutionContext = (options: ILocalToolExecutionCont
         if (rootDir) {
           await resolveCanonicalExistingLocalPath(resolved, targetPath, rootDir);
         }
+        if (fileChangeObserver) {
+          await fileChangeObserver.beforeFileChange({
+            kind: 'delete',
+            path: plugins.path.join(
+              await resolveCanonicalProspectivePath(plugins.path.dirname(resolved)),
+              plugins.path.basename(resolved),
+            ),
+          });
+          throwIfAborted(abortSignal);
+        }
         await runBeforeLocalOperation(options.beforeOperation, abortSignal);
         await plugins.fs.promises.rm(resolved, { recursive: false, force: false });
         throwIfAborted(abortSignal);