farming-code 2.2.8 → 2.2.12

package/backend/terminal-screen-worker-pool.js

@@ -1,6 +1,8 @@
 const TerminalScreenWorker = require('./terminal-screen-worker');
 
 const DEFAULT_POOL_SIZE = 3;
+const DEFAULT_RETRY_DELAY_MS = 250;
+const MAX_RETRY_DELAY_MS = 5000;
 
 function normalizePoolSize(value) {
   const parsed = Number(value);
@@ -21,6 +23,11 @@ class TerminalScreenWorkerPool {
     this.waiters = [];
     this.readyWaiters = [];
     this.pendingStarts = 0;
+    this.consecutiveStartFailures = 0;
+    this.retryDelayMs = Number.isFinite(options.retryDelayMs)
+      ? Math.max(0, Number(options.retryDelayMs))
+      : DEFAULT_RETRY_DELAY_MS;
+    this.retryTimer = null;
     this.disposed = false;
 
     this.ensureCapacity();
@@ -32,6 +39,7 @@ class TerminalScreenWorkerPool {
       idle: this.idle.length,
       pendingStarts: this.pendingStarts,
       waiters: this.waiters.length,
+      consecutiveStartFailures: this.consecutiveStartFailures,
     };
   }
 
@@ -60,7 +68,7 @@ class TerminalScreenWorkerPool {
     const worker = this.idle.shift();
     if (worker) {
       this.ensureCapacity();
-      return this.prepareWorker(worker, options);
+      return this.prepareCheckedWorker(worker, options);
     }
 
     return new Promise((resolve, reject) => {
@@ -70,7 +78,7 @@ class TerminalScreenWorkerPool {
   }
 
   ensureCapacity() {
-    if (this.disposed || this.size <= 0) {
+    if (this.disposed || this.size <= 0 || this.retryTimer) {
       return;
     }
 
@@ -82,9 +90,11 @@ class TerminalScreenWorkerPool {
 
   startWorker() {
     this.pendingStarts += 1;
+    let failed = false;
     this.createReadyWorker()
       .then((worker) => {
         this.pendingStarts -= 1;
+        this.consecutiveStartFailures = 0;
         if (this.disposed) {
           worker.dispose().catch(() => {});
           return;
@@ -92,18 +102,41 @@ class TerminalScreenWorkerPool {
         this.deliverWorker(worker);
       })
       .catch((error) => {
+        failed = true;
         this.pendingStarts -= 1;
+        this.consecutiveStartFailures += 1;
         const waiter = this.waiters.shift();
         if (waiter) {
           waiter.reject(error);
         }
+        if (this.pendingStarts === 0 && this.idle.length < this.size) {
+          const readyWaiters = this.readyWaiters.splice(0);
+          readyWaiters.forEach(({ reject }) => reject(error));
+        }
       })
       .finally(() => {
         this.notifyReadyWaiters();
-        this.ensureCapacity();
+        if (failed) {
+          this.scheduleCapacityRetry();
+        } else {
+          this.ensureCapacity();
+        }
       });
   }
 
+  scheduleCapacityRetry() {
+    if (this.disposed || this.size <= 0 || this.retryTimer) return;
+    const delayMs = Math.min(
+      MAX_RETRY_DELAY_MS,
+      this.retryDelayMs * Math.max(1, 2 ** Math.min(this.consecutiveStartFailures - 1, 5)),
+    );
+    this.retryTimer = setTimeout(() => {
+      this.retryTimer = null;
+      this.ensureCapacity();
+    }, delayMs);
+    this.retryTimer.unref?.();
+  }
+
   async createReadyWorker() {
     const worker = new this.WorkerClass(this.workerOptions);
     try {
@@ -122,12 +155,28 @@ class TerminalScreenWorkerPool {
   async prepareWorker(worker, options = {}) {
     const cols = Number(options.cols || this.workerOptions.cols || 80);
     const rows = Number(options.rows || this.workerOptions.rows || 30);
-    if (Number.isFinite(cols) && cols > 0 && Number.isFinite(rows) && rows > 0) {
-      await worker.resize(cols, rows);
+    if (typeof options.runtimeEpoch === 'string') {
+      await worker.setRuntimeEpoch(options.runtimeEpoch, cols, rows);
+    } else if (Number.isFinite(cols) && cols > 0 && Number.isFinite(rows) && rows > 0) {
+      await worker.resize(cols, rows, 1);
     }
     return worker;
   }
 
+  async prepareCheckedWorker(worker, options = {}) {
+    try {
+      return await this.prepareWorker(worker, options);
+    } catch (error) {
+      try {
+        await worker.dispose();
+      } catch {
+        // best effort
+      }
+      this.ensureCapacity();
+      throw error;
+    }
+  }
+
   deliverWorker(worker) {
     const waiter = this.waiters.shift();
     if (!waiter) {
@@ -136,7 +185,7 @@ class TerminalScreenWorkerPool {
       return;
     }
 
-    this.prepareWorker(worker, waiter.options).then(waiter.resolve, waiter.reject);
+    this.prepareCheckedWorker(worker, waiter.options).then(waiter.resolve, waiter.reject);
   }
 
   notifyReadyWaiters() {
@@ -153,6 +202,10 @@ class TerminalScreenWorkerPool {
     }
 
     this.disposed = true;
+    if (this.retryTimer) {
+      clearTimeout(this.retryTimer);
+      this.retryTimer = null;
+    }
     const error = new Error('Terminal screen worker pool is disposed');
     this.waiters.splice(0).forEach(({ reject }) => reject(error));
     this.readyWaiters.splice(0).forEach(({ reject }) => reject(error));

package/backend/terminal-screen-worker-thread.js

@@ -1,20 +1,17 @@
 const { parentPort, workerData } = require('worker_threads');
 const TerminalScreenState = require('./terminal-screen-state');
 
-const PREVIEW_FLUSH_INTERVAL_MS = 120;
-const MAX_PENDING_DATA_BYTES = 128 * 1024;
+const PREVIEW_FLUSH_INTERVAL_MS = 50;
 
 const screenState = new TerminalScreenState(workerData || {});
-let pendingData = '';
-let previewFlushTimer = null;
+let runtimeEpoch = typeof workerData.runtimeEpoch === 'string' ? workerData.runtimeEpoch : '';
+let appliedOutputSeq = 0;
+let appliedStateRevision = 0;
 let lastPreviewText = '';
 let lastTitle = '';
 let lastSnapshotFingerprint = '';
 let messageQueue = Promise.resolve();
-
-function byteLength(value) {
-  return Buffer.byteLength(String(value || ''), 'utf8');
-}
+let previewFlushTimer = null;
 
 function postPreview(state) {
   const previewText = state.previewText || '';
@@ -39,86 +36,129 @@ function postPreview(state) {
   });
 }
 
-async function flushPending(options = {}) {
-  if (previewFlushTimer) {
-    clearTimeout(previewFlushTimer);
-    previewFlushTimer = null;
-  }
-
-  if (pendingData) {
-    const data = pendingData;
-    pendingData = '';
-    const state = await screenState.write(data);
-    if (options.emitPreview !== false) {
-      postPreview(state);
-    }
-    return screenState.getState({ includeRenderOutput: options.includeRenderOutput });
-  }
+function withOutputSeq(state) {
+  return {
+    ...state,
+    runtimeEpoch,
+    outputSeq: appliedOutputSeq,
+    stateRevision: appliedStateRevision,
+  };
+}
 
-  const state = screenState.getState({ includeRenderOutput: options.includeRenderOutput });
+function currentState(options = {}) {
+  const state = screenState.getState({
+    includeRenderOutput: options.includeRenderOutput,
+    refreshPreview: options.refreshPreview,
+  });
   if (options.emitPreview) {
     postPreview(state);
   }
-  return state;
+  return withOutputSeq(state);
 }
 
-function scheduleFlush() {
-  if (previewFlushTimer) {
-    return;
-  }
-
+function schedulePreview() {
+  if (previewFlushTimer) return;
   previewFlushTimer = setTimeout(() => {
-    flushPending({ includeRenderOutput: false, emitPreview: true }).catch((error) => {
-      parentPort.postMessage({
-        type: 'error',
-        message: error.message,
-      });
-    });
+    previewFlushTimer = null;
+    postPreview(currentState({
+      includeRenderOutput: false,
+      refreshPreview: true,
+    }));
   }, PREVIEW_FLUSH_INTERVAL_MS);
+  if (typeof previewFlushTimer.unref === 'function') previewFlushTimer.unref();
 }
 
-async function appendData(data) {
-  const text = String(data || '');
-  if (!text) return;
-
-  if (pendingData && byteLength(pendingData) + byteLength(text) > MAX_PENDING_DATA_BYTES) {
-    await flushPending({ includeRenderOutput: false, emitPreview: false });
+function assertNextRevision(stateRevision, transition) {
+  if (!Number.isFinite(stateRevision) || stateRevision !== appliedStateRevision + 1) {
+    throw new Error(
+      `Terminal screen ${transition} revision gap: expected ${appliedStateRevision + 1}, received ${stateRevision}`,
+    );
   }
+}
 
-  if (byteLength(text) > MAX_PENDING_DATA_BYTES) {
-    const state = await screenState.write(text);
-    postPreview(state);
-    return;
+async function appendEntries(rawEntries) {
+  const entries = Array.isArray(rawEntries) ? rawEntries : [];
+  if (entries.length === 0) return;
+
+  let expectedRevision = appliedStateRevision + 1;
+  let nextOutputSeq = appliedOutputSeq;
+  let data = '';
+  for (const entry of entries) {
+    const stateRevision = Number(entry && entry.stateRevision);
+    if (!Number.isFinite(stateRevision) || stateRevision !== expectedRevision) {
+      throw new Error(
+        `Terminal screen append revision gap: expected ${expectedRevision}, received ${entry && entry.stateRevision}`,
+      );
+    }
+    const text = String((entry && entry.data) || '');
+    if (!text) {
+      throw new Error(`Terminal screen append revision ${stateRevision} has no data`);
+    }
+    const outputSeq = Number(entry && entry.outputSeq);
+    if (!Number.isFinite(outputSeq) || outputSeq !== nextOutputSeq + 1) {
+      throw new Error(
+        `Terminal screen output sequence gap: expected ${nextOutputSeq + 1}, received ${entry && entry.outputSeq}`,
+      );
+    }
+    data += text;
+    nextOutputSeq = outputSeq;
+    expectedRevision += 1;
   }
 
-  pendingData += text;
-  scheduleFlush();
+  const state = await screenState.write(data);
+  appliedStateRevision = expectedRevision - 1;
+  appliedOutputSeq = nextOutputSeq;
+  schedulePreview();
+  return state;
 }
 
 async function handleRequest(message) {
   switch (message.type) {
     case 'append':
-      await appendData(message.data);
-      return null;
+      await appendEntries(message.entries);
+      return currentState({
+        includeRenderOutput: false,
+        refreshPreview: false,
+      });
+    case 'set-runtime-epoch':
+      runtimeEpoch = typeof message.runtimeEpoch === 'string' ? message.runtimeEpoch : '';
+      appliedOutputSeq = 0;
+      appliedStateRevision = 0;
+      if (Number.isFinite(message.cols) && Number.isFinite(message.rows)) {
+        screenState.resize(message.cols, message.rows);
+      }
+      return currentState({ includeRenderOutput: false });
     case 'resize': {
-      await flushPending({ includeRenderOutput: false, emitPreview: false });
-      const state = screenState.resize(message.cols, message.rows);
+      assertNextRevision(message.stateRevision, 'resize');
+      screenState.resize(message.cols, message.rows);
+      appliedStateRevision = message.stateRevision;
+      const state = screenState.getState({ includeRenderOutput: true });
       postPreview(state);
-      return state;
+      return withOutputSeq(state);
     }
     case 'clear': {
-      await flushPending({ includeRenderOutput: false, emitPreview: false });
+      assertNextRevision(message.stateRevision, 'clear');
       const state = await screenState.clearBuffer();
+      const outputSeq = Number(message.outputSeq);
+      if (!Number.isFinite(outputSeq) || outputSeq !== appliedOutputSeq) {
+        throw new Error(
+          `Terminal screen clear output sequence mismatch: expected ${appliedOutputSeq}, received ${message.outputSeq}`,
+        );
+      }
+      appliedStateRevision = message.stateRevision;
       postPreview(state);
-      return state;
+      return withOutputSeq(state);
     }
     case 'get-state':
-      return flushPending({
+      return currentState({
         includeRenderOutput: message.includeRenderOutput !== false,
         emitPreview: false,
       });
     case 'dispose':
-      await flushPending({ includeRenderOutput: false, emitPreview: false });
+      if (previewFlushTimer) {
+        clearTimeout(previewFlushTimer);
+        previewFlushTimer = null;
+      }
       screenState.dispose();
       return { disposed: true };
     default:

package/backend/terminal-screen-worker.js

@@ -6,6 +6,7 @@ const { Worker } = require('worker_threads');
 const APPEND_FLUSH_INTERVAL_MS = 16;
 const MAX_PENDING_APPEND_BYTES = 128 * 1024;
 const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
+const DEFAULT_STATE_REQUEST_HARD_TIMEOUT_MS = 5000;
 const PACKAGED_WORKER_FILE = 'terminal-screen-worker-thread.pkg.js';
 const SOURCE_WORKER_FILE = 'terminal-screen-worker-thread.js';
 
@@ -28,11 +29,17 @@ class TerminalScreenWorker extends EventEmitter {
     super();
     this.nextRequestId = 1;
     this.pendingRequests = new Map();
-    this.pendingAppendData = '';
+    this.pendingAppendEntries = [];
+    this.pendingAppendBytes = 0;
+    this.pendingAppendWaiters = [];
     this.appendFlushTimer = null;
+    this.stateRequestInFlight = null;
     this.requestTimeoutMs = Number.isFinite(options.requestTimeoutMs)
       ? Math.max(1, Math.floor(options.requestTimeoutMs))
       : DEFAULT_REQUEST_TIMEOUT_MS;
+    this.stateRequestHardTimeoutMs = Number.isFinite(options.stateRequestHardTimeoutMs)
+      ? Math.max(1, Math.floor(options.stateRequestHardTimeoutMs))
+      : Math.min(this.requestTimeoutMs, DEFAULT_STATE_REQUEST_HARD_TIMEOUT_MS);
     this.failed = false;
     this.disposed = false;
     const workerFile = resolveWorkerFile();
@@ -40,6 +47,7 @@ class TerminalScreenWorker extends EventEmitter {
     const workerData = { ...options };
     delete workerData.WorkerClass;
     delete workerData.requestTimeoutMs;
+    delete workerData.stateRequestHardTimeoutMs;
     this.worker = new WorkerClass(path.join(__dirname, workerFile), {
       workerData,
     });
@@ -79,7 +87,7 @@ class TerminalScreenWorker extends EventEmitter {
     });
 
     this.worker.on('error', (error) => {
-      this.emit('error', error);
+      this.handleWorkerFailure(error);
     });
 
     this.worker.on('exit', (code) => {
@@ -87,26 +95,34 @@ class TerminalScreenWorker extends EventEmitter {
         return;
       }
 
-      const error = new Error(`Terminal screen worker exited unexpectedly with code ${code}`);
-      this.failed = true;
-      this.pendingRequests.forEach(({ reject, timer }) => {
-        clearTimeout(timer);
-        reject(error);
-      });
-      this.pendingRequests.clear();
-      this.emit('error', error);
+      this.handleWorkerFailure(new Error(`Terminal screen worker exited unexpectedly with code ${code}`));
     });
   }
 
-  handlePostMessageFailure(error) {
+  handleWorkerFailure(error) {
     if (this.disposed) return;
+    const failure = error instanceof Error ? error : new Error(String(error));
+    const shouldEmit = !this.failed;
     this.failed = true;
     if (this.appendFlushTimer) {
       clearTimeout(this.appendFlushTimer);
       this.appendFlushTimer = null;
     }
-    this.pendingAppendData = '';
-    this.emit('error', error instanceof Error ? error : new Error(String(error)));
+    this.pendingAppendEntries = [];
+    this.pendingAppendBytes = 0;
+    this.pendingAppendWaiters.splice(0).forEach(({ reject }) => reject(failure));
+    this.pendingRequests.forEach(({ reject, timer }) => {
+      clearTimeout(timer);
+      reject(failure);
+    });
+    this.pendingRequests.clear();
+    if (shouldEmit && this.listenerCount('error') > 0) {
+      this.emit('error', failure);
+    }
+  }
+
+  handlePostMessageFailure(error) {
+    this.handleWorkerFailure(error);
   }
 
   postWorkerMessage(message, options = {}) {
@@ -125,20 +141,24 @@ class TerminalScreenWorker extends EventEmitter {
       clearTimeout(this.appendFlushTimer);
       this.appendFlushTimer = null;
     }
-    if (this.disposed || this.failed || !this.pendingAppendData) {
-      this.pendingAppendData = '';
+    if (this.disposed || this.failed || this.pendingAppendEntries.length === 0) {
+      this.pendingAppendEntries = [];
+      this.pendingAppendBytes = 0;
       return;
     }
 
-    const data = this.pendingAppendData;
-    this.pendingAppendData = '';
-    this.postWorkerMessage({
-      type: 'append',
-      data,
-    });
+    const entries = this.pendingAppendEntries;
+    const waiters = this.pendingAppendWaiters;
+    this.pendingAppendEntries = [];
+    this.pendingAppendBytes = 0;
+    this.pendingAppendWaiters = [];
+    this.request('append', { entries }, { flushAppend: false }).then(
+      state => waiters.forEach(({ resolve }) => resolve(state)),
+      error => waiters.forEach(({ reject }) => reject(error)),
+    );
   }
 
-  request(type, payload = {}) {
+  request(type, payload = {}, options = {}) {
     if (this.disposed) {
       return Promise.reject(new Error('Terminal screen worker is disposed'));
     }
@@ -146,7 +166,9 @@ class TerminalScreenWorker extends EventEmitter {
       return Promise.reject(new Error('Terminal screen worker is not available'));
     }
 
-    this.flushAppend();
+    if (options.flushAppend !== false) {
+      this.flushAppend();
+    }
     if (this.failed) {
       return Promise.reject(new Error('Terminal screen worker is not available'));
     }
@@ -154,12 +176,15 @@ class TerminalScreenWorker extends EventEmitter {
     this.nextRequestId += 1;
 
     return new Promise((resolve, reject) => {
+      const requestTimeoutMs = Number.isFinite(options.timeoutMs)
+        ? Math.max(1, Math.floor(options.timeoutMs))
+        : this.requestTimeoutMs;
       const timer = setTimeout(() => {
         this.pendingRequests.delete(requestId);
         const error = new Error(`Terminal screen worker request timed out: ${type}`);
         error.code = 'ETIMEDOUT';
         reject(error);
-      }, this.requestTimeoutMs);
+      }, requestTimeoutMs);
       if (typeof timer.unref === 'function') timer.unref();
 
       this.pendingRequests.set(requestId, { resolve, reject, timer });
@@ -172,56 +197,108 @@ class TerminalScreenWorker extends EventEmitter {
       } catch (error) {
         this.pendingRequests.delete(requestId);
         clearTimeout(timer);
-        this.failed = true;
+        this.handlePostMessageFailure(error);
         reject(error);
       }
     });
   }
 
-  append(data) {
+  append(data, stateRevision, outputSeq = null) {
     if (this.disposed || this.failed) {
-      return;
+      return Promise.reject(new Error('Terminal screen worker is not available'));
     }
 
     const text = String(data || '');
     if (!text) return;
+    if (!Number.isFinite(stateRevision)) {
+      this.handlePostMessageFailure(new Error('Terminal screen append requires a finite state revision'));
+      return Promise.reject(new Error('Terminal screen append requires a finite state revision'));
+    }
 
     if (
-      this.pendingAppendData &&
-      byteLength(this.pendingAppendData) + byteLength(text) > MAX_PENDING_APPEND_BYTES
+      this.pendingAppendEntries.length > 0 &&
+      this.pendingAppendBytes + byteLength(text) > MAX_PENDING_APPEND_BYTES
     ) {
       this.flushAppend();
     }
 
-    if (byteLength(text) > MAX_PENDING_APPEND_BYTES) {
-      this.postWorkerMessage({
-        type: 'append',
-        data: text,
-      });
-      return;
-    }
-
-    this.pendingAppendData += text;
-    if (this.appendFlushTimer) {
-      return;
-    }
+    return new Promise((resolve, reject) => {
+      this.pendingAppendEntries.push({ data: text, stateRevision, outputSeq });
+      this.pendingAppendBytes += byteLength(text);
+      this.pendingAppendWaiters.push({ resolve, reject });
+      if (byteLength(text) > MAX_PENDING_APPEND_BYTES) {
+        this.flushAppend();
+        return;
+      }
+      if (this.appendFlushTimer) return;
+      this.appendFlushTimer = setTimeout(() => {
+        this.flushAppend();
+      }, APPEND_FLUSH_INTERVAL_MS);
+    });
+  }
 
-    this.appendFlushTimer = setTimeout(() => {
-      this.flushAppend();
-    }, APPEND_FLUSH_INTERVAL_MS);
+  resize(cols, rows, stateRevision) {
+    return this.request('resize', { cols, rows, stateRevision });
   }
 
-  resize(cols, rows) {
-    return this.request('resize', { cols, rows });
+  setRuntimeEpoch(runtimeEpoch, cols, rows) {
+    return this.request('set-runtime-epoch', { runtimeEpoch, cols, rows });
   }
 
-  clear() {
-    return this.request('clear');
+  clear(stateRevision, outputSeq = null) {
+    return this.request('clear', { stateRevision, outputSeq });
   }
 
   getState(options = {}) {
-    return this.request('get-state', {
-      includeRenderOutput: options.includeRenderOutput !== false,
+    if (!this.stateRequestInFlight) {
+      const request = this.request('get-state', {
+        // A full checkpoint can satisfy callers that only need metadata too,
+        // while the reverse would make coalescing unsafe.
+        includeRenderOutput: true,
+      }, {
+        // Caller deadlines are deliberately softer than this shared deadline.
+        // A timed-out caller may stop waiting, but the single-flight itself
+        // must never poison every later checkpoint for the generic 30s worker
+        // request timeout. Crossing this hard deadline means the authoritative
+        // reducer can no longer prove progress, so fail it closed.
+        timeoutMs: this.stateRequestHardTimeoutMs,
+      });
+      const sharedRequest = request.catch((error) => {
+        if (error && error.code === 'ETIMEDOUT') {
+          this.handleWorkerFailure(error);
+        }
+        throw error;
+      }).finally(() => {
+        if (this.stateRequestInFlight === sharedRequest) {
+          this.stateRequestInFlight = null;
+        }
+      });
+      this.stateRequestInFlight = sharedRequest;
+    }
+
+    const sharedRequest = this.stateRequestInFlight;
+    const timeoutMs = Number.isFinite(options.timeoutMs)
+      ? Math.max(1, Math.floor(options.timeoutMs))
+      : null;
+    if (timeoutMs === null) return sharedRequest;
+
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        const error = new Error('Terminal screen worker request timed out: get-state');
+        error.code = 'ETIMEDOUT';
+        reject(error);
+      }, timeoutMs);
+      if (typeof timer.unref === 'function') timer.unref();
+      sharedRequest.then(
+        (state) => {
+          clearTimeout(timer);
+          resolve(state);
+        },
+        (error) => {
+          clearTimeout(timer);
+          reject(error);
+        },
+      );
     });
   }
 
@@ -239,7 +316,11 @@ class TerminalScreenWorker extends EventEmitter {
       clearTimeout(this.appendFlushTimer);
       this.appendFlushTimer = null;
     }
-    this.pendingAppendData = '';
+    this.pendingAppendEntries = [];
+    this.pendingAppendBytes = 0;
+    this.pendingAppendWaiters.splice(0).forEach(({ reject }) => {
+      reject(new Error('Terminal screen worker is disposed'));
+    });
     this.disposed = true;
     this.pendingRequests.forEach(({ reject, timer }) => {
       clearTimeout(timer);
@@ -253,3 +334,4 @@ class TerminalScreenWorker extends EventEmitter {
 module.exports = TerminalScreenWorker;
 module.exports.resolveWorkerFile = resolveWorkerFile;
 module.exports.DEFAULT_REQUEST_TIMEOUT_MS = DEFAULT_REQUEST_TIMEOUT_MS;
+module.exports.DEFAULT_STATE_REQUEST_HARD_TIMEOUT_MS = DEFAULT_STATE_REQUEST_HARD_TIMEOUT_MS;