@mengruo/dsh-vision-toolkit 0.1.2 → 0.1.4

package/lib/runtime.js

@@ -107,6 +107,10 @@ export class Semaphore {
     get idle() {
         return this.active === 0 && this.waiters.length === 0;
     }
+    /** Free slots still claimable without queuing. */
+    get available() {
+        return Math.max(0, this.limit - this.active);
+    }
     /** Acquire one slot, aborting while queued when `signal` fires. */
     async acquire(signal, permits = 1) {
         if (signal.aborted)
@@ -161,7 +165,7 @@ export class Semaphore {
     }
 }
 const REGION_PATTERN = /^\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*$/;
-const MAX_TIMEOUT_MS = 600_000;
+const MAX_TIMEOUT_SECONDS = 600;
 const FORMAT_BY_EXTENSION = new Map([
     ['.png', 'png'],
     ['.jpg', 'jpeg'],
@@ -170,6 +174,31 @@ const FORMAT_BY_EXTENSION = new Map([
     ['.webp', 'webp'],
 ]);
 const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/;
+/**
+ * Error codes a provider retries against the SAME provider within its
+ * `attempts` budget. Only transient failures are worth re-requesting: a
+ * timeout may clear on the next attempt and a 5xx / network drop is usually
+ * ephemeral. Deterministic failures (auth, quota, rate_limit, invalid_request,
+ * region, tos) must fail over to the next provider immediately instead of
+ * re-requesting a backend that cannot succeed with the same input.
+ */
+const RETRYABLE_CODES = new Set(['timeout', 'server', 'network']);
+/** Resolve as soon as `signal` aborts (or immediately when already aborted). */
+function untilAbort(signal) {
+    if (signal.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => signal.addEventListener('abort', () => resolve(), { once: true }));
+}
+/** Sleep for `ms`, resolving early when `signal` aborts. */
+function abortableSleep(ms, signal) {
+    if (signal.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => {
+        const onAbort = () => { clearTimeout(timer); resolve(); };
+        const timer = setTimeout(() => { signal.removeEventListener('abort', onAbort); resolve(); }, ms);
+        signal.addEventListener('abort', onAbort, { once: true });
+    });
+}
 function integerInRange(value, fallback, minimum, maximum, name) {
     const resolved = value ?? fallback;
     if (!Number.isInteger(resolved) || resolved < minimum || resolved > maximum) {
@@ -309,19 +338,29 @@ export function parseRegion(region) {
 export class VisionToolkitRuntime {
     ctx;
     config;
+    readableStorageDirs;
     semaphores = new Map();
     glanceCache = new WeakMap();
     providerGates = new Map();
     adapter;
-    constructor(ctx, config, adapter) {
+    constructor(ctx, config, adapter, readableStorageDirs = []) {
         this.ctx = ctx;
         this.config = config;
+        this.readableStorageDirs = readableStorageDirs;
         this.adapter = adapter ?? new UpstreamAdapter(ctx, config);
     }
     /** Pinned and prepared upstream identity. */
     get upstreamVersion() {
         return this.adapter.versionInfo;
     }
+    /** Per-session cap on concurrent tool operations. */
+    get sessionMaxConcurrency() {
+        return this.config.sessionMaxConcurrency;
+    }
+    /** Shared storage root belonging to this immutable runtime generation. */
+    get storageDirectory() {
+        return this.config.storageDir;
+    }
     /** Stable identity for persisted image descriptions produced by this runtime. */
     get evidenceFingerprint() {
         return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim());
@@ -347,17 +386,13 @@ export class VisionToolkitRuntime {
             glance: (request, options) => this.glanceWithEnv(request, options),
         });
     }
-    timeout(options) {
-        const value = options.timeoutMs ?? this.operationTimeoutMs();
-        if (!Number.isInteger(value) || value < 1000 || value > MAX_TIMEOUT_MS) {
-            throw new VisionToolkitError('input', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`);
+    /** Global hard timeout (ms) for one tool invocation, honoring the per-call override. */
+    hardTimeoutMs(options) {
+        const seconds = options.timeoutSeconds ?? this.config.hardTimeoutSeconds;
+        if (!Number.isInteger(seconds) || seconds < 1 || seconds > MAX_TIMEOUT_SECONDS) {
+            throw new VisionToolkitError('input', `timeoutSeconds must be an integer between 1 and ${MAX_TIMEOUT_SECONDS}`);
         }
-        return value;
-    }
-    /** Overall operation budget: the slowest enabled provider's timeout, else the global default. */
-    operationTimeoutMs() {
-        const providers = this.config.providers.filter(provider => provider.enabled);
-        return providers.length === 0 ? this.config.timeoutMs : Math.max(...providers.map(provider => provider.timeoutMs));
+        return seconds * 1000;
     }
     operationError(tool, error, deadline, phase = 'execution') {
         if (deadline.cancelled) {
@@ -370,17 +405,44 @@ export class VisionToolkitRuntime {
             return error;
         return new VisionToolkitError('runtime', `${tool}: execution failed`, { cause: error });
     }
-    semaphore(options) {
+    /** Per-session concurrency gate; callers acquire without queuing (excess is rejected). */
+    sessionGate(options) {
         const key = options.sessionId ?? `workspace:${options.workspace}`;
-        const value = this.semaphores.get(key) ?? new Semaphore(this.config.concurrency);
+        const value = this.semaphores.get(key) ?? new Semaphore(this.config.sessionMaxConcurrency);
         this.semaphores.set(key, value);
         return { key, value };
     }
+    /** Live concurrency accounting for the calling session across the enabled provider pool. */
+    concurrencyStatus(options) {
+        const gate = this.sessionGate(options);
+        const sessionFree = gate.value.available;
+        const models = this.config.providers
+            .filter(provider => provider.enabled)
+            .map(provider => {
+            const modelGate = this.providerGate(provider);
+            return {
+                name: provider.name,
+                concurrency: provider.concurrency,
+                inUse: provider.concurrency - modelGate.available,
+                free: modelGate.available,
+            };
+        });
+        const modelFree = models.reduce((sum, model) => sum + model.free, 0);
+        return {
+            available: Math.min(sessionFree, modelFree),
+            sessionMax: this.config.sessionMaxConcurrency,
+            sessionInUse: this.config.sessionMaxConcurrency - sessionFree,
+            sessionFree,
+            modelFree,
+            models,
+        };
+    }
     async runOperation(tool, options, action, permits = 1) {
-        const timeoutMs = this.timeout(options);
-        const semaphore = this.semaphore(options);
+        const hardTimeoutMs = this.hardTimeoutMs(options);
+        const startedAt = Date.now();
+        const deadlineAt = startedAt + hardTimeoutMs;
         const metrics = {
-            startedAt: Date.now(),
+            startedAt,
             queueMs: 0,
             upstreamMs: 0,
             imageBytes: 0,
@@ -389,51 +451,30 @@ export class VisionToolkitRuntime {
             cacheHits: 0,
             usedVisionService: false,
         };
-        let acquired = false;
-        const queueDeadline = createDeadline(options.signal, timeoutMs);
-        try {
-            await semaphore.value.acquire(queueDeadline.signal, permits);
-            acquired = true;
-            if (queueDeadline.signal.aborted)
-                throw this.operationError(tool, undefined, queueDeadline, 'queue');
-            metrics.queueMs = Date.now() - metrics.startedAt;
-        }
-        catch (error) {
-            metrics.queueMs = Date.now() - metrics.startedAt;
-            const classified = this.operationError(tool, error, queueDeadline, 'queue');
-            if (acquired) {
-                semaphore.value.release(permits);
-                acquired = false;
-            }
-            this.ctx.logger.warn('dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d', tool, classified.code, Date.now() - metrics.startedAt, metrics.queueMs, metrics.upstreamMs, metrics.imageCount, metrics.imageBytes, metrics.imagePixels, metrics.cacheHits);
-            throw classified;
+        const gate = this.sessionGate(options);
+        if (!gate.value.tryAcquire(permits)) {
+            throw new VisionToolkitError('capacity', `${tool}: exceeded the session concurrency limit`);
         }
-        finally {
-            queueDeadline.cleanup();
-            if (!acquired && semaphore.value.idle)
-                this.semaphores.delete(semaphore.key);
-        }
-        const executionDeadline = createDeadline(options.signal, timeoutMs);
+        const executionDeadline = createDeadline(options.signal, hardTimeoutMs);
         try {
             if (executionDeadline.signal.aborted)
                 throw this.operationError(tool, undefined, executionDeadline);
-            const value = await action({ signal: executionDeadline.signal, metrics });
+            const value = await action({ signal: executionDeadline.signal, metrics, deadlineAt });
             if (executionDeadline.signal.aborted)
                 throw this.operationError(tool, undefined, executionDeadline);
-            this.ctx.logger.info('dsh-vision-toolkit tool=%s outcome=ok totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d model=%s', tool, Date.now() - metrics.startedAt, metrics.queueMs, metrics.upstreamMs, metrics.imageCount, metrics.imageBytes, metrics.imagePixels, metrics.cacheHits, metrics.usedVisionService ? this.config.provider.model : 'local');
+            this.ctx.logger.info('dsh-vision-toolkit tool=%s outcome=ok totalMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d model=%s', tool, Date.now() - metrics.startedAt, metrics.upstreamMs, metrics.imageCount, metrics.imageBytes, metrics.imagePixels, metrics.cacheHits, metrics.usedVisionService ? this.config.provider.model : 'local');
             return value;
         }
         catch (error) {
             const classified = this.operationError(tool, error, executionDeadline);
-            this.ctx.logger.warn('dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d', tool, classified.code, Date.now() - metrics.startedAt, metrics.queueMs, metrics.upstreamMs, metrics.imageCount, metrics.imageBytes, metrics.imagePixels, metrics.cacheHits);
+            this.ctx.logger.warn('dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d', tool, classified.code, Date.now() - metrics.startedAt, metrics.upstreamMs, metrics.imageCount, metrics.imageBytes, metrics.imagePixels, metrics.cacheHits);
             throw classified;
         }
         finally {
-            if (acquired)
-                semaphore.value.release(permits);
+            gate.value.release(permits);
             executionDeadline.cleanup();
-            if (semaphore.value.idle)
-                this.semaphores.delete(semaphore.key);
+            if (gate.value.idle)
+                this.semaphores.delete(gate.key);
         }
     }
     /** Highest-priority enabled provider, falling back to the first entry. */
@@ -500,12 +541,12 @@ export class VisionToolkitRuntime {
         return entry.env;
     }
     pathPolicy(workspace) {
-        return createPathPolicy(workspace, this.config.allowedDirs);
+        return createPathPolicy(workspace, this.config.allowedDirs, this.config.storageDir, this.readableStorageDirs);
     }
     async compressedImageRoot(policy) {
-        const root = join(policy.workspace, '.dsh-vision-toolkit', 'tmp', 'compressed-images');
-        let current = policy.workspace;
-        for (const segment of ['.dsh-vision-toolkit', 'tmp', 'compressed-images']) {
+        const root = join(policy.storageRoot, 'tmp', 'compressed-images');
+        let current = policy.storageRoot;
+        for (const segment of ['tmp', 'compressed-images']) {
             current = join(current, segment);
             try {
                 await mkdir(current, { mode: 0o700 });
@@ -518,13 +559,13 @@ export class VisionToolkitRuntime {
             if (info.isSymbolicLink() || !info.isDirectory()) {
                 throw new VisionToolkitError('path', `compressed-image cache path is not a real directory: ${current}`);
             }
-            if (!isWithin(policy.workspace, current)) {
-                throw new VisionToolkitError('path', `compressed-image cache path escaped the workspace: ${current}`);
+            if (!isWithin(policy.storageRoot, current)) {
+                throw new VisionToolkitError('path', `compressed-image cache path escaped plugin storage: ${current}`);
             }
         }
         const canonical = await realpath(root);
-        if (!isWithin(policy.workspace, canonical)) {
-            throw new VisionToolkitError('path', 'compressed-image cache resolved outside the workspace');
+        if (!isWithin(policy.storageRoot, canonical)) {
+            throw new VisionToolkitError('path', 'compressed-image cache resolved outside plugin storage');
         }
         return canonical;
     }
@@ -757,56 +798,268 @@ export class VisionToolkitRuntime {
         return this.autoCompressImage(image, policy, operation, first.maxImageBytes, first.maxImagePixels);
     }
     /**
-     * Run one online-vision upstream command across the enabled provider pool in
-     * priority order. A provider is skipped when its size limits or concurrency
-     * are exhausted, retried up to its attempt count, and the next provider
-     * takes over on failure. Throws once every provider has failed.
+     * Hedge-based failover across the enabled provider pool. The highest-priority
+     * provider runs first; when one of its requests crosses t1 it keeps running
+     * while the next provider starts in parallel. A provider whose cumulative
+     * request time reaches t2 is terminated. A 429 provider is parked and moved
+     * past immediately; parked providers are revisited at a 10s cadence once
+     * every other provider is exhausted. The result always prefers the earliest
+     * (highest-priority) provider.
      */
-    async runVisionWithFailover(tool, args, images, operation, pool) {
+    async runVisionHedge(tool, args, images, operation, pool) {
         if (pool.length === 0) {
             throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
         }
-        let lastError;
-        let attempted = false;
-        for (const { provider, env } of pool) {
-            const fits = images.every(image => image.bytes <= provider.maxImageBytes && image.width * image.height <= provider.maxImagePixels);
-            if (!fits)
-                continue;
-            const gate = this.providerGate(provider);
-            if (!gate.tryAcquire())
-                continue;
-            attempted = true;
-            try {
-                for (let attempt = 1; attempt <= provider.attempts; attempt++) {
-                    const attemptDeadline = createDeadline(operation.signal, provider.timeoutMs);
-                    try {
-                        return await this.runUpstream(tool, args, { signal: attemptDeadline.signal, metrics: operation.metrics }, env);
+        // Providers that cannot accept the image size are skipped entirely.
+        const eligible = pool.filter(({ provider }) => images.every(image => image.bytes <= provider.maxImageBytes && image.width * image.height <= provider.maxImagePixels));
+        if (eligible.length === 0) {
+            throw new VisionToolkitError('capacity', `${tool}: no enabled vision provider accepts the image size`);
+        }
+        const tasks = eligible.map((entry, index) => {
+            let settle;
+            const settled = new Promise(resolve => { settle = resolve; });
+            return {
+                index,
+                entry,
+                cumulativeMs: 0,
+                status: 'idle',
+                hedged: false,
+                launched: false,
+                settled,
+                settle,
+                abort: new AbortController(),
+            };
+        });
+        const n = tasks.length;
+        const launch = (from) => {
+            for (let i = from; i < n; i++) {
+                const task = tasks[i];
+                if (task === undefined || task.launched || task.status !== 'idle')
+                    continue;
+                task.launched = true;
+                void this.runProviderTask(tool, args, operation, task, () => launch(i + 1));
+                return;
+            }
+        };
+        launch(0);
+        const settleAny = (subset) => Promise.race([...subset.map(task => task.settled), untilAbort(operation.signal)]);
+        while (true) {
+            if (operation.signal.aborted)
+                break;
+            const running = tasks.filter(task => task.status === 'running');
+            const successIndex = tasks.findIndex(task => task.status === 'succeeded');
+            if (successIndex >= 0) {
+                const blocking = running.filter(task => task.index < successIndex);
+                if (blocking.length === 0) {
+                    for (const task of running)
+                        task.abort.abort();
+                    return tasks[successIndex].result;
+                }
+                await settleAny(blocking);
+            }
+            else {
+                if (running.length === 0)
+                    break;
+                await settleAny(running);
+            }
+        }
+        // No success from the main pass. Revisit parked (429) providers at a 10s cadence.
+        if (!operation.signal.aborted) {
+            const parked = tasks.filter(task => task.status === 'ratelimited');
+            if (parked.length > 0) {
+                const revisited = await this.revisitRateLimited(tool, args, operation, parked);
+                if (revisited !== undefined)
+                    return revisited;
+            }
+        }
+        const success = tasks.find(task => task.status === 'succeeded');
+        if (success !== undefined)
+            return success.result;
+        if (operation.signal.aborted) {
+            throw new VisionToolkitError('timeout', `${tool}: timed out`);
+        }
+        const firstError = tasks.find(task => task.status === 'failed' || task.status === 'ratelimited');
+        if (firstError?.error !== undefined)
+            throw firstError.error;
+        throw new VisionToolkitError('service', `${tool}: all vision providers failed`);
+    }
+    /** Remaining request budget (ms) for one provider: the tighter of its t2 and the global deadline. */
+    providerRequestBudget(task, operation) {
+        const t2Remaining = task.entry.provider.t2Seconds * 1000 - task.cumulativeMs;
+        const globalRemaining = operation.deadlineAt - Date.now();
+        return Math.max(0, Math.min(t2Remaining, globalRemaining));
+    }
+    /**
+     * Advance to the next provider after one provider reached a terminal
+     * failure. This is what makes failover work for FAST failures too: the
+     * hedge timer only launches the next provider when the current one is SLOW
+     * (crosses t1), so a quick auth/5xx/network failure must explicitly launch
+     * the successor. Never advances when a higher-priority provider superseded
+     * this task, when the whole operation was cancelled, or when the global
+     * deadline has too little room left for another request. `launch` is
+     * idempotent, so an earlier hedge timer cannot cause a double launch.
+     */
+    advanceAfterFailure(task, operation, launchNext) {
+        if (task.abort.signal.aborted || operation.signal.aborted)
+            return;
+        if (operation.deadlineAt - Date.now() < this.config.minAvailableSeconds * 1000)
+            return;
+        launchNext();
+    }
+    /**
+     * Run one provider to a terminal state: retryable errors retry within
+     * `attempts`, a single request crossing t1 hedges the next provider, and the
+     * provider is terminated once its cumulative time reaches t2. A 429 parks the
+     * provider and moves on. No request is issued once the remaining budget drops
+     * below the configured minimum available time.
+     */
+    async runProviderTask(tool, args, operation, task, launchNext) {
+        const { provider, env } = task.entry;
+        const gate = this.providerGate(provider);
+        if (!gate.tryAcquire()) {
+            task.status = 'failed';
+            task.error = new VisionToolkitError('capacity', `${tool}: ${provider.name} has no free concurrency slot`);
+            task.settle();
+            launchNext();
+            return;
+        }
+        task.status = 'running';
+        try {
+            let attempt = 0;
+            const minAvailableMs = this.config.minAvailableSeconds * 1000;
+            while (true) {
+                const budget = this.providerRequestBudget(task, operation);
+                if (budget < minAvailableMs) {
+                    task.status = 'failed';
+                    task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} has insufficient remaining time`);
+                    this.advanceAfterFailure(task, operation, launchNext);
+                    return;
+                }
+                const reqDeadline = createDeadline(AbortSignal.any([operation.signal, task.abort.signal]), budget);
+                const hedgeMs = Math.min(provider.t1Seconds * 1000, budget);
+                let hedgeTimer;
+                if (!task.hedged) {
+                    hedgeTimer = setTimeout(() => {
+                        task.hedged = true;
+                        launchNext();
+                    }, hedgeMs);
+                }
+                const started = Date.now();
+                try {
+                    const result = await this.runUpstream(tool, args, { signal: reqDeadline.signal, metrics: operation.metrics }, env);
+                    if (hedgeTimer !== undefined)
+                        clearTimeout(hedgeTimer);
+                    task.cumulativeMs += Date.now() - started;
+                    task.status = 'succeeded';
+                    task.result = result;
+                    return;
+                }
+                catch (error) {
+                    if (hedgeTimer !== undefined)
+                        clearTimeout(hedgeTimer);
+                    task.cumulativeMs += Date.now() - started;
+                    if (task.abort.signal.aborted) {
+                        task.status = 'failed';
+                        task.error = new VisionToolkitError('cancelled', `${tool}: superseded by a higher-priority provider`);
+                        return;
+                    }
+                    if (operation.signal.aborted) {
+                        task.status = 'failed';
+                        task.error = new VisionToolkitError('timeout', `${tool}: timed out`);
+                        return;
                     }
-                    catch (error) {
-                        if (operation.signal.aborted)
-                            throw error;
-                        const classified = error instanceof VisionToolkitError
-                            ? error
-                            : new VisionToolkitError('service', `${tool}: request failed`, { cause: error });
-                        lastError = attemptDeadline.timedOut && classified.code === 'cancelled'
-                            ? new VisionToolkitError('timeout', `${tool}: ${provider.name} request timed out after ${provider.timeoutMs}ms`, { cause: classified })
-                            : classified;
+                    const classified = error instanceof VisionToolkitError
+                        ? error
+                        : new VisionToolkitError('service', `${tool}: request failed`, { cause: error });
+                    if (reqDeadline.timedOut) {
+                        task.status = 'failed';
+                        task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} exhausted its t2 budget`);
+                        this.advanceAfterFailure(task, operation, launchNext);
+                        return;
                     }
-                    finally {
-                        attemptDeadline.cleanup();
+                    if (classified.code === 'rate_limit') {
+                        task.status = 'ratelimited';
+                        task.error = classified;
+                        launchNext();
+                        return;
                     }
+                    if (RETRYABLE_CODES.has(classified.code) && attempt + 1 < provider.attempts) {
+                        attempt += 1;
+                        continue;
+                    }
+                    task.status = 'failed';
+                    task.error = classified;
+                    this.advanceAfterFailure(task, operation, launchNext);
+                    return;
+                }
+                finally {
+                    reqDeadline.cleanup();
                 }
-            }
-            finally {
-                gate.release();
             }
         }
-        if (!attempted) {
-            throw new VisionToolkitError('capacity', `${tool}: no enabled vision provider accepts the image size or has a free concurrency slot`);
+        finally {
+            gate.release();
+            task.settle();
         }
-        if (lastError instanceof VisionToolkitError)
-            throw lastError;
-        throw new VisionToolkitError('service', `${tool}: all vision providers failed`, { cause: lastError });
+    }
+    /**
+     * Revisit parked (429) providers in priority order at a 10s cadence until one
+     * succeeds or every provider exhausts its budget. Returns a success result or
+     * `undefined` when the global deadline or minimum available time stops the loop.
+     */
+    async revisitRateLimited(tool, args, operation, parked) {
+        const minAvailableMs = this.config.minAvailableSeconds * 1000;
+        while (!operation.signal.aborted) {
+            let anyRevisitable = false;
+            for (const task of parked) {
+                if (operation.signal.aborted)
+                    break;
+                if (task.status !== 'ratelimited')
+                    continue;
+                const budget = this.providerRequestBudget(task, operation);
+                if (budget < minAvailableMs)
+                    continue;
+                anyRevisitable = true;
+                await abortableSleep(Math.min(10_000, budget), operation.signal);
+                if (operation.signal.aborted)
+                    break;
+                const { provider, env } = task.entry;
+                const gate = this.providerGate(provider);
+                if (!gate.tryAcquire())
+                    continue;
+                const reqDeadline = createDeadline(operation.signal, Math.min(provider.t2Seconds * 1000 - task.cumulativeMs, operation.deadlineAt - Date.now()));
+                const started = Date.now();
+                try {
+                    const result = await this.runUpstream(tool, args, { signal: reqDeadline.signal, metrics: operation.metrics }, env);
+                    task.cumulativeMs += Date.now() - started;
+                    task.status = 'succeeded';
+                    task.result = result;
+                    return result;
+                }
+                catch (error) {
+                    task.cumulativeMs += Date.now() - started;
+                    if (reqDeadline.timedOut) {
+                        task.status = 'failed';
+                        task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} exhausted its t2 budget`);
+                        continue;
+                    }
+                    const classified = error instanceof VisionToolkitError
+                        ? error
+                        : new VisionToolkitError('service', `${tool}: request failed`, { cause: error });
+                    if (classified.code === 'rate_limit')
+                        continue;
+                    task.status = 'failed';
+                    task.error = classified;
+                }
+                finally {
+                    reqDeadline.cleanup();
+                    gate.release();
+                }
+            }
+            if (!anyRevisitable)
+                break;
+        }
+        return undefined;
     }
     async glanceCacheKey(request, images, pool, signal) {
         const imageFingerprints = await Promise.all(images.map(async (image) => {
@@ -950,7 +1203,7 @@ export class VisionToolkitRuntime {
                     return cached.result;
                 }
             }
-            const result = await this.runVisionWithFailover('glance', [
+            const result = await this.runVisionHedge('glance', [
                 ...images.map(image => image.path),
                 ...(request.region !== undefined ? ['--region', request.region] : []),
                 ...(request.ocr === true ? ['--ocr'] : []),
@@ -997,7 +1250,7 @@ export class VisionToolkitRuntime {
         }
         const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation);
         this.accountImage(image, operation);
-        const result = await this.runVisionWithFailover(tool, [
+        const result = await this.runVisionHedge(tool, [
             image.path,
             request.target,
             ...(request.region !== undefined ? ['--region', request.region] : []),
@@ -1306,9 +1559,6 @@ export class VisionToolkitRuntime {
             const overlap = request.overlap === undefined
                 ? undefined
                 : integerInRange(request.overlap, request.overlap, 0, 10000, 'long_screenshot_ocr.overlap');
-            const chunkTimeoutSeconds = finiteInRange(request.chunkTimeoutSeconds ?? Math.min(180, Math.max(1, Math.ceil(this.timeout(options) / 1000))), 1, 600, 'long_screenshot_ocr.chunkTimeoutSeconds');
-            if (chunkTimeoutSeconds === undefined)
-                throw new VisionToolkitError('input', 'long_screenshot_ocr chunk timeout is required');
             if (request.prompt !== undefined && request.prompt.trim().length === 0) {
                 throw new VisionToolkitError('input', 'long_screenshot_ocr.prompt must not be empty when provided');
             }
@@ -1351,13 +1601,13 @@ export class VisionToolkitRuntime {
                     '--jobs',
                     String(jobs),
                     '--timeout',
-                    String(chunkTimeoutSeconds),
+                    String(this.config.hardTimeoutSeconds),
                     ...(splitOnly ? ['--split-only'] : []),
                     ...(request.resume === true ? ['--resume'] : []),
                 ];
                 const result = splitOnly
                     ? await this.runUpstream('long_screenshot_ocr', ocrArgs, operation)
-                    : await this.runVisionWithFailover('long_screenshot_ocr', ocrArgs, [image], operation, pool);
+                    : await this.runVisionHedge('long_screenshot_ocr', ocrArgs, [image], operation, pool);
                 const reported = result.stdout.trim();
                 const expectedReported = splitOnly ? stagedManifest : stagedOutput;
                 if (reported !== expectedReported) {
@@ -1689,6 +1939,18 @@ export class VisionToolkitRuntime {
             }
         });
     }
+    async writableDirectoryCheck(path, label) {
+        const probe = join(path, `.vision-toolkit-health-${randomUUID()}`);
+        try {
+            await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' });
+            await rm(probe, { force: true });
+            return { status: 'ok', detail: `${label} is writable: ${path}` };
+        }
+        catch {
+            await rm(probe, { force: true }).catch(() => { });
+            return { status: 'error', detail: `${label} is not writable: ${path}` };
+        }
+    }
     /** Health: inspect local readiness, and optionally probe one provider's `/models` plus one real multimodal request. */
     async health(testConnection, options, testModel = false, provider) {
         return this.runOperation('vision_toolkit_health', options, async (operation) => {
@@ -1712,6 +1974,29 @@ export class VisionToolkitRuntime {
                     throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled');
                 chrome = { status: 'error', detail: 'Chrome availability probe failed' };
             }
+            let resolvedCredential;
+            let credential;
+            try {
+                resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
+                    ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
+                    : await this.ctx.credentials.resolve(this.config.provider.credential);
+                credential = resolvedCredential === undefined
+                    ? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
+                    : { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` };
+            }
+            catch {
+                credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` };
+            }
+            let artifactDirectory;
+            try {
+                // allowedDirs are session input roots; they do not affect output readiness.
+                const policy = await createPathPolicy(options.workspace, [], this.config.storageDir);
+                artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory');
+            }
+            catch {
+                artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' };
+            }
+            const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory');
             let service = {
                 status: 'not_tested',
                 detail: 'Connection was not tested; use the per-provider API test',
@@ -1785,7 +2070,7 @@ export class VisionToolkitRuntime {
                     }
                     if (testModel) {
                         try {
-                            const attemptDeadline = createDeadline(operation.signal, target.timeoutMs);
+                            const attemptDeadline = createDeadline(operation.signal, target.t2Seconds * 1000);
                             try {
                                 const result = await this.runUpstream('glance', [VISION_MODEL_TEST_IMAGE, '-q', VISION_MODEL_TEST_PROMPT], { signal: attemptDeadline.signal, metrics: operation.metrics }, entry.env);
                                 if (result.stdout.trim().length === 0) {
@@ -1809,7 +2094,7 @@ export class VisionToolkitRuntime {
                     }
                 }
             }
-            const checks = { python, dependencies, chrome, service, model };
+            const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model };
             const healthy = Object.values(checks).every(check => check.status !== 'error');
             return {
                 pluginVersion: PLUGIN_VERSION,