@mengruo/dsh-vision-toolkit 0.1.3 → 0.1.5

package/lib/runtime.js

@@ -15,6 +15,7 @@ import { isBuiltInFreeVisionProvider } from "./config.js";
 import { BUILT_IN_FREE_VISION_KEY } from "./defaults.js";
 import { evidenceRuntimeFingerprint } from "./evidence-cache.js";
 import { VisionToolkitError } from "./errors.js";
+import { ObjectStorageClient, splitObjectStorageCredential } from "./object-storage.js";
 import { assertDistinctOutput, commitStagedDirectory, commitStagedOutput, createPathPolicy, createStagedDirectory, createStagedOutput, isWithin, resolveHtmlFile, resolveInputFile, resolveOutputDirectory, resolveOutputFile, seedStagedDirectory, } from "./paths.js";
 import { parseCropOutput, parseDominantColorsOutput, parseExtractForegroundOutput, parseHtmlScreenshotOutput, parseLocationOutput, parsePixelDiffOutput, parseTraceOutput, UpstreamAdapter, } from "./upstream.js";
 import { PLUGIN_VERSION } from "./version.js";
@@ -174,8 +175,25 @@ const FORMAT_BY_EXTENSION = new Map([
     ['.webp', 'webp'],
 ]);
 const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/;
-/** Error codes a provider retries within its attempt budget (429 is handled separately). */
-const RETRYABLE_CODES = new Set(['service', 'timeout']);
+/** MIME type for one analyzed image format, used when uploading to object storage. */
+function imageMimeType(format) {
+    switch (format) {
+        case 'png': return 'image/png';
+        case 'jpeg': return 'image/jpeg';
+        case 'gif': return 'image/gif';
+        case 'webp': return 'image/webp';
+        default: return 'application/octet-stream';
+    }
+}
+/**
+ * Error codes a provider retries against the SAME provider within its
+ * `attempts` budget. Only transient failures are worth re-requesting: a
+ * timeout may clear on the next attempt and a 5xx / network drop is usually
+ * ephemeral. Deterministic failures (auth, quota, rate_limit, invalid_request,
+ * region, tos) must fail over to the next provider immediately instead of
+ * re-requesting a backend that cannot succeed with the same input.
+ */
+const RETRYABLE_CODES = new Set(['timeout', 'server', 'network']);
 /** Resolve as soon as `signal` aborts (or immediately when already aborted). */
 function untilAbort(signal) {
     if (signal.aborted)
@@ -331,13 +349,15 @@ export function parseRegion(region) {
 export class VisionToolkitRuntime {
     ctx;
     config;
+    readableStorageDirs;
     semaphores = new Map();
     glanceCache = new WeakMap();
     providerGates = new Map();
     adapter;
-    constructor(ctx, config, adapter) {
+    constructor(ctx, config, adapter, readableStorageDirs = []) {
         this.ctx = ctx;
         this.config = config;
+        this.readableStorageDirs = readableStorageDirs;
         this.adapter = adapter ?? new UpstreamAdapter(ctx, config);
     }
     /** Pinned and prepared upstream identity. */
@@ -348,6 +368,10 @@ export class VisionToolkitRuntime {
     get sessionMaxConcurrency() {
         return this.config.sessionMaxConcurrency;
     }
+    /** Shared storage root belonging to this immutable runtime generation. */
+    get storageDirectory() {
+        return this.config.storageDir;
+    }
     /** Stable identity for persisted image descriptions produced by this runtime. */
     get evidenceFingerprint() {
         return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim());
@@ -478,6 +502,7 @@ export class VisionToolkitRuntime {
             VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
             VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
             ...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
+            ...(provider.stream ? { VISION_STREAM: '1' } : {}),
             VISION_USER_AGENT: provider.userAgent,
             LANG: this.config.language,
         };
@@ -493,6 +518,8 @@ export class VisionToolkitRuntime {
                 protocol: provider.protocol,
                 anthropicThinking: provider.anthropicThinking,
                 userAgent: provider.userAgent,
+                stream: provider.stream,
+                uploadViaUrl: provider.uploadViaUrl,
             })
                 ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
                 : await this.ctx.credentials.resolve(provider.credential);
@@ -528,12 +555,12 @@ export class VisionToolkitRuntime {
         return entry.env;
     }
     pathPolicy(workspace) {
-        return createPathPolicy(workspace, this.config.allowedDirs);
+        return createPathPolicy(workspace, this.config.allowedDirs, this.config.storageDir, this.readableStorageDirs);
     }
     async compressedImageRoot(policy) {
-        const root = join(policy.workspace, '.dsh-vision-toolkit', 'tmp', 'compressed-images');
-        let current = policy.workspace;
-        for (const segment of ['.dsh-vision-toolkit', 'tmp', 'compressed-images']) {
+        const root = join(policy.storageRoot, 'tmp', 'compressed-images');
+        let current = policy.storageRoot;
+        for (const segment of ['tmp', 'compressed-images']) {
             current = join(current, segment);
             try {
                 await mkdir(current, { mode: 0o700 });
@@ -546,13 +573,13 @@ export class VisionToolkitRuntime {
             if (info.isSymbolicLink() || !info.isDirectory()) {
                 throw new VisionToolkitError('path', `compressed-image cache path is not a real directory: ${current}`);
             }
-            if (!isWithin(policy.workspace, current)) {
-                throw new VisionToolkitError('path', `compressed-image cache path escaped the workspace: ${current}`);
+            if (!isWithin(policy.storageRoot, current)) {
+                throw new VisionToolkitError('path', `compressed-image cache path escaped plugin storage: ${current}`);
             }
         }
         const canonical = await realpath(root);
-        if (!isWithin(policy.workspace, canonical)) {
-            throw new VisionToolkitError('path', 'compressed-image cache resolved outside the workspace');
+        if (!isWithin(policy.storageRoot, canonical)) {
+            throw new VisionToolkitError('path', 'compressed-image cache resolved outside plugin storage');
         }
         return canonical;
     }
@@ -746,6 +773,83 @@ export class VisionToolkitRuntime {
         operation.metrics.imageBytes += image.bytes;
         operation.metrics.imagePixels += image.width * image.height;
     }
+    /** Resolve the configured object storage into a usable client, or undefined. */
+    async resolveObjectStorageClient() {
+        const objectStorage = this.config.objectStorage;
+        if (objectStorage.credential === undefined || objectStorage.endpoint.length === 0 || objectStorage.bucket.length === 0) {
+            return undefined;
+        }
+        let resolved;
+        try {
+            resolved = await this.ctx.credentials.resolve(objectStorage.credential);
+        }
+        catch {
+            resolved = undefined;
+        }
+        if (resolved === undefined)
+            return undefined;
+        let accessKeyId;
+        let secretAccessKey;
+        try {
+            const split = splitObjectStorageCredential(resolved.value);
+            accessKeyId = split.accessKeyId;
+            secretAccessKey = split.secretAccessKey;
+        }
+        catch (error) {
+            throw new VisionToolkitError('config', 'object storage credential is malformed', { cause: error });
+        }
+        const settings = {
+            endpoint: objectStorage.endpoint,
+            bucket: objectStorage.bucket,
+            accessKeyId,
+            secretAccessKey,
+            ...(objectStorage.publicBase === undefined ? {} : { publicBase: objectStorage.publicBase }),
+        };
+        return new ObjectStorageClient(settings);
+    }
+    /**
+     * Upload the prepared images to object storage and return their URLs plus a
+     * cleanup callback, when the primary provider opts into URL transfer and
+     * object storage is configured. Returns undefined otherwise (base64 path).
+     */
+    async maybeTransferImages(pool, images, operation) {
+        const primary = pool[0];
+        if (primary === undefined || primary.provider.uploadViaUrl !== true)
+            return undefined;
+        const client = await this.resolveObjectStorageClient();
+        if (client === undefined) {
+            throw new VisionToolkitError('config', 'uploadViaUrl is enabled but object storage is not configured');
+        }
+        const keys = [];
+        const urls = [];
+        try {
+            for (const image of images) {
+                if (operation.signal.aborted)
+                    throw new VisionToolkitError('cancelled', 'vision image upload cancelled');
+                const uploaded = await client.uploadImage(image.path, imageMimeType(image.format));
+                keys.push(uploaded.key);
+                urls.push(uploaded.url);
+            }
+        }
+        catch (error) {
+            await Promise.allSettled(keys.map(key => client.deleteObject(key)));
+            throw error;
+        }
+        return {
+            urls,
+            cleanup: async () => {
+                await Promise.allSettled(keys.map(key => client.deleteObject(key)));
+            },
+        };
+    }
+    /** Settings "test storage" probe: upload → head → delete a tiny marker object. */
+    async testObjectStorage() {
+        const client = await this.resolveObjectStorageClient();
+        if (client === undefined) {
+            throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)');
+        }
+        return client.test();
+    }
     /** Stable gate key for one provider's in-flight request cap. */
     providerGate(provider) {
         const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`;
@@ -876,6 +980,23 @@ export class VisionToolkitRuntime {
         const globalRemaining = operation.deadlineAt - Date.now();
         return Math.max(0, Math.min(t2Remaining, globalRemaining));
     }
+    /**
+     * Advance to the next provider after one provider reached a terminal
+     * failure. This is what makes failover work for FAST failures too: the
+     * hedge timer only launches the next provider when the current one is SLOW
+     * (crosses t1), so a quick auth/5xx/network failure must explicitly launch
+     * the successor. Never advances when a higher-priority provider superseded
+     * this task, when the whole operation was cancelled, or when the global
+     * deadline has too little room left for another request. `launch` is
+     * idempotent, so an earlier hedge timer cannot cause a double launch.
+     */
+    advanceAfterFailure(task, operation, launchNext) {
+        if (task.abort.signal.aborted || operation.signal.aborted)
+            return;
+        if (operation.deadlineAt - Date.now() < this.config.minAvailableSeconds * 1000)
+            return;
+        launchNext();
+    }
     /**
      * Run one provider to a terminal state: retryable errors retry within
      * `attempts`, a single request crossing t1 hedges the next provider, and the
@@ -902,6 +1023,7 @@ export class VisionToolkitRuntime {
                 if (budget < minAvailableMs) {
                     task.status = 'failed';
                     task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} has insufficient remaining time`);
+                    this.advanceAfterFailure(task, operation, launchNext);
                     return;
                 }
                 const reqDeadline = createDeadline(AbortSignal.any([operation.signal, task.abort.signal]), budget);
@@ -943,6 +1065,7 @@ export class VisionToolkitRuntime {
                     if (reqDeadline.timedOut) {
                         task.status = 'failed';
                         task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} exhausted its t2 budget`);
+                        this.advanceAfterFailure(task, operation, launchNext);
                         return;
                     }
                     if (classified.code === 'rate_limit') {
@@ -957,6 +1080,7 @@ export class VisionToolkitRuntime {
                     }
                     task.status = 'failed';
                     task.error = classified;
+                    this.advanceAfterFailure(task, operation, launchNext);
                     return;
                 }
                 finally {
@@ -1058,6 +1182,8 @@ export class VisionToolkitRuntime {
                 protocol: env.VISION_API_PROTOCOL,
                 anthropicThinking: env.VISION_ANTHROPIC_THINKING,
                 sslVerify: env.VISION_SSL_VERIFY ?? null,
+                stream: env.VISION_STREAM === '1',
+                uploadViaUrl: provider.uploadViaUrl,
                 userAgent: env.VISION_USER_AGENT,
                 credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
                 maxImageBytes: provider.maxImageBytes,
@@ -1170,25 +1296,34 @@ export class VisionToolkitRuntime {
                     return cached.result;
                 }
             }
-            const result = await this.runVisionHedge('glance', [
-                ...images.map(image => image.path),
-                ...(request.region !== undefined ? ['--region', request.region] : []),
-                ...(request.ocr === true ? ['--ocr'] : []),
-                ...(request.query !== undefined ? ['-q', request.query] : []),
-            ], images, operation, pool);
-            const answer = result.stdout.trim();
-            if (answer.length === 0)
-                throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
-            const value = {
-                images,
-                mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
-                answer,
-                truncated: false,
-            };
-            if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
-                this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value });
+            const transfer = request.region === undefined
+                ? await this.maybeTransferImages(pool, images, operation)
+                : undefined;
+            try {
+                const result = await this.runVisionHedge('glance', [
+                    ...(transfer !== undefined ? transfer.urls : images.map(image => image.path)),
+                    ...(transfer === undefined && request.region !== undefined ? ['--region', request.region] : []),
+                    ...(request.ocr === true ? ['--ocr'] : []),
+                    ...(request.query !== undefined ? ['-q', request.query] : []),
+                ], images, operation, pool);
+                const answer = result.stdout.trim();
+                if (answer.length === 0)
+                    throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
+                const value = {
+                    images,
+                    mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
+                    answer,
+                    truncated: false,
+                };
+                if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
+                    this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value });
+                }
+                return value;
+            }
+            finally {
+                if (transfer !== undefined)
+                    await transfer.cleanup();
             }
-            return value;
         });
     }
     validateLocations(elements, width, height) {
@@ -1217,14 +1352,25 @@ export class VisionToolkitRuntime {
         }
         const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation);
         this.accountImage(image, operation);
-        const result = await this.runVisionHedge(tool, [
-            image.path,
-            request.target,
-            ...(request.region !== undefined ? ['--region', request.region] : []),
-        ], [image], operation, pool);
-        const elements = parseLocationOutput(result.stdout);
-        this.validateLocations(elements, image.width, image.height);
-        return { image, elements };
+        const transfer = request.region === undefined
+            ? await this.maybeTransferImages(pool, [image], operation)
+            : undefined;
+        try {
+            const result = await this.runVisionHedge(tool, transfer !== undefined
+                ? [transfer.urls[0], request.target, '--size', `${image.width}x${image.height}`]
+                : [
+                    image.path,
+                    request.target,
+                    ...(request.region !== undefined ? ['--region', request.region] : []),
+                ], [image], operation, pool);
+            const elements = parseLocationOutput(result.stdout);
+            this.validateLocations(elements, image.width, image.height);
+            return { image, elements };
+        }
+        finally {
+            if (transfer !== undefined)
+                await transfer.cleanup();
+        }
     }
     /** ground: locate one named target and return pixel boxes. */
     async ground(request, options) {
@@ -1906,6 +2052,18 @@ export class VisionToolkitRuntime {
             }
         });
     }
+    async writableDirectoryCheck(path, label) {
+        const probe = join(path, `.vision-toolkit-health-${randomUUID()}`);
+        try {
+            await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' });
+            await rm(probe, { force: true });
+            return { status: 'ok', detail: `${label} is writable: ${path}` };
+        }
+        catch {
+            await rm(probe, { force: true }).catch(() => { });
+            return { status: 'error', detail: `${label} is not writable: ${path}` };
+        }
+    }
     /** Health: inspect local readiness, and optionally probe one provider's `/models` plus one real multimodal request. */
     async health(testConnection, options, testModel = false, provider) {
         return this.runOperation('vision_toolkit_health', options, async (operation) => {
@@ -1929,6 +2087,29 @@ export class VisionToolkitRuntime {
                     throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled');
                 chrome = { status: 'error', detail: 'Chrome availability probe failed' };
             }
+            let resolvedCredential;
+            let credential;
+            try {
+                resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
+                    ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
+                    : await this.ctx.credentials.resolve(this.config.provider.credential);
+                credential = resolvedCredential === undefined
+                    ? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
+                    : { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` };
+            }
+            catch {
+                credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` };
+            }
+            let artifactDirectory;
+            try {
+                // allowedDirs are session input roots; they do not affect output readiness.
+                const policy = await createPathPolicy(options.workspace, [], this.config.storageDir);
+                artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory');
+            }
+            catch {
+                artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' };
+            }
+            const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory');
             let service = {
                 status: 'not_tested',
                 detail: 'Connection was not tested; use the per-provider API test',
@@ -2026,7 +2207,7 @@ export class VisionToolkitRuntime {
                     }
                 }
             }
-            const checks = { python, dependencies, chrome, service, model };
+            const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model };
             const healthy = Object.values(checks).every(check => check.status !== 'error');
             return {
                 pluginVersion: PLUGIN_VERSION,