@phamkhachoabk/dsh-vision-describe 0.1.0

package/README.md

@@ -0,0 +1,37 @@
+# @phamkhachoabk/dsh-vision-describe
+
+Service Definition for whole-image understanding: `ctx.visionDescribe`, the
+engine contract, the result schema, and the fallback ladder.
+
+This is the half OCR cannot answer. `ctx.ocr` recovers the text *in* an image;
+this recovers what the image *is* — layout, icons, chart shapes, colors,
+whether a dialog is an error. A consumer asks both and hands a text-only model
+the two answers side by side.
+
+## Using it
+
+```ts
+const spec = ctx.visionDescribe.resolve({ prompt: 'name the error dialog' })
+const [result] = await ctx.visionDescribe.describe([input], spec, signal)
+```
+
+`resolve()` is the one defaulting step: engines only ever see a complete
+`VisionDescribeSpec`. `describe()` never throws for a failed image — an image
+no engine could describe comes back with `status: 'failed'`, so an auxiliary
+capability cannot break a caller's turn.
+
+## Providing an engine
+
+Subclass `VisionDescribeEngine` and register the instance:
+
+```ts
+ctx.effect(() => ctx.visionDescribe.registerEngine(engine))
+```
+
+Engines are ordered by `tier`, lowest first, and the ladder stops at the first
+one whose attempt comes back `ok`. An engine that cannot run here answers
+`unusable` from `probe()` and is recorded as `skipped` rather than called.
+
+Today at most one engine is ever ready on a given machine, because each
+provider is gated to the platform its runtime supports. The ladder exists so
+that adding a provider for another platform is a registration, not a redesign.

package/lib/engine.d.ts

@@ -0,0 +1,45 @@
+/** The provider side of the vision-describe seam. @module @phamkhachoabk/dsh-vision-describe */
+import type { EngineAvailability, VisionDescribeAttempt, VisionDescribeInput, VisionDescribeSpec } from './types.ts';
+/**
+ * One vision-description backend. Providers subclass this and register the
+ * instance with `ctx.visionDescribe.registerEngine()`.
+ *
+ * Engines are ordered by {@link tier}: 1 is preferred. Today at most one
+ * engine is ever `ready` on a given machine (each provider is gated to the
+ * platform its runtime supports), but the ladder exists so a future provider
+ * for another platform, or a second local model, slots in without a redesign.
+ */
+export declare abstract class VisionDescribeEngine {
+    /** Stable identity; appears in results, cache keys and user-facing notices. */
+    abstract readonly id: string;
+    /** Preference rank; 1 is tried first. Ties break on `id` for determinism. */
+    abstract readonly tier: number;
+    /**
+     * Opaque build identity of this engine. It participates in the cache key, so
+     * changing the backend invalidates cached results without any explicit
+     * invalidation step.
+     */
+    abstract readonly version: string;
+    /**
+     * Whether this engine can run here. Callers treat every failure alike, so a
+     * missing runtime and an unsupported platform both answer `unusable`.
+     * @param signal - cancellation for any probe work.
+     * @returns readiness, or the reason this engine is unusable.
+     */
+    abstract probe(signal?: AbortSignal): Promise<EngineAvailability>;
+    /**
+     * Describe a batch of images in natural language.
+     * @param inputs - images to describe, in caller order.
+     * @param spec - a fully resolved request; engines apply no defaults.
+     * @param signal - cancellation for the whole batch.
+     * @returns one attempt per input, in the same order.
+     */
+    abstract describe(inputs: readonly VisionDescribeInput[], spec: VisionDescribeSpec, signal: AbortSignal): Promise<readonly VisionDescribeAttempt[]>;
+    /**
+     * Optional background preparation, such as starting a persistent model
+     * process so a user's first image does not pay for its startup. Failures
+     * are advisory.
+     * @param signal - cancellation for the warm-up.
+     */
+    warmUp?(signal?: AbortSignal): Promise<void>;
+}

package/lib/engine.js

@@ -0,0 +1,12 @@
+/** The provider side of the vision-describe seam. @module @phamkhachoabk/dsh-vision-describe */
+/**
+ * One vision-description backend. Providers subclass this and register the
+ * instance with `ctx.visionDescribe.registerEngine()`.
+ *
+ * Engines are ordered by {@link tier}: 1 is preferred. Today at most one
+ * engine is ever `ready` on a given machine (each provider is gated to the
+ * platform its runtime supports), but the ladder exists so a future provider
+ * for another platform, or a second local model, slots in without a redesign.
+ */
+export class VisionDescribeEngine {
+}

package/lib/index.d.ts

@@ -0,0 +1,70 @@
+/** Vision-description capability seam (`ctx.visionDescribe`). @module @phamkhachoabk/dsh-vision-describe */
+import { Context, Service } from '@deepseek-ai/cordis';
+import z from '@deepseek-ai/schemastery';
+import { VisionDescribeEngine } from './engine.ts';
+import type { VisionDescribeInput, VisionDescribeRequest, VisionDescribeResult, VisionDescribeSpec } from './types.ts';
+export { VisionDescribeEngine } from './engine.ts';
+export * from './types.ts';
+declare module '@deepseek-ai/cordis' {
+    interface Context {
+        visionDescribe: VisionDescribeService;
+    }
+}
+/**
+ * Deployment defaults for {@link VisionDescribeService.resolve}. These are the
+ * plugin's `Config`, not constants: a deployment changes them from `cordis.yml`.
+ */
+export interface VisionDescribeServiceConfig {
+    perImageTimeoutMs: number;
+    prompt: string;
+}
+/**
+ * Registry and fallback pipeline for vision-description engines.
+ *
+ * A registry rather than an abstract seam because the capability has several
+ * providers competing in a defined order, the same shape `ctx.ocr` uses.
+ */
+export declare class VisionDescribeService extends Service {
+    private readonly config;
+    /** Deployment defaults; the one place request defaulting is configured. */
+    static Config: z<VisionDescribeServiceConfig>;
+    private readonly registrations;
+    constructor(ctx: Context, config: VisionDescribeServiceConfig);
+    /**
+     * Add one engine to the ladder.
+     * @param engine - the provider's engine instance.
+     * @returns an idempotent disposer that removes exactly this registration.
+     */
+    registerEngine(engine: VisionDescribeEngine): () => void;
+    /**
+     * Identity of the whole registered ladder. It participates in cache keys, so
+     * adding, removing or upgrading any engine invalidates cached results with
+     * no explicit invalidation step.
+     * @returns a stable string over every registered engine and its version.
+     */
+    identity(): string;
+    /** Registered engines in ladder order, most preferred first. */
+    engines(): readonly VisionDescribeEngine[];
+    /**
+     * Turn a caller's wishes into a complete spec. Defaulting lives here, at the
+     * package boundary, so no engine ever sees a partially specified request.
+     * @param request - optional caller overrides.
+     * @returns a fully resolved spec.
+     */
+    resolve(request?: VisionDescribeRequest): VisionDescribeSpec;
+    /** Warm every ready engine that offers it. Failures are advisory and logged. */
+    warmUp(signal?: AbortSignal): Promise<void>;
+    private availabilityOf;
+    /**
+     * Describe every image, advancing through the engine ladder until one
+     * succeeds. Never throws for a describe failure: an image no engine could
+     * describe comes back as a `failed` result so the caller can carry on.
+     * @param inputs - images to describe.
+     * @param spec - a resolved spec from {@link resolve}.
+     * @param signal - cancellation for the whole call.
+     * @returns one result per input, in input order.
+     */
+    describe(inputs: readonly VisionDescribeInput[], spec: VisionDescribeSpec, signal: AbortSignal): Promise<readonly VisionDescribeResult[]>;
+    private finish;
+}
+export default VisionDescribeService;

package/lib/index.js

@@ -0,0 +1,191 @@
+/** Vision-description capability seam (`ctx.visionDescribe`). @module @phamkhachoabk/dsh-vision-describe */
+import { Context, Service } from '@deepseek-ai/cordis';
+import z from '@deepseek-ai/schemastery';
+import { VisionDescribeEngine } from "./engine.js";
+export { VisionDescribeEngine } from "./engine.js";
+export * from "./types.js";
+/** Default prompt asked of every engine, steering it toward what a text-only model needs. */
+const DEFAULT_PROMPT = 'Describe this image for someone who cannot see it. Cover what kind of '
+    + 'image it is (photo, screenshot, diagram, UI, chart, document…), its overall layout, and any '
+    + 'visually significant detail a caption would otherwise miss — colors, icons, highlighted or '
+    + 'error states, chart shapes, spatial relationships. Do not transcribe long passages of text '
+    + 'verbatim; a separate OCR pass already does that.';
+/**
+ * Registry and fallback pipeline for vision-description engines.
+ *
+ * A registry rather than an abstract seam because the capability has several
+ * providers competing in a defined order, the same shape `ctx.ocr` uses.
+ */
+export class VisionDescribeService extends Service {
+    config;
+    /** Deployment defaults; the one place request defaulting is configured. */
+    static Config = z.object({
+        perImageTimeoutMs: z.number().step(1).min(100).default(60_000)
+            .description('Budget one engine gets for one image.'),
+        prompt: z.string().default(DEFAULT_PROMPT)
+            .description('Instruction sent to every engine, steering what it attends to.'),
+    });
+    registrations = new Map();
+    constructor(ctx, config) {
+        super(ctx, 'visionDescribe');
+        this.config = config;
+    }
+    /**
+     * Add one engine to the ladder.
+     * @param engine - the provider's engine instance.
+     * @returns an idempotent disposer that removes exactly this registration.
+     */
+    registerEngine(engine) {
+        if (this.registrations.has(engine.id)) {
+            throw new Error(`visionDescribe: engine ${engine.id} is already registered`);
+        }
+        const registration = { engine };
+        this.registrations.set(engine.id, registration);
+        return () => {
+            if (this.registrations.get(engine.id) === registration) {
+                this.registrations.delete(engine.id);
+            }
+        };
+    }
+    /**
+     * Identity of the whole registered ladder. It participates in cache keys, so
+     * adding, removing or upgrading any engine invalidates cached results with
+     * no explicit invalidation step.
+     * @returns a stable string over every registered engine and its version.
+     */
+    identity() {
+        return this.engines().map(engine => `${engine.id}@${engine.version}`).join(',');
+    }
+    /** Registered engines in ladder order, most preferred first. */
+    engines() {
+        return [...this.registrations.values()]
+            .map(registration => registration.engine)
+            .sort((left, right) => left.tier - right.tier || left.id.localeCompare(right.id));
+    }
+    /**
+     * Turn a caller's wishes into a complete spec. Defaulting lives here, at the
+     * package boundary, so no engine ever sees a partially specified request.
+     * @param request - optional caller overrides.
+     * @returns a fully resolved spec.
+     */
+    resolve(request = {}) {
+        return {
+            perImageTimeoutMs: request.perImageTimeoutMs ?? this.config.perImageTimeoutMs,
+            prompt: request.prompt ?? this.config.prompt,
+        };
+    }
+    /** Warm every ready engine that offers it. Failures are advisory and logged. */
+    async warmUp(signal) {
+        await Promise.all(this.engines().map(async (engine) => {
+            if (engine.warmUp === undefined)
+                return;
+            const availability = await this.availabilityOf(engine, signal);
+            if (availability.kind !== 'ready')
+                return;
+            try {
+                await engine.warmUp(signal);
+            }
+            catch (error) {
+                this.ctx.logger.warn(`visionDescribe: warm-up of ${engine.id} failed: ${String(error)}`);
+            }
+        }));
+    }
+    availabilityOf(engine, signal) {
+        const registration = this.registrations.get(engine.id);
+        if (registration === undefined) {
+            return Promise.resolve({ kind: 'unusable', reason: 'engine is not registered' });
+        }
+        registration.availability ??= engine.probe(signal).catch((error) => ({
+            kind: 'unusable',
+            reason: String(error),
+        }));
+        return registration.availability;
+    }
+    /**
+     * Describe every image, advancing through the engine ladder until one
+     * succeeds. Never throws for a describe failure: an image no engine could
+     * describe comes back as a `failed` result so the caller can carry on.
+     * @param inputs - images to describe.
+     * @param spec - a resolved spec from {@link resolve}.
+     * @param signal - cancellation for the whole call.
+     * @returns one result per input, in input order.
+     */
+    async describe(inputs, spec, signal) {
+        const started = Date.now();
+        const pending = new Map(inputs.map(input => [input.attachmentId, { input, records: [] }]));
+        for (const engine of this.engines()) {
+            const remaining = [...pending.values()].filter(entry => entry.accepted === undefined);
+            if (remaining.length === 0 || signal.aborted)
+                break;
+            const availability = await this.availabilityOf(engine, signal);
+            if (availability.kind !== 'ready') {
+                for (const entry of remaining) {
+                    entry.records.push({
+                        engine: engine.id, outcome: 'skipped', ms: 0, detail: availability.reason,
+                    });
+                }
+                continue;
+            }
+            let attempts;
+            try {
+                attempts = await engine.describe(remaining.map(entry => entry.input), spec, signal);
+            }
+            catch (error) {
+                // An engine that throws is a broken engine, not a failed image; the
+                // ladder continues so one bad provider cannot deny the whole feature.
+                for (const entry of remaining) {
+                    entry.records.push({
+                        engine: engine.id, outcome: 'error', ms: 0, detail: String(error),
+                    });
+                }
+                continue;
+            }
+            for (const attempt of attempts) {
+                const entry = pending.get(attempt.attachmentId);
+                if (entry === undefined)
+                    continue;
+                entry.records.push({
+                    engine: engine.id,
+                    outcome: attempt.outcome,
+                    ms: attempt.ms,
+                    ...(attempt.detail === undefined ? {} : { detail: attempt.detail }),
+                });
+                if (attempt.outcome === 'ok') {
+                    entry.accepted = attempt;
+                    entry.acceptedEngine = engine.id;
+                }
+            }
+        }
+        const totalMs = Date.now() - started;
+        return inputs.map(input => this.finish(pending.get(input.attachmentId), totalMs));
+    }
+    finish(entry, totalMs) {
+        const attempted = entry.records;
+        const accepted = entry.accepted;
+        if (accepted === undefined || entry.acceptedEngine === undefined) {
+            return {
+                attachmentId: entry.input.attachmentId,
+                status: 'failed',
+                engine: '',
+                attemptedEngines: attempted,
+                fallback: 'exhausted',
+                description: '',
+                timings: { totalMs, engineMs: attempted.reduce((sum, record) => sum + record.ms, 0) },
+                warnings: [],
+            };
+        }
+        // `none` only when the very first engine tried carried the result.
+        const fallback = attempted[0]?.engine === entry.acceptedEngine ? 'none' : 'used';
+        return {
+            attachmentId: entry.input.attachmentId,
+            status: 'ok',
+            engine: entry.acceptedEngine,
+            attemptedEngines: attempted,
+            fallback,
+            description: accepted.description,
+            timings: { totalMs, engineMs: accepted.ms },
+            warnings: accepted.warnings,
+        };
+    }
+}
+export default VisionDescribeService;

package/lib/schema.d.ts

@@ -0,0 +1,8 @@
+/** Durable-boundary validation for cached results. @module @phamkhachoabk/dsh-vision-describe */
+import { z } from 'zod';
+import type { VisionDescribeResult } from './types.ts';
+/**
+ * Validates every cached record. The cache is a durable boundary, so a record
+ * written by an older build is rejected rather than trusted.
+ */
+export declare const visionDescribeResultSchema: z.ZodType<VisionDescribeResult>;

package/lib/schema.js

@@ -0,0 +1,22 @@
+/** Durable-boundary validation for cached results. @module @phamkhachoabk/dsh-vision-describe */
+import { z } from 'zod';
+const outcome = z.enum(['ok', 'empty', 'error', 'timeout', 'skipped']);
+/**
+ * Validates every cached record. The cache is a durable boundary, so a record
+ * written by an older build is rejected rather than trusted.
+ */
+export const visionDescribeResultSchema = z.object({
+    attachmentId: z.string(),
+    status: z.enum(['ok', 'failed']),
+    engine: z.string(),
+    attemptedEngines: z.array(z.object({
+        engine: z.string(),
+        outcome,
+        ms: z.number(),
+        detail: z.string().optional(),
+    })),
+    fallback: z.enum(['none', 'used', 'exhausted']),
+    description: z.string(),
+    timings: z.object({ totalMs: z.number(), engineMs: z.number() }),
+    warnings: z.array(z.string()),
+});

package/lib/types.d.ts

@@ -0,0 +1,69 @@
+/** Types of the vision-description capability seam. @module @phamkhachoabk/dsh-vision-describe */
+import type { AttachmentId } from '@deepseek-ai/dsh-attachment';
+/** Why one engine's attempt on one image ended. */
+export type VisionDescribeOutcome = 'ok' | 'empty' | 'error' | 'timeout' | 'skipped';
+export interface VisionDescribeAttemptRecord {
+    engine: string;
+    outcome: VisionDescribeOutcome;
+    ms: number;
+    /** Present for `error` and `timeout`; the pipeline surfaces it in warnings. */
+    detail?: string;
+}
+/** One engine's answer for one image, before the pipeline decides to accept it. */
+export interface VisionDescribeAttempt {
+    attachmentId: AttachmentId;
+    outcome: VisionDescribeOutcome;
+    ms: number;
+    /** Free-form natural-language account of what the image shows. */
+    description: string;
+    warnings: readonly string[];
+    detail?: string;
+}
+export type VisionDescribeStatus = 'ok' | 'failed';
+export type VisionDescribeFallback = 'none' | 'used' | 'exhausted';
+/** The pipeline's answer for one image. */
+export interface VisionDescribeResult {
+    attachmentId: AttachmentId;
+    status: VisionDescribeStatus;
+    /** Engine that produced `description`; empty when every engine failed. */
+    engine: string;
+    attemptedEngines: readonly VisionDescribeAttemptRecord[];
+    fallback: VisionDescribeFallback;
+    description: string;
+    timings: {
+        totalMs: number;
+        engineMs: number;
+    };
+    warnings: readonly string[];
+}
+/** One image handed to the pipeline. */
+export interface VisionDescribeInput {
+    attachmentId: AttachmentId;
+    /** Absolute host path readable by the provider's execution world. */
+    path: string;
+}
+/**
+ * A caller's wishes, every field optional. Turning this into a
+ * {@link VisionDescribeSpec} is an explicit `resolve()` step at the package
+ * boundary, never a hidden `??` inside the pipeline.
+ */
+export interface VisionDescribeRequest {
+    perImageTimeoutMs?: number;
+    /** Instruction steering what the engine attends to; engines apply their own default. */
+    prompt?: string;
+}
+/** A fully resolved request. Engines receive only this. */
+export interface VisionDescribeSpec {
+    perImageTimeoutMs: number;
+    prompt: string;
+}
+/**
+ * Availability of one engine. A missing runtime and an unsupported platform
+ * are deliberately indistinguishable, so consumers have one fail-closed path.
+ */
+export type EngineAvailability = {
+    kind: 'ready';
+} | {
+    kind: 'unusable';
+    reason: string;
+};

package/lib/types.js

@@ -0,0 +1,2 @@
+/** Types of the vision-description capability seam. @module @phamkhachoabk/dsh-vision-describe */
+export {};

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@phamkhachoabk/dsh-vision-describe",
+  "version": "0.1.0",
+  "description": "Vision description capability seam for DeepSeek Harness: ctx.visionDescribe, the engine contract, and the fallback pipeline for whole-image semantic understanding",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./schema": {
+      "types": "./lib/schema.d.ts",
+      "default": "./lib/schema.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/",
+    "README.md"
+  ],
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "zod": "^4.4.3"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "^4.0.2",
+    "@deepseek-ai/dsh-attachment": "^0.1.5-rc.1",
+    "@deepseek-ai/schemastery": "^3.18.2"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.2",
+    "@deepseek-ai/dsh-attachment": "^0.1.5-rc.1",
+    "@deepseek-ai/schemastery": "^3.18.2"
+  }
+}