dodo-mcp 1.0.0

package/dist/services/multimodal/contracts.js

@@ -0,0 +1,59 @@
+import { z } from 'zod';
+import { DesktopActionSchema } from '../desktop/protocol.js';
+export const Id = z.string().regex(/^[a-z]+_[a-z0-9]{8,64}$/).max(100);
+export const Path = z.string().min(1).max(1024);
+export const Hash = z.string().regex(/^sha256:[a-f0-9]{64}$/);
+export const Key = z.string().min(8).max(128);
+export const Crop = z.object({ left: z.number().int().min(0).max(20000), top: z.number().int().min(0).max(20000), width: z.number().int().min(1).max(20000), height: z.number().int().min(1).max(20000) }).strict();
+export const AssetInfo = z.object({ assetId: Id, kind: z.enum(['image', 'audio', 'transcript']), mimeType: z.string(), bytes: z.number().int().nonnegative(), sha256: Hash, createdAt: z.number(), expiresAt: z.number(), timeSec: z.number().nonnegative().optional(), endSec: z.number().nonnegative().optional() });
+export const Segment = z.object({ startSec: z.number().nonnegative(), endSec: z.number().nonnegative(), text: z.string().max(4000) }).strict().refine(s => s.endSec >= s.startSec, 'end must follow start');
+export const Transcript = z.object({ source: z.enum(['whisper.cpp', 'sidecar_subtitles']), language: z.string(), mediaId: Id, startSec: z.number(), endSec: z.number(), segments: z.array(Segment).max(10000), truncated: z.boolean() });
+export const BrowserAction = z.discriminatedUnion('kind', [
+    z.object({ kind: z.literal('click'), selector: z.string().min(1).max(500) }).strict(),
+    z.object({ kind: z.literal('fill'), selector: z.string().min(1).max(500), text: z.string().max(4000) }).strict(),
+    z.object({ kind: z.literal('press'), selector: z.string().min(1).max(500), key: z.enum(['Enter', 'Tab', 'Escape', 'Space', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', 'Backspace']) }).strict(),
+    z.object({ kind: z.literal('select'), selector: z.string().min(1).max(500), value: z.string().max(500) }).strict(),
+    z.object({ kind: z.literal('scroll'), deltaY: z.number().int().min(-2000).max(2000) }).strict(),
+    z.object({ kind: z.literal('media'), selector: z.string().min(1).max(500), control: z.enum(['play', 'pause', 'seek', 'mute']), timeSec: z.number().min(0).max(86400).optional() }).strict(),
+    z.object({ kind: z.literal('navigate'), url: z.string().min(1).max(2048) }).strict(),
+]);
+export const GameAction = z.discriminatedUnion('kind', [
+    z.object({ kind: z.literal('wait'), durationMs: z.number().int().min(0).max(2000).default(200) }).strict(),
+    z.object({ kind: z.literal('keys'), keys: z.array(z.string().regex(/^(?:[a-z0-9]|ArrowUp|ArrowDown|ArrowLeft|ArrowRight|Space|Enter|Escape)$/)).min(1).max(4), holdMs: z.number().int().min(0).max(500).default(0) }).strict(),
+    z.object({ kind: z.literal('click'), x: z.number().min(0).max(2000), y: z.number().min(0).max(2000) }).strict(),
+]);
+export const WorkflowStep = z.object({
+    instruction: z.string().min(1).max(1000),
+    expectedBefore: z.string().min(1).max(300), expectedAfter: z.string().max(300).default(''),
+    action: z.discriminatedUnion('target', [
+        z.object({ target: z.literal('browser'), action: BrowserAction }).strict(),
+        z.object({ target: z.literal('desktop'), action: DesktopActionSchema }).strict(),
+        z.object({ target: z.literal('manual') }).strict(),
+    ]),
+    evidence: z.array(z.object({ source: z.enum(['observation', 'media', 'user_note']), reference: z.string().max(150), timeSec: z.number().nonnegative().optional() }).strict()).max(5).default([]),
+}).strict();
+export const Workflow = z.object({ name: z.string().min(1).max(100), goal: z.string().min(1).max(1500), tags: z.array(z.string().min(1).max(40)).max(10).default([]), steps: z.array(WorkflowStep).min(1).max(30) }).strict();
+export const WorkflowRecord = Workflow.extend({ workflowId: Id, revision: Hash, createdAt: z.number(), updatedAt: z.number(), provenance: z.literal('client_authored_untrusted_steps_not_permissions') });
+export const WorkerSpec = z.object({
+    operation: z.enum(['probe', 'frames', 'audio', 'transcribe', 'speak']),
+    input: z.string(), directory: z.string(), ffmpeg: z.string(), ffprobe: z.string(),
+    whisper: z.string().optional(), model: z.string().optional(), say: z.string().optional(),
+    speechProgram: z.string().optional(), speechKind: z.enum(['macos-say', 'windows-sapi', 'espeak-ng']).optional(),
+    startSec: z.number().min(0).max(86400), durationSec: z.number().positive().max(600),
+    times: z.array(z.number().min(0).max(86400)).max(8), maxEdge: z.number().int().min(320).max(1600),
+    language: z.string().regex(/^(auto|[a-z]{2,3})$/), voice: z.string().max(80).optional(),
+}).strict();
+export const WorkerResult = z.object({
+    operation: WorkerSpec.shape.operation,
+    metadata: z.object({ durationSec: z.number().nonnegative().nullable(), streams: z.array(z.object({ type: z.string(), codec: z.string(), width: z.number().optional(), height: z.number().optional() })).max(20) }).optional(),
+    files: z.array(z.object({ name: z.string().regex(/^out-[0-9]+\.(?:jpg|wav|json)$/), kind: AssetInfo.shape.kind, mimeType: z.string(), timeSec: z.number().optional(), endSec: z.number().optional() })).max(10),
+    notes: z.array(z.string().max(500)).max(10),
+}).strict();
+export const MediaJobResult = z.object({ jobId: z.string(), mediaId: Id.nullable(), operation: WorkerSpec.shape.operation, status: z.string(), exitCode: z.number().nullable(), sandboxed: z.string().nullable(), assets: z.array(AssetInfo), metadata: WorkerResult.shape.metadata, notes: z.array(z.string()), error: z.string().nullable() });
+export const ImageView = z.object({ width: z.number(), height: z.number(), sourceWidth: z.number(), sourceHeight: z.number(), crop: Crop.nullable(), scaleX: z.number(), scaleY: z.number(), sourceOffsetX: z.number(), sourceOffsetY: z.number() });
+export const ScreenFrame = z.object({ observationId: Id, windowId: z.number(), appId: z.string(), pid: z.number(), snapshotId: z.string(), expiresAt: z.number(), capturedAt: z.number(), asset: AssetInfo, view: ImageView,
+    difference: z.object({ comparable: z.boolean(), changedFraction: z.number().nullable(), note: z.string() }).nullable(),
+});
+export const PlaybackState = z.object({ selector: z.string(), currentTime: z.number().nonnegative(), duration: z.number().nonnegative().nullable(), paused: z.boolean(), readyState: z.number().int().min(0).max(4), errorCode: z.number().int().nullable() });
+export const BrowserObservation = z.object({ sessionId: Id, observationId: Id, url: z.string(), title: z.string(), text: z.string(), elements: z.array(z.object({ selector: z.string(), tag: z.string(), role: z.string(), name: z.string(), disabled: z.boolean(), type: z.string() })).max(100), truncated: z.boolean(), media: z.array(PlaybackState).max(8).default([]), expiresAt: z.number(), asset: AssetInfo, console: z.array(z.string()).max(30), network: z.array(z.string()).max(30), note: z.string() });
+//# sourceMappingURL=contracts.js.map

package/dist/services/multimodal/contracts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contracts.js","sourceRoot":"","sources":["../../../src/services/multimodal/contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AACvE,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAChD,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;AAC9D,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAC9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAEpN,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAEvT,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;AAE5M,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;AAEzO,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACxD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IACrF,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAChH,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IACvM,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAClH,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAC/F,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3L,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;CACrF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACrD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1G,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,0EAA0E,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAC9N,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;CAChH,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACxC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1F,MAAM,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACrC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1E,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE;QAChF,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;KACnD,CAAC;IACF,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CACjM,CAAC,CAAC,MAAM,EAAE,CAAC;AACZ,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAE9N,MAAM,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,iDAAiD,CAAC,EAAE,CAAC,CAAC;AAE1M,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IACtE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACjF,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxF,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/G,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC;IACnF,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACjG,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxF,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS;IACrC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7N,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IAC/M,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;CAC5C,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACjV,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACrP,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS;IACzN,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvH,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC/P,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC"}

package/dist/services/multimodal/gameService.js

@@ -0,0 +1,95 @@
+import { DodoError } from '../../errors.js';
+import { newId } from '../../util/hash.js';
+import { DesktopActionSchema } from '../desktop/protocol.js';
+import { MediaStorage, actorKey } from './storage.js';
+import { ScreenService } from './images.js';
+import { BrowserService } from './browserService.js';
+export class GameService {
+    storage;
+    screen;
+    browser;
+    games = new Map();
+    constructor(storage, screen, browser) {
+        this.storage = storage;
+        this.screen = screen;
+        this.browser = browser;
+    }
+    get(actor, id) { this.storage.check(); const g = this.games.get(id); if (!g || g.owner !== actorKey(actor) || g.expiresAt < Date.now())
+        throw new DodoError('NOT_FOUND', 'unknown/expired game session'); return g; }
+    async open(actor, opts, check) {
+        if ((opts.browserId === undefined) === (opts.windowId === undefined))
+            throw new DodoError('INVALID_INPUT', 'select exactly one browserId or permitted desktop windowId');
+        if (this.games.size >= 8)
+            throw new DodoError('RESOURCE_LIMIT', 'game session limit reached');
+        const game = { id: newId('game'), owner: actorKey(actor), observationId: '', steps: 0, expiresAt: Date.now() + 900000, busy: false, ...(opts.browserId ? { browserId: opts.browserId } : {}), ...(opts.windowId ? { windowId: opts.windowId } : {}) };
+        this.games.set(game.id, game);
+        try {
+            return await this.observe(actor, game.id, check);
+        }
+        catch (err) {
+            this.games.delete(game.id);
+            throw err;
+        }
+    }
+    async observe(actor, id, check) {
+        const g = this.get(actor, id);
+        if (g.busy)
+            throw new DodoError('RESOURCE_LIMIT', 'game session busy');
+        const observation = g.browserId ? await this.browser.observe(actor, g.browserId) : (await this.screen.observe(actor, { windowId: g.windowId, maxEdge: 1280, count: 1, intervalMs: 0 }, check)).frames[0];
+        g.observationId = observation.observationId;
+        return { gameId: g.id, steps: g.steps, expiresAt: g.expiresAt, mode: g.browserId ? 'isolated_browser' : 'permitted_desktop', observation,
+            note: 'The client model chooses moves. No background autoplay or win/score claim. Browser key holds are bounded; native desktop uses existing discrete actions only.' };
+    }
+    async step(actor, id, observationId, action, check, gate) {
+        const g = this.get(actor, id);
+        if (g.busy)
+            throw new DodoError('RESOURCE_LIMIT', 'game session busy');
+        if (g.observationId !== observationId)
+            throw new DodoError('STALE_WORKSPACE', 'use the latest game observation');
+        if (g.steps >= 1000)
+            throw new DodoError('RESOURCE_LIMIT', 'session move budget reached; open a new reviewed session');
+        if (!g.browserId && action.kind === 'keys' && (action.keys.length !== 1 || action.holdMs !== 0))
+            throw new DodoError('NOT_SUPPORTED', 'desktop games support one discrete key per step; simultaneous/held keys are available only in isolated browser games');
+        g.busy = true;
+        let dispatched = false;
+        try {
+            if (g.browserId) {
+                gate();
+                const observation = await this.browser.gameStep(actor, g.browserId, observationId, action, check);
+                g.steps++;
+                g.observationId = observation.observationId;
+                return { gameId: id, steps: g.steps, observation, note: 'Input dispatched and an after-frame captured; outcome must be evaluated by the client model.' };
+            }
+            const frame = this.screen.frame(actor, observationId);
+            check();
+            if (action.kind === 'wait')
+                await new Promise(resolve => setTimeout(resolve, action.durationMs));
+            else {
+                const names = { ArrowUp: 'up', ArrowDown: 'down', ArrowLeft: 'left', ArrowRight: 'right', Space: 'space', Enter: 'enter', Escape: 'escape' };
+                const native = DesktopActionSchema.parse(action.kind === 'click' ? { kind: 'click', x: action.x, y: action.y } : { kind: 'key', key: names[action.keys[0]] ?? action.keys[0], modifiers: [] });
+                await this.storage.services.desktop.action(frame.snapshotId, native, `game-${g.id}-${g.steps}-${observationId}`, actor, policy => { check(); gate(policy); });
+                dispatched = true;
+            }
+            check();
+            const observation = (await this.screen.observe(actor, { windowId: g.windowId, maxEdge: 1280, count: 1, intervalMs: 0 }, check)).frames[0];
+            g.steps++;
+            g.observationId = observation.observationId;
+            return { gameId: id, steps: g.steps, observation, note: 'Existing desktop snapshot/foreground/permission checks enforced. This is a discrete move, not low-latency continuous control.' };
+        }
+        catch (err) {
+            // Once input was posted, a failed after-observation must never make the step retryable.
+            if (dispatched)
+                throw new DodoError('RECOVERY_REQUIRED', 'game input was posted but its outcome could not be observed; inspect before another move');
+            throw err;
+        }
+        finally {
+            g.busy = false;
+        }
+    }
+    closeSession(actor, id) { this.get(actor, id); this.games.delete(id); return { closed: true, gameId: id }; }
+    sweep() { for (const [id, game] of this.games)
+        if (!game.busy && game.expiresAt <= Date.now())
+            this.games.delete(id); }
+    close() { this.games.clear(); }
+}
+//# sourceMappingURL=gameService.js.map

package/dist/services/multimodal/gameService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gameService.js","sourceRoot":"","sources":["../../../src/services/multimodal/gameService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAc,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAIrD,MAAM,OAAO,WAAW;IAEO;IAAwC;IAAwC;IAD5F,KAAK,GAAG,IAAI,GAAG,EAAgB,CAAC;IACjD,YAA6B,OAAqB,EAAmB,MAAqB,EAAmB,OAAuB;QAAvG,YAAO,GAAP,OAAO,CAAc;QAAmB,WAAM,GAAN,MAAM,CAAe;QAAmB,YAAO,GAAP,OAAO,CAAgB;IAAG,CAAC;IAChI,GAAG,CAAC,KAAY,EAAE,EAAU,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE;QAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,8BAA8B,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5O,KAAK,CAAC,IAAI,CAAC,KAAY,EAAE,IAAuE,EAAE,KAAiB;QACjH,IAAI,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,eAAe,EAAE,4DAA4D,CAAC,CAAC;QACzK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,4BAA4B,CAAC,CAAC;QAC9F,MAAM,IAAI,GAAS,EAAE,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC5P,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAAC,MAAM,GAAG,CAAC;QAAC,CAAC;IAClH,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,KAAY,EAAE,EAAU,EAAE,KAAiB;QACvD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAAC,IAAI,CAAC,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC;QACtG,MAAM,WAAW,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAS,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC3M,CAAC,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;QAC5C,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,mBAAmB,EAAE,WAAW;YACtI,IAAI,EAAE,+JAA+J,EAAE,CAAC;IAC5K,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,KAAY,EAAE,EAAU,EAAE,aAAqB,EAAE,MAAuB,EAAE,KAAiB,EAAE,IAAsC;QAC5I,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAAC,IAAI,CAAC,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC;QACtG,IAAI,CAAC,CAAC,aAAa,KAAK,aAAa;YAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,EAAE,iCAAiC,CAAC,CAAC;QACjH,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,0DAA0D,CAAC,CAAC;QACvH,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,eAAe,EAAE,sHAAsH,CAAC,CAAC;QAC9P,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;QAAC,IAAI,UAAU,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC;YACH,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;gBAChB,IAAI,EAAE,CAAC;gBAAC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;gBAC1G,CAAC,CAAC,KAAK,EAAE,CAAC;gBAAC,CAAC,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;gBACvD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,8FAA8F,EAAE,CAAC;YAC3J,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YAAC,KAAK,EAAE,CAAC;YAC/D,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;gBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;iBAC5F,CAAC;gBACJ,MAAM,KAAK,GAA2B,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;gBACrK,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;gBACjM,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI,aAAa,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9J,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;YACD,KAAK,EAAE,CAAC;YACR,MAAM,WAAW,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAS,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;YAC5I,CAAC,CAAC,KAAK,EAAE,CAAC;YAAC,CAAC,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;YACvD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,+HAA+H,EAAE,CAAC;QAC5L,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wFAAwF;YACxF,IAAI,UAAU;gBAAE,MAAM,IAAI,SAAS,CAAC,mBAAmB,EAAE,0FAA0F,CAAC,CAAC;YACrJ,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IAC/B,CAAC;IACD,YAAY,CAAC,KAAY,EAAE,EAAU,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3H,KAAK,KAAK,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvH,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;CAChC"}

package/dist/services/multimodal/images.js

@@ -0,0 +1,107 @@
+import sharp from 'sharp';
+import { DodoError } from '../../errors.js';
+import { digestOf, newId } from '../../util/hash.js';
+import { MediaStorage, actorKey } from './storage.js';
+import { ScreenFrame } from './contracts.js';
+const OPTIONS = { limitInputPixels: 40_000_000, failOn: 'warning' };
+export async function imageView(bytes, maxEdge, crop) {
+    if (bytes.length > 16 * 1024 * 1024)
+        throw new DodoError('FILE_TOO_LARGE', 'image input exceeds 16 MiB');
+    const image = sharp(bytes, OPTIONS);
+    const metadata = await image.metadata();
+    if (!['jpeg', 'png', 'webp', 'gif'].includes(metadata.format ?? '') || !metadata.width || !metadata.height)
+        throw new DodoError('UNSUPPORTED_ENCODING', 'use PNG/JPEG/WebP/GIF; animated files use the first frame');
+    const width = metadata.width, height = metadata.height;
+    if (crop && (crop.left + crop.width > width || crop.top + crop.height > height))
+        throw new DodoError('INVALID_INPUT', 'crop is outside source image pixels');
+    if (crop)
+        image.extract(crop);
+    // No super-resolution claims. Pixel geometry remains explicit for subsequent actions.
+    const converted = await image.resize({ width: maxEdge, height: maxEdge, fit: 'inside', withoutEnlargement: !crop }).jpeg({ quality: 80 }).toBuffer({ resolveWithObject: true });
+    return { bytes: converted.data, view: { width: converted.info.width, height: converted.info.height, sourceWidth: width, sourceHeight: height, crop: crop ?? null,
+            scaleX: (crop?.width ?? width) / converted.info.width, scaleY: (crop?.height ?? height) / converted.info.height, sourceOffsetX: crop?.left ?? 0, sourceOffsetY: crop?.top ?? 0 } };
+}
+export async function imageDifference(before, after) {
+    const normalize = (b) => sharp(b, OPTIONS).resize(64, 64, { fit: 'fill' }).removeAlpha().toColourspace('srgb').raw().toBuffer();
+    const [a, b] = await Promise.all([normalize(before), normalize(after)]);
+    let changed = 0;
+    for (let i = 0; i < Math.min(a.length, b.length); i += 3)
+        if (Math.max(Math.abs(a[i] - b[i]), Math.abs(a[i + 1] - b[i + 1]), Math.abs(a[i + 2] - b[i + 2])) > 24)
+            changed++;
+    return changed / 4096;
+}
+export class ScreenService {
+    storage;
+    frames = new Map();
+    busy = false;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    sweep() { for (const [id, f] of this.frames)
+        if (f.frame.expiresAt <= Date.now())
+            this.frames.delete(id); }
+    frame(actor, id) {
+        this.storage.check();
+        this.sweep();
+        const value = this.frames.get(id);
+        if (!value || value.owner !== actorKey(actor))
+            throw new DodoError('NOT_FOUND', 'screen observation expired or belongs to another client');
+        this.storage.get(actor, value.frame.asset.assetId);
+        return value.frame;
+    }
+    async observe(actor, opts, check) {
+        this.storage.check();
+        if (this.busy)
+            throw new DodoError('RESOURCE_LIMIT', 'screen observation already in progress');
+        if ((opts.count - 1) * opts.intervalMs > 3000)
+            throw new DodoError('RESOURCE_LIMIT', 'screen sample burst is limited to 3 seconds');
+        this.busy = true;
+        try {
+            const out = [];
+            let previous = opts.previousId;
+            for (let i = 0; i < opts.count; i++) {
+                if (i)
+                    await new Promise(resolve => setTimeout(resolve, opts.intervalMs));
+                this.storage.check();
+                check();
+                const desktop = this.storage.services.desktop, policy = desktop.policy(), policyDigest = digestOf(policy);
+                const captured = await desktop.capture(opts.windowId, opts.maxEdge, false, actor);
+                check();
+                const raw = Buffer.from(captured.image, 'base64'), converted = await imageView(raw, opts.maxEdge, opts.crop);
+                const expiresAt = Math.min(captured.data.expiresAt, Date.now() + 30000);
+                const guard = () => { this.storage.check(); const now = desktop.policy(); if (now.mode === 'off' || digestOf(now) !== policyDigest || expiresAt <= Date.now())
+                    throw new DodoError('FORBIDDEN', 'desktop permission/snapshot changed; capture again'); };
+                const key = digestOf({ windowId: captured.data.windowId, pid: captured.data.pid, appId: captured.data.appId, bounds: captured.data.bounds, view: converted.view });
+                let difference = null;
+                if (previous) {
+                    this.frame(actor, previous);
+                    const prior = this.frames.get(previous);
+                    const comparable = prior.comparisonKey === key;
+                    difference = { comparable, changedFraction: comparable ? await imageDifference(this.storage.get(actor, prior.frame.asset.assetId).bytes, converted.bytes) : null,
+                        note: comparable ? 'Fraction of changed cells on a 64x64 RGB grid; pixel difference is not semantic understanding or task success.' : 'Window identity/geometry/crop changed; images are not comparable.' };
+                }
+                guard();
+                const asset = this.storage.put(actor, 'image', 'image/jpeg', converted.bytes, { ttlMs: Math.max(1, expiresAt - Date.now()), guard });
+                const frame = { observationId: newId('obs'), windowId: captured.data.windowId, appId: captured.data.appId, pid: captured.data.pid, snapshotId: captured.data.snapshotId, expiresAt, capturedAt: Date.now(), asset, view: converted.view, difference };
+                this.sweep();
+                if (this.frames.size >= 32)
+                    this.frames.delete(this.frames.keys().next().value);
+                this.frames.set(frame.observationId, { frame, owner: actorKey(actor), comparisonKey: key, policyDigest });
+                out.push(frame);
+                previous = frame.observationId;
+            }
+            return { frames: out, note: 'Images are returned to the client model. This is an explicit bounded burst, not continuous background surveillance. Crop coordinates map back using view.scale/offset; desktop_action uses original snapshot image coordinates.' };
+        }
+        finally {
+            this.busy = false;
+        }
+    }
+    async readFile(actor, file, maxEdge, crop) {
+        const input = this.storage.services.wfs.readFileBytes(file, Math.min(this.storage.services.limits.readFileBytes, 16 * 1024 * 1024));
+        const image = await imageView(input.bytes, maxEdge, crop);
+        const guard = () => { const p = this.storage.services.wfs.resolve(input.rel); this.storage.services.wfs.assertRegularFileForDirectAccess(p); };
+        return { asset: this.storage.put(actor, 'image', 'image/jpeg', image.bytes, { guard }), view: image.view };
+    }
+    close() { this.frames.clear(); }
+}
+//# sourceMappingURL=images.js.map

package/dist/services/multimodal/images.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"images.js","sourceRoot":"","sources":["../../../src/services/multimodal/images.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAc,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,WAAW,EAAqC,MAAM,gBAAgB,CAAC;AAGhF,MAAM,OAAO,GAAG,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,EAAE,SAAkB,EAAE,CAAC;AAC7E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,OAAe,EAAE,IAAe;IAC7E,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI;QAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,4BAA4B,CAAC,CAAC;IACzG,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC7E,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,sBAAsB,EAAE,2DAA2D,CAAC,CAAC;IACrN,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IACvD,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,eAAe,EAAE,qCAAqC,CAAC,CAAC;IAC7J,IAAI,IAAI;QAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,sFAAsF;IACtF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;IAChL,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI;YAC9J,MAAM,EAAE,CAAC,IAAI,EAAE,KAAK,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,IAAI,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACvL,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc,EAAE,KAAa;IACjE,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IACxI,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAAC,IAAI,OAAO,GAAG,CAAC,CAAC;IACzF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,GAAG,EAAE;YAAE,OAAO,EAAE,CAAC;IAClL,OAAO,OAAO,GAAG,IAAI,CAAC;AACxB,CAAC;AAGD,MAAM,OAAO,aAAa;IAGK;IAFpB,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC7C,IAAI,GAAG,KAAK,CAAC;IACrB,YAA6B,OAAqB;QAArB,YAAO,GAAP,OAAO,CAAc;IAAG,CAAC;IAC9C,KAAK,KAAW,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;YAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzH,KAAK,CAAC,KAAY,EAAE,EAAU;QAC5B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAAC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,yDAAyD,CAAC,CAAC;QAC3I,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAAC,OAAO,KAAK,CAAC,KAAK,CAAC;IACzE,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,KAAY,EAAE,IAA4I,EAAE,KAAiB;QACzL,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAAC,IAAI,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,wCAAwC,CAAC,CAAC;QACrH,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,gBAAgB,EAAE,6CAA6C,CAAC,CAAC;QACpI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,GAAG,GAAkB,EAAE,CAAC;YAAC,IAAI,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;YAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,CAAC;oBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC1E,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAAC,KAAK,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC1G,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;gBAAC,KAAK,EAAE,CAAC;gBAC3F,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7G,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACxE,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,YAAY,IAAI,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;oBAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,oDAAoD,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzP,MAAM,GAAG,GAAG,QAAQ,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;gBACnK,IAAI,UAAU,GAA8B,IAAI,CAAC;gBACjD,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;oBAAC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;oBACtE,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,KAAK,GAAG,CAAC;oBAC/C,UAAU,GAAG,EAAE,UAAU,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;wBAC9J,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,gHAAgH,CAAC,CAAC,CAAC,mEAAmE,EAAE,CAAC;gBAChN,CAAC;gBACD,KAAK,EAAE,CAAC;gBAAC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9I,MAAM,KAAK,GAAgB,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC;gBACnQ,IAAI,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE;oBAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC,CAAC;gBAC/F,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,CAAC;gBAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAAC,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC;YAC7J,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,iOAAiO,EAAE,CAAC;QAClQ,CAAC;gBAAS,CAAC;YAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAAC,CAAC;IAClC,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,KAAY,EAAE,IAAY,EAAE,OAAe,EAAE,IAAe;QACzE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;QACpI,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/I,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7G,CAAC;IACD,KAAK,KAAW,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;CACvC"}

package/dist/services/multimodal/mediaService.js

@@ -0,0 +1,279 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DodoError } from '../../errors.js';
+import { newId } from '../../util/hash.js';
+import { resolveTrustedExecutable } from '../../platform/execResolve.js';
+import { findSpeechEngine } from '../../platform/speech.js';
+import { MediaStorage, actorKey } from './storage.js';
+import { WorkerSpec, WorkerResult, Transcript } from './contracts.js';
+export function parseSubtitles(text) {
+    const lines = text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n').split('\n');
+    const result = [];
+    const seconds = (value) => { const numbers = value.replace(',', '.').split(':').map(Number); return numbers.reduce((a, v) => a * 60 + v, 0); };
+    const timestamp = /((?:\d{1,3}:)?\d{2}:\d{2}[.,]\d{3})\s+-->\s+((?:\d{1,3}:)?\d{2}:\d{2}[.,]\d{3})/;
+    for (let i = 0; i < lines.length; i++) {
+        const match = timestamp.exec(lines[i]);
+        if (!match)
+            continue;
+        const startSec = seconds(match[1]), endSec = seconds(match[2]);
+        const content = [];
+        while (++i < lines.length && lines[i].trim() !== '')
+            content.push(lines[i]);
+        if (!Number.isFinite(startSec) || !Number.isFinite(endSec) || startSec < 0 || endSec < startSec || endSec > 86400)
+            throw new DodoError('INVALID_INPUT', 'invalid subtitle timestamps');
+        result.push({ startSec, endSec, text: content.join('\n').replace(/<[^>]*>/g, '').slice(0, 4000) });
+        if (result.length > 10000)
+            throw new DodoError('RESOURCE_LIMIT', 'subtitle cue limit exceeded');
+    }
+    if (!result.length)
+        throw new DodoError('INVALID_INPUT', 'no valid SRT/WebVTT cues found');
+    return result.sort((a, b) => a.startSec - b.startSec);
+}
+export class MediaService {
+    storage;
+    sources = new Map();
+    jobs = new Map();
+    pendingOpens = 0;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    program(name) { try {
+        return resolveTrustedExecutable(name, this.storage.services.wfs.root, { allowBatch: false });
+    }
+    catch {
+        return null;
+    } }
+    capabilities() {
+        const defaultModel = path.join(this.storage.configDir, 'models', 'ggml-tiny.bin');
+        return { ffmpeg: this.program('ffmpeg') !== null, ffprobe: this.program('ffprobe') !== null, whisper: this.program('whisper-cli') !== null || this.program('whisper-cpp') !== null,
+            defaultModelPresent: this.modelExists(defaultModel), modelLocation: 'DODO config directory/models/ggml-tiny.bin, or a guarded workspace modelFile', speechSynthesis: findSpeechEngine(this.storage.services.wfs.root) !== undefined,
+            limits: { sourceBytes: 512 * 1024 * 1024, cachedSourceBytes: 1024 * 1024 * 1024, framesPerCall: 8, audioSeconds: 120, transcriptionSeconds: 600 },
+            notes: ['No automatic model download or microphone/system audio capture.', 'Tools operate on local workspace media and synthesized speech. Cloud analysis depends on the connected client.'] };
+    }
+    modelExists(file) { try {
+        const st = fs.lstatSync(file);
+        return st.isFile() && !st.isSymbolicLink() && st.nlink === 1 && st.size > 1000000 && st.size <= 512 * 1024 * 1024;
+    }
+    catch {
+        return false;
+    } }
+    requireProgram(name) { const result = this.program(name); if (!result)
+        throw new DodoError('NOT_SUPPORTED', `${name} is not on the trusted PATH; install the local media dependency first`); return result; }
+    source(actor, id) {
+        this.storage.check();
+        const result = this.sources.get(id);
+        if (!result || result.owner !== actorKey(actor) || result.expiresAt < Date.now())
+            throw new DodoError('NOT_FOUND', 'unknown/expired media source for this client');
+        this.storage.services.wfs.resolve(result.sourcePath, { allowMissing: true });
+        return result;
+    }
+    launch(actor, operation, mediaId, input, extras = {}) {
+        this.storage.check();
+        if (this.jobs.size >= 32)
+            throw new DodoError('RESOURCE_LIMIT', 'media job limit reached; close completed media handles');
+        const ffmpeg = this.requireProgram('ffmpeg'), ffprobe = this.requireProgram('ffprobe');
+        const workerUrls = [new URL('./mediaWorker.js', import.meta.url), new URL('../../../dist/services/multimodal/mediaWorker.js', import.meta.url)];
+        const worker = workerUrls.map(url => fileURLToPath(url)).find(p => fs.existsSync(p));
+        if (!worker)
+            throw new DodoError('NOT_SUPPORTED', 'media worker is not built; run npm run build');
+        const directory = this.storage.directory();
+        try {
+            const spec = WorkerSpec.parse({ operation, input, directory, ffmpeg, ffprobe, startSec: 0, durationSec: 30, times: [], maxEdge: 1280, language: 'auto', ...extras });
+            const specFile = path.join(directory, 'spec.json');
+            fs.writeFileSync(specFile, JSON.stringify(spec), { mode: 0o600, flag: 'wx' });
+            const s = this.storage.services;
+            const job = s.jobs.start({ workspaceId: s.workspaceId, epoch: s.epoch, principal: actor.grantId, kind: 'exec', program: 'node', args: [worker, specFile], cwdRel: '.', timeoutMs: operation === 'transcribe' ? 600000 : 180000, stdin: false, network: false });
+            this.jobs.set(job.jobId, { owner: actorKey(actor), mediaId, directory, operation, sandboxed: job.sandboxed, notes: [], expiresAt: Date.now() + 1800000 });
+            return job.jobId;
+        }
+        catch (err) {
+            this.storage.removeDirectory(directory);
+            throw err;
+        }
+    }
+    async open(actor, file, waitMs) {
+        this.storage.check();
+        this.requireProgram('ffprobe');
+        this.requireProgram('ffmpeg');
+        if (this.sources.size + this.pendingOpens >= 8)
+            throw new DodoError('RESOURCE_LIMIT', 'at most eight open media sources');
+        this.pendingOpens++;
+        const directory = this.storage.directory();
+        let source;
+        try {
+            const copied = await this.storage.copySource(file, directory);
+            source = { id: newId('media'), owner: actorKey(actor), directory, ...copied, expiresAt: Date.now() + 1800000 };
+            this.sources.set(source.id, source);
+            const jobId = this.launch(actor, 'probe', source.id, source.path);
+            await this.storage.services.jobs.waitForExit(jobId, waitMs);
+            return { mediaId: source.id, sourcePath: source.sourcePath, sourceHash: source.hash, bytes: source.size, expiresAt: source.expiresAt, job: this.report(actor, jobId) };
+        }
+        catch (err) {
+            if (source) {
+                this.sources.delete(source.id);
+                this.storage.releaseSource(source.size);
+            }
+            this.storage.removeDirectory(directory);
+            throw err;
+        }
+        finally {
+            this.pendingOpens--;
+        }
+    }
+    async extract(actor, id, kind, opts) {
+        const source = this.source(actor, id);
+        if (kind === 'frames' && !opts.times.length)
+            throw new DodoError('INVALID_INPUT', 'frames requires explicit timestamps');
+        if (kind === 'audio' && opts.durationSec > 120)
+            throw new DodoError('RESOURCE_LIMIT', 'audio excerpt cap is 120 seconds');
+        const jobId = this.launch(actor, kind, source.id, source.path, { times: opts.times, startSec: opts.startSec, durationSec: opts.durationSec, maxEdge: opts.maxEdge });
+        await this.storage.services.jobs.waitForExit(jobId, opts.waitMs);
+        return this.report(actor, jobId);
+    }
+    async transcribe(actor, id, opts) {
+        const source = this.source(actor, id);
+        const whisper = this.program('whisper-cli') ?? this.program('whisper-cpp');
+        if (!whisper)
+            throw new DodoError('NOT_SUPPORTED', 'install whisper.cpp (whisper-cli/whisper-cpp) on the trusted PATH');
+        let model = path.join(this.storage.configDir, 'models', 'ggml-tiny.bin');
+        if (opts.modelFile) {
+            const resolved = this.storage.services.wfs.resolve(opts.modelFile);
+            this.storage.services.wfs.assertRegularFileForDirectAccess(resolved);
+            model = resolved.abs;
+        }
+        else {
+            const parent = path.dirname(model);
+            try {
+                if (fs.lstatSync(parent).isSymbolicLink())
+                    throw new DodoError('PATH_DENIED', 'model directory must not be a symlink');
+            }
+            catch (err) {
+                if (err instanceof DodoError)
+                    throw err;
+            }
+        }
+        if (!this.modelExists(model))
+            throw new DodoError('NOT_SUPPORTED', 'local GGML Whisper model missing or invalid; supply modelFile or install models/ggml-tiny.bin in the DODO config directory');
+        const jobId = this.launch(actor, 'transcribe', source.id, source.path, { whisper, model, startSec: opts.startSec, durationSec: opts.durationSec, language: opts.language });
+        await this.storage.services.jobs.waitForExit(jobId, opts.waitMs);
+        return this.report(actor, jobId);
+    }
+    async synthesize(actor, text, voice, waitMs) {
+        const speech = findSpeechEngine(this.storage.services.wfs.root);
+        if (!speech)
+            throw new DodoError('NOT_SUPPORTED', 'no working local speech engine or voice; run dodo setup');
+        this.requireProgram('ffmpeg');
+        this.requireProgram('ffprobe');
+        const directory = this.storage.directory(), input = path.join(directory, 'speech.txt');
+        // Disable embedded speech directives; never put private input text in argv or job logs.
+        fs.writeFileSync(input, text.replace(/\[\[/g, '[ ['), { flag: 'wx', mode: 0o600 });
+        let jobId;
+        try {
+            jobId = this.launch(actor, 'speak', null, input, { speechProgram: speech.program, speechKind: speech.kind, ...(voice ? { voice } : {}) });
+        }
+        catch (err) {
+            this.storage.removeDirectory(directory);
+            throw err;
+        }
+        this.jobs.get(jobId).inputDirectory = directory;
+        await this.storage.services.jobs.waitForExit(jobId, waitMs);
+        return this.report(actor, jobId);
+    }
+    report(actor, jobId) {
+        this.storage.check();
+        const record = this.jobs.get(jobId);
+        if (!record || record.owner !== actorKey(actor) || record.expiresAt < Date.now())
+            throw new DodoError('NOT_FOUND', 'unknown media job for this client');
+        if (record.mediaId)
+            this.source(actor, record.mediaId);
+        const job = this.storage.services.jobs.getJobChecked(jobId, this.storage.services.workspaceId);
+        let error = null;
+        if (job.status === 'exited' && job.exitCode === 0 && record.assets === undefined) {
+            try {
+                const result = WorkerResult.parse(JSON.parse(this.storage.resultFile(record.directory, 'result.json', 65536).toString('utf8')));
+                if (result.operation !== record.operation)
+                    throw new Error('operation mismatch');
+                const assets = [];
+                for (const output of result.files) {
+                    let bytes = this.storage.resultFile(record.directory, output.name);
+                    if (output.kind === 'transcript')
+                        bytes = Buffer.from(JSON.stringify(Transcript.parse({ ...JSON.parse(bytes.toString('utf8')), mediaId: record.mediaId })));
+                    const guard = record.mediaId ? () => { this.source(actor, record.mediaId); } : undefined;
+                    assets.push(this.storage.put(actor, output.kind, output.mimeType, bytes, { ...(output.timeSec !== undefined ? { timeSec: output.timeSec } : {}), ...(output.endSec !== undefined ? { endSec: output.endSec } : {}), ...(guard ? { guard } : {}) }));
+                }
+                record.assets = assets;
+                record.metadata = result.metadata;
+                record.notes = result.notes;
+                this.storage.removeDirectory(record.directory);
+                if (record.inputDirectory)
+                    this.storage.removeDirectory(record.inputDirectory);
+            }
+            catch (err) {
+                error = `Output unavailable/invalid: ${err instanceof DodoError ? err.code : 'INVALID_OUTPUT'}. Inspect the original job log; do not assume the media was read.`;
+            }
+        }
+        else if (job.status !== 'running' && !(job.status === 'exited' && job.exitCode === 0))
+            error = 'Media processor did not complete successfully; inspect job_output. No automatic retry or sandbox downgrade.';
+        return { jobId, mediaId: record.mediaId, operation: record.operation, status: error ? 'failed' : job.status, exitCode: job.exitCode, sandboxed: record.sandboxed, assets: record.assets ?? [], ...(record.metadata ? { metadata: record.metadata } : {}), notes: record.notes, error };
+    }
+    subtitles(actor, id, file, language) {
+        this.source(actor, id);
+        const input = this.storage.services.wfs.readTextFile(file, Math.min(this.storage.services.limits.readFileBytes, 2 * 1024 * 1024));
+        const segments = parseSubtitles(input.text), transcript = Transcript.parse({ source: 'sidecar_subtitles', language, mediaId: id, startSec: segments[0].startSec, endSec: Math.max(...segments.map(s => s.endSec)), segments, truncated: false });
+        return this.storage.put(actor, 'transcript', 'application/json', Buffer.from(JSON.stringify(transcript)), { guard: () => { this.source(actor, id); this.storage.services.wfs.resolve(file, { allowMissing: true }); } });
+    }
+    search(actor, assetId, query, limit) {
+        const asset = this.storage.get(actor, assetId);
+        if (asset.meta.kind !== 'transcript')
+            throw new DodoError('INVALID_INPUT', 'search needs a transcript asset');
+        const transcript = Transcript.parse(JSON.parse(asset.bytes.toString('utf8'))), terms = query.toLowerCase().trim().split(/\s+/).filter(Boolean);
+        const matches = transcript.segments.filter(s => terms.every(t => s.text.toLowerCase().includes(t)));
+        return { mediaId: transcript.mediaId, source: transcript.source, matches: matches.slice(0, limit), totalMatches: matches.length, truncated: matches.length > limit, note: 'Literal transcript search only. No visual scene understanding is claimed for unsampled frames.' };
+    }
+    async closeSource(actor, id) {
+        const source = this.source(actor, id), jobs = [...this.jobs].filter(([, j]) => j.mediaId === id);
+        for (const [jobId] of jobs) {
+            this.storage.services.jobs.cancel(jobId, this.storage.services.workspaceId, 100);
+            await this.storage.services.jobs.waitForExit(jobId, 1000);
+        }
+        for (const [jobId, job] of jobs) {
+            this.storage.removeDirectory(job.directory);
+            this.jobs.delete(jobId);
+        }
+        this.sources.delete(id);
+        this.storage.releaseSource(source.size);
+        this.storage.removeDirectory(source.directory);
+        return { closed: true, mediaId: id };
+    }
+    async sweep() {
+        const expiredSources = [...this.sources.values()].filter(s => s.expiresAt <= Date.now());
+        const expiredIds = new Set(expiredSources.map(s => s.id));
+        for (const [id, job] of this.jobs)
+            if (job.expiresAt <= Date.now() || (job.mediaId !== null && expiredIds.has(job.mediaId))) {
+                this.storage.services.jobs.cancel(id, this.storage.services.workspaceId, 100);
+                await this.storage.services.jobs.waitForExit(id, 2000);
+                this.storage.removeDirectory(job.directory);
+                if (job.inputDirectory)
+                    this.storage.removeDirectory(job.inputDirectory);
+                this.jobs.delete(id);
+            }
+        for (const source of expiredSources) {
+            this.sources.delete(source.id);
+            this.storage.releaseSource(source.size);
+            this.storage.removeDirectory(source.directory);
+        }
+    }
+    async close() {
+        for (const [jobId] of this.jobs) {
+            try {
+                this.storage.services.jobs.cancel(jobId, this.storage.services.workspaceId, 100);
+            }
+            catch { /* already finished */ }
+        }
+        await Promise.all([...this.jobs.keys()].map(id => this.storage.services.jobs.waitForExit(id, 2000).catch(() => undefined)));
+        this.jobs.clear();
+        this.sources.clear();
+    }
+}
+//# sourceMappingURL=mediaService.js.map