comisai 1.0.4

package/node_modules/@comis/daemon/dist/wiring/setup-channels-media.js

@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Media pipeline assembly: per-platform resolvers, CompositeResolver,
+ * resolveAttachment callback, preprocessMessage callback (link understanding,
+ * vision gating, workspace persistence), and audioPreflight callback.
+ * Extracted from setup-channels.ts to isolate the media pipeline assembly
+ * (~300 lines) into a single-concern module.
+ * @module
+ */
+import { randomUUID } from "node:crypto";
+import { isVisionCapable } from "@comis/agent";
+import { createWhatsAppResolver, createSlackResolver, createIMessageResolver, audioPreflight, } from "@comis/channels";
+import { createCompositeResolver, createMediaPersistenceService, preprocessMessage, sanitizeImageForApi, createVisionProviderRegistry, selectVisionProvider, } from "@comis/skills";
+import { getModel } from "@mariozechner/pi-ai";
+import { safePath } from "@comis/core";
+import os from "node:os";
+// ---------------------------------------------------------------------------
+// Build function
+// ---------------------------------------------------------------------------
+/**
+ * Assemble the complete media pipeline: per-platform resolvers,
+ * CompositeResolver, resolveAttachment, preprocessMessage, and audioPreflight.
+ * @param deps - Media pipeline dependencies
+ * @returns Composite resolver, attachment resolver, message preprocessor, audio preflight
+ */
+export async function buildMediaPipeline(deps) {
+    const { container, channelsLogger, adaptersByType, tgPlugin, linePlugin, ssrfFetcher, linkRunner, transcriber, maxMediaBytes, defaultAgentId, } = deps;
+    // -- Media file persistence --
+    const persistenceConfig = container.config.integrations.media.persistence;
+    const persistenceEnabled = persistenceConfig.enabled && !!deps.workspaceDirs;
+    // Create per-agent persistence services (each agent saves to its own workspace)
+    const agentPersistenceServices = new Map();
+    if (persistenceEnabled && deps.workspaceDirs) {
+        for (const [agentId, wsDir] of deps.workspaceDirs) {
+            agentPersistenceServices.set(agentId, createMediaPersistenceService({
+                workspaceDir: wsDir,
+                logger: channelsLogger,
+                maxBytes: persistenceConfig.maxFileBytes,
+            }));
+        }
+        channelsLogger.info({ agentCount: agentPersistenceServices.size }, "Media file persistence enabled");
+    }
+    // Determine if the default agent's model supports vision input.
+    const agents = container.config.agents;
+    const defaultAgentConfig = Object.values(agents)[0];
+    let defaultModelVisionCapable = false;
+    if (defaultAgentConfig) {
+        try {
+            /* eslint-disable @typescript-eslint/no-explicit-any -- pi-ai getModel requires KnownProvider/KnownModel, config stores flexible strings */
+            const resolvedModel = getModel(defaultAgentConfig.provider, defaultAgentConfig.model);
+            /* eslint-enable @typescript-eslint/no-explicit-any */
+            if (resolvedModel) {
+                defaultModelVisionCapable = isVisionCapable(resolvedModel);
+            }
+        }
+        catch {
+            // Model resolution failed -- assume no vision, use text-description fallback
+            channelsLogger.debug("Model resolution for vision check failed, defaulting to text-description path");
+        }
+    }
+    channelsLogger.debug({ defaultModelVisionCapable }, "Vision capability check complete");
+    // Build video description callback from VisionProvider registry
+    const visionConfig = container.config.integrations.media.vision;
+    let describeVideoCallback;
+    if (visionConfig.enabled) {
+        const visionRegistry = createVisionProviderRegistry({
+            secretManager: container.secretManager,
+            config: visionConfig,
+        });
+        const videoProvider = selectVisionProvider(visionRegistry, "video");
+        if (videoProvider?.describeVideo) {
+            const videoTimeoutMs = visionConfig.videoTimeoutMs;
+            const maxDescChars = visionConfig.videoMaxDescriptionChars;
+            describeVideoCallback = async (video, mimeType, prompt) => {
+                return videoProvider.describeVideo({
+                    video,
+                    prompt,
+                    mimeType,
+                });
+            };
+            channelsLogger.debug({ provider: videoProvider.id, videoTimeoutMs, maxDescChars }, "Video description callback wired");
+        }
+        else {
+            channelsLogger.debug("No video-capable vision provider found, video description disabled");
+        }
+    }
+    // Helper: attempt to get a secret, return undefined if not found
+    const getSecret = (name) => {
+        try {
+            return container.secretManager.get(name);
+        }
+        catch {
+            return undefined;
+        }
+    };
+    const channelConfig = container.config.channels;
+    // Build per-platform resolvers for platforms that have adapters.
+    // Discord and Signal resolvers are NOT registered by scheme. Their HTTPS
+    // URLs go through the SSRF fallback.
+    const platformResolvers = [];
+    const whatsappAdapter = adaptersByType.get("whatsapp");
+    if (whatsappAdapter && "getRawMessage" in whatsappAdapter) {
+        platformResolvers.push(createWhatsAppResolver({
+            getRawMessage: (id) => whatsappAdapter.getRawMessage(id),
+            maxBytes: maxMediaBytes,
+            logger: channelsLogger,
+        }));
+    }
+    const slackToken = channelConfig?.slack.botToken || getSecret("SLACK_BOT_TOKEN");
+    if (adaptersByType.has("slack") && slackToken) {
+        platformResolvers.push(createSlackResolver({ botToken: slackToken, maxBytes: maxMediaBytes, logger: channelsLogger }));
+    }
+    if (adaptersByType.has("imessage")) {
+        platformResolvers.push(createIMessageResolver({
+            allowedBasePaths: [safePath(safePath(safePath(os.homedir(), "Library"), "Messages"), "Attachments")],
+            maxBytes: maxMediaBytes,
+            logger: channelsLogger,
+        }));
+    }
+    // Telegram: resolver created from plugin handle (exposes Grammy Bot + botToken via closure)
+    if (tgPlugin) {
+        platformResolvers.push(tgPlugin.createResolver({ ssrfFetcher, maxBytes: maxMediaBytes, logger: channelsLogger }));
+    }
+    // LINE: resolver created from plugin handle (exposes BlobClient via closure)
+    if (linePlugin) {
+        platformResolvers.push(linePlugin.createResolver({ maxBytes: maxMediaBytes, logger: channelsLogger }));
+    }
+    // Create CompositeResolver with all per-platform resolvers + SSRF fallback
+    const compositeResolver = createCompositeResolver({
+        resolvers: platformResolvers,
+        ssrfFetcher,
+        maxBytes: maxMediaBytes,
+        logger: channelsLogger,
+    });
+    channelsLogger.debug({
+        resolverCount: platformResolvers.length,
+        schemes: compositeResolver.schemes,
+    }, "CompositeResolver initialized");
+    // Resolve attachment callback for media preprocessor and preflight
+    const resolveAttachment = async (att) => {
+        const result = await compositeResolver.resolve(att);
+        if (!result.ok) {
+            channelsLogger.warn({ url: att.url, err: result.error.message, hint: "Check platform resolver and network connectivity", errorKind: "network" }, "Media resolution failed");
+            return null;
+        }
+        return result.value.buffer;
+    };
+    // Build preprocessMessage callback (wraps link understanding + media resolution)
+    const preprocessMessageCallback = async (msg) => {
+        // Per-channel media processing config (all default to true when absent)
+        // Exclude healthCheck key from lookup -- it's not a channel adapter entry
+        const channelEntry = msg.channelType !== "healthCheck"
+            ? container.config.channels[msg.channelType]
+            : undefined;
+        const channelMediaConfig = channelEntry && "mediaProcessing" in channelEntry ? channelEntry.mediaProcessing : undefined;
+        // 1. Link understanding (enrich text with link content)
+        let enrichedMsg = msg;
+        if (msg.text && linkRunner && channelMediaConfig?.understandLinks !== false) {
+            const linkResult = await linkRunner.processMessage(msg.text);
+            if (linkResult.enrichedText !== msg.text) {
+                enrichedMsg = { ...msg, text: linkResult.enrichedText };
+            }
+        }
+        // 2. Media preprocessing with vision gating
+        if (enrichedMsg.attachments && enrichedMsg.attachments.length > 0) {
+            const hasImages = enrichedMsg.attachments.some((a) => a.type === "image" || a.mimeType?.startsWith("image/"));
+            // Vision gating: use native image content blocks when model supports vision,
+            // fall back to text-description via imageAnalyzer when it does not.
+            // Also respect per-channel analyzeImages toggle.
+            const imagesEnabled = channelMediaConfig?.analyzeImages !== false;
+            const visionAvailable = hasImages && defaultModelVisionCapable && imagesEnabled;
+            // Wrap resolveAttachment to intercept buffers for workspace persistence
+            const persistedFiles = [];
+            const effectiveResolve = persistenceEnabled
+                ? async (att) => {
+                    const buffer = await resolveAttachment(att);
+                    if (buffer) {
+                        // Classify attachment for subdirectory routing
+                        const mediaKind = att.mimeType?.startsWith("image/") || att.type === "image" ? "image"
+                            : att.mimeType?.startsWith("video/") || att.type === "video" ? "video"
+                                : att.mimeType?.startsWith("audio/") || att.type === "audio" ? "audio"
+                                    : "document";
+                        // Determine which agent's workspace to use
+                        const agentId = enrichedMsg.metadata?._agentId ?? defaultAgentId;
+                        const svc = agentPersistenceServices.get(agentId)
+                            ?? agentPersistenceServices.get(defaultAgentId);
+                        if (svc) {
+                            try {
+                                const persistResult = await svc.persist(buffer, {
+                                    mimeType: att.mimeType,
+                                    fileName: att.fileName,
+                                    mediaKind: mediaKind,
+                                });
+                                if (persistResult.ok) {
+                                    persistedFiles.push(persistResult.value);
+                                    channelsLogger.info({ relativePath: persistResult.value.relativePath, sizeBytes: persistResult.value.sizeBytes, mediaKind }, "Media file persisted to workspace");
+                                }
+                                else {
+                                    channelsLogger.warn({ err: persistResult.error.message, mediaKind, hint: "File persistence failed; message processing continues", errorKind: "resource" }, "Media file persistence failed");
+                                }
+                            }
+                            catch (e) {
+                                channelsLogger.warn({ err: e instanceof Error ? e.message : String(e), mediaKind, hint: "File persistence threw; message processing continues", errorKind: "resource" }, "Media file persistence error");
+                            }
+                        }
+                    }
+                    return buffer;
+                }
+                : resolveAttachment;
+            // Per-channel processor gating: disable processors when channel config says false
+            const audioEnabled = channelMediaConfig?.transcribeAudio !== false;
+            const videosEnabled = channelMediaConfig?.describeVideos !== false;
+            const documentsEnabled = channelMediaConfig?.extractDocuments !== false;
+            const result = await preprocessMessage({
+                // Gate auto-transcription on global config + per-channel toggle
+                transcriber: container.config.integrations.media.transcription.autoTranscribe && audioEnabled
+                    ? transcriber
+                    : undefined,
+                // Pass imageAnalyzer ONLY when vision is NOT available AND images are enabled
+                imageAnalyzer: visionAvailable ? undefined : (imagesEnabled ? deps.imageAnalyzer : undefined),
+                resolveAttachment: effectiveResolve,
+                maxMediaBytes,
+                logger: channelsLogger,
+                // Vision-direct path: sanitize images for API injection
+                visionAvailable,
+                sanitizeImage: visionAvailable ? async (buffer, mimeType) => {
+                    return sanitizeImageForApi(buffer, mimeType);
+                } : undefined,
+                // Video description via Gemini (or other video-capable provider) — gated per channel
+                describeVideo: videosEnabled ? describeVideoCallback : undefined,
+                maxVideoDescriptionChars: visionConfig.videoMaxDescriptionChars,
+                // Document extraction pipeline — gated per channel
+                fileExtractor: documentsEnabled ? deps.fileExtractor : undefined,
+                fileExtractionConfig: deps.fileExtractionConfig ? {
+                    maxTotalChars: deps.fileExtractionConfig.maxTotalChars,
+                } : undefined,
+            }, enrichedMsg);
+            // Emit file extraction events
+            for (const fe of result.fileExtractions) {
+                container.eventBus.emit("media:file_extracted", {
+                    fileName: fe.fileName,
+                    mimeType: fe.mimeType,
+                    chars: fe.extractedChars,
+                    truncated: fe.truncated,
+                    durationMs: fe.durationMs,
+                    timestamp: Date.now(),
+                });
+            }
+            // Store memory entries linking persisted files to text descriptions
+            if (persistedFiles.length > 0 && deps.memoryAdapter) {
+                const agentId = enrichedMsg.metadata?._agentId ?? defaultAgentId;
+                const tenantId = deps.tenantId ?? "default";
+                for (const pf of persistedFiles) {
+                    const kindLabel = pf.mediaKind === "image" ? "Photo"
+                        : pf.mediaKind === "video" ? "Video"
+                            : pf.mediaKind === "audio" ? "Audio"
+                                : "Document";
+                    // Build concise memory content with file path at the START (survives truncation)
+                    const senderInfo = enrichedMsg.senderId ?? "unknown";
+                    const channelType = enrichedMsg.channelType ?? "unknown";
+                    const content = `File: ${pf.relativePath} | [${kindLabel} received] From: ${senderInfo} via ${channelType}`;
+                    const entryId = randomUUID();
+                    try {
+                        const storeResult = await deps.memoryAdapter.store({
+                            id: entryId,
+                            tenantId,
+                            agentId,
+                            userId: senderInfo,
+                            content,
+                            trustLevel: "learned",
+                            source: { who: senderInfo, channel: channelType },
+                            tags: ["media-file", pf.mediaKind],
+                            createdAt: Date.now(),
+                        });
+                        if (storeResult.ok && deps.embeddingQueue) {
+                            deps.embeddingQueue.enqueue(entryId, content);
+                        }
+                        if (!storeResult.ok) {
+                            channelsLogger.warn({ err: storeResult.error.message, relativePath: pf.relativePath, hint: "Memory store failed for persisted file", errorKind: "resource" }, "Media memory entry store failed");
+                        }
+                    }
+                    catch (e) {
+                        channelsLogger.warn({ err: e instanceof Error ? e.message : String(e), relativePath: pf.relativePath, hint: "Memory store threw for persisted file", errorKind: "resource" }, "Media memory entry store error");
+                    }
+                    // Emit event regardless of memory store success
+                    container.eventBus.emit("media:file_persisted", {
+                        relativePath: pf.relativePath,
+                        mimeType: pf.mimeType,
+                        sizeBytes: pf.sizeBytes,
+                        mediaKind: pf.mediaKind,
+                        agentId,
+                        timestamp: Date.now(),
+                    });
+                }
+                channelsLogger.debug?.({ count: persistedFiles.length, paths: persistedFiles.map(f => f.relativePath) }, "Media persistence batch complete");
+            }
+            // Inject imageContents into message metadata for executor consumption
+            if (result.imageContents && result.imageContents.length > 0) {
+                channelsLogger.debug({ imageContentCount: result.imageContents.length }, "Vision-direct imageContents injected into message metadata");
+                return {
+                    ...result.message,
+                    metadata: {
+                        ...result.message.metadata,
+                        imageContents: result.imageContents,
+                    },
+                };
+            }
+            return result.message;
+        }
+        return enrichedMsg;
+    };
+    // Build audioPreflight callback (wraps transcriber + resolveAttachment)
+    const botNames = Object.values(container.config.agents)
+        .map((a) => a.name)
+        .filter((n) => typeof n === "string" && n.length > 0);
+    const preflightFn = transcriber
+        ? async (msg) => {
+            return audioPreflight({
+                transcriber,
+                resolveAttachment,
+                botNames,
+                logger: channelsLogger,
+            }, msg);
+        }
+        : undefined;
+    return {
+        compositeResolver,
+        resolveAttachment,
+        preprocessMessage: preprocessMessageCallback,
+        audioPreflight: preflightFn,
+    };
+}

package/node_modules/@comis/daemon/dist/wiring/setup-channels.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Channel setup orchestrator: bootstraps adapters (via setup-channels-adapters),
+ * assembles media pipeline (via setup-channels-media), wires cron delivery
+ * listener, voice response pipeline, and creates the ChannelManager.
+ * Decomposed from a single 910-line file into three focused modules:
+ * - setup-channels.ts (this file) -- orchestration (~300L)
+ * - setup-channels-adapters.ts -- per-platform adapter bootstrap (~200L)
+ * - setup-channels-media.ts -- media pipeline assembly (~300L)
+ * @module
+ */
+import type { AppContainer, ChannelPort, ChannelPluginPort, NormalizedMessage, SessionKey, TranscriptionPort, TTSPort, ImageAnalysisPort, FileExtractionPort, FileExtractionConfig, MemoryPort, QueueConfig } from "@comis/core";
+import type { ComisLogger } from "@comis/infra";
+import type { AgentExecutor, createSessionLifecycle, ActiveRunRegistry } from "@comis/agent";
+import type { createSessionStore } from "@comis/memory";
+import { type CommandQueue } from "@comis/agent";
+import { type ChannelManager, type ApprovalNotifier } from "@comis/channels";
+import type { MediaResolverPort } from "@comis/core";
+import { type SsrfGuardedFetcher } from "@comis/skills";
+import type { RpcCall } from "@comis/skills";
+import type { LinkRunner, AudioConverter, MediaTempManager, MediaSemaphore } from "@comis/skills";
+import type { ExecutionLogEntry } from "@comis/scheduler";
+import type { LifecycleReactor } from "@comis/channels";
+/** All services produced by the channel bootstrap phase. */
+export interface ChannelsResult {
+    /** Channel adapters keyed by platform type (telegram, discord, etc.). */
+    adaptersByType: Map<string, ChannelPort>;
+    /** Channel lifecycle manager (optional -- undefined when no adapters enabled). */
+    channelManager?: ChannelManager;
+    /** Composite media resolver routing to per-platform resolvers (optional -- undefined when no ssrfFetcher). */
+    compositeResolver?: MediaResolverPort;
+    /** Attachment resolver callback for media URL resolution -- used by RPC handlers. */
+    resolveAttachment: (url: string) => Promise<Buffer | null>;
+    /** Lifecycle reactors created per eligible adapter (for shutdown cleanup). */
+    lifecycleReactors: LifecycleReactor[];
+    /** Approval notifier for forwarding approval events to chat channels (optional -- undefined when no adapters enabled). */
+    approvalNotifier?: ApprovalNotifier;
+    /** Full plugin objects keyed by channel type for capabilities RPC */
+    channelPlugins: Map<string, ChannelPluginPort>;
+    /** The command queue instance for parent session TTL extension during graph execution. */
+    commandQueue?: CommandQueue;
+}
+/** Dependencies for channel adapter bootstrap. */
+export interface ChannelsDeps {
+    /** Bootstrap output: config, event bus, secret manager. */
+    container: AppContainer;
+    /** Per-agent executor instances keyed by agentId. */
+    executors: Map<string, AgentExecutor>;
+    /** Default agent ID from routing config. */
+    defaultAgentId: string;
+    /** Shared session manager across all agents. */
+    sessionManager: ReturnType<typeof createSessionLifecycle>;
+    /** Session persistence store (for getResetTriggers). */
+    sessionStore: ReturnType<typeof createSessionStore>;
+    /** Root logger (for cron delivery logs). */
+    logger: ComisLogger;
+    /** Module-bound logger for channels subsystem. */
+    channelsLogger: ComisLogger;
+    /** Link understanding runner for message text enrichment. */
+    linkRunner: LinkRunner;
+    /** SSRF-guarded fetcher for media downloads. */
+    ssrfFetcher: SsrfGuardedFetcher;
+    /** STT transcriber for audio preflight (optional -- config/key may be missing). */
+    transcriber?: TranscriptionPort;
+    /** Maximum media file size in bytes for inbound pre-check. */
+    maxMediaBytes: number;
+    /** Tool assembler passed through to channel-manager deps. Options.sessionKey threads
+     *  the inbound session's persistent FileStateTracker via SessionTrackerRegistry. Cron
+     *  delivery path (L370) intentionally omits options -- cron is heartbeat-style, no
+     *  conversation session. */
+    assembleToolsForAgent?: (agentId: string, options?: {
+        sessionKey?: import("@comis/core").SessionKey;
+    }) => Promise<any[]>;
+    /** TTS adapter for voice response pipeline (optional -- TTS may not be configured). */
+    ttsAdapter?: TTSPort;
+    /** Audio converter for MP3-to-OGG/Opus conversion (optional -- ffmpeg may be absent). */
+    audioConverter?: AudioConverter;
+    /** Media temp manager for scratch files. */
+    mediaTempManager?: MediaTempManager;
+    /** Media semaphore for concurrency limiting. */
+    mediaSemaphore?: MediaSemaphore;
+    /** Optional image analyzer for text-description fallback when model lacks vision capability. */
+    imageAnalyzer?: ImageAnalysisPort;
+    /** File extractor for document attachment processing (optional -- documents skipped when absent). */
+    fileExtractor?: FileExtractionPort;
+    /** File extraction config for budget limits and feature flags. */
+    fileExtractionConfig?: FileExtractionConfig;
+    /** Per-agent workspace directory paths (for media file persistence). */
+    workspaceDirs?: Map<string, string>;
+    /** Default agent workspace directory path. */
+    defaultWorkspaceDir?: string;
+    /** Memory adapter for storing media file references. */
+    memoryAdapter?: MemoryPort;
+    /** Default tenant ID for memory storage. */
+    tenantId?: string;
+    /** Embedding queue for new memory entries (optional). */
+    embeddingQueue?: {
+        enqueue(id: string, content: string): void;
+    };
+    /** Queue configuration for per-session serialization. When enabled, creates a CommandQueue for the ChannelManager. */
+    queueConfig?: QueueConfig;
+    /** Delivery queue for crash-safe persistence */
+    deliveryQueue?: import("@comis/core").DeliveryQueuePort;
+    /** Optional active run registry for SDK-native steer+followup inbound routing */
+    activeRunRegistry?: ActiveRunRegistry;
+    /** RPC call dispatcher for /config chat commands (deferred dispatch -- safe to pass before wireDispatch). */
+    rpcCall?: RpcCall;
+    /** Optional callback for task extraction after successful agent execution (gated by config.scheduler.tasks.enabled). */
+    onTaskExtraction?: (conversationText: string, sessionKey: string, agentId: string) => Promise<void>;
+    /** Optional callback fired after each successful inbound message processing. Used by restart continuation tracker. */
+    onMessageProcessed?: (msg: NormalizedMessage, channelType: string) => void;
+    /** Optional approval gate for /approve and /deny chat commands in inbound pipeline (APPR-CHAT). */
+    approvalGate?: import("@comis/core").ApprovalGate;
+    /** Per-agent PI session adapters for session stats/destroy in slash commands (CMD-WIRE). */
+    piSessionAdapters?: Map<string, {
+        getSessionStats(key: SessionKey): {
+            messageCount: number;
+            createdAt?: number;
+            tokens?: {
+                input: number;
+                output: number;
+                cacheRead: number;
+                cacheWrite: number;
+                total: number;
+            };
+            userMessages?: number;
+            assistantMessages?: number;
+            toolCalls?: number;
+            toolResults?: number;
+            cost?: number;
+        } | undefined;
+        destroySession(key: SessionKey): Promise<void>;
+    }>;
+    /** Per-agent cost trackers for /usage and /status cost data (CMD-WIRE). */
+    costTrackers?: Map<string, {
+        getByProvider(): Array<{
+            provider: string;
+            model: string;
+            totalTokens: number;
+            totalCost: number;
+            callCount: number;
+        }>;
+        getBySession(key: string): {
+            totalTokens: number;
+            totalCost: number;
+        };
+    }>;
+    /** Per-agent cron execution trackers for enriched JSONL entries. */
+    cronExecutionTrackers?: Map<string, {
+        record(entry: ExecutionLogEntry): Promise<void>;
+    }>;
+}
+/**
+ * Bootstrap all enabled channel adapters from config, wire cron delivery and
+ * tool audit event listeners, and create + start the ChannelManager.
+ * This is the largest single extraction due to 8 platform
+ * adapter blocks, each with credential resolution, validation, and plugin
+ * creation.
+ * @param deps - Channel bootstrap dependencies
+ * @returns Channel adapters map and optional channel manager
+ */
+export declare function setupChannels(deps: ChannelsDeps): Promise<ChannelsResult>;