@2en/clawly-plugins 1.26.0-beta.0 → 1.26.0

package/tools/clawly-send-image.ts

@@ -0,0 +1,228 @@
+/**
+ * Agent tool: clawly_send_image — send an image to the user.
+ *
+ * Accepts a URL (https/http) or a local file path (or arrays of either).
+ * URLs are downloaded and cached locally. Magic-byte validation ensures
+ * only real images are served. Injects a MEDIA:<path> assistant message
+ * via chat.inject so the mobile client renders inline images.
+ */
+
+import crypto from 'node:crypto'
+import fsp from 'node:fs/promises'
+import path from 'node:path'
+
+import {fileTypeFromBuffer, fileTypeFromFile} from 'file-type'
+
+import {injectAssistantMessage, resolveSessionKey} from '../gateway/inject'
+import type {PluginApi} from '../types'
+
+const TOOL_NAME = 'clawly_send_image'
+const MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024 // 50 MB
+
+const parameters: Record<string, unknown> = {
+  type: 'object',
+  required: ['source'],
+  properties: {
+    source: {
+      oneOf: [
+        {type: 'string', description: 'Image URL (https://...) or local file path'},
+        {
+          type: 'array',
+          items: {type: 'string'},
+          description: 'Array of image URLs or local file paths',
+        },
+      ],
+    },
+    caption: {type: 'string', description: 'Optional caption text'},
+  },
+}
+
+// ── Image MIME allowlist (aligned with expo-image, excluding SVG) ────
+
+const IMAGE_MIMES = new Set([
+  'image/jpeg',
+  'image/png',
+  'image/apng',
+  'image/gif',
+  'image/webp',
+  'image/avif',
+  'image/heic',
+  'image/bmp',
+  'image/x-icon',
+])
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+function downloadsDir(api: PluginApi): string {
+  return path.join(api.runtime.state.resolveStateDir(), 'clawly', 'downloads')
+}
+
+function extFromUrl(url: string): string {
+  try {
+    const pathname = new URL(url).pathname
+    const ext = path.extname(pathname).toLowerCase()
+    if (
+      ext &&
+      ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.avif', '.heic', '.ico'].includes(ext)
+    )
+      return ext
+  } catch {}
+  return '.jpg'
+}
+
+function localPath(source: string, ext: string, api: PluginApi): string {
+  const hash = crypto.createHash('sha256').update(source).digest('hex')
+  return path.join(downloadsDir(api), `${hash}${ext}`)
+}
+
+async function fileExists(p: string): Promise<boolean> {
+  try {
+    await fsp.access(p)
+    return true
+  } catch {
+    return false
+  }
+}
+
+function formatMediaMessage(filePaths: string[], caption?: string): string {
+  const parts: string[] = []
+  if (caption) parts.push(caption)
+  for (const p of filePaths) parts.push(`MEDIA:${p}`)
+  return parts.join('\n')
+}
+
+// ── Download ─────────────────────────────────────────────────────────
+
+/** Downloads an image, validates it, and saves with the detected extension. Returns the final path or throws. */
+const DOWNLOAD_TIMEOUT_MS = 30_000
+
+async function downloadImage(url: string, api: PluginApi): Promise<string> {
+  const dir = downloadsDir(api)
+  await fsp.mkdir(dir, {recursive: true})
+
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), DOWNLOAD_TIMEOUT_MS)
+  try {
+    const res = await fetch(url, {redirect: 'follow', signal: controller.signal})
+    if (!res.ok) throw new Error(`download failed: HTTP ${res.status}`)
+
+    const contentLength = res.headers.get('content-length')
+    if (contentLength && Number(contentLength) > MAX_DOWNLOAD_BYTES) {
+      throw new Error(`file too large (${contentLength} bytes, max ${MAX_DOWNLOAD_BYTES})`)
+    }
+
+    const chunks: Uint8Array[] = []
+    let totalSize = 0
+    const reader = res.body?.getReader()
+    if (!reader) throw new Error('no response body')
+
+    while (true) {
+      const {done, value} = await reader.read()
+      if (done) break
+      totalSize += value.byteLength
+      if (totalSize > MAX_DOWNLOAD_BYTES) {
+        reader.cancel()
+        throw new Error(`file too large (exceeded ${MAX_DOWNLOAD_BYTES} bytes)`)
+      }
+      chunks.push(value)
+    }
+
+    const buffer = Buffer.concat(chunks)
+
+    const type = await fileTypeFromBuffer(buffer)
+    if (!type || !IMAGE_MIMES.has(type.mime)) {
+      throw new Error('downloaded file is not a recognized image format')
+    }
+
+    const dest = localPath(url, `.${type.ext}`, api)
+    await fsp.writeFile(dest, buffer)
+    api.logger.info(`${TOOL_NAME}: downloaded ${url} -> ${dest} (${buffer.length} bytes)`)
+    return dest
+  } finally {
+    clearTimeout(timeout)
+  }
+}
+
+// ── Tool registration ────────────────────────────────────────────────
+
+export function registerSendImageTool(api: PluginApi) {
+  api.registerTool({
+    name: TOOL_NAME,
+    description:
+      'Send one or more images to the user. Accepts a single URL/path or an array of URLs/paths. Images are displayed inline in the chat.',
+    parameters,
+    async execute(_toolCallId, params) {
+      // Normalize source to string[]
+      let sources: string[]
+      if (typeof params.source === 'string') {
+        sources = [params.source.trim()]
+      } else if (Array.isArray(params.source)) {
+        sources = params.source
+          .filter((s): s is string => typeof s === 'string')
+          .map((s) => s.trim())
+          .filter(Boolean)
+      } else {
+        sources = []
+      }
+
+      if (sources.length === 0) {
+        return {content: [{type: 'text', text: JSON.stringify({error: 'source is required'})}]}
+      }
+
+      const caption = typeof params.caption === 'string' ? params.caption.trim() : undefined
+
+      const resolveSource = async (source: string): Promise<string> => {
+        if (source.startsWith('https://') || source.startsWith('http://')) {
+          // Check cache with URL-derived extension (best guess)
+          const guessExt = extFromUrl(source)
+          const cached = localPath(source, guessExt, api)
+          if (await fileExists(cached)) {
+            api.logger.info(`${TOOL_NAME}: serving cached ${cached}`)
+            return cached
+          }
+
+          return await downloadImage(source, api)
+        }
+
+        // Local file source
+        if (!(await fileExists(source))) {
+          throw new Error(`${source}: file not found`)
+        }
+
+        const type = await fileTypeFromFile(source)
+        if (!type || !IMAGE_MIMES.has(type.mime)) {
+          throw new Error(`${source}: not a recognized image format`)
+        }
+
+        // Copy to downloads dir so the outbound endpoint can serve it
+        const dest = localPath(source, `.${type.ext}`, api)
+        if (!(await fileExists(dest))) {
+          const dir = path.dirname(dest)
+          await fsp.mkdir(dir, {recursive: true})
+          await fsp.copyFile(source, dest)
+        }
+        return dest
+      }
+
+      try {
+        const resolved = await Promise.all(sources.map(resolveSource))
+
+        // Inject MEDIA: lines as an assistant message so they appear in the chat
+        const mediaMessage = formatMediaMessage(resolved, caption)
+        const sessionKey = await resolveSessionKey('clawly', api)
+        await injectAssistantMessage({sessionKey, message: mediaMessage}, api)
+        api.logger.info(`${TOOL_NAME}: injected ${resolved.length} image(s) into session`)
+
+        return {
+          content: [{type: 'text', text: JSON.stringify({sent: true, count: resolved.length})}],
+        }
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        api.logger.error(`${TOOL_NAME}: ${msg}`)
+        return {content: [{type: 'text', text: JSON.stringify({error: msg})}]}
+      }
+    },
+  })
+
+  api.logger.info(`tool: registered ${TOOL_NAME} agent tool`)
+}

package/tools/index.ts

@@ -1,10 +1,12 @@
 import type {PluginApi} from '../types'
 import {registerIsUserOnlineTool} from './clawly-is-user-online'
 import {registerSendAppPushTool} from './clawly-send-app-push'
+import {registerSendImageTool} from './clawly-send-image'
 import {registerSendMessageTool} from './clawly-send-message'
 
 export function registerTools(api: PluginApi) {
   registerIsUserOnlineTool(api)
   registerSendAppPushTool(api)
+  registerSendImageTool(api)
   registerSendMessageTool(api)
 }