@kortix/sandbox 0.4.1

package/opencode/tools/image-gen.ts

@@ -0,0 +1,342 @@
+import { tool } from "@opencode-ai/plugin";
+import Replicate from "replicate";
+import { writeFileSync, mkdirSync, readFileSync, existsSync } from "fs";
+import { resolve, basename, extname, dirname } from "path";
+
+const GENERATE_MODEL = "black-forest-labs/flux-schnell";
+const EDIT_MODEL = "black-forest-labs/flux-redux-dev";
+const UPSCALE_MODEL = "recraft-ai/recraft-crisp-upscale";
+const REMOVE_BG_MODEL = "bria/remove-background";
+
+type Action = "generate" | "edit" | "upscale" | "remove_bg";
+
+interface FileOutput {
+  url(): string | URL;
+  blob(): Promise<Blob>;
+  [Symbol.iterator](): Iterator<Uint8Array>;
+}
+
+function ensureDir(filePath: string): void {
+  mkdirSync(dirname(filePath), { recursive: true });
+}
+
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 60);
+}
+
+function generateFilename(
+  action: Action,
+  prompt?: string,
+  ext = ".webp",
+): string {
+  const ts = Date.now();
+  const label = prompt ? slugify(prompt) : action;
+  return `${label}-${ts}${ext}`;
+}
+
+async function outputToBytes(
+  output: unknown,
+): Promise<{ bytes: Buffer; url: string }> {
+  if (output && typeof output === "object" && "url" in output) {
+    const fo = output as FileOutput;
+    const blob = await fo.blob();
+    const urlVal = fo.url();
+    return {
+      bytes: Buffer.from(await blob.arrayBuffer()),
+      url: typeof urlVal === "string" ? urlVal : urlVal.toString(),
+    };
+  }
+
+  if (Array.isArray(output) && output.length > 0) {
+    return outputToBytes(output[0]);
+  }
+
+  if (typeof output === "string") {
+    if (output.startsWith("data:")) {
+      const b64 = output.split(",")[1] ?? "";
+      return { bytes: Buffer.from(b64, "base64"), url: "" };
+    }
+    if (output.startsWith("http")) {
+      const res = await fetch(output);
+      return {
+        bytes: Buffer.from(await res.arrayBuffer()),
+        url: output,
+      };
+    }
+  }
+
+  throw new Error("Unexpected model output format");
+}
+
+function loadImageAsBase64(imagePath: string): string {
+  const abs = resolve(imagePath);
+  if (!existsSync(abs)) throw new Error(`Image not found: ${abs}`);
+  const bytes = readFileSync(abs);
+  const ext = extname(abs).toLowerCase();
+  const mimeMap: Record<string, string> = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+  };
+  const mime = mimeMap[ext] ?? "image/png";
+  return `data:${mime};base64,${bytes.toString("base64")}`;
+}
+
+const SIZE_TO_ASPECT: Record<string, string> = {
+  "1024x1024": "1:1",
+  "1536x1024": "3:2",
+  "1024x1536": "2:3",
+  "1792x1024": "16:9",
+  "1024x1792": "9:16",
+};
+
+const QUALITY_MAP: Record<string, number> = {
+  low: 60,
+  medium: 80,
+  high: 95,
+};
+
+async function runGenerate(
+  replicate: Replicate,
+  prompt: string,
+  size: string,
+  quality: string,
+  outputDir: string,
+): Promise<{ path: string; url: string }> {
+  const aspectRatio = SIZE_TO_ASPECT[size] ?? "1:1";
+  const outputQuality = QUALITY_MAP[quality] ?? 80;
+
+  const output = await replicate.run(GENERATE_MODEL, {
+    input: {
+      prompt,
+      num_outputs: 1,
+      aspect_ratio: aspectRatio,
+      output_format: "webp",
+      output_quality: outputQuality,
+    },
+  });
+
+  const { bytes, url } = await outputToBytes(output);
+  const filename = generateFilename("generate", prompt, ".webp");
+  const outPath = resolve(outputDir, filename);
+  ensureDir(outPath);
+  writeFileSync(outPath, bytes);
+  return { path: outPath, url };
+}
+
+async function runEdit(
+  replicate: Replicate,
+  prompt: string,
+  imagePath: string,
+  size: string,
+  quality: string,
+  outputDir: string,
+): Promise<{ path: string; url: string }> {
+  const aspectRatio = SIZE_TO_ASPECT[size] ?? "1:1";
+  const outputQuality = QUALITY_MAP[quality] ?? 80;
+  const imageDataUrl = loadImageAsBase64(imagePath);
+
+  const output = await replicate.run(EDIT_MODEL, {
+    input: {
+      prompt,
+      redux_image: imageDataUrl,
+      num_outputs: 1,
+      aspect_ratio: aspectRatio,
+      output_format: "webp",
+      output_quality: outputQuality,
+    },
+  });
+
+  const { bytes, url } = await outputToBytes(output);
+  const filename = generateFilename("edit", prompt, ".webp");
+  const outPath = resolve(outputDir, filename);
+  ensureDir(outPath);
+  writeFileSync(outPath, bytes);
+  return { path: outPath, url };
+}
+
+async function runUpscale(
+  replicate: Replicate,
+  imagePath: string,
+  outputDir: string,
+): Promise<{ path: string; url: string }> {
+  const imageDataUrl = loadImageAsBase64(imagePath);
+
+  const output = await replicate.run(UPSCALE_MODEL, {
+    input: { image: imageDataUrl },
+  });
+
+  const { bytes, url } = await outputToBytes(output);
+  const base = basename(imagePath, extname(imagePath));
+  const filename = `${base}-upscaled-${Date.now()}.webp`;
+  const outPath = resolve(outputDir, filename);
+  ensureDir(outPath);
+  writeFileSync(outPath, bytes);
+  return { path: outPath, url };
+}
+
+async function runRemoveBg(
+  replicate: Replicate,
+  imagePath: string,
+  outputDir: string,
+): Promise<{ path: string; url: string }> {
+  const imageDataUrl = loadImageAsBase64(imagePath);
+
+  const output = await replicate.run(REMOVE_BG_MODEL, {
+    input: { image: imageDataUrl },
+  });
+
+  const { bytes, url } = await outputToBytes(output);
+  const base = basename(imagePath, extname(imagePath));
+  const filename = `${base}-nobg-${Date.now()}.png`;
+  const outPath = resolve(outputDir, filename);
+  ensureDir(outPath);
+  writeFileSync(outPath, bytes);
+  return { path: outPath, url };
+}
+
+function friendlyError(e: unknown): string {
+  const msg = e instanceof Error ? e.message : String(e);
+
+  if (msg.includes("moderation") || msg.includes("safety"))
+    return "Content was blocked by the safety filter. Try a different prompt.";
+  if (msg.includes("rate") || msg.includes("429"))
+    return "Rate limited. Wait a moment and try again.";
+  if (msg.includes("invalid") && msg.includes("image"))
+    return "The input image format is not supported. Use PNG, JPEG, or WebP.";
+  if (msg.includes("timeout"))
+    return "The operation timed out. Try again or use a simpler prompt.";
+
+  return msg;
+}
+
+export default tool({
+  description:
+    "Generate, edit, upscale, or remove backgrounds from images using AI models via Replicate. " +
+    "Actions: 'generate' (text-to-image via Flux Schnell), 'edit' (modify existing image with prompt via Flux Redux), " +
+    "'upscale' (enhance resolution via Recraft Crisp Upscale), 'remove_bg' (remove background via BRIA RMBG 2.0). " +
+    "Requires REPLICATE_API_TOKEN. You MUST specify output_dir to control where files are saved.",
+  args: {
+    action: tool.schema
+      .string()
+      .describe(
+        "Action to perform: 'generate', 'edit', 'upscale', or 'remove_bg'",
+      ),
+    prompt: tool.schema
+      .string()
+      .optional()
+      .describe(
+        "Text prompt for generate/edit. Required for 'generate' and 'edit'. " +
+          "Be specific and descriptive for best results.",
+      ),
+    image_path: tool.schema
+      .string()
+      .optional()
+      .describe(
+        "Path to input image. Required for 'edit', 'upscale', and 'remove_bg'. " +
+          "Supports PNG, JPEG, WebP.",
+      ),
+    size: tool.schema
+      .string()
+      .optional()
+      .describe(
+        "Image size for generate/edit: '1024x1024' (1:1, default), '1536x1024' (3:2 landscape), " +
+          "'1024x1536' (2:3 portrait), '1792x1024' (16:9 wide), '1024x1792' (9:16 tall)",
+      ),
+    quality: tool.schema
+      .string()
+      .optional()
+      .describe("Image quality: 'low', 'medium' (default), or 'high'"),
+    output_dir: tool.schema
+      .string()
+      .describe(
+        "Output directory where the generated image will be saved. The directory will be created if it doesn't exist. Required.",
+      ),
+  },
+  async execute(args, _context) {
+    const token = process.env.REPLICATE_API_TOKEN;
+    if (!token) return "Error: REPLICATE_API_TOKEN not set.";
+
+    const action = args.action as Action;
+    if (!["generate", "edit", "upscale", "remove_bg"].includes(action))
+      return `Error: Invalid action '${action}'. Use 'generate', 'edit', 'upscale', or 'remove_bg'.`;
+
+    if ((action === "generate" || action === "edit") && !args.prompt)
+      return `Error: 'prompt' is required for '${action}' action.`;
+
+    if (
+      (action === "edit" || action === "upscale" || action === "remove_bg") &&
+      !args.image_path
+    )
+      return `Error: 'image_path' is required for '${action}' action.`;
+
+    if (!args.output_dir)
+      return "Error: 'output_dir' is required. Specify where to save the output.";
+
+    const replicate = new Replicate({ auth: token });
+    const outputDir = resolve(args.output_dir);
+    const size = args.size ?? "1024x1024";
+    const quality = args.quality ?? "medium";
+
+    try {
+      let result: { path: string; url: string };
+
+      switch (action) {
+        case "generate":
+          result = await runGenerate(
+            replicate,
+            args.prompt!,
+            size,
+            quality,
+            outputDir,
+          );
+          break;
+        case "edit":
+          result = await runEdit(
+            replicate,
+            args.prompt!,
+            args.image_path!,
+            size,
+            quality,
+            outputDir,
+          );
+          break;
+        case "upscale":
+          result = await runUpscale(replicate, args.image_path!, outputDir);
+          break;
+        case "remove_bg":
+          result = await runRemoveBg(replicate, args.image_path!, outputDir);
+          break;
+      }
+
+      return JSON.stringify(
+        {
+          success: true,
+          action,
+          output_path: result.path,
+          replicate_url: result.url,
+          ...(args.prompt && { prompt: args.prompt }),
+          ...(args.image_path && { input_image: args.image_path }),
+        },
+        null,
+        2,
+      );
+    } catch (e) {
+      return JSON.stringify(
+        {
+          success: false,
+          action,
+          error: friendlyError(e),
+        },
+        null,
+        2,
+      );
+    }
+  },
+});

package/opencode/tools/image-search.ts

@@ -0,0 +1,190 @@
+import { tool } from "@opencode-ai/plugin";
+import Replicate from "replicate";
+
+const SERPER_IMAGES_URL = "https://google.serper.dev/images";
+const MOONDREAM_MODEL =
+  "lucataco/moondream2:72ccb656353c348c1385df54b237eeb7bfa874bf11486cf0b9473e691b662d31";
+const MOONDREAM_PROMPT =
+  "Describe this image in detail. Include any text visible in the image.";
+const IMAGE_DOWNLOAD_TIMEOUT_MS = 15_000;
+
+interface SerperImage {
+  imageUrl: string;
+  title?: string;
+  link?: string;
+  imageWidth?: number;
+  imageHeight?: number;
+}
+
+interface SerperResponse {
+  images?: SerperImage[];
+  searchParameters?: Record<string, unknown>;
+}
+
+interface EnrichedImage {
+  url: string;
+  title: string;
+  source: string;
+  width: number;
+  height: number;
+  description: string;
+}
+
+function extractImages(data: SerperResponse): EnrichedImage[] {
+  return (data.images ?? []).map((img) => ({
+    url: img.imageUrl,
+    title: img.title ?? "",
+    source: img.link ?? "",
+    width: img.imageWidth ?? 0,
+    height: img.imageHeight ?? 0,
+    description: "",
+  }));
+}
+
+async function describeImage(
+  replicate: Replicate,
+  imageUrl: string,
+): Promise<string> {
+  try {
+    const res = await fetch(imageUrl, {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      },
+      signal: AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS),
+      redirect: "follow",
+    });
+
+    if (!res.ok) return "";
+    const contentType = res.headers.get("content-type") ?? "";
+    if (!contentType.startsWith("image/")) return "";
+
+    const imageBytes = await res.arrayBuffer();
+    const b64 = Buffer.from(imageBytes).toString("base64");
+    const dataUrl = `data:${contentType};base64,${b64}`;
+
+    const output: unknown = await replicate.run(MOONDREAM_MODEL, {
+      input: { image: dataUrl, prompt: MOONDREAM_PROMPT },
+    });
+
+    if (typeof output === "string") return output.trim();
+    if (output && typeof output === "object" && Symbol.iterator in output) {
+      return Array.from(output as Iterable<unknown>)
+        .map(String)
+        .join("")
+        .trim();
+    }
+    return "";
+  } catch {
+    return "";
+  }
+}
+
+async function enrichImages(images: EnrichedImage[]): Promise<EnrichedImage[]> {
+  const replicateToken = process.env.REPLICATE_API_TOKEN;
+  if (!replicateToken || images.length === 0) return images;
+
+  const replicate = new Replicate({ auth: replicateToken });
+
+  return Promise.all(
+    images.map(async (img) => {
+      try {
+        const description = await describeImage(replicate, img.url);
+        return { ...img, description: description || img.description };
+      } catch {
+        return img;
+      }
+    }),
+  );
+}
+
+export default tool({
+  description:
+    "Search for images using the Serper Google Images API. " +
+    "Returns image URLs with titles, source pages, dimensions, and AI-generated descriptions. " +
+    "When REPLICATE_API_TOKEN is set, enriches results with Moondream2 vision descriptions. " +
+    "Supports batch queries separated by |||. " +
+    "Use specific descriptive queries including topic/brand names for best results.",
+  args: {
+    query: tool.schema
+      .string()
+      .describe(
+        "Image search query. For batch, separate with ||| (e.g. 'cats ||| dogs')",
+      ),
+    num_results: tool.schema
+      .number()
+      .optional()
+      .describe("Images per query (1-100). Default: 12"),
+    enrich: tool.schema
+      .boolean()
+      .optional()
+      .describe(
+        "Enrich images with AI descriptions via Moondream2. Requires REPLICATE_API_TOKEN. Default: true",
+      ),
+  },
+  async execute(args, _context) {
+    const apiKey = process.env.SERPER_API_KEY;
+    if (!apiKey) return "Error: SERPER_API_KEY not set.";
+
+    const numResults = Math.max(1, Math.min(args.num_results ?? 12, 100));
+    const shouldEnrich = args.enrich !== false;
+    const queries = args.query
+      .split("|||")
+      .map((q) => q.trim())
+      .filter(Boolean);
+    if (queries.length === 0) return "Error: empty query.";
+
+    const headers = {
+      "X-API-KEY": apiKey,
+      "Content-Type": "application/json",
+    };
+
+    try {
+      if (queries.length === 1) {
+        const res = await fetch(SERPER_IMAGES_URL, {
+          method: "POST",
+          headers,
+          body: JSON.stringify({ q: queries[0], num: numResults }),
+        });
+        if (!res.ok)
+          return `Error: Serper API returned ${res.status}: ${await res.text()}`;
+
+        const data = (await res.json()) as SerperResponse;
+        let images = extractImages(data);
+
+        if (images.length === 0) return `No images found for: '${queries[0]}'`;
+        if (shouldEnrich) images = await enrichImages(images);
+
+        return JSON.stringify(
+          { query: queries[0], total: images.length, images },
+          null,
+          2,
+        );
+      }
+
+      const payload = queries.map((q) => ({ q, num: numResults }));
+      const res = await fetch(SERPER_IMAGES_URL, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(payload),
+      });
+      if (!res.ok)
+        return `Error: Serper API returned ${res.status}: ${await res.text()}`;
+
+      const data = await res.json();
+      const dataArr: SerperResponse[] = Array.isArray(data) ? data : [data];
+
+      const results = await Promise.all(
+        dataArr.map(async (d, i) => {
+          let images = extractImages(d);
+          if (shouldEnrich) images = await enrichImages(images);
+          return { query: queries[i], total: images.length, images };
+        }),
+      );
+
+      return JSON.stringify({ batch_mode: true, results }, null, 2);
+    } catch (e) {
+      return `Error: ${String(e)}`;
+    }
+  },
+});

package/opencode/tools/memory-get.ts

@@ -0,0 +1,168 @@
+/**
+ * memory-get — Read a specific memory file by path.
+ *
+ * Mirrors OpenClaw's `memory_get` tool:
+ * - Path validation (restricted to .kortix/ directory)
+ * - Symlink rejection (prevents path traversal)
+ * - Line range slicing support
+ * - Non-Markdown rejection
+ *
+ * Provides safe, structured access to memory files without
+ * needing bash or the generic read tool.
+ */
+
+import { tool } from "@opencode-ai/plugin"
+import { readFile, lstat, realpath } from "node:fs/promises"
+import * as path from "node:path"
+
+const BASE_PATH = "/workspace/.kortix"
+
+const ALLOWED_EXTENSIONS = new Set([
+  ".md",
+  ".txt",
+  ".json",
+  ".yaml",
+  ".yml",
+  ".toml",
+])
+
+function isSubPath(parent: string, child: string): boolean {
+  const relative = path.relative(parent, child)
+  return !relative.startsWith("..") && !path.isAbsolute(relative)
+}
+
+export default tool({
+  description:
+    "Read a specific memory file by path. " +
+    "Restricted to files under workspace/.kortix/ for security. " +
+    "Supports reading full files or specific line ranges. " +
+    "Use memory_search to find relevant files first, then memory_get to read the full content.",
+  args: {
+    path: tool.schema.string().describe(
+      "Path to the memory file. Can be absolute (e.g., '/workspace/.kortix/MEMORY.md') " +
+        "or relative to .kortix/ (e.g., 'MEMORY.md', 'memory/decisions.md', 'journal/2025-01-15.md'). " +
+        "Must be under workspace/.kortix/.",
+    ),
+    start_line: tool.schema
+      .number()
+      .optional()
+      .describe("Start reading from this line number (1-indexed). Default: 1"),
+    lines: tool.schema
+      .number()
+      .optional()
+      .describe(
+        "Number of lines to read. Default: all. Useful for large files.",
+      ),
+  },
+  async execute(args, _ctx) {
+    // Resolve path — support both absolute and relative
+    let filePath = args.path
+    if (!path.isAbsolute(filePath)) {
+      filePath = path.join(BASE_PATH, filePath)
+    }
+    filePath = path.resolve(filePath)
+
+    // Security: must be under BASE_PATH
+    if (!isSubPath(BASE_PATH, filePath)) {
+      return JSON.stringify(
+        {
+          error: "Access denied",
+          message: `Path must be under ${BASE_PATH}. Got: ${args.path}`,
+          allowed_paths: [
+            "MEMORY.md",
+            "memory/*.md",
+            "journal/*.md",
+            "knowledge/*.md",
+            "sessions/*.md",
+          ],
+        },
+        null,
+        2,
+      )
+    }
+
+    // Security: reject symlinks that point outside BASE_PATH
+    try {
+      const stats = await lstat(filePath)
+      if (stats.isSymbolicLink()) {
+        const resolved = await realpath(filePath)
+        if (!isSubPath(BASE_PATH, resolved)) {
+          return JSON.stringify(
+            {
+              error: "Access denied",
+              message: "Symlinks pointing outside memory directory are not allowed.",
+            },
+            null,
+            2,
+          )
+        }
+      }
+    } catch {
+      return JSON.stringify(
+        {
+          error: "File not found",
+          message: `No file at: ${args.path}`,
+          suggestion: "Use memory_search to find available memory files.",
+        },
+        null,
+        2,
+      )
+    }
+
+    // Validate extension
+    const ext = path.extname(filePath).toLowerCase()
+    if (!ALLOWED_EXTENSIONS.has(ext)) {
+      return JSON.stringify(
+        {
+          error: "Invalid file type",
+          message: `Only Markdown and text files are allowed. Got: ${ext}`,
+          allowed: Array.from(ALLOWED_EXTENSIONS),
+        },
+        null,
+        2,
+      )
+    }
+
+    // Read the file
+    try {
+      const content = await readFile(filePath, "utf-8")
+      const allLines = content.split("\n")
+      const totalLines = allLines.length
+
+      // Apply line range if specified
+      const startLine = Math.max(1, args.start_line ?? 1)
+      const endLine = args.lines
+        ? Math.min(startLine + args.lines - 1, totalLines)
+        : totalLines
+      const sliced = allLines.slice(startLine - 1, endLine)
+
+      // Calculate relative path for display
+      const relativePath = path.relative(BASE_PATH, filePath)
+
+      return JSON.stringify(
+        {
+          path: relativePath,
+          absolute_path: filePath,
+          total_lines: totalLines,
+          showing: {
+            start: startLine,
+            end: endLine,
+            count: sliced.length,
+          },
+          content: sliced.join("\n"),
+        },
+        null,
+        2,
+      )
+    } catch (e) {
+      return JSON.stringify(
+        {
+          error: "Read failed",
+          message: `Failed to read ${args.path}: ${e instanceof Error ? e.message : String(e)}`,
+        },
+        null,
+        2,
+      )
+    }
+  },
+})