@nomac/cli 0.15.0 → 0.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nomac/cli",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "A real cloud Mac for AI agents: Xcode, the iOS Simulator, signing, TestFlight and App Store publishing, by the minute, over MCP, SSH and HTTPS.",
   "license": "UNLICENSED",
   "type": "module",

package/skill/SKILL.md

@@ -22,7 +22,8 @@ alone; or `npx simulator-use cloud start` from a terminal
 (https://nomac.app/simulator). `phone_app_run` builds from a git link, or from
 the project here: in `nomac mcp`, `local_path` sends the folder (only what
 changed); on the hosted server, `phone_upload_link` answers one command that
-sends it.
+sends it. `phone_ui_look` with `screenshot_file` saves the screen as a
+full-size PNG (in `nomac mcp`, a file here; hosted, a download link).
 
 ## Cloud Mac setup and workflow
 

package/src/phone-tools.json

@@ -160,6 +160,10 @@
      "description": "Also return a JPEG sized to points",
      "type": "boolean"
     },
+    "screenshot_file": {
+     "description": "Also save the screen at full resolution as a PNG, answered as a download link (open it before the phone stops)",
+     "type": "boolean"
+    },
     "full": {
      "description": "Also save the whole accessibility tree to a JSON file",
      "type": "boolean"

package/src/phone-tools.mjs

@@ -4,8 +4,12 @@
 // each call goes through the REST API with the account's key, as the hosted tools do (apps/web/lib/phone-mcp.ts).
 
 import { randomBytes } from "node:crypto";
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
+import { createWriteStream, readFileSync } from "node:fs";
+import { mkdtemp, rm, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import { api } from "./api.mjs";
 import { projectRoot, syncApp, syncFolder } from "./phone-sync.mjs";
 
@@ -47,6 +51,51 @@ function phoneReply(answer, tool, key) {
   return { content, ...(result.isError ? { isError: true } : {}) };
 }
 
+/** A file link in a phone's answer (fileLinkUrl in apps/web/lib/phones.ts), never the sentence's period. */
+const FILE_LINK = /https?:\/\/[^\s"'<>]+?\/api\/v1\/simulator\/files\/([A-Za-z0-9_-]+)\.[A-Za-z0-9_-]+/g;
+const KEEP_BYTES = 200 * 1024 * 1024;
+
+/** The file's name as its link carries it (the payload's n), safe for a folder. */
+function linkName(payload) {
+  try {
+    const name = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")).n;
+    return typeof name === "string" ? basename(name).replace(/[^A-Za-z0-9._-]/g, "_").replace(/^\.+/, "") || undefined : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * The files phone_ui_look saved on the phone (screenshot_file's PNG, full's tree), saved on this machine as
+ * simulator-use's own tool saves them: each link in the answer becomes the file's path. One that cannot be saved
+ * stays a link.
+ */
+async function keepFiles(reply) {
+  const links = new Map();
+  for (const c of reply.content) if (c.type === "text") for (const [link, payload] of c.text.matchAll(FILE_LINK)) links.set(link, payload);
+  if (!links.size) return reply;
+  const dir = await mkdtemp(join(tmpdir(), "nomac-phone-"));
+  const saved = new Map();
+  for (const [link, payload] of links) {
+    const file = join(dir, linkName(payload) ?? `file-${saved.size + 1}`);
+    try {
+      const response = await fetch(link, { signal: AbortSignal.timeout(120_000) });
+      if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
+      let size = 0;
+      await pipeline(Readable.fromWeb(response.body).map(chunk => {
+        size += chunk.length;
+        if (size > KEEP_BYTES) throw new Error("larger than a screenshot or a tree");
+        return chunk;
+      }), createWriteStream(file));
+      saved.set(link, file);
+    } catch {
+      await unlink(file).catch(() => undefined);
+    }
+  }
+  if (!saved.size) await rm(dir, { recursive: true, force: true });
+  return { ...reply, content: reply.content.map(c => (c.type === "text" ? { ...c, text: [...saved].reduce((text, [link, file]) => text.split(link).join(file), c.text) } : c)) };
+}
+
 /** Fetches a git repository or unpacks an upload into the phone's workspace, as the hosted tools do before a run. */
 async function place(phone, input, args, tool, key, fits) {
   const { git, ref, secret, upload, name } = input;
@@ -95,7 +144,8 @@ const simulatorTool = (listed, bare) => async input => {
   const placed = await place(phone, { git, ref, secret, upload, name }, args, listed, key, fits);
   if (placed) return placed;
   if ((git || upload || sent) && !fits(1)) return asText({ state: "running", request_key: key, ...(sent ? { sent } : {}), next: `The project is on the phone. Call ${listed} again with the same arguments and request_key ${key} to run it${sent ? " (only what changed is sent again)" : ""}.` });
-  const reply = phoneReply(await api("POST", `${phones}/${phone}/tools/${bare}`, { body: args, idempotencyKey: key }), listed, key);
+  let reply = phoneReply(await api("POST", `${phones}/${phone}/tools/${bare}`, { body: args, idempotencyKey: key }), listed, key);
+  if (bare === "ui_look" && (args.screenshot_file || args.full) && !reply.isError) reply = await keepFiles(reply);
   if (!sent) return reply;
   // What was sent comes first, as the local tool's note does; after an error, which stays the answer's first part.
   return { ...reply, content: reply.isError ? [...reply.content, { type: "text", text: sent }] : [{ type: "text", text: sent }, ...reply.content] };
@@ -167,9 +217,16 @@ const LOCAL_PATH = {
   phone_tests_run: "Instead of git or upload: the project's folder on this machine (or its .xcodeproj or .xcworkspace), sent to the phone first, only the files it lacks; path is then a folder inside it",
   phone_app_control: "install: a .app built for the Simulator on this machine, sent to the phone first",
 };
+/** What phone_ui_look saves comes to this machine, as a file. */
+const SAVED_HERE = { screenshot_file: "Also save the screen at full resolution to a PNG file on this machine; the answer gives its path" };
+const properties = (name, inputSchema) => ({
+  ...inputSchema.properties,
+  ...(name === "phone_ui_look" ? Object.fromEntries(Object.entries(SAVED_HERE).map(([key, description]) => [key, { ...inputSchema.properties[key], description }])) : {}),
+  ...(LOCAL_PATH[name] ? { local_path: { type: "string", minLength: 1, maxLength: 4096, description: LOCAL_PATH[name] } } : {}),
+});
 export const PHONE_TOOLS = LISTED.map(({ name, description, inputSchema }) => ({
   name,
   description: LOCAL_PATH[name] ? `${description} Here, local_path sends ${name === "phone_app_control" ? "a .app" : "a project folder"} from this machine in the same call.` : description,
-  inputSchema: LOCAL_PATH[name] ? { ...inputSchema, properties: { ...inputSchema.properties, local_path: { type: "string", minLength: 1, maxLength: 4096, description: LOCAL_PATH[name] } } } : inputSchema,
+  inputSchema: { ...inputSchema, properties: properties(name, inputSchema) },
   handler: name === "phone" ? cloud : name === "phone_upload_link" ? uploadLink : simulatorTool(name, bareOf(name)),
 }));