@pmndre/pine 0.0.0-stage → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +226 -2
  3. package/bin/pine.js +27 -0
  4. package/docs/SKILL.md +234 -0
  5. package/docs/references/agents.md +454 -0
  6. package/docs/references/coding.md +311 -0
  7. package/docs/references/computer.md +212 -0
  8. package/docs/references/configuration.md +310 -0
  9. package/docs/references/events.md +267 -0
  10. package/docs/references/mcp.md +420 -0
  11. package/docs/references/operations.md +394 -0
  12. package/docs/references/overview.md +195 -0
  13. package/docs/references/security.md +896 -0
  14. package/docs/references/setup.md +385 -0
  15. package/docs/references/sources.md +772 -0
  16. package/docs/references/stack.md +121 -0
  17. package/docs/references/tui.md +575 -0
  18. package/examples/home/.env.example +7 -0
  19. package/examples/home/USER.md +4 -0
  20. package/examples/home/agents/main/agent.json +4 -0
  21. package/examples/home/agents/main/workspace/AGENTS.md +14 -0
  22. package/examples/home/agents/main/workspace/MEMORY.md +4 -0
  23. package/examples/home/agents/main/workspace/SOUL.md +4 -0
  24. package/examples/home/models.json +62 -0
  25. package/examples/home/pine.json +34 -0
  26. package/examples/home/skills/README.txt +7 -0
  27. package/examples/launchd/com.pine.daemon.plist +32 -0
  28. package/examples/systemd/pine.service +22 -0
  29. package/mcp/browser.d.ts +99 -0
  30. package/mcp/browser.js +496 -0
  31. package/mcp/commands.d.ts +20 -0
  32. package/mcp/commands.js +88 -0
  33. package/mcp/computer.d.ts +139 -0
  34. package/mcp/computer.js +698 -0
  35. package/mcp/computer.swift +989 -0
  36. package/mcp/lib.d.ts +23 -0
  37. package/mcp/lib.js +52 -0
  38. package/mcp/web.d.ts +6 -0
  39. package/mcp/web.js +140 -0
  40. package/package.json +32 -4
  41. package/src/access.d.ts +112 -0
  42. package/src/access.js +438 -0
  43. package/src/actions.d.ts +75 -0
  44. package/src/actions.js +167 -0
  45. package/src/admin.d.ts +167 -0
  46. package/src/admin.js +625 -0
  47. package/src/agent-admin.d.ts +24 -0
  48. package/src/agent-admin.js +55 -0
  49. package/src/agents.d.ts +540 -0
  50. package/src/agents.js +2776 -0
  51. package/src/background.d.ts +29 -0
  52. package/src/background.js +100 -0
  53. package/src/cli.d.ts +2 -0
  54. package/src/cli.js +777 -0
  55. package/src/computer.d.ts +30 -0
  56. package/src/computer.js +32 -0
  57. package/src/config.d.ts +396 -0
  58. package/src/config.js +549 -0
  59. package/src/dev/claude-model.d.ts +37 -0
  60. package/src/dev/claude-model.js +204 -0
  61. package/src/env-file.d.ts +4 -0
  62. package/src/env-file.js +40 -0
  63. package/src/goals.d.ts +133 -0
  64. package/src/goals.js +316 -0
  65. package/src/http.d.ts +46 -0
  66. package/src/http.js +509 -0
  67. package/src/limiter.d.ts +32 -0
  68. package/src/limiter.js +84 -0
  69. package/src/login.d.ts +23 -0
  70. package/src/login.js +130 -0
  71. package/src/manual.d.ts +61 -0
  72. package/src/manual.js +443 -0
  73. package/src/mcp-config.d.ts +49 -0
  74. package/src/mcp-config.js +182 -0
  75. package/src/models.d.ts +90 -0
  76. package/src/models.js +274 -0
  77. package/src/net.d.ts +50 -0
  78. package/src/net.js +150 -0
  79. package/src/pine.d.ts +32 -0
  80. package/src/pine.js +134 -0
  81. package/src/programs.d.ts +123 -0
  82. package/src/programs.js +412 -0
  83. package/src/router.d.ts +56 -0
  84. package/src/router.js +164 -0
  85. package/src/sandbox-fs.d.ts +34 -0
  86. package/src/sandbox-fs.js +186 -0
  87. package/src/sandbox-linux.d.ts +96 -0
  88. package/src/sandbox-linux.js +297 -0
  89. package/src/sandbox-setup.d.ts +70 -0
  90. package/src/sandbox-setup.js +216 -0
  91. package/src/security.d.ts +238 -0
  92. package/src/security.js +660 -0
  93. package/src/skills.d.ts +67 -0
  94. package/src/skills.js +163 -0
  95. package/src/snapshots.d.ts +68 -0
  96. package/src/snapshots.js +306 -0
  97. package/src/source-types.d.ts +61 -0
  98. package/src/source-types.js +1 -0
  99. package/src/sources/command.d.ts +4 -0
  100. package/src/sources/command.js +77 -0
  101. package/src/sources/folder.d.ts +6 -0
  102. package/src/sources/folder.js +116 -0
  103. package/src/sources/ftp.d.ts +4 -0
  104. package/src/sources/ftp.js +307 -0
  105. package/src/sources/http.d.ts +4 -0
  106. package/src/sources/http.js +201 -0
  107. package/src/sources/mqtt.d.ts +18 -0
  108. package/src/sources/mqtt.js +97 -0
  109. package/src/sources/telegram.d.ts +22 -0
  110. package/src/sources/telegram.js +283 -0
  111. package/src/sources/webhook.d.ts +4 -0
  112. package/src/sources/webhook.js +52 -0
  113. package/src/sources.d.ts +344 -0
  114. package/src/sources.js +966 -0
  115. package/src/store.d.ts +99 -0
  116. package/src/store.js +277 -0
  117. package/src/timers.d.ts +83 -0
  118. package/src/timers.js +318 -0
  119. package/src/tools.d.ts +11 -0
  120. package/src/tools.js +1297 -0
  121. package/src/tui/app.d.ts +305 -0
  122. package/src/tui/app.js +3195 -0
  123. package/src/tui/client.d.ts +236 -0
  124. package/src/tui/client.js +131 -0
  125. package/src/tui/code-pane.d.ts +68 -0
  126. package/src/tui/code-pane.js +535 -0
  127. package/src/tui/highlight.d.ts +8 -0
  128. package/src/tui/highlight.js +86 -0
  129. package/src/tui/picture.d.ts +206 -0
  130. package/src/tui/picture.js +801 -0
  131. package/src/tui/run.d.ts +8 -0
  132. package/src/tui/run.js +97 -0
  133. package/src/tui/theme.d.ts +32 -0
  134. package/src/tui/theme.js +128 -0
  135. package/src/tui/undo.d.ts +5 -0
  136. package/src/tui/undo.js +50 -0
  137. package/src/tui/voice.d.ts +49 -0
  138. package/src/tui/voice.js +155 -0
  139. package/src/tui/voice.swift +223 -0
  140. package/src/types.d.ts +139 -0
  141. package/src/types.js +1 -0
  142. package/src/undo.d.ts +224 -0
  143. package/src/undo.js +465 -0
  144. package/src/update.d.ts +35 -0
  145. package/src/update.js +130 -0
  146. package/src/util.d.ts +65 -0
  147. package/src/util.js +171 -0
  148. package/src/watching.d.ts +20 -0
  149. package/src/watching.js +68 -0
  150. package/src/workspace-view.d.ts +72 -0
  151. package/src/workspace-view.js +221 -0
@@ -0,0 +1,698 @@
1
+ // Bundled MCP server "computer": operate the apps on the owner's Mac.
2
+ //
3
+ // One tool set, two ways to work:
4
+ //
5
+ // - A model that reads: app_snapshot gives a window as an outline taken from the accessibility
6
+ // tree, in which every control that can be used has a reference; app_press, app_set_text and
7
+ // app_menu act on references and menu names. Nothing has to be seen.
8
+ // - A model that sees: app_screenshot gives a picture of a window or of the screen; app_click,
9
+ // app_drag and app_scroll take positions in that picture, which are turned into points on the
10
+ // screen here (a picture is smaller than a Retina window, and a second display has its own
11
+ // place on the plane).
12
+ // - For both: app_type and app_key use the keyboard, app_open starts or brings forward, apps lists.
13
+ //
14
+ // The work on the machine is done by a helper, computer.swift, that this server builds with
15
+ // `swiftc` on first use and keeps in pine's home. Why a compiled helper and not JXA through
16
+ // `osascript`: the accessibility API is a C API of out-parameters and CoreFoundation types that
17
+ // JXA's bridge reaches only in parts and unreliably; its other route, System Events, makes one
18
+ // Apple event per property (a window of a few hundred controls takes many seconds) and needs a
19
+ // third grant (Automation) on top. A small Swift program has neither problem, needs no dependency
20
+ // besides Apple's command line tools, which a Mac that has git already has, and one that has not
21
+ // is told what to install. The helper decides nothing: which app may be operated and what a
22
+ // result says is here.
23
+ //
24
+ // Who may use it is the daemon's business (src/agents.ts): only the owner's own conversations of
25
+ // an agent whose reach is the account get this server, and the owner is asked once per app in a
26
+ // conversation. What this server adds is that an action lands in the app that was named: a
27
+ // reference belongs to it, a position lies in one of its windows, and keys are sent to its process alone.
28
+ // It does not click or type in the app pine itself runs in, where pine's questions are answered.
29
+ //
30
+ // Everything a screen shows is outside content; pine marks sessions that use it as tainted.
31
+ //
32
+ // options: { images?, imageSize?, block?: string[] }
33
+ import { execFile, spawn } from "node:child_process";
34
+ import { createHash } from "node:crypto";
35
+ import fs from "node:fs";
36
+ import os from "node:os";
37
+ import path from "node:path";
38
+ import readline from "node:readline";
39
+ import { options, serve } from "./lib.js";
40
+ import { APP_LOOK, WHOLE_SCREEN } from "../src/computer.js";
41
+ const APP = { type: "string", description: "The app's name, as apps lists it. Example: TextEdit" };
42
+ const X = { type: "number", description: "Pixels from the left edge of the last picture" };
43
+ const Y = { type: "number", description: "Pixels from the top edge of the last picture" };
44
+ const tool = (name, description, properties = {}, required = []) => ({
45
+ name,
46
+ description,
47
+ inputSchema: { type: "object", properties, required },
48
+ ...(APP_LOOK.includes(name) ? { annotations: { readOnlyHint: true } } : {}),
49
+ });
50
+ /** Written for a small local model: short, concrete, one thing each. */
51
+ export const TOOLS = [
52
+ tool("apps", "List the apps that are running on your owner's Mac, with their windows. Start here: it shows what is open and what each app is called."),
53
+ tool("app_open", "Open an app, or bring it to the front when it is already running.", { app: { type: "string", description: "The app's name. Example: Safari" } }, ["app"]),
54
+ tool("app_snapshot", "Read a window of an app as text: an outline of everything in it, with what it says. Every control that can be used has a reference like [ref=e12], for app_press and app_set_text. Take a new one after something has changed.", { app: APP, window: { type: "string", description: "Which window, when the app has several: its number or a part of its title. Default: the front one" } }, ["app"]),
55
+ tool("app_press", "Press a control by its reference from app_snapshot: a button, a checkbox, a link, a tab, a row of a list.", { app: APP, ref: { type: "string", description: "The reference from app_snapshot. Example: e12" } }, ["app", "ref"]),
56
+ tool("app_set_text", "Put text into a text field by its reference from app_snapshot. It replaces what the field held. A search or address field acts only on the return key: press it with app_key afterwards.", { app: APP, ref: { type: "string", description: "The reference from app_snapshot. Example: e12" }, text: { type: "string", description: "The text the field should hold" } }, ["app", "ref", "text"]),
57
+ tool("app_menu", 'Pick an entry from an app\'s menu bar. Example: path "File > Save". Without path it lists the menus; with the name of a menu it lists what is in it.', { app: APP, path: { type: "string", description: 'Menu names from the menu bar down, separated by " > ". Example: Format > Font > Bold' } }, ["app"]),
58
+ tool("app_screenshot", "Take a picture of an app's window, or of the whole screen when no app is given. For a model that can see pictures; to read text, app_snapshot is better. The answer says how large the picture is: app_click, app_drag and app_scroll take positions in it.", { app: APP, window: { type: "string", description: "Which window, when the app has several: its number or a part of its title. Default: the front one" }, screen: { type: "number", description: "Without app: which screen, when there are several. Default 1, the main one" } }),
59
+ tool("app_click", 'Click at a position in the last picture from app_screenshot. count 2 is a double click. button "right" opens the context menu.', { app: APP, x: X, y: Y, button: { type: "string", enum: ["left", "right"], description: "Default left" }, count: { type: "number", description: "1, 2 or 3 clicks. Default 1" } }, ["app", "x", "y"]),
60
+ tool("app_drag", "Press the mouse button at one position in the last picture, move to another and let go: to move something, to select, to pull a slider.", { app: APP, x: X, y: Y, to_x: { type: "number", description: "Where to let go: pixels from the left edge of the last picture" }, to_y: { type: "number", description: "Where to let go: pixels from the top edge of the last picture" } }, ["app", "x", "y", "to_x", "to_y"]),
61
+ tool("app_scroll", "Turn the scroll wheel over a position in the last picture.", { app: APP, x: X, y: Y, direction: { type: "string", enum: ["up", "down", "left", "right"] }, amount: { type: "number", description: "How many lines. Default 5" } }, ["app", "x", "y", "direction"]),
62
+ tool("app_type", "Type text on the keyboard into an app, where its cursor is. Put the cursor into the field first (app_press or app_click). A line break in the text presses the return key. The app may correct what is typed, as it does for a person; app_set_text writes exactly.", { app: APP, text: { type: "string", description: "The text to type" } }, ["app", "text"]),
63
+ tool("app_key", 'Press a key or a shortcut in an app. Examples: "return", "tab", "escape", "down", "cmd+s", "cmd+shift+z". Several, separated by spaces, are pressed one after another.', { app: APP, keys: { type: "string", description: "Example: cmd+a" } }, ["app", "keys"]),
64
+ ];
65
+ /** What needs eyes: left out for a model that cannot see. */
66
+ export const PICTURE_TOOLS = ["app_screenshot", "app_click", "app_drag", "app_scroll"];
67
+ /** macOS only; with a stand-in for the helper, wherever that runs. */
68
+ export const available = (o, platform = process.platform) => platform === "darwin" || Boolean(o.helper);
69
+ export function unavailable(platform = process.platform) {
70
+ const system = platform === "linux" ? "Linux" : platform === "win32" ? "Windows" : platform;
71
+ return `App control is not available on this system (${system}). It works on macOS only. Tell your owner that; do not look for another way to click or type in apps.`;
72
+ }
73
+ export function toolsOffered(o, platform = process.platform) {
74
+ if (!available(o, platform))
75
+ return [];
76
+ return TOOLS.filter((t) => !(o.block ?? []).includes(t.name) && (o.images !== false || !PICTURE_TOOLS.includes(t.name)));
77
+ }
78
+ const ROLES = {
79
+ AXButton: "button", AXCheckBox: "checkbox", AXRadioButton: "radio button", AXPopUpButton: "pop-up button", AXMenuButton: "menu button", AXTextField: "text field",
80
+ AXTextArea: "text area", AXStaticText: "text", AXImage: "image", AXLink: "link", AXGroup: "group", AXScrollArea: "scroll area", AXTable: "table", AXOutline: "list",
81
+ AXRow: "row", AXCell: "cell", AXToolbar: "toolbar", AXTabGroup: "tabs", AXSlider: "slider", AXComboBox: "combo box", AXHeading: "heading", AXWebArea: "web page",
82
+ AXList: "list", AXSheet: "dialog", AXSplitGroup: "group", AXDisclosureTriangle: "disclosure triangle", AXMenuItem: "menu entry", AXMenu: "menu", AXIncrementor: "stepper",
83
+ AXProgressIndicator: "progress bar", AXBusyIndicator: "busy indicator", AXColorWell: "color well", AXDateField: "date field", AXPopover: "popover", AXWindow: "window",
84
+ };
85
+ const SUBROLES = {
86
+ AXCloseButton: "close button", AXMinimizeButton: "minimize button", AXZoomButton: "zoom button", AXFullScreenButton: "full screen button", AXSearchField: "search field",
87
+ AXSecureTextField: "password field", AXTabButton: "tab", AXDialog: "dialog", AXSystemDialog: "dialog", AXToggle: "switch", AXSwitch: "switch",
88
+ };
89
+ /** A role as a word a person would use. */
90
+ export function roleName(role = "", subrole = "") {
91
+ return SUBROLES[subrole] ?? ROLES[role] ?? (role.replace(/^AX/, "").replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase() || "thing");
92
+ }
93
+ /** Containers that say nothing by themselves: what is in them moves up a level. */
94
+ const FLAT = new Set(["AXGroup", "AXSplitGroup", "AXScrollArea", "AXLayoutArea", "AXCell", "AXUnknown", "AXGenericElement"]);
95
+ const TICKED = new Set(["AXCheckBox", "AXRadioButton"]);
96
+ const oneLine = (s, max) => {
97
+ const t = s.replace(/\s+/g, " ").trim();
98
+ return t.length > max ? `${t.slice(0, max)}…` : t;
99
+ };
100
+ const quote = (s, max = 200) => `"${oneLine(s, max).replace(/"/g, "'")}"`;
101
+ /** What somebody would call a control: its role, and its name if it has one. */
102
+ const called = (n) => `${roleName(n.role, n.subrole)}${n.title ? ` ${quote(n.title, 60)}` : ""}`;
103
+ /**
104
+ * A window as an indented outline. Controls that can be used carry their reference; containers
105
+ * without a name are left out and their content moves up, so that a small model reads a few
106
+ * dozen lines and not the app's whole view hierarchy.
107
+ */
108
+ export function outline(root, limit = 24_000) {
109
+ const visit = (n, parent) => {
110
+ const label = oneLine(n.title || n.desc || (n.role === "AXStaticText" ? String(n.value ?? "") : "") || (n.ref ? (n.help ?? "") : ""), 100_000);
111
+ const parts = (n.children ?? []).map((c) => visit(c, label || parent)).filter((lines) => lines.length);
112
+ const kids = parts.flat();
113
+ if (n.role === "AXStaticText") {
114
+ const text = String(n.value ?? n.title ?? n.desc ?? "");
115
+ // A button's own caption, said again by the text inside it.
116
+ if (!text.trim() || oneLine(text, 100_000) === parent)
117
+ return kids;
118
+ return text.includes("\n") || text.length > 200 ? ["- text:", ...block(text, n.more), ...kids] : [`- text ${quote(text)}`, ...kids];
119
+ }
120
+ const value = n.value !== undefined && !TICKED.has(n.role) ? n.value : undefined;
121
+ if (!n.ref && !label && value === undefined) {
122
+ // A container of one thing is that thing.
123
+ if (FLAT.has(n.role) || parts.length <= 1)
124
+ return kids;
125
+ }
126
+ const states = [
127
+ ...(TICKED.has(n.role) && n.value !== undefined ? [n.value === 1 ? "checked" : n.value === 2 ? "mixed" : "not checked"] : []),
128
+ ...(n.focused ? ["focused"] : []),
129
+ ...(n.selected ? ["selected"] : []),
130
+ ...(n.disabled && n.ref ? ["disabled"] : []),
131
+ ...(n.edit && value === undefined ? [n.placeholder ? `empty, shows ${quote(n.placeholder, 60)}` : "empty"] : []),
132
+ ];
133
+ let head = `- ${roleName(n.role, n.subrole)}${label ? ` ${quote(label)}` : ""}${n.ref ? ` [ref=${n.ref}]` : ""}${states.length ? ` (${states.join(", ")})` : ""}`;
134
+ const long = typeof value === "string" && (value.includes("\n") || value.length > 200);
135
+ if (value !== undefined && !long && String(value) !== label)
136
+ head += `: ${typeof value === "string" ? quote(value) : value}`;
137
+ return [long ? `${head}, holds:` : head, ...(long ? block(String(value), n.more) : []), ...kids.map((l) => ` ${l}`)];
138
+ };
139
+ const block = (text, more) => [...text.replace(/\n+$/, "").split("\n").map((l) => ` ${l}`), ...(more ? [` [${more} more characters are not shown]`] : [])];
140
+ const lines = (root.children ?? []).flatMap((c) => visit(c, oneLine(root.title ?? "", 100_000)));
141
+ let out = "";
142
+ for (let i = 0; i < lines.length; i++) {
143
+ if (out.length + lines[i].length > limit)
144
+ return `${out}[cut: ${lines.length - i} more lines. The window holds more than fits here; scroll it or open a smaller part, then take a new app_snapshot]`;
145
+ out += `${lines[i]}\n`;
146
+ }
147
+ return out.trimEnd() || "(the window is empty)";
148
+ }
149
+ // ---- keys, windows, positions --------------------------------------------------------------------
150
+ const MODS = {
151
+ cmd: "cmd", command: "cmd", meta: "cmd", super: "cmd", win: "cmd", "⌘": "cmd", ctrl: "ctrl", control: "ctrl", "⌃": "ctrl",
152
+ alt: "alt", option: "alt", opt: "alt", "⌥": "alt", shift: "shift", "⇧": "shift", fn: "fn",
153
+ };
154
+ const KEYS = ["return", "tab", "space", "delete", "forwarddelete", "escape", "left", "right", "up", "down", "home", "end", "pageup", "pagedown", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12"];
155
+ const KEY_ALIASES = {
156
+ enter: "return", esc: "escape", backspace: "delete", del: "forwarddelete", arrowleft: "left", arrowright: "right", arrowup: "up", arrowdown: "down", pgup: "pageup", pgdn: "pagedown", pgdown: "pagedown", spacebar: "space", "↵": "return", "⎋": "escape", "⇥": "tab", "⌫": "delete",
157
+ };
158
+ /** "cmd+shift+s return" as the keys to press, in order. Throws a sentence for the model when something is not a key. */
159
+ export function parseKeys(text) {
160
+ const parts = String(text ?? "").trim().split(/[\s,]+/).filter(Boolean);
161
+ if (!parts.length)
162
+ throw new Error('Say which key: for example "return", "tab" or "cmd+s".');
163
+ return parts.map((part) => {
164
+ // "cmd++" is cmd and the plus key.
165
+ const pieces = part === "+" ? ["+"] : part.endsWith("++") ? [...part.slice(0, -2).split("+"), "+"] : part.split("+");
166
+ const last = pieces.pop() ?? "";
167
+ const mods = pieces.map((m) => {
168
+ const mod = MODS[m.toLowerCase()];
169
+ if (!mod)
170
+ throw new Error(`"${m}" in "${part}" is not a modifier key. Use cmd, ctrl, alt, shift or fn.`);
171
+ return mod;
172
+ });
173
+ const name = last.toLowerCase().replace(/[-_]/g, "");
174
+ const key = KEY_ALIASES[name] ?? (KEYS.includes(name) ? name : [...last].length === 1 ? last.toLowerCase() : undefined);
175
+ if (!key)
176
+ throw new Error(`"${last}" is not a key. Use one character, or one of: ${KEYS.join(", ")}. To type text, use app_type.`);
177
+ return { key, mods: [...new Set(mods)] };
178
+ });
179
+ }
180
+ const windowLine = (w) => `${w.index} ${quote(w.title || "(no title)", 80)}${w.minimized ? " (minimized)" : ""}`;
181
+ /** The window somebody means: by number, or by a part of its title. Nothing asked for: none, and the helper takes the front one. */
182
+ export function pickWindow(windows, want) {
183
+ if (want === undefined || want === null || String(want).trim() === "")
184
+ return undefined;
185
+ const text = String(want).trim();
186
+ const found = /^\d+$/.test(text) ? windows.find((w) => w.index === Number(text)) : (windows.find((w) => w.title.toLowerCase() === text.toLowerCase()) ?? windows.find((w) => w.title.toLowerCase().includes(text.toLowerCase())));
187
+ if (!found)
188
+ throw new Error(windows.length ? `There is no window "${text}". The windows are: ${windows.map(windowLine).join(", ")}.` : "The app has no window open.");
189
+ return found;
190
+ }
191
+ /**
192
+ * A position in a picture as a point on the screen. The picture may be smaller than what it
193
+ * shows (a Retina window has twice the pixels, and pictures are scaled down for the model), and
194
+ * what it shows may lie anywhere on the plane of the displays, so both are a matter of ratio and offset.
195
+ */
196
+ export function toScreen(p, px, py) {
197
+ const round = (n) => Math.round(n * 10) / 10;
198
+ return { x: round(p.x + ((px + 0.5) * p.w) / p.width), y: round(p.y + ((py + 0.5) * p.h) / p.height) };
199
+ }
200
+ class HelperError extends Error {
201
+ code;
202
+ constructor(code, message) {
203
+ super(message);
204
+ this.code = code;
205
+ }
206
+ }
207
+ export const BUILDING = "pine is building its helper for app control. That happens once and takes about half a minute. Try again in a moment.";
208
+ /** What the agent is told when macOS has not been told to let pine in. Plain words, and where to click. */
209
+ export function grantText(kind, host, shown) {
210
+ const who = host ? `"${host}", the app pine runs in` : "the app pine runs in (the terminal app pine was started from)";
211
+ if (kind === "accessibility") {
212
+ return `macOS does not let pine operate apps yet. Your owner has to allow it, once: open System Settings > Privacy & Security > Accessibility and switch on ${who}.${shown ? " macOS has just put a window on their screen that leads there." : ""} Nothing was done. Tell your owner this, and try again when they say it is done.`;
213
+ }
214
+ return `macOS does not let pine take pictures of the screen yet. Your owner has to allow it, once: open System Settings > Privacy & Security > Screen & System Audio Recording (on older systems: Screen Recording) and switch on ${who}. macOS then wants that app quit and opened again, so pine has to be started again too.${shown ? " macOS has just put a window on their screen that leads there." : ""} Tell your owner this. Until then you can read windows with app_snapshot, which needs no picture.`;
215
+ }
216
+ /**
217
+ * The tools, on top of a helper that is started when the first one is used. Calls run one after
218
+ * another: there is one mouse and one keyboard.
219
+ */
220
+ export function computer(o, io) {
221
+ let child;
222
+ let starting;
223
+ /** Requests a helper has not answered yet, with the helper they were sent to. */
224
+ const waiting = new Map();
225
+ let ids = 0;
226
+ /** The app pine runs in, once asked for. */
227
+ let host;
228
+ /** The last picture the model was given, and what it showed. */
229
+ let picture;
230
+ /** macOS shows its own request once for each grant; after that it is the agent who says it. */
231
+ const prompted = new Set();
232
+ let queue = Promise.resolve();
233
+ const wire = (c) => {
234
+ let last = "";
235
+ c.stderr?.on("data", (d) => (last = `${last}${d}`.slice(-600)));
236
+ readline.createInterface({ input: c.stdout }).on("line", (line) => {
237
+ try {
238
+ const msg = JSON.parse(line);
239
+ waiting.get(msg.id)?.answer(msg);
240
+ }
241
+ catch {
242
+ // not an answer
243
+ }
244
+ });
245
+ c.stdin?.on("error", () => { });
246
+ c.on("exit", () => {
247
+ if (child === c)
248
+ child = undefined;
249
+ const why = last.trim().split("\n").at(-1);
250
+ // Only what was asked of this helper: a new one may be at work already.
251
+ for (const w of [...waiting.values()])
252
+ if (w.to === c)
253
+ w.answer({ ok: false, code: "ended", error: `pine's helper for app control stopped${why ? `: ${why.slice(0, 300)}` : ""}. Try once more.` });
254
+ });
255
+ };
256
+ const start = () => {
257
+ if (child)
258
+ return Promise.resolve(child);
259
+ starting ??= io.launch().then((c) => {
260
+ wire(c);
261
+ child = c;
262
+ starting = undefined;
263
+ return c;
264
+ }, (e) => {
265
+ starting = undefined;
266
+ throw e;
267
+ });
268
+ // A call must not wait longer than a tool call may take. The build goes on, and the next call finds it done.
269
+ let timer;
270
+ const late = new Promise((_, reject) => (timer = setTimeout(() => reject(new Error(BUILDING)), io.patience ?? 20_000)));
271
+ starting.catch(() => { });
272
+ return Promise.race([starting, late]).finally(() => clearTimeout(timer));
273
+ };
274
+ const stop = () => {
275
+ const c = child;
276
+ child = undefined;
277
+ c?.kill();
278
+ };
279
+ const ask = async (cmd, params = {}, ms = 20_000) => {
280
+ const c = await start();
281
+ return new Promise((resolve, reject) => {
282
+ const id = ++ids;
283
+ const timer = setTimeout(() => {
284
+ waiting.delete(id);
285
+ // Whatever it hangs on, a new helper does not.
286
+ if (child === c)
287
+ stop();
288
+ reject(new Error("The app did not answer in time. It may be busy or showing a dialog. Look at it again (app_snapshot) before you go on."));
289
+ }, ms);
290
+ waiting.set(id, {
291
+ to: c,
292
+ answer(msg) {
293
+ clearTimeout(timer);
294
+ waiting.delete(id);
295
+ if (msg.ok)
296
+ resolve(msg);
297
+ else
298
+ reject(new HelperError(String(msg.code ?? "failed"), String(msg.error ?? "")));
299
+ },
300
+ });
301
+ c.stdin.write(`${JSON.stringify({ ...params, id, cmd })}\n`);
302
+ });
303
+ };
304
+ const hostApp = async () => {
305
+ if (host === undefined)
306
+ host = (await ask("status")).host ?? null;
307
+ return host ?? undefined;
308
+ };
309
+ /** A failure of the helper as a sentence the agent can act on. */
310
+ const explain = async (e) => {
311
+ if (!(e instanceof HelperError))
312
+ return e instanceof Error ? e : new Error(String(e));
313
+ if (e.code === "accessibility" || e.code === "screen") {
314
+ let shown = false;
315
+ let name;
316
+ try {
317
+ name = (await hostApp())?.name;
318
+ if (!prompted.has(e.code)) {
319
+ prompted.add(e.code);
320
+ await ask("status", { ask: e.code });
321
+ shown = true;
322
+ }
323
+ }
324
+ catch {
325
+ // the sentence below still says what to do
326
+ }
327
+ // A helper that was started before the owner allowed it does not see the grant: the next call starts a new one.
328
+ stop();
329
+ return new Error(grantText(e.code, name, shown));
330
+ }
331
+ if (e.code === "ended")
332
+ return new Error(e.message);
333
+ if (e.code === "gone")
334
+ return new Error("That reference does not exist any more: the window has changed. Take a new app_snapshot and use the references from it.");
335
+ if (e.code === "not_running")
336
+ return new Error("The app is not running any more. Open it again with app_open.");
337
+ if (e.code === "minimized")
338
+ return new Error("That window is put away in the Dock, so there is nothing to take a picture of. Bring the app to the front with app_open first.");
339
+ return new Error(e.message ? `${e.message[0].toUpperCase()}${e.message.slice(1)}${/[.!?]$/.test(e.message) ? "" : "."}` : "That did not work.");
340
+ };
341
+ const appName = (args) => {
342
+ const name = typeof args.app === "string" ? args.app.trim() : "";
343
+ if (!name)
344
+ throw new Error("Say which app: app is its name, as apps lists it.");
345
+ return name;
346
+ };
347
+ /** The running app a call names. `acting`: the call clicks, types or presses, which pine does not do in the app it runs in itself. */
348
+ const find = async (args, acting) => {
349
+ const name = appName(args);
350
+ const a = await ask("find", { app: name });
351
+ if (!a.running)
352
+ throw new Error(`${name} is not running. Open it with app_open, or look at what is running with apps.`);
353
+ if (acting && a.host)
354
+ throw new Error(`${a.name} is the app pine itself runs in. pine does not click, type or press there: that is where your owner answers pine's questions. You can look at it (app_snapshot${o.images === false ? "" : ", app_screenshot"}). To run a command, use your own shell.`);
355
+ return a;
356
+ };
357
+ const toFront = async (a, window) => {
358
+ const r = await ask("activate", { pid: a.pid, ...(window ? { ref: window } : {}) });
359
+ if (!r.front)
360
+ throw new Error(`${a.name} could not be brought to the front, so nothing was done in it. Another app may be holding the screen (a dialog, a full screen app). Tell your owner.`);
361
+ };
362
+ /** That a point on the screen is in the app that was named, and what is there. */
363
+ const landsIn = async (a, at) => {
364
+ const there = await ask("at", at);
365
+ if (there.pid === a.pid || there.wpid === a.pid)
366
+ return there.pid === a.pid ? there : {};
367
+ const other = there.name ?? there.wname;
368
+ throw new Error(other
369
+ ? `At that position is ${other}, not ${a.name}, so nothing was done. Take a new app_screenshot to see where things are now. To act in ${other}, name it as app.`
370
+ : `Nothing of ${a.name} is at that position, so nothing was done. Take a new app_screenshot to see where things are now.`);
371
+ };
372
+ /** A position in the last picture as a point on the screen, in the app that was named. */
373
+ const place = async (a, x, y, what = "x and y") => {
374
+ if (!picture)
375
+ throw new Error(`${what} are positions in a picture, and you have not taken one yet. Take one with app_screenshot first. Without pictures, use app_snapshot and app_press.`);
376
+ const px = Number(x);
377
+ const py = Number(y);
378
+ if (!Number.isFinite(px) || !Number.isFinite(py))
379
+ throw new Error(`${what} must be numbers: pixels in the last picture, from its top left corner.`);
380
+ if (px < 0 || py < 0 || px >= picture.width || py >= picture.height)
381
+ throw new Error(`That position is outside the last picture, which is ${picture.width} by ${picture.height} pixels: x goes from 0 to ${picture.width - 1}, y from 0 to ${picture.height - 1}.`);
382
+ if (picture.pid !== undefined && picture.pid !== a.pid)
383
+ throw new Error(`The last picture shows ${picture.app}, not ${a.name}. Take a picture of ${a.name} first (app_screenshot), then give positions in that.`);
384
+ let shown = picture;
385
+ if (picture.ref) {
386
+ // The window may have been moved since the picture was taken.
387
+ try {
388
+ const f = await ask("frame", { ref: picture.ref });
389
+ // What the picture shows starts where it started in the window: beside it, when a menu hung out of it.
390
+ shown = { ...picture, x: f.x + picture.dx, y: f.y + picture.dy };
391
+ }
392
+ catch {
393
+ throw new Error("The window in the last picture is gone. Take a new app_screenshot.");
394
+ }
395
+ }
396
+ return toScreen(shown, px, py);
397
+ };
398
+ const windowsOf = async (a) => (await ask("windows", { pid: a.pid })).windows ?? [];
399
+ const others = (all, shown) => {
400
+ const rest = all.filter((w) => w.index !== shown?.index);
401
+ return rest.length ? ` Other windows: ${rest.map(windowLine).join(", ")}.` : "";
402
+ };
403
+ const tools = {
404
+ async apps() {
405
+ const r = await ask("apps");
406
+ const list = r.apps ?? [];
407
+ const lines = [...list.filter((a) => a.front), ...list.filter((a) => !a.front)].map((a) => {
408
+ const wins = a.windows;
409
+ const marks = [a.front ? "in front" : "", a.host ? "pine runs in it" : ""].filter(Boolean).join(", ");
410
+ return `- ${a.name}${marks ? ` (${marks})` : ""}${wins === undefined ? "" : wins.length ? `: ${wins.length === 1 ? "window" : "windows"} ${wins.map(windowLine).join(", ")}` : ": no window"}`;
411
+ });
412
+ const screens = r.displays ?? [];
413
+ const note = r.accessibility === false ? `\n\nThe windows are not listed: ${grantText("accessibility", (await hostApp())?.name, false)}` : "";
414
+ return `Running apps:\n${lines.join("\n") || "(none)"}${screens.length > 1 ? `\n\nScreens: ${screens.map((s) => `${s.index} (${s.w} by ${s.h}${s.main ? ", main" : ""})`).join(", ")}` : ""}${note}`;
415
+ },
416
+ async app_open(args) {
417
+ const name = appName(args).replace(/\.app$/i, "");
418
+ if (/[/\\]/.test(name))
419
+ throw new Error("Give the app's name, for example Safari, not a path to it.");
420
+ let a;
421
+ try {
422
+ a = await ask("open", { app: name }, 25_000);
423
+ }
424
+ catch (e) {
425
+ if (e instanceof HelperError && e.code === "not_found")
426
+ throw new Error(`No app called "${name}" was found on this Mac. apps lists what is running.`);
427
+ throw e;
428
+ }
429
+ const was = a.launched ? "was opened" : "was already running";
430
+ try {
431
+ await ask("activate", { pid: a.pid });
432
+ }
433
+ catch (e) {
434
+ throw new Error(`${a.name} ${was}, but it could not be brought to the front. ${(await explain(e)).message}`);
435
+ }
436
+ // Its windows are listed where macOS lets pine see them; where not, the next call says what is missing.
437
+ const all = await windowsOf(a).catch(() => undefined);
438
+ return `${a.name} ${was} and is in front${a.name.toLowerCase() === name.toLowerCase() ? "" : ` (you asked for "${name}"; in other calls either name works)`}.${all ? ` ${all.length ? `Its ${all.length === 1 ? "window" : "windows"}: ${all.map(windowLine).join(", ")}` : "It has no window open"}.` : ""} Read a window with app_snapshot${o.images === false ? "" : ", or look at it with app_screenshot"}.`;
439
+ },
440
+ async app_snapshot(args) {
441
+ const a = await find(args, false);
442
+ const want = pickWindow(await windowsOf(a), args.window);
443
+ const r = await ask("tree", { pid: a.pid, ...(want ? { window: want.index } : {}) });
444
+ const all = r.windows ?? [];
445
+ if (!r.window || !r.root)
446
+ return `${a.name} has no window open.${others(all)} Its menu bar can still be used (app_menu), and app_open brings it to the front.`;
447
+ const shown = r.window;
448
+ return `${a.name}, window ${windowLine(shown)}.${others(all, shown)}\nControls with [ref=...] can be used: app_press presses one, app_set_text writes into a text field.\n\n${outline(r.root)}${r.cut ? "\n[The window holds more controls than are listed here.]" : ""}`;
449
+ },
450
+ async app_press(args) {
451
+ const a = await find(args, true);
452
+ const ref = String(args.ref ?? "").trim().replace(/^\[?ref=|\]$/g, "");
453
+ if (!ref)
454
+ throw new Error("Say which control: ref is its reference from app_snapshot, for example e12.");
455
+ const it = await ask("whose", { ref });
456
+ if (it.pid !== a.pid)
457
+ throw new Error(`The reference ${ref} is not a control of ${a.name}. Take an app_snapshot of ${a.name} and use a reference from it.`);
458
+ const r = await ask("press", { ref });
459
+ if (!r.done) {
460
+ // Nothing to press by name: click in its middle, as a person would.
461
+ await toFront(a);
462
+ await landsIn(a, { x: r.x, y: r.y });
463
+ await ask("click", { x: r.x, y: r.y });
464
+ return `Clicked the ${called(it)} in ${a.name}. Take a new app_snapshot to see what changed.`;
465
+ }
466
+ return `${r.selected ? "Selected" : "Pressed"} the ${called(it)} in ${a.name}.${r.unsure ? " The app has not answered yet, which often means a dialog has opened." : ""} Take a new app_snapshot to see what changed.`;
467
+ },
468
+ async app_set_text(args) {
469
+ const a = await find(args, true);
470
+ const ref = String(args.ref ?? "").trim().replace(/^\[?ref=|\]$/g, "");
471
+ if (!ref)
472
+ throw new Error("Say which field: ref is its reference from app_snapshot, for example e12.");
473
+ if (typeof args.text !== "string")
474
+ throw new Error("Say what to write: text is what the field should hold.");
475
+ const it = await ask("whose", { ref });
476
+ if (it.pid !== a.pid)
477
+ throw new Error(`The reference ${ref} is not a control of ${a.name}. Take an app_snapshot of ${a.name} and use a reference from it.`);
478
+ const r = await ask("set", { ref, text: args.text });
479
+ if (!r.done) {
480
+ // A field that cannot be written into directly: click it, select what is there, type over it.
481
+ await toFront(a);
482
+ await landsIn(a, { x: r.x, y: r.y });
483
+ await ask("click", { x: r.x, y: r.y });
484
+ await ask("key", { key: "a", mods: ["cmd"], pid: a.pid });
485
+ await ask("type", { text: args.text, pid: a.pid });
486
+ }
487
+ return `The ${called(it)} in ${a.name} now holds ${quote(args.text, 80)}.`;
488
+ },
489
+ async app_menu(args) {
490
+ const names = String(args.path ?? "").split(">").map((s) => s.trim()).filter(Boolean);
491
+ // Looking into a menu changes nothing; picking from one is an action like any other.
492
+ const a = await find(args, false);
493
+ const same = (title, name) => title.replace(/(…|\.\.\.)$/, "").trim().toLowerCase() === name.replace(/(…|\.\.\.)$/, "").trim().toLowerCase();
494
+ const show = (items) => items.map((i) => `- ${i.title}${i.mark ? ` (${i.mark === "✓" ? "ticked" : i.mark})` : ""}${i.disabled ? " (not available now)" : ""}${i.submenu ? " >" : ""}${i.shortcut ? ` ${i.shortcut}` : ""}`).join("\n");
495
+ // The first entry of the menu bar is the Apple menu, which belongs to the system.
496
+ let items = ((await ask("menu", { pid: a.pid })).items ?? []).slice(1);
497
+ if (!names.length)
498
+ return `The menus of ${a.name}:\n${show(items) || "(none)"}\nGive path to look into one or to pick an entry. Example: path "File > Save".`;
499
+ // An app in the background greys out most of its menus: they say what can be done once it is in front.
500
+ await toFront(a);
501
+ let entry;
502
+ for (let i = 0; i < names.length; i++) {
503
+ entry = items.find((x) => same(x.title, names[i]));
504
+ const where = i === 0 ? `the menu bar of ${a.name}` : `"${names.slice(0, i).join(" > ")}"`;
505
+ if (!entry)
506
+ throw new Error(`There is no "${names[i]}" in ${where}. There is:\n${show(items) || "(nothing)"}`);
507
+ if (i < names.length - 1 || entry.submenu) {
508
+ if (!entry.submenu)
509
+ throw new Error(`"${names.slice(0, i + 1).join(" > ")}" is an entry, not a menu: there is nothing below it.`);
510
+ items = (await ask("menu", { pid: a.pid, ref: entry.ref })).items ?? [];
511
+ }
512
+ }
513
+ if (entry.submenu)
514
+ return `"${names.join(" > ")}" in ${a.name} holds:\n${show(items) || "(nothing)"}`;
515
+ if (entry.disabled)
516
+ throw new Error(`"${names.join(" > ")}" is not available right now (it is greyed out). Something has to be selected or open first.`);
517
+ await find(args, true);
518
+ const r = await ask("press", { ref: entry.ref });
519
+ return `Picked "${names.join(" > ")}" in ${a.name}.${r.unsure ? " The app has not answered yet, which often means a dialog has opened." : ""}`;
520
+ },
521
+ async app_screenshot(args) {
522
+ const size = { edge: Math.max(320, Math.min(4000, Number(o.imageSize) || 1280)), area: 1_150_000, dir: io.scratch };
523
+ let r;
524
+ let what;
525
+ let of = {};
526
+ if (typeof args.app === "string" && args.app.trim()) {
527
+ const a = await find(args, false);
528
+ const all = await windowsOf(a);
529
+ const w = pickWindow(all, args.window) ?? all.find((x) => x.main && !x.minimized) ?? all.find((x) => !x.minimized) ?? all[0];
530
+ if (!w)
531
+ throw new Error(`${a.name} has no window open, so there is nothing to take a picture of. app_open brings it to the front.`);
532
+ const shoot = () => ask("shot", { ...size, pid: a.pid, ref: w.ref });
533
+ try {
534
+ r = await shoot();
535
+ }
536
+ catch (e) {
537
+ if (!(e instanceof HelperError) || e.code !== "behind")
538
+ throw e;
539
+ // Something hangs over the window (a sheet, a bubble), so the picture is of that part of the
540
+ // screen, and the screen shows the app only when the app is in front.
541
+ await toFront(a, w.ref);
542
+ r = await shoot();
543
+ }
544
+ what = `${a.name}, window ${windowLine(w)}${r.more ? ", and what is open over it" : ""}`;
545
+ of = { pid: a.pid, app: a.name, ref: w.ref, dx: r.x - (r.wx ?? r.x), dy: r.y - (r.wy ?? r.y) };
546
+ }
547
+ else {
548
+ r = await ask("shot", { ...size, ...(args.screen !== undefined ? { display: Number(args.screen) } : {}) });
549
+ what = `${WHOLE_SCREEN}${r.displays > 1 ? ` ${r.display} of ${r.displays}` : ""}`;
550
+ }
551
+ let bytes;
552
+ try {
553
+ bytes = fs.readFileSync(r.file);
554
+ }
555
+ finally {
556
+ fs.rmSync(r.file, { force: true });
557
+ }
558
+ picture = { width: r.width, height: r.height, x: r.x, y: r.y, w: r.w, h: r.h, dx: 0, dy: 0, ...of };
559
+ const text = `Picture of ${what}: ${r.width} by ${r.height} pixels. For app_click, app_drag and app_scroll, x and y are pixels in this picture, counted from its top left corner (x from 0 to ${r.width - 1}, y from 0 to ${r.height - 1}).`;
560
+ return [{ type: "text", text }, { type: "image", data: bytes.toString("base64"), mimeType: bytes[0] === 0x89 ? "image/png" : "image/jpeg" }];
561
+ },
562
+ async app_click(args) {
563
+ const a = await find(args, true);
564
+ const at = await place(a, args.x, args.y);
565
+ await toFront(a, picture?.ref);
566
+ const there = await landsIn(a, at);
567
+ const count = Math.max(1, Math.min(3, Math.round(Number(args.count) || 1)));
568
+ const right = args.button === "right";
569
+ await ask("click", { ...at, count, button: right ? "right" : "left" });
570
+ return `${right ? "Right-clicked" : count === 2 ? "Double-clicked" : count === 3 ? "Triple-clicked" : "Clicked"}${there.role ? ` the ${called(there)}` : ""} at ${Math.round(Number(args.x))}, ${Math.round(Number(args.y))} in ${a.name}. Take a new app_screenshot to see what changed.`;
571
+ },
572
+ async app_drag(args) {
573
+ const a = await find(args, true);
574
+ const from = await place(a, args.x, args.y);
575
+ const to = await place(a, args.to_x, args.to_y, "to_x and to_y");
576
+ await toFront(a, picture?.ref);
577
+ await landsIn(a, from);
578
+ await ask("drag", { ...from, x2: to.x, y2: to.y });
579
+ return `Dragged from ${Math.round(Number(args.x))}, ${Math.round(Number(args.y))} to ${Math.round(Number(args.to_x))}, ${Math.round(Number(args.to_y))} in ${a.name}. Take a new app_screenshot to see what changed.`;
580
+ },
581
+ async app_scroll(args) {
582
+ const a = await find(args, true);
583
+ const at = await place(a, args.x, args.y);
584
+ const lines = Math.max(1, Math.min(100, Math.round(Number(args.amount) || 5)));
585
+ const turn = { up: { dx: 0, dy: lines }, down: { dx: 0, dy: -lines }, left: { dx: lines, dy: 0 }, right: { dx: -lines, dy: 0 } };
586
+ const d = turn[String(args.direction)];
587
+ if (!d)
588
+ throw new Error("direction is up, down, left or right.");
589
+ await toFront(a, picture?.ref);
590
+ await landsIn(a, at);
591
+ await ask("scroll", { ...at, ...d });
592
+ return `Scrolled ${args.direction} by ${lines} lines in ${a.name}. Take a new app_screenshot to see where you are.`;
593
+ },
594
+ async app_type(args) {
595
+ const a = await find(args, true);
596
+ if (typeof args.text !== "string" || !args.text)
597
+ throw new Error("Say what to type: text is the text.");
598
+ await toFront(a);
599
+ // The keys are sent to this app and to no other, whatever comes to the front meanwhile.
600
+ await ask("type", { text: args.text, pid: a.pid }, 25_000);
601
+ return `Typed ${[...args.text].length} characters into ${a.name}.`;
602
+ },
603
+ async app_key(args) {
604
+ const keys = parseKeys(args.keys);
605
+ const a = await find(args, true);
606
+ await toFront(a);
607
+ for (const k of keys)
608
+ await ask("key", { ...k, pid: a.pid });
609
+ return `Pressed ${keys.map((k) => [...k.mods, k.key].join("+")).join(", then ")} in ${a.name}.`;
610
+ },
611
+ };
612
+ return {
613
+ call(name, args) {
614
+ const run = async () => {
615
+ const fn = tools[name];
616
+ if (!fn || !toolsOffered(o, "darwin").some((t) => t.name === name))
617
+ throw new Error(`The tool ${name} is not available.`);
618
+ try {
619
+ return await fn(args ?? {});
620
+ }
621
+ catch (e) {
622
+ throw await explain(e);
623
+ }
624
+ };
625
+ const next = queue.then(run, run);
626
+ queue = next.catch(() => { });
627
+ return next;
628
+ },
629
+ close: stop,
630
+ };
631
+ }
632
+ // ---- the helper ----------------------------------------------------------------------------------
633
+ export const HELPER_SOURCE = path.join(import.meta.dirname, "computer.swift");
634
+ export const NO_COMPILER = "App control needs Apple's command line developer tools, once, to build its helper, and they are not installed on this Mac. Your owner installs them by typing this into a terminal: xcode-select --install (macOS then shows a window to confirm; it takes a few minutes). Tell your owner this, and try again when they say it is done.";
635
+ const runProgram = (file, args, ms) => new Promise((resolve) => {
636
+ execFile(file, args, { timeout: ms, maxBuffer: 4 * 1024 * 1024 }, (err, stdout, stderr) => resolve({ code: err ? (typeof err.code === "number" ? err.code : 1) : 0, stdout: String(stdout), stderr: String(stderr) || (err ? err.message : "") }));
637
+ });
638
+ /**
639
+ * The helper as a program: built from its source the first time, then found again by the
640
+ * source's hash, so an update of pine builds a new one. Rejects with a sentence for the agent.
641
+ */
642
+ export async function buildHelper(dir, source = HELPER_SOURCE, run = runProgram) {
643
+ const file = path.join(dir, `pine-computer-${createHash("sha256").update(fs.readFileSync(source)).digest("hex").slice(0, 12)}`);
644
+ if (fs.existsSync(file))
645
+ return file;
646
+ // /usr/bin/swiftc exists on every Mac, but without the tools it only opens a window that offers to install them.
647
+ const tools = await run("/usr/bin/xcode-select", ["-p"], 10_000);
648
+ if (tools.code !== 0 || !fs.existsSync(tools.stdout.trim()))
649
+ throw new Error(NO_COMPILER);
650
+ fs.mkdirSync(dir, { recursive: true });
651
+ const tmp = `${file}.${process.pid}.tmp`;
652
+ const built = await run("/usr/bin/swiftc", ["-swift-version", "5", "-O", "-o", tmp, source], 300_000);
653
+ if (built.code !== 0 || !fs.existsSync(tmp)) {
654
+ fs.rmSync(tmp, { force: true });
655
+ const why = built.stderr.split("\n").filter((l) => /error/i.test(l)).slice(0, 3).join(" | ") || built.stderr.trim().split("\n").slice(-2).join(" | ");
656
+ throw new Error(`pine could not build its helper for app control (${why.slice(0, 500) || "the compiler gave no reason"}). Tell your owner: Apple's command line developer tools may be missing a part or need an update; "xcode-select --install" in a terminal installs them.`);
657
+ }
658
+ // Whole or not at all: another conversation may be starting the same program at this moment.
659
+ fs.renameSync(tmp, file);
660
+ for (const old of fs.readdirSync(dir))
661
+ if (old.startsWith("pine-computer-") && path.join(dir, old) !== file && !old.endsWith(".tmp"))
662
+ fs.rmSync(path.join(dir, old), { force: true });
663
+ return file;
664
+ }
665
+ if (import.meta.main) {
666
+ const o = options();
667
+ if (!available(o)) {
668
+ serve({ name: "pine-computer", version: "1.0.0", instructions: unavailable() }, [], () => unavailable());
669
+ }
670
+ else {
671
+ const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "pine-computer-"));
672
+ const dir = o.helperDir ?? path.join(os.homedir(), "Library", "Caches", "pine-computer");
673
+ let built;
674
+ const program = () => (built ??= buildHelper(dir).catch((e) => ((built = undefined), Promise.reject(e))));
675
+ const server = computer(o, {
676
+ scratch,
677
+ launch: async () => {
678
+ const cmd = o.helper ? (Array.isArray(o.helper) ? o.helper : [o.helper]) : [await program()];
679
+ const child = spawn(cmd[0], cmd.slice(1), { stdio: ["pipe", "pipe", "pipe"] });
680
+ return new Promise((resolve, reject) => {
681
+ child.once("spawn", () => resolve(child));
682
+ child.once("error", (e) => reject(new Error(`pine's helper for app control could not be started (${e.message}). Tell your owner.`)));
683
+ });
684
+ },
685
+ });
686
+ // Built while the conversation is still about something else, so that the first call does not wait for it.
687
+ if (!o.helper)
688
+ program().catch(() => { });
689
+ // However this process ends, the helper ends with it and no picture is left behind.
690
+ process.on("exit", () => {
691
+ server.close();
692
+ fs.rmSync(scratch, { recursive: true, force: true });
693
+ });
694
+ process.on("SIGTERM", () => process.exit(0));
695
+ process.on("SIGINT", () => process.exit(0));
696
+ serve({ name: "pine-computer", version: "1.0.0" }, toolsOffered(o).map((t) => ({ ...t, handler: (args) => server.call(t.name, args) })));
697
+ }
698
+ }