@ycheung5/pi-computer-use 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GLOSSARY.md +38 -0
- package/LICENSE +21 -0
- package/README.md +165 -2
- package/assets/accessibility.cs +208 -0
- package/assets/browser-indicators.cs +37 -0
- package/assets/capture.cs +215 -0
- package/assets/clipboard.cs +197 -0
- package/assets/controls.cs +249 -0
- package/assets/indicator-preview.png +0 -0
- package/assets/indicators.cs +252 -0
- package/assets/input-timing.cs +42 -0
- package/assets/node-LICENSE.txt +2732 -0
- package/assets/ocr.cs +86 -0
- package/assets/ownership.cs +59 -0
- package/assets/pointer.cs +241 -0
- package/assets/puppeteer-LICENSE.txt +202 -0
- package/assets/readme-banner.svg +33 -0
- package/assets/stop.cs +197 -0
- package/assets/text.cs +279 -0
- package/assets/wgc.cs +167 -0
- package/assets/window-management.cs +217 -0
- package/assets/window-marker.cs +23 -0
- package/assets/window-references.cs +80 -0
- package/assets/windows.cs +127 -0
- package/dist/accessibility.d.ts +29 -0
- package/dist/accessibility.js +258 -0
- package/dist/browser-actions.d.ts +22 -0
- package/dist/browser-actions.js +246 -0
- package/dist/browser-indicators.d.ts +23 -0
- package/dist/browser-indicators.js +69 -0
- package/dist/browser-profiles.d.ts +10 -0
- package/dist/browser-profiles.js +174 -0
- package/dist/browser.d.ts +11 -0
- package/dist/browser.js +2071 -0
- package/dist/capture.d.ts +74 -0
- package/dist/capture.js +199 -0
- package/dist/client.d.ts +39 -0
- package/dist/client.js +384 -0
- package/dist/desktop.d.ts +20 -0
- package/dist/desktop.js +42 -0
- package/dist/extension.d.ts +4 -0
- package/dist/extension.js +409 -0
- package/dist/input-timing.d.ts +24 -0
- package/dist/input-timing.js +70 -0
- package/dist/launcher.d.ts +6 -0
- package/dist/launcher.js +85 -0
- package/dist/native.d.ts +11 -0
- package/dist/native.js +146 -0
- package/dist/ocr.d.ts +52 -0
- package/dist/ocr.js +246 -0
- package/dist/ownership.d.ts +4 -0
- package/dist/ownership.js +24 -0
- package/dist/pointer.d.ts +128 -0
- package/dist/pointer.js +198 -0
- package/dist/protocol.d.ts +15 -0
- package/dist/protocol.js +1 -0
- package/dist/runtime.d.ts +8 -0
- package/dist/runtime.js +72 -0
- package/dist/settings.d.ts +85 -0
- package/dist/settings.js +449 -0
- package/dist/stop.d.ts +24 -0
- package/dist/stop.js +156 -0
- package/dist/text.d.ts +120 -0
- package/dist/text.js +212 -0
- package/dist/waits.d.ts +19 -0
- package/dist/waits.js +262 -0
- package/dist/window-management.d.ts +25 -0
- package/dist/window-management.js +167 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +553 -0
- package/docs/README.md +55 -0
- package/docs/adr/0001-preserve-reference-behavior.md +9 -0
- package/docs/adr/0002-window-lifetime-references.md +11 -0
- package/docs/adr/0003-exclusive-desktop-control.md +11 -0
- package/docs/adr/0004-pi-first-acceptance.md +9 -0
- package/docs/adr/0005-default-apps-and-workspace-reuse.md +7 -0
- package/docs/agents/domain.md +9 -0
- package/docs/agents/issue-tracker.md +10 -0
- package/docs/agents/triage-labels.md +9 -0
- package/docs/browser.md +113 -0
- package/docs/implementation-plan.md +153 -0
- package/docs/indicators.md +53 -0
- package/docs/native-interop-research.md +17 -0
- package/docs/ownership.md +25 -0
- package/docs/pi-extension-research.md +57 -0
- package/docs/pointer.md +77 -0
- package/docs/runtime.md +42 -0
- package/docs/settings.md +33 -0
- package/docs/spec.md +230 -0
- package/docs/stopping.md +83 -0
- package/docs/text.md +99 -0
- package/docs/tickets.md +85 -0
- package/docs/validation.md +1420 -0
- package/docs/windows.md +171 -0
- package/package.json +134 -4
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export declare function updateBrowserIndicator(input: {
|
|
2
|
+
mode: "mark" | "hide" | "show" | "clear";
|
|
3
|
+
point?: {
|
|
4
|
+
x: number;
|
|
5
|
+
y: number;
|
|
6
|
+
};
|
|
7
|
+
}): {
|
|
8
|
+
active: boolean;
|
|
9
|
+
tabBadge: boolean;
|
|
10
|
+
clickMarker: boolean;
|
|
11
|
+
hidden?: undefined;
|
|
12
|
+
ariaHidden?: undefined;
|
|
13
|
+
pointerEvents?: undefined;
|
|
14
|
+
shadowClosed?: undefined;
|
|
15
|
+
} | {
|
|
16
|
+
active: boolean;
|
|
17
|
+
tabBadge: boolean;
|
|
18
|
+
clickMarker: boolean;
|
|
19
|
+
hidden: boolean;
|
|
20
|
+
ariaHidden: boolean;
|
|
21
|
+
pointerEvents: string;
|
|
22
|
+
shadowClosed: boolean;
|
|
23
|
+
};
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// Runs only in this extension's isolated browser realm. Keep the function self-contained.
|
|
2
|
+
export function updateBrowserIndicator(input) {
|
|
3
|
+
const world = globalThis;
|
|
4
|
+
let cue = world.__piComputerUseCue;
|
|
5
|
+
if (input.mode === "clear") {
|
|
6
|
+
cue?.host.remove();
|
|
7
|
+
cue?.icon.remove();
|
|
8
|
+
delete world.__piComputerUseCue;
|
|
9
|
+
return { active: false, tabBadge: false, clickMarker: false };
|
|
10
|
+
}
|
|
11
|
+
if (!cue?.host.isConnected &&
|
|
12
|
+
input.mode === "mark" &&
|
|
13
|
+
document.documentElement) {
|
|
14
|
+
cue?.icon.remove();
|
|
15
|
+
const host = document.createElement("div");
|
|
16
|
+
host.setAttribute("aria-hidden", "true");
|
|
17
|
+
host.setAttribute("role", "presentation");
|
|
18
|
+
host.style.cssText =
|
|
19
|
+
"all:initial!important;position:fixed!important;left:0!important;top:0!important;width:0!important;height:0!important;overflow:visible!important;pointer-events:none!important;z-index:2147483647!important";
|
|
20
|
+
const root = host.attachShadow({ mode: "closed" });
|
|
21
|
+
const marker = document.createElementNS("http://www.w3.org/2000/svg", "svg");
|
|
22
|
+
marker.setAttribute("viewBox", "0 0 28 34");
|
|
23
|
+
marker.setAttribute("aria-hidden", "true");
|
|
24
|
+
marker.style.cssText =
|
|
25
|
+
"position:fixed;width:28px;height:34px;pointer-events:none;display:none;filter:drop-shadow(0 0 3px #2d9cff)";
|
|
26
|
+
const arrow = document.createElementNS("http://www.w3.org/2000/svg", "path");
|
|
27
|
+
arrow.setAttribute("d", "M2 2 L2 25 L8 20 L13 31 L18 28 L13 18 L23 18 Z");
|
|
28
|
+
arrow.setAttribute("fill", "#2d9cff");
|
|
29
|
+
arrow.setAttribute("stroke", "white");
|
|
30
|
+
arrow.setAttribute("stroke-width", "1.5");
|
|
31
|
+
marker.append(arrow);
|
|
32
|
+
root.append(marker);
|
|
33
|
+
document.documentElement.append(host);
|
|
34
|
+
const icon = document.createElement("link");
|
|
35
|
+
icon.rel = "icon";
|
|
36
|
+
icon.type = "image/svg+xml";
|
|
37
|
+
icon.href =
|
|
38
|
+
"data:image/svg+xml," +
|
|
39
|
+
encodeURIComponent('<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="6" fill="#247fd9"/><text x="16" y="23" font-family="sans-serif" font-size="23" text-anchor="middle" fill="white">π</text></svg>');
|
|
40
|
+
(document.head ?? document.documentElement).append(icon);
|
|
41
|
+
cue = { host, marker, icon };
|
|
42
|
+
world.__piComputerUseCue = cue;
|
|
43
|
+
}
|
|
44
|
+
if (!cue)
|
|
45
|
+
return { active: false, tabBadge: false, clickMarker: false };
|
|
46
|
+
if (input.mode === "hide")
|
|
47
|
+
cue.host.style.setProperty("display", "none", "important");
|
|
48
|
+
else if (input.mode === "show")
|
|
49
|
+
cue.host.style.removeProperty("display");
|
|
50
|
+
else {
|
|
51
|
+
cue.host.style.removeProperty("display");
|
|
52
|
+
if (input.point &&
|
|
53
|
+
Number.isFinite(input.point.x) &&
|
|
54
|
+
Number.isFinite(input.point.y)) {
|
|
55
|
+
cue.marker.style.left = `${input.point.x}px`;
|
|
56
|
+
cue.marker.style.top = `${input.point.y}px`;
|
|
57
|
+
cue.marker.style.display = "block";
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return {
|
|
61
|
+
active: cue.host.isConnected,
|
|
62
|
+
tabBadge: cue.icon.isConnected,
|
|
63
|
+
clickMarker: cue.marker.style.display !== "none",
|
|
64
|
+
hidden: cue.host.style.display === "none",
|
|
65
|
+
ariaHidden: cue.host.getAttribute("aria-hidden") === "true",
|
|
66
|
+
pointerEvents: cue.host.style.pointerEvents,
|
|
67
|
+
shadowClosed: cue.host.shadowRoot === null,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface BrowserProfile {
|
|
2
|
+
path: string;
|
|
3
|
+
kept: boolean;
|
|
4
|
+
lock: string;
|
|
5
|
+
token: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function acquireBrowserProfile(app: string, executable: string, kept: boolean): Promise<BrowserProfile>;
|
|
8
|
+
export declare function recordProfileBrowser(profile: BrowserProfile, pid: number | undefined): Promise<void>;
|
|
9
|
+
export declare function releaseBrowserProfile(profile: BrowserProfile, browserPid?: number): Promise<void>;
|
|
10
|
+
export declare function browserIdleMs(): number;
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { access, mkdir, mkdtemp, readdir, readFile, rename, rm, writeFile, } from "node:fs/promises";
|
|
3
|
+
import { join, resolve } from "node:path";
|
|
4
|
+
const alive = (pid) => {
|
|
5
|
+
try {
|
|
6
|
+
process.kill(pid, 0);
|
|
7
|
+
return true;
|
|
8
|
+
}
|
|
9
|
+
catch (error) {
|
|
10
|
+
return error.code !== "ESRCH";
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
const busy = (detail) => {
|
|
14
|
+
throw new Error(`BROWSER_PROFILE_BUSY: ${detail}; close the owning automation browser before retrying`);
|
|
15
|
+
};
|
|
16
|
+
async function ownerAt(lock) {
|
|
17
|
+
try {
|
|
18
|
+
const value = JSON.parse(await readFile(join(lock, "owner.json"), "utf8"));
|
|
19
|
+
if (value &&
|
|
20
|
+
typeof value === "object" &&
|
|
21
|
+
"token" in value &&
|
|
22
|
+
typeof value.token === "string" &&
|
|
23
|
+
"workerPid" in value &&
|
|
24
|
+
Number.isSafeInteger(value.workerPid) &&
|
|
25
|
+
Number(value.workerPid) > 0 &&
|
|
26
|
+
"phase" in value &&
|
|
27
|
+
["launching", "running"].includes(String(value.phase))) {
|
|
28
|
+
const owner = value;
|
|
29
|
+
if (owner.browserPid !== undefined &&
|
|
30
|
+
(!Number.isSafeInteger(owner.browserPid) || owner.browserPid <= 0))
|
|
31
|
+
return undefined;
|
|
32
|
+
return owner;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch { }
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
function stale(owner) {
|
|
39
|
+
if (!owner || alive(owner.workerPid))
|
|
40
|
+
return false;
|
|
41
|
+
if (owner.browserPid === undefined)
|
|
42
|
+
return false; // A crash during launch has an unknown browser owner.
|
|
43
|
+
return !alive(owner.browserPid);
|
|
44
|
+
}
|
|
45
|
+
async function claim(lock) {
|
|
46
|
+
const token = randomUUID();
|
|
47
|
+
try {
|
|
48
|
+
await access(`${lock}.recover`);
|
|
49
|
+
busy("profile recovery is active or incomplete");
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
if (error.code !== "ENOENT")
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
await mkdir(lock);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (error.code !== "EEXIST")
|
|
60
|
+
throw error;
|
|
61
|
+
const owner = await ownerAt(lock);
|
|
62
|
+
if (!stale(owner))
|
|
63
|
+
busy(owner?.browserPid && alive(owner.browserPid)
|
|
64
|
+
? "profile browser is still alive (possibly orphaned)"
|
|
65
|
+
: "live, unknown or incomplete profile ownership");
|
|
66
|
+
const recovery = `${lock}.recover`;
|
|
67
|
+
try {
|
|
68
|
+
await mkdir(recovery);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
busy("another profile recovery is active or incomplete");
|
|
72
|
+
}
|
|
73
|
+
try {
|
|
74
|
+
const confirmed = await ownerAt(lock);
|
|
75
|
+
if (!stale(confirmed) || confirmed?.token !== owner?.token)
|
|
76
|
+
busy("profile ownership changed during recovery");
|
|
77
|
+
const abandoned = `${lock}.abandoned-${token}`;
|
|
78
|
+
await rename(lock, abandoned);
|
|
79
|
+
try {
|
|
80
|
+
await mkdir(lock);
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
await rm(abandoned, { recursive: true, force: true });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
await rm(recovery, { recursive: true, force: true });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const owner = { token, workerPid: process.pid, phase: "launching" };
|
|
91
|
+
await writeFile(join(lock, "owner.json"), JSON.stringify(owner), {
|
|
92
|
+
flag: "wx",
|
|
93
|
+
});
|
|
94
|
+
return token;
|
|
95
|
+
}
|
|
96
|
+
export async function acquireBrowserProfile(app, executable, kept) {
|
|
97
|
+
const local = process.env.LOCALAPPDATA;
|
|
98
|
+
if (!local)
|
|
99
|
+
throw new Error("BROWSER_SETUP_FAILED: Windows LOCALAPPDATA is required");
|
|
100
|
+
const root = resolve(process.env.PI_CU_BROWSER_PROFILE_ROOT ??
|
|
101
|
+
join(local, "pi-computer-use", "browser"));
|
|
102
|
+
await mkdir(root, { recursive: true });
|
|
103
|
+
// Only sweep our recorded temporary leases after both recorded processes
|
|
104
|
+
// are gone; unknown launches and live orphans remain untouched.
|
|
105
|
+
for (const entry of await readdir(root, { withFileTypes: true })) {
|
|
106
|
+
if (!entry.isDirectory() ||
|
|
107
|
+
!/^temporary-[A-Za-z0-9]+\.lock$/.test(entry.name))
|
|
108
|
+
continue;
|
|
109
|
+
const lock = join(root, entry.name);
|
|
110
|
+
if (!stale(await ownerAt(lock)))
|
|
111
|
+
continue;
|
|
112
|
+
try {
|
|
113
|
+
const token = await claim(lock);
|
|
114
|
+
await releaseBrowserProfile({
|
|
115
|
+
path: lock.slice(0, -5),
|
|
116
|
+
kept: false,
|
|
117
|
+
lock,
|
|
118
|
+
token,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
if (!(error instanceof Error &&
|
|
123
|
+
error.message.startsWith("BROWSER_PROFILE_BUSY")))
|
|
124
|
+
throw error;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const identity = createHash("sha256")
|
|
128
|
+
.update(`${app}\0${resolve(executable).toLowerCase()}`)
|
|
129
|
+
.digest("hex")
|
|
130
|
+
.slice(0, 24);
|
|
131
|
+
const path = kept
|
|
132
|
+
? join(root, `kept-${app}-${identity}`)
|
|
133
|
+
: await mkdtemp(join(root, "temporary-"));
|
|
134
|
+
const lock = `${path}.lock`;
|
|
135
|
+
try {
|
|
136
|
+
const token = await claim(lock);
|
|
137
|
+
await mkdir(path, { recursive: true });
|
|
138
|
+
return { path, kept, lock, token };
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
if (!kept)
|
|
142
|
+
await rm(path, { recursive: true, force: true });
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
export async function recordProfileBrowser(profile, pid) {
|
|
147
|
+
if (!pid)
|
|
148
|
+
busy("launched browser did not expose its owning process");
|
|
149
|
+
const owner = await ownerAt(profile.lock);
|
|
150
|
+
if (owner?.token !== profile.token)
|
|
151
|
+
busy("profile ownership changed before launch completed");
|
|
152
|
+
await writeFile(join(profile.lock, "owner.json"), JSON.stringify({ ...owner, browserPid: pid, phase: "running" }));
|
|
153
|
+
}
|
|
154
|
+
export async function releaseBrowserProfile(profile, browserPid) {
|
|
155
|
+
const owner = await ownerAt(profile.lock);
|
|
156
|
+
if (owner?.token !== profile.token)
|
|
157
|
+
busy("profile ownership changed during cleanup");
|
|
158
|
+
if (browserPid && alive(browserPid))
|
|
159
|
+
busy("browser is still alive; profile retained and blocked");
|
|
160
|
+
if (!profile.kept)
|
|
161
|
+
await rm(profile.path, {
|
|
162
|
+
recursive: true,
|
|
163
|
+
force: true,
|
|
164
|
+
maxRetries: 10,
|
|
165
|
+
retryDelay: 100,
|
|
166
|
+
});
|
|
167
|
+
await rm(profile.lock, { recursive: true, force: true });
|
|
168
|
+
}
|
|
169
|
+
export function browserIdleMs() {
|
|
170
|
+
const value = Number(process.env.PI_CU_BROWSER_IDLE_MS ?? 600000);
|
|
171
|
+
if (!Number.isSafeInteger(value) || value < 1000 || value > 86400000)
|
|
172
|
+
throw new Error("BROWSER_SETTINGS: PI_CU_BROWSER_IDLE_MS must be 1000–86400000 milliseconds");
|
|
173
|
+
return value;
|
|
174
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { WorkerResult } from "./protocol.js";
|
|
2
|
+
export declare function configureBrowser(settings: {
|
|
3
|
+
browserApp: string;
|
|
4
|
+
browserVisible: boolean;
|
|
5
|
+
browserSignIn: boolean;
|
|
6
|
+
}): Promise<void>;
|
|
7
|
+
export declare function isBrowserRequest(args: Record<string, unknown>): boolean;
|
|
8
|
+
export declare function clearBrowserIndicators(): Promise<void>;
|
|
9
|
+
export declare function browserListing(): Promise<Record<string, unknown>>;
|
|
10
|
+
export declare function browse(action: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<WorkerResult>;
|
|
11
|
+
export declare function disposeBrowser(): Promise<void>;
|