@ycheung5/pi-computer-use 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/assets/input-timing.cs +42 -0
- package/assets/pointer.cs +16 -4
- package/assets/text.cs +47 -17
- package/dist/browser-actions.d.ts +1 -0
- package/dist/browser-actions.js +34 -1
- package/dist/client.d.ts +1 -0
- package/dist/client.js +70 -17
- package/dist/extension.js +3 -3
- package/dist/input-timing.d.ts +24 -0
- package/dist/input-timing.js +70 -0
- package/dist/native.js +1 -0
- package/dist/pointer.d.ts +20 -0
- package/dist/pointer.js +24 -0
- package/dist/text.d.ts +20 -0
- package/dist/text.js +33 -4
- package/dist/worker.js +14 -1
- package/docs/README.md +4 -3
- package/docs/implementation-plan.md +15 -1
- package/docs/pointer.md +12 -0
- package/docs/runtime.md +1 -1
- package/docs/spec.md +71 -2
- package/docs/text.md +22 -2
- package/docs/tickets.md +8 -7
- package/docs/validation.md +55 -7
- package/package.json +1 -1
package/dist/pointer.d.ts
CHANGED
|
@@ -34,6 +34,16 @@ export declare const pointerTools: {
|
|
|
34
34
|
minItems: number;
|
|
35
35
|
maxItems: number;
|
|
36
36
|
};
|
|
37
|
+
holdMs?: {
|
|
38
|
+
type: string;
|
|
39
|
+
minimum: number;
|
|
40
|
+
maximum: number;
|
|
41
|
+
} | undefined;
|
|
42
|
+
timeoutMs?: {
|
|
43
|
+
type: string;
|
|
44
|
+
minimum: number;
|
|
45
|
+
maximum: number;
|
|
46
|
+
} | undefined;
|
|
37
47
|
windowRef: {
|
|
38
48
|
type: string;
|
|
39
49
|
minLength: number;
|
|
@@ -82,6 +92,16 @@ export declare const pointerTools: {
|
|
|
82
92
|
type: string;
|
|
83
93
|
default: boolean;
|
|
84
94
|
};
|
|
95
|
+
holdMs?: {
|
|
96
|
+
type: string;
|
|
97
|
+
minimum: number;
|
|
98
|
+
maximum: number;
|
|
99
|
+
} | undefined;
|
|
100
|
+
timeoutMs?: {
|
|
101
|
+
type: string;
|
|
102
|
+
minimum: number;
|
|
103
|
+
maximum: number;
|
|
104
|
+
} | undefined;
|
|
85
105
|
windowRef: {
|
|
86
106
|
type: string;
|
|
87
107
|
minLength: number;
|
package/dist/pointer.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { pressTiming, timingMilliseconds } from "./input-timing.js";
|
|
1
2
|
import { confirmInputCleanup, nativeDesktop, retainInput } from "./native.js";
|
|
2
3
|
const integer = { type: "integer" };
|
|
3
4
|
const target = {
|
|
@@ -20,6 +21,9 @@ export const pointerTools = ["click", "double_click", "move", "drag"].map((actio
|
|
|
20
21
|
additionalProperties: false,
|
|
21
22
|
properties: {
|
|
22
23
|
...target,
|
|
24
|
+
...(action === "click"
|
|
25
|
+
? { holdMs: timingMilliseconds, timeoutMs: timingMilliseconds }
|
|
26
|
+
: {}),
|
|
23
27
|
...(action === "drag"
|
|
24
28
|
? {
|
|
25
29
|
path: {
|
|
@@ -84,6 +88,7 @@ export function validatePointer(action, args) {
|
|
|
84
88
|
"includeOffscreen",
|
|
85
89
|
...(action === "move" ? [] : ["button", "dispatch"]),
|
|
86
90
|
]),
|
|
91
|
+
...(action === "click" ? ["holdMs", "timeoutMs"] : []),
|
|
87
92
|
...(action === "scroll" ? ["deltaX", "deltaY", "browserTab"] : []),
|
|
88
93
|
]);
|
|
89
94
|
if (args.windowRef !== undefined &&
|
|
@@ -115,6 +120,25 @@ export function validatePointer(action, args) {
|
|
|
115
120
|
if (args.dispatch !== undefined &&
|
|
116
121
|
!["auto", "background", "foreground"].includes(String(args.dispatch)))
|
|
117
122
|
return bad("unsupported dispatch");
|
|
123
|
+
if (action === "click") {
|
|
124
|
+
try {
|
|
125
|
+
pressTiming(args, 90_000);
|
|
126
|
+
}
|
|
127
|
+
catch (cause) {
|
|
128
|
+
return {
|
|
129
|
+
ok: false,
|
|
130
|
+
action,
|
|
131
|
+
error: cause instanceof Error ? cause.message : String(cause),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
if ((args.holdMs !== undefined || args.timeoutMs !== undefined) &&
|
|
135
|
+
(args.dispatch === "background" || args.activate === false))
|
|
136
|
+
return {
|
|
137
|
+
ok: false,
|
|
138
|
+
action,
|
|
139
|
+
error: "TIMING_UNSUPPORTED: click timing requires native foreground dispatch without explicit activate: false",
|
|
140
|
+
};
|
|
141
|
+
}
|
|
118
142
|
if (action === "drag") {
|
|
119
143
|
if (!Array.isArray(args.path) ||
|
|
120
144
|
args.path.length < 2 ||
|
package/dist/text.d.ts
CHANGED
|
@@ -24,6 +24,16 @@ export declare const textTools: ({
|
|
|
24
24
|
type: string;
|
|
25
25
|
default: boolean;
|
|
26
26
|
};
|
|
27
|
+
inputDelayMs: {
|
|
28
|
+
type: string;
|
|
29
|
+
minimum: number;
|
|
30
|
+
maximum: number;
|
|
31
|
+
};
|
|
32
|
+
timeoutMs: {
|
|
33
|
+
type: string;
|
|
34
|
+
minimum: number;
|
|
35
|
+
maximum: number;
|
|
36
|
+
};
|
|
27
37
|
windowRef: {
|
|
28
38
|
type: string;
|
|
29
39
|
minLength: number;
|
|
@@ -55,6 +65,16 @@ export declare const textTools: ({
|
|
|
55
65
|
type: string;
|
|
56
66
|
additionalProperties: boolean;
|
|
57
67
|
properties: {
|
|
68
|
+
holdMs: {
|
|
69
|
+
type: string;
|
|
70
|
+
minimum: number;
|
|
71
|
+
maximum: number;
|
|
72
|
+
};
|
|
73
|
+
timeoutMs: {
|
|
74
|
+
type: string;
|
|
75
|
+
minimum: number;
|
|
76
|
+
maximum: number;
|
|
77
|
+
};
|
|
58
78
|
keys: {
|
|
59
79
|
type: string;
|
|
60
80
|
items: {
|
package/dist/text.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { pressTiming, textTiming, timingMilliseconds } from "./input-timing.js";
|
|
1
2
|
import { compileNative } from "./native.js";
|
|
2
3
|
import { executePointer } from "./pointer.js";
|
|
3
4
|
const target = {
|
|
@@ -28,6 +29,8 @@ export const textTools = [
|
|
|
28
29
|
default: "preserve",
|
|
29
30
|
},
|
|
30
31
|
restoreClipboard: { type: "boolean", default: true },
|
|
32
|
+
inputDelayMs: timingMilliseconds,
|
|
33
|
+
timeoutMs: timingMilliseconds,
|
|
31
34
|
},
|
|
32
35
|
required: ["text"],
|
|
33
36
|
},
|
|
@@ -39,6 +42,8 @@ export const textTools = [
|
|
|
39
42
|
additionalProperties: false,
|
|
40
43
|
properties: {
|
|
41
44
|
...target,
|
|
45
|
+
holdMs: timingMilliseconds,
|
|
46
|
+
timeoutMs: timingMilliseconds,
|
|
42
47
|
keys: {
|
|
43
48
|
type: "array",
|
|
44
49
|
items: { type: "string", minLength: 1 },
|
|
@@ -94,8 +99,15 @@ export function validateText(action, args) {
|
|
|
94
99
|
const allowed = new Set([
|
|
95
100
|
...Object.keys(target),
|
|
96
101
|
...(action === "type_text"
|
|
97
|
-
? [
|
|
98
|
-
|
|
102
|
+
? [
|
|
103
|
+
"text",
|
|
104
|
+
"method",
|
|
105
|
+
"focusMode",
|
|
106
|
+
"restoreClipboard",
|
|
107
|
+
"inputDelayMs",
|
|
108
|
+
"timeoutMs",
|
|
109
|
+
]
|
|
110
|
+
: ["keys", "dispatch", "holdMs", "timeoutMs"]),
|
|
99
111
|
]);
|
|
100
112
|
if (args.windowRef !== undefined &&
|
|
101
113
|
(typeof args.windowRef !== "string" || !args.windowRef.trim()))
|
|
@@ -118,6 +130,12 @@ export function validateText(action, args) {
|
|
|
118
130
|
args.text.length > 1000000 ||
|
|
119
131
|
args.text.includes("\0"))
|
|
120
132
|
return bad("text must be nonempty, without NUL, at most 1000000 UTF-16 units");
|
|
133
|
+
try {
|
|
134
|
+
textTiming(args, 90_000);
|
|
135
|
+
}
|
|
136
|
+
catch (cause) {
|
|
137
|
+
return error(String(cause instanceof Error ? cause.message : cause));
|
|
138
|
+
}
|
|
121
139
|
if (args.method !== undefined &&
|
|
122
140
|
!["auto", "clipboard", "sendinput", "background"].includes(String(args.method)))
|
|
123
141
|
return bad("unsupported method");
|
|
@@ -129,6 +147,15 @@ export function validateText(action, args) {
|
|
|
129
147
|
return bad("restoreClipboard must be boolean");
|
|
130
148
|
}
|
|
131
149
|
else {
|
|
150
|
+
try {
|
|
151
|
+
pressTiming(args, 90_000);
|
|
152
|
+
}
|
|
153
|
+
catch (cause) {
|
|
154
|
+
return error(String(cause instanceof Error ? cause.message : cause));
|
|
155
|
+
}
|
|
156
|
+
if ((args.holdMs !== undefined || args.timeoutMs !== undefined) &&
|
|
157
|
+
(args.dispatch === "background" || args.browserTab !== undefined))
|
|
158
|
+
return error("TIMING_UNSUPPORTED: keypress timing requires native foreground dispatch");
|
|
132
159
|
if (!Array.isArray(args.keys) ||
|
|
133
160
|
args.keys.length < 1 ||
|
|
134
161
|
args.keys.length > 4 ||
|
|
@@ -155,8 +182,10 @@ export function validateText(action, args) {
|
|
|
155
182
|
return bad("unsupported key");
|
|
156
183
|
}
|
|
157
184
|
}
|
|
158
|
-
if (main !== 1)
|
|
159
|
-
return bad("
|
|
185
|
+
if (main !== 1 && !(main === 0 && args.keys.length === 1))
|
|
186
|
+
return bad("one main key or one standalone generic modifier is required");
|
|
187
|
+
if (main === 0 && args.browserTab !== undefined)
|
|
188
|
+
return error("TIMING_UNSUPPORTED: standalone modifiers require native foreground dispatch");
|
|
160
189
|
if (args.dispatch !== undefined &&
|
|
161
190
|
!["foreground", "background"].includes(String(args.dispatch)))
|
|
162
191
|
return bad("unsupported dispatch");
|
package/dist/worker.js
CHANGED
|
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { createInterface } from "node:readline";
|
|
3
3
|
import { controlTools, executeAccessibility, observationTools, } from "./accessibility.js";
|
|
4
4
|
import { browse, browserListing, clearBrowserIndicators, configureBrowser, disposeBrowser, isBrowserRequest, } from "./browser.js";
|
|
5
|
-
import { browserActionTools } from "./browser-actions.js";
|
|
5
|
+
import { browserActionRefusal, browserActionTools } from "./browser-actions.js";
|
|
6
6
|
import { screenshot, screenshotParameters } from "./capture.js";
|
|
7
7
|
import { disposeDesktop, listWindows, listWindowsParameters, retireDesktop, } from "./desktop.js";
|
|
8
8
|
import { runNativeOperation } from "./native.js";
|
|
@@ -249,6 +249,13 @@ async function dispatch(request, signal) {
|
|
|
249
249
|
process.stdout.write(`${JSON.stringify({ id: request.id, result: { ok: false, action: request.action, error: "INVALID_ARGUMENTS: windowRef requires a nonempty native-window reference and cannot target desktop/browser channel" } })}\n`);
|
|
250
250
|
return;
|
|
251
251
|
}
|
|
252
|
+
const browserRefusal = isBrowserRequest(request.args)
|
|
253
|
+
? browserActionRefusal(request.action, request.args)
|
|
254
|
+
: undefined;
|
|
255
|
+
if (browserRefusal) {
|
|
256
|
+
process.stdout.write(`${JSON.stringify({ id: request.id, result: { ok: false, action: request.action, error: browserRefusal } })}\n`);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
252
259
|
const denial = await settings.authorize(request.action);
|
|
253
260
|
if (denial) {
|
|
254
261
|
process.stdout.write(`${JSON.stringify({ id: request.id, result: { ok: false, action: request.action, error: denial } })}\n`);
|
|
@@ -343,6 +350,8 @@ async function dispatch(request, signal) {
|
|
|
343
350
|
// stale/disabled/covered-point error can trigger replay.
|
|
344
351
|
if ((request.args.dispatch === "background" ||
|
|
345
352
|
(request.action === "click" &&
|
|
353
|
+
request.args.holdMs === undefined &&
|
|
354
|
+
request.args.timeoutMs === undefined &&
|
|
346
355
|
request.args.elementId &&
|
|
347
356
|
(request.args.button ?? "left") === "left" &&
|
|
348
357
|
!request.args.activate &&
|
|
@@ -520,6 +529,10 @@ input.on("line", (line) => {
|
|
|
520
529
|
});
|
|
521
530
|
input.on("close", () => {
|
|
522
531
|
clearInterval(presentationPoll);
|
|
532
|
+
// EOF is shutdown, including when a call deliberately disabled its deadline.
|
|
533
|
+
for (const request of requests.values())
|
|
534
|
+
if (request.action !== "operation_end")
|
|
535
|
+
void stopOperation(request.operationId).catch((error) => process.stderr.write(`${String(error)}\n`));
|
|
523
536
|
process.stdout.write(`${JSON.stringify({ event: "shutdown_state", phase: "started" })}\n`);
|
|
524
537
|
void queue
|
|
525
538
|
.then(async () => {
|
package/docs/README.md
CHANGED
|
@@ -5,11 +5,12 @@ Read the row matching the task; follow section links before reading whole docume
|
|
|
5
5
|
| Task | Read |
|
|
6
6
|
| --- | --- |
|
|
7
7
|
| Requirements and scope | [Spec: solution](spec.md#solution), [user stories](spec.md#user-stories), [out of scope](spec.md#out-of-scope) |
|
|
8
|
+
| Modifier taps, holds and timing controls | [Input timing spec](spec.md#input-timing-and-modifier-taps), [delivery](implementation-plan.md#input-timing-delivery), [scoped acceptance](validation.md#input-timing-and-modifier-tap-acceptance) |
|
|
8
9
|
| Implementation contracts | [Spec: implementation decisions](spec.md#implementation-decisions) |
|
|
9
10
|
| Delivery, design review and feasibility | [Design revision](implementation-plan.md#design-revision), [plan: detailed milestones](implementation-plan.md#detailed-milestones), [feasibility gates](implementation-plan.md#feasibility-gates) |
|
|
10
11
|
| Test design and release validation | [Spec: testing decisions](spec.md#testing-decisions), [plan: coverage matrix](implementation-plan.md#coverage-matrix) |
|
|
11
12
|
| Pi acceptance, phased testing and agent recovery | [Current phases](implementation-plan.md#detailed-milestones), [baseline audit](validation.md#baseline-identity-and-schema-evidence), [guidance inventory](validation.md#model-visible-guidance-inventory), [scenario catalog](validation.md#alignment-scenario-catalog) |
|
|
12
|
-
| Release checks, capability coverage and pending acceptance | [0.1.0 release preparation](validation.md#first-release-preparation), [current release audit](validation.md#final-release-audit), [capability coverage](validation.md#capability-coverage), [validation history](validation.md) |
|
|
13
|
+
| Release checks, capability coverage and pending acceptance | [0.1.1 preparation and release notes](validation.md#011-release-preparation), [verified 0.1.0 npm release](validation.md#npm-publication-receipt), [0.1.0 release preparation](validation.md#first-release-preparation), [current release audit](validation.md#final-release-audit), [capability coverage](validation.md#capability-coverage), [validation history](validation.md) |
|
|
13
14
|
| Pi interfaces and sources | [Pi extension research](pi-extension-research.md) (research context; current spec takes precedence over earlier proposals) |
|
|
14
15
|
| Domain terminology | [Glossary](../GLOSSARY.md) |
|
|
15
16
|
| Current Pi-first acceptance scope | [ADR 0004](adr/0004-pi-first-acceptance.md) |
|
|
@@ -27,8 +28,8 @@ Read the row matching the task; follow section links before reading whole docume
|
|
|
27
28
|
| Scoped native document and visual agent acceptance | [Document workflow](validation.md#native-document-workflow), [handback continuation](validation.md#native-cancellation-and-handback-continuation), [opt-in visual runner](windows.md#scoped-visual-agent-check), [A04 evidence](validation.md#visual-ocr-association-preflight) |
|
|
28
29
|
| Lifetime-bound native targeting and stale-reference behavior | [Window references](windows.md#window-lifetime-references), [accepted design](implementation-plan.md#window-lifetime-design) |
|
|
29
30
|
| Native interop alternatives and tradeoffs | [Native interop research](native-interop-research.md) |
|
|
30
|
-
| Guarded desktop pointer and dedicated input validation | [Pointer actions](pointer.md), [popup/menu/cross-process drop and restoration acceptance](validation.md#scoped-pointer-and-restoration-acceptance) |
|
|
31
|
-
| Text, chords, clipboard preservation and validation | [Text entry](text.md), [chord/editor-focus acceptance](validation.md#pi-chords-and-editor-focus-acceptance), [clipboard acceptance](validation.md#pi-clipboard-contention-and-privacy-acceptance) |
|
|
31
|
+
| Guarded desktop pointer and dedicated input validation | [Pointer actions](pointer.md), [single-click holds/deadlines](pointer.md#single-click-holds-and-deadlines), [popup/menu/cross-process drop and restoration acceptance](validation.md#scoped-pointer-and-restoration-acceptance) |
|
|
32
|
+
| Text, chords, keypress timing, clipboard preservation and validation | [Text entry](text.md), [keypress timing](text.md#keypress-holds-and-deadlines), [text pacing](text.md#sendinput-pacing-and-deadlines), [chord/editor-focus acceptance](validation.md#pi-chords-and-editor-focus-acceptance), [clipboard acceptance](validation.md#pi-clipboard-contention-and-privacy-acceptance) |
|
|
32
33
|
| Personal settings, commands and tool loading | [Settings](settings.md) |
|
|
33
34
|
| Browser reads/actions, dedicated profiles, idle cleanup and fixtures | [Browser](browser.md) |
|
|
34
35
|
| Domain-document consumer rules | [Domain docs](agents/domain.md) |
|
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
# Pi computer-use implementation plan
|
|
2
2
|
|
|
3
|
-
Status:
|
|
3
|
+
Status: 0.1.0 is published with its recorded acceptance limits. Post-release delivery is owned by the [current queue](tickets.md#execution-queue), including the agreed input-timing change below. [ADR 0004](adr/0004-pi-first-acceptance.md) remains binding; DSH/manual parity runs are not prerequisites.
|
|
4
4
|
|
|
5
5
|
Requirements and behavioral contracts are authoritative in the [specification](spec.md). Use the [documentation index](README.md) to load only relevant context.
|
|
6
6
|
|
|
7
|
+
## Input timing delivery
|
|
8
|
+
|
|
9
|
+
The [input timing/modifier-tap spec](spec.md#input-timing-and-modifier-taps) is the only requirement authority for this first post-release change. It extends existing tools. All three slices are implemented and accepted at the agreed seams, merged into main through PR #44 and prepared for 0.1.1. The [composed receipt](validation.md#input-timing-and-modifier-tap-acceptance) owns the source scope, final checks, live results and fixture-failure dispositions. Published npm 0.1.0 remains unchanged.
|
|
10
|
+
|
|
11
|
+
1. Extend existing action schemas/normalizers and public tool/worker tests for standalone generic modifiers, timing values, route capability refusal and unchanged defaults.
|
|
12
|
+
2. Keep per-call duration/deadline calculation in reusable request handling rather than duplicate it in host adapters. Thread effective policy through Pi/worker/native execution so omitted, explicit positive and disabled execution deadlines remain consistent. Reuse controlled request peers for long-duration/overflow/cancellation checks without real long sleeps; do not add a general scheduler or test-only production API.
|
|
13
|
+
3. Extend native text/pointer execution with cancellable, representable pacing/hold waits and release acknowledgments using current input ownership. Keep each hold within one call and retain target/focus/protection checks. Reject unsupported explicit backend combinations before dispatch.
|
|
14
|
+
4. Run affected configured regressions and package checks; extend the current owned text/pointer fixtures for one grouped live acceptance of actual down/up effects, exact text and newly affected cleanup. Exercise both host request paths only where changed forwarding requires it. Reuse accepted stop/display/profile evidence and retain any observed failures.
|
|
15
|
+
5. Reconcile the existing topic docs/tool descriptions with the implemented fields and record validation. The second input-coverage change and any release/version bump need their own delivery work; do not expand this slice or mark it complete from portable mocks alone.
|
|
16
|
+
|
|
17
|
+
The delivery is complete for #41 keyboard, #42 text and #43 mouse. Reuse the linked acceptance; do not rerun historical release checks. Future broader input coverage and any merge/version/publication remain separate work.
|
|
18
|
+
|
|
19
|
+
Existing code locations for implementation navigation: `src/text.ts`, `src/pointer.ts` and `src/browser-actions.ts` own routed argument contracts; `src/extension.ts`, `src/client.ts`, `src/native.ts` and `src/worker.ts` own forwarding, deadlines and lifecycle; `assets/text.cs` and `assets/pointer.cs` own native effects/release. Prefer existing `tests/text.test.ts`, `tests/pointer.test.ts`, `tests/worker.test.ts`, input-preparation/extension tests and native text/pointer fixtures. These are delivery pointers, not a new shared-module hierarchy.
|
|
20
|
+
|
|
7
21
|
## Structure
|
|
8
22
|
|
|
9
23
|
Keep flat, responsibility-named TypeScript modules in `src/`, tests in `tests/`, and packaged assets in `assets/`. Keep host-specific launcher and Windows interop logic separate from reusable orchestration through modules, without pre-creating nested directories. The [repository layout](README.md#repository-layout) is the navigation guide.
|
package/docs/pointer.md
CHANGED
|
@@ -14,6 +14,18 @@ A missing cleanup acknowledgment retains desktop ownership and blocks replacemen
|
|
|
14
14
|
|
|
15
15
|
Confirmed activation and successful input/background operations remember the exact target HWND/PID for this operation. Later targetless observations and input use that identity after foreground is returned. Explicit targets, browser requests, element references and explicit desktop scope take precedence. Settlement clears the remembered target; a stale identity fails rather than guessing the user's current foreground. Screenshots retain their existing full-desktop versus window-capture semantics.
|
|
16
16
|
|
|
17
|
+
## Single-click holds and deadlines
|
|
18
|
+
|
|
19
|
+
The source candidate adds optional `holdMs` and `timeoutMs` to single `click` for existing left/right/middle buttons. Omitted or zero hold releases immediately. Explicit timing uses native foreground dispatch, including auto element clicks; explicit `dispatch: background`, `activate: false` and browser targets refuse before input. With both fields omitted, existing routing and activation rules above apply. Double-click and drag do not accept timing fields.
|
|
20
|
+
|
|
21
|
+
Both request layers reuse the [keypress timing policy](text.md#keypress-holds-and-deadlines): positive timeout is authoritative, zero disables only the execution deadline, and omission adds an explicit hold to the ordinary budget. Safe whole nonnegative milliseconds have no product cap; invalid values and derived overflow refuse before input. Native waits use bounded interruptible intervals and check target, foreground, point, protection and pointer interference throughout a positive hold. Zero/omitted holds retain immediate release without a new post-down target check that could reject a dialog opened by the click.
|
|
22
|
+
|
|
23
|
+
Results report effective `timing` (`holdMs`, `timeoutMs`, `deadlinePolicy`), `inputDispatched`, interruption and `cleanupConfirmed`. Dispatch remains distinct from an observed application effect. Release runs before pointer/foreground restoration. Unconfirmed release retains ownership and blocks dependent work; timeout, cancellation and partial input never trigger replay.
|
|
24
|
+
|
|
25
|
+
The existing owned fixture prepares scoped acceptance with `npm run test:native:pointer:physical -- --list --case=timing` (safe planning only). The Pi-only `--case=timing` run requires a build, Windows Node and `PI_CU_DEDICATED_INPUT_DESKTOP=1` on the authorized idle test desktop. It observes actual button messages/timestamps and OS state for default click/restoration, zero hold, timed left/right/middle, a covering-target error, shorter deadline and cancellation of a representable very long hold with deadline disabled. Failure or unconfirmed cleanup retains the owner and fixture. This case does not rerun the old physical baseline or physical Esc demonstration.
|
|
26
|
+
|
|
27
|
+
Portable worker/peer checks verify schema, route refusals, two-layer timing forwarding, default/positive/disabled deadlines, long values, overflow, shutdown and ownership retention without sleeping for long requested durations. Native compilation and the scoped actual fixture effects passed; see the [composed receipt](validation.md#input-timing-and-modifier-tap-acceptance). Portable results alone do not establish physical input timing.
|
|
28
|
+
|
|
17
29
|
## Validation
|
|
18
30
|
|
|
19
31
|
- `npm test`: worker argument validation and the public worker lifecycle contract that prevents retirement while cleanup is unconfirmed. These portable tests do not prove Windows input delivery.
|
package/docs/runtime.md
CHANGED
|
@@ -12,7 +12,7 @@ A checksum mismatch fails explicitly; remove only the indicated runtime cache an
|
|
|
12
12
|
|
|
13
13
|
From WSL2, a bounded fixed-variable `cmd.exe /u /d` query discovers the Windows data directory and architecture. UTF-16LE decoding preserves Unicode, and environment values are never interpolated into commands. Discovery is shared with personal settings and cached for subsequent launcher calls. `wslpath` translates the mounted runtime cache and the compiled worker location, including WSL UNC paths. The launcher executes that same private Windows Node directly with argument arrays, keeping protocol stdout separate from bootstrap diagnostics; it never sends a Linux PID to a Windows process API. Windows executable interop and `cmd.exe`/`wslpath` must be available; browser discovery and some native fixtures also use Windows PowerShell. A failed bootstrap reports `WSL_BOOTSTRAP_FAILED` with its cause.
|
|
14
14
|
|
|
15
|
-
The runtime and worker start on first tool execution, not extension registration. Restoring tool exposure reads personal settings at session startup; in WSL2 this performs the small shared environment lookup but does not provision or start the worker. The cross-host settings fixture measured 39 ms for this first lookup on the tested host (a prior PowerShell query took about 1.2 seconds); this is measured overhead, not a zero-cost startup claim. The worker stays available until session shutdown/reload. Startup provisioning is cancelled on shutdown, even if a provider fails to observe the abort signal. Worker timeouts reject pending work and retire the worker; a later call can start a replacement. Input is never automatically replayed.
|
|
15
|
+
The runtime and worker start on first tool execution, not extension registration. Restoring tool exposure reads personal settings at session startup; in WSL2 this performs the small shared environment lookup but does not provision or start the worker. The cross-host settings fixture measured 39 ms for this first lookup on the tested host (a prior PowerShell query took about 1.2 seconds); this is measured overhead, not a zero-cost startup claim. The worker stays available until session shutdown/reload. Startup provisioning is cancelled on shutdown, even if a provider fails to observe the abort signal. Worker timeouts reject pending work and retire the worker when ownership permits; a later call can start a replacement. Keypress, single-click and explicit SendInput text execution deadlines first request cooperative cancellation and retain the cleanup reply during an independent five-second grace. Optional hold/pacing and positive/disabled deadline policies cross both request layers; see [keypress timing](text.md#keypress-holds-and-deadlines), [text pacing](text.md#sendinput-pacing-and-deadlines) and [single-click timing](pointer.md#single-click-holds-and-deadlines). Worker shutdown also signals active operations when an execution deadline is disabled. Input is never automatically replayed.
|
|
16
16
|
|
|
17
17
|
## Interfaces for implementation
|
|
18
18
|
|
package/docs/spec.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Pi Windows Computer Use
|
|
2
2
|
|
|
3
|
-
Status: agreed scope. Original product specification: [GitHub issue #1](https://github.com/ycheung5/pi-computer-use/issues/1).
|
|
3
|
+
Status: agreed scope. Original product specification: [GitHub issue #1](https://github.com/ycheung5/pi-computer-use/issues/1). Version 0.1.0 is published and its product/release owners are completed. The post-release [input timing and modifier-tap specification](#input-timing-and-modifier-taps) is merged into main for 0.1.1; it is not part of published 0.1.0. [ADR 0004](adr/0004-pi-first-acceptance.md) supersedes mandatory DSH parity; historical alignment records are not current release gates.
|
|
4
4
|
|
|
5
5
|
## Problem Statement
|
|
6
6
|
|
|
@@ -156,6 +156,75 @@ Accepted product decision, 2026-11-01; scoped control (#31) and presentation/dis
|
|
|
156
156
|
|
|
157
157
|
## Further Notes
|
|
158
158
|
|
|
159
|
-
|
|
159
|
+
Version 0.1.0 is published with its scoped acceptance and documented limitations. The input-timing change below is merged into main for 0.1.1 with scoped acceptance recorded. Follow the [current outcome map and queue](tickets.md#delivery-outcomes-and-ownership), [input-timing delivery plan](implementation-plan.md#input-timing-delivery) and [validation receipts](validation.md#npm-publication-receipt). Each runnable scenario proceeds through baseline, diagnosis, targeted repair and affected regressions; final reconciliation and optimization follow the required outcomes. Pi functional and preservation requirements remain binding; removed DSH comparison gates must not be reintroduced through historical notes.
|
|
160
160
|
|
|
161
161
|
Issue tracker: [ycheung5/pi-computer-use](https://github.com/ycheung5/pi-computer-use). See [tracker configuration](agents/issue-tracker.md) for operations and [triage labels](agents/triage-labels.md) for status vocabulary.
|
|
162
|
+
|
|
163
|
+
## Input timing and modifier taps
|
|
164
|
+
|
|
165
|
+
Status: implemented, accepted and merged through PR #44 for 0.1.1, 2026-11-10; [GitHub specification #40](https://github.com/ycheung5/pi-computer-use/issues/40). The following template is the canonical first-change requirement set. See the [composed receipt](validation.md#input-timing-and-modifier-tap-acceptance). These new parameters are not in the unchanged published npm 0.1.0.
|
|
166
|
+
|
|
167
|
+
### Problem Statement
|
|
168
|
+
|
|
169
|
+
Pi users cannot tap Ctrl, Alt or Shift alone, request a timed key/button press, or adjust the fixed SendInput typing delay. Long requested input durations also conflict with execution deadlines inside the Windows worker and at the Pi boundary. Agents need explicit per-call controls that preserve existing behavior when omitted and retain stopping, cleanup and non-replay guarantees. This addresses input flexibility; the cause of the user's whole-task slowness has not been measured.
|
|
170
|
+
|
|
171
|
+
### Solution
|
|
172
|
+
|
|
173
|
+
Extend the existing keypress, click and text-entry tools with standalone generic modifier taps and opt-in per-call timing. Every press/hold completes and releases inside one tool call. Agents can choose a typing interval, hold duration and execution timeout, including disabling that execution timeout. Keep the existing Pi extension and Windows worker; add no new tool family, MCP server or persistent held-input state.
|
|
174
|
+
|
|
175
|
+
### User Stories
|
|
176
|
+
|
|
177
|
+
1. As an agent, I want to tap Ctrl alone, so that applications can respond to a standalone modifier.
|
|
178
|
+
2. As an agent, I want to tap Alt or Shift alone, so that modifier-only interactions are available.
|
|
179
|
+
3. As a user, I want existing shortcuts unchanged, so that upgrading preserves current workflows.
|
|
180
|
+
4. As an agent, I want to hold a supported key or chord for a requested duration, so that I can perform a long press.
|
|
181
|
+
5. As an agent, I want to hold a supported mouse button at the selected target, so that I can perform a long mouse press.
|
|
182
|
+
6. As a user, I want each held input released before its call completes, so that the model cannot leave a key down while thinking.
|
|
183
|
+
7. As an agent, I want an explicit SendInput typing interval, so that I can adapt pacing to an application's behavior.
|
|
184
|
+
8. As a user, I want the existing 50 ms typing interval when omitted, so that the compatibility baseline remains available.
|
|
185
|
+
9. As an agent, I want zero typing delay where appropriate, so that I can request unpaced dispatch explicitly.
|
|
186
|
+
10. As a user, I want no arbitrary duration ceiling, so that the extension does not guess how long my task should take.
|
|
187
|
+
11. As an agent, I want a per-call execution timeout, so that I can choose how long to wait for that operation.
|
|
188
|
+
12. As an agent, I want zero timeout to disable the execution deadline, so that an intentional long operation is not cut short by that deadline.
|
|
189
|
+
13. As an agent, I want default deadlines to accommodate explicit timing, so that accepted long-duration requests can finish.
|
|
190
|
+
14. As a user, I want Esc and cancellation to interrupt waits, so that increasing durations does not prevent stopping.
|
|
191
|
+
15. As a user, I want input released after errors, shutdown or cancellation, so that failures do not leave controls held.
|
|
192
|
+
16. As a user, I want uncertain cleanup to retain ownership and block dependent work, so that subsequent input does not compound uncertainty.
|
|
193
|
+
17. As an agent, I want unsupported method/parameter combinations rejected before input, so that my requested method is not silently changed.
|
|
194
|
+
18. As an agent, I want actual timing settings and dispatch status reported, so that I can distinguish configuration from observed task success.
|
|
195
|
+
19. As a user, I want partial or stopped input never automatically replayed, so that changing timing cannot duplicate actions.
|
|
196
|
+
20. As a maintainer, I want native Windows and WSL2 callers to share the same contract, so that host routing does not change timing semantics.
|
|
197
|
+
21. As a maintainer, I want focused regression and owned-fixture checks, so that this update does not reopen accepted release tests.
|
|
198
|
+
22. As a user, I want broader keyboard/mouse coverage delivered separately, so that the first change stays small and reviewable.
|
|
199
|
+
|
|
200
|
+
### Implementation Decisions
|
|
201
|
+
|
|
202
|
+
- Extend existing input schemas, route validation, Pi integration, request/deadline handling and native input execution. Preserve the current module responsibilities and independent emergency-stop helper. No new dependencies, public test API, general scheduler or persistent down/up API.
|
|
203
|
+
- Accept one standalone generic Ctrl/Control, Alt/Option or Shift modifier in keypress. Existing supported ordinary keys/chords remain supported. Multiple modifiers without a main key are not newly promised. Existing aliases and malformed/duplicate-key validation remain applicable.
|
|
204
|
+
- Add optional holdMs to keypress and single click only. Omission preserves current immediate release; zero explicitly requests immediate release. For a chord, hold its pressed main key and modifiers together, then release all held inputs. For a click, press the selected supported button at the resolved target, wait, and release. Existing targeting, foreground intent and restoration rules continue to apply. No new auto-repeat guarantee is implied by a held key.
|
|
205
|
+
- Add optional inputDelayMs to type_text only with explicitly selected sendinput. Omission preserves the existing 50 ms pacing. Zero requests no intentional inter-unit delay. Keep current Unicode/newline/tab handling; the unit is the actual dispatched UTF-16 input unit, with the existing newline normalization. This is not a promise of exact application processing time or universal correctness at lower intervals.
|
|
206
|
+
- Add optional timeoutMs to type_text, keypress and single click only. A positive value supplies the requested execution deadline; zero disables that call's execution deadline through all relevant input execution layers. An explicit shorter timeout is honored, not silently extended, and may end partial input with the existing cleanup/non-replay result.
|
|
207
|
+
- With timeoutMs omitted, ordinary calls retain current deadline behavior. When holdMs or inputDelayMs is explicitly supplied, derive sufficient default execution budgets from the requested hold/pacing duration plus existing execution overhead allowances. Both the native-helper request and outer Pi-worker request must accommodate it; an inner 40-second or outer 90-second timer must not silently override the accepted contract.
|
|
208
|
+
- Timing arguments are non-negative safe whole milliseconds, without arbitrary product ceilings. Reject negative, fractional, non-finite, unsafe or otherwise unrepresentable values and unrepresentable derived durations before input. Long waits must not wrap, clamp to a short timer, or accidentally mean infinity because of an underlying API integer limit. Zero has the parameter-specific meanings above.
|
|
209
|
+
- Disabling the execution deadline does not disable explicit cancellation, shutdown handling, emergency stopping, target/protection checks or cleanup acknowledgments. Existing startup, target-resolution and cleanup safeguards remain independent. Timing is best effort, not a hard real-time guarantee.
|
|
210
|
+
- Validate backend/method capability before sending input. The first slice requires native foreground timed holds and explicit native SendInput pacing. A selected browser/background/quiet route must either honor the requested semantics or return a clear unsupported-combination error before input. Do not silently ignore a parameter, switch an explicit method, or escalate an explicit background request. Existing auto routing remains unchanged when new parameters are omitted; its resolution with new parameters must honor them or refuse before dispatch. This slice does not promise universal browser/background long-press support.
|
|
211
|
+
- Preserve exclusive control, visible local border, target identity/focus/occlusion checks, protection during tool gaps, clipboard preservation and human handback. Cancellation/error/shutdown must release agent-held input, or explicitly report unconfirmed cleanup and block dependent work. No automatic replay or forced termination of an uncertain input owner.
|
|
212
|
+
- Report the effective requested timing/deadline policy, whether input was dispatched or interrupted, and existing cleanup confirmation. A successful dispatch does not establish an observed application effect. Existing result fields retain their meaning.
|
|
213
|
+
- Keep the current Pi package and tool names. The published 0.1.0 remains unchanged; implementation and any later publication are separate work.
|
|
214
|
+
|
|
215
|
+
### Testing Decisions
|
|
216
|
+
|
|
217
|
+
- Reuse the confirmed public Pi-tool and worker request/cancellation interfaces as the highest practical seams. Extend existing schema, input-preparation, text, pointer and worker/transport tests; do not add a new framework or public timing-test interface.
|
|
218
|
+
- Test externally visible validation, routing, effects and cleanup results rather than private state. Cover modifier taps, existing chords, omitted/zero/positive timing, representable large values and derived overflow, invalid values, unsupported routes and no input before refusal.
|
|
219
|
+
- Use existing controllable worker peers and owned fixtures to test omitted/positive/disabled execution deadlines at both request layers, a deliberately shorter timeout, cancellation during a long requested interval and release/ownership uncertainty. Use short deterministic simulations for very long values and timer-overflow cases; tests must not sleep for the full requested durations.
|
|
220
|
+
- Extend the existing native text and pointer fixtures for one grouped scoped Windows acceptance: standalone modifier down/up events, a timed key/chord and mouse press/release, explicit typing intervals with exact Unicode readback, default pacing regression, and stop/error cleanup of newly held inputs. Observe events/effects independently of returned dispatch success. Reuse existing stop/control evidence; repeat a physical Esc demonstration only if the changed mechanism or a failure specifically requires it.
|
|
221
|
+
- Check relevant Pi/WSL2-to-Windows and native request forwarding so no layer drops a field or restores a hidden default deadline. Keep these checks scoped to changed behavior; no new paid-model reset series, DSH comparison or unrelated display/browser/profile acceptance.
|
|
222
|
+
- Run configured portable checks and actual package/documentation validation for implementation. Record unrun live prerequisites and limitations honestly. Passing mocks do not establish real Windows input effects; exact-text verification is required before claiming lower pacing correct.
|
|
223
|
+
|
|
224
|
+
### Out of Scope
|
|
225
|
+
|
|
226
|
+
Broader named-key coverage, Windows keys, left/right modifiers, numpad/media keys, mouse side buttons and modifier-click/drag are deferred to the second change. Also excluded: raw virtual-key/scan-code input, holds across calls, separate key-down/up tools, double-click/drag timing controls, privileged or device-specific integrations, secure-desktop bypass, changing default typing pace, global timeout removal, a new MCP adapter, universal device/application coverage and diagnosing or claiming a fix for unmeasured whole-task latency.
|
|
227
|
+
|
|
228
|
+
### Further Notes
|
|
229
|
+
|
|
230
|
+
The user confirmed this first-change scope after choosing a two-change delivery. Existing 0.1.0 acceptance remains closed and reusable. This spec describes the implementation merged into main for 0.1.1; the unchanged npm 0.1.0 does not accept the new timing parameters. Readable key names remain the agent-facing interface. Required input release and uncertain-cleanup safeguards qualify every duration/timeout option. Future broader input coverage must be specified and accepted separately rather than silently added here.
|
package/docs/text.md
CHANGED
|
@@ -2,15 +2,35 @@
|
|
|
2
2
|
|
|
3
3
|
The candidate acquires [exclusive control](spec.md#exclusive-control) before physical text entry. Suppressed physical events do not invalidate typing; foreign injected or unconsumed events still do. Pi chord/focus has [scoped acceptance](validation.md#pi-chords-and-editor-focus-acceptance); #31 control/handback and #27 clipboard outcomes retain their accepted receipts.
|
|
4
4
|
|
|
5
|
-
`type_text` accepts `text`, `method: auto | clipboard | sendinput | background`, `focusMode: preserve | main`, `restoreClipboard` (default true), and native target options. Browser options retain browser-first routing. `keypress` accepts a `keys` array with
|
|
5
|
+
`type_text` accepts `text`, `method: auto | clipboard | sendinput | background`, `focusMode: preserve | main`, `restoreClipboard` (default true), and native target options. Browser options retain browser-first routing. `keypress` accepts a `keys` array with one main key and optional Ctrl/Alt/Shift modifiers, or a single standalone generic Ctrl/Control, Alt/Option or Shift. Windows/Cmd/Meta keys are blocked; unsupported names/layout mappings fail without input. Explicit background keypress supports one printable character only.
|
|
6
6
|
|
|
7
7
|
Auto text first tries the focused native Edit/RichEdit control using bounded `EM_REPLACESEL`, preserving its selection. This quiet path handles Unicode without clipboard or physical keys; multiline text, main-focus selection and unsupported controls require physical entry. An uncertain edit-message result is never retried as clipboard input. Background WM_CHAR delivery is explicitly unverified. Read-only/disabled controls refuse mutation.
|
|
8
8
|
|
|
9
9
|
The operation's last target is reused after foreground restoration. Fresh Win32 focus takes precedence over remembered native focus. Only an agent-established focus is remembered; subsequent click/focus/target changes invalidate it. Remembered controls must retain their HWND/PID/root and unchanged input-history stamp. An intervening user input with unavailable fresh focus returns `TARGET_FOCUS_CHANGED`; it does not guess the old field. Main focus explicitly chooses the largest enabled, keyboard-focusable, on-screen writable Edit/Document control. After an explicit or remembered focus request, physical text entry waits up to one second for foreground and the requested field to agree. The wait observes without repeating focus or text, checks cancellation and identity, and refuses with `TARGET_FOCUS_CHANGED` if focus cannot be confirmed. An explicit field request cannot be replaced by remembered focus. Successful confirmation reports `focusConfirmed` and `inputFocusHandle`. Optional native edit `verifyValue` is capped at 200 characters and excludes password edits. Physical entry does not read password values.
|
|
10
10
|
|
|
11
|
+
## Keypress holds and deadlines
|
|
12
|
+
|
|
13
|
+
On integration/input-timing-40 (not published npm 0.1.0), the #41 keyboard slice adds `holdMs` and `timeoutMs` to `keypress`. Omitted or zero `holdMs` releases immediately. A positive hold keeps the main key and its modifiers down together, then releases every agent-held key within the call. Immediate presses release without post-dispatch focus checks, preserving shortcuts that move focus or open menus. Positive holds retain the existing keypress identity, foreground, protection and interference checks while permitting focus changes between controls in the same window. Held keys do not promise auto-repeat. Timing and standalone modifiers require native foreground input; browser and explicit background timing requests return `TIMING_UNSUPPORTED` before input. Existing background printable-character behavior remains available with timing omitted.
|
|
14
|
+
|
|
15
|
+
A positive `timeoutMs` requests that call's execution deadline; an explicit shorter deadline can interrupt a hold. Zero disables only the execution deadline. Cancellation, emergency stopping, shutdown, target/focus checks, physical protection and cleanup still apply. Omitted timing keeps the ordinary native-helper 40-second and Pi-worker 90-second budgets. An explicit hold with timeout omitted adds its duration to those existing overhead allowances at each layer. Values must be finite, non-negative safe whole milliseconds; derived overflow is refused before dispatch. Long timers and native waits use bounded chunks without wrapping.
|
|
16
|
+
|
|
17
|
+
Results report `timing: { holdMs, timeoutMs, deadlinePolicy }` with `default`, `derived`, `explicit` or `disabled` policy, `inputDispatched`, `interrupted` and existing `cleanupConfirmed`. The native result describes its effective helper budget; a transport deadline result describes the expiring request layer. Execution expiry requests cooperative cancellation and gives cleanup acknowledgment an independent five-second grace. A missing acknowledgment retains ownership and blocks dependent input; it never authorizes killing an uncertain input owner. Dispatch and requested timings do not establish an application effect. There is no automatic replay.
|
|
18
|
+
|
|
19
|
+
Portable tests cover schema/route refusals, aliases, invalid/large/overflow values, both request layers, explicit short deadlines, cancellation races, shutdown and ownership uncertainty using controlled peers. These tests establish forwarding and lifecycle behavior, not Windows input effects. Scoped Windows acceptance is recorded in the [composed receipt](validation.md#input-timing-and-modifier-tap-acceptance). Prepare its safe plan with `npm run test:native:text -- --timing --plan`. The grouped owned-fixture run is `npm run test:native:text -- --timing` on an authorized Windows desktop with `PI_CU_DEDICATED_INPUT_DESKTOP=1` after building. It observes down/up messages and durations independently for aliases, immediate release, a key/chord hold, default/zero/timed Tab focus changes, foreground-loss release, short deadlines and cancellation with a disabled deadline; it checks OS key state and released control. It neither accesses the clipboard nor repeats the old chord suite. Follow the [WSL runtime prerequisites](runtime.md#verification) before running from a WSL worktree. Timing is best effort. The final keyboard-only run used `--timing --phase=keyboard`; `--phase=text` selects text cases and omission selects all. The two fixture failures/corrections are retained in the receipt.
|
|
20
|
+
|
|
21
|
+
## SendInput pacing and deadlines
|
|
22
|
+
|
|
23
|
+
Explicit native `method: sendinput` accepts `inputDelayMs` and `timeoutMs`. Omitted pacing remains 50 ms; zero requests no intentional delay. Each delay follows one dispatched UTF-16 unit (CRLF is one Enter); a surrogate pair remains two UTF-16 units. Values are non-negative safe whole milliseconds. Timing fields on auto, clipboard, background or browser routes return `TIMING_UNSUPPORTED` before input, including explicit zero. Omitted fields preserve those routes.
|
|
24
|
+
|
|
25
|
+
Text reuses the [keypress execution deadline and cleanup contract](#keypress-holds-and-deadlines): positive timeout is honored unchanged, zero disables only execution expiry, and omitted timeout adds explicit pacing multiplied by the normalized unit count to each layer's ordinary overhead. Both the product and derived budgets must be representable, even when execution expiry is disabled. Long pacing waits check cancellation, target identity, focus and protection at least every 25 ms. They never wrap to a short wait. A stopped or partial entry is never replayed.
|
|
26
|
+
|
|
27
|
+
Results expose effective `inputDelayMs`, `timing: { inputDelayMs, timeoutMs, deadlinePolicy }`, `inputDispatched`, `interrupted`, `inputUnitsDispatched` and cleanup confirmation. The unit count records complete dispatched down/up pairs, not observed text. Exact text still requires independent observation.
|
|
28
|
+
|
|
29
|
+
The existing `--timing` plan includes exact Unicode/CRLF/tab readback at omitted (50 ms), zero and 25 ms pacing, a shorter text deadline, focus-change interruption, and cancellation during a `MAX_SAFE_INTEGER` delay with the execution deadline disabled. The owned fixture enables tab insertion only during text cases, then restores ordinary Tab focus behavior for keypress cases. Portable tests verify safe values, routing, budgets at both request layers, host forwarding, shutdown and uncertain cleanup with controlled peers. Windows compilation, exact-text and interruption acceptance passed in the named owned fixture; see the [composed receipt](validation.md#input-timing-and-modifier-tap-acceptance). No lower interval is claimed universally correct.
|
|
30
|
+
|
|
11
31
|
## Physical entry and cleanup
|
|
12
32
|
|
|
13
|
-
Explicit `sendinput` text uses KEYEVENTF_UNICODE down/up pairs for ordinary text, Enter key pairs for LF or CR (CRLF becomes one Enter), and Tab key pairs for tabs. Whitespace-only strings are valid text. These keys retain the target application's normal Enter/Tab behavior. A cancellable
|
|
33
|
+
Explicit `sendinput` text uses KEYEVENTF_UNICODE down/up pairs for ordinary text, Enter key pairs for LF or CR (CRLF becomes one Enter), and Tab key pairs for tabs. Whitespace-only strings are valid text. These keys retain the target application's normal Enter/Tab behavior. A cancellable gap follows each dispatched unit (50 ms when omitted): both unpaced delivery and a 10 ms gap were observed to corrupt appended text in modern Notepad despite successful dispatch. The pinned DSH reference instead batches Unicode events without this per-character delay. Pacing and control-character handling changed together in Pi, so the successful regression does not establish that 50 ms is necessary or isolate the cause. See the [Pi text validation scope](validation.md#unicode-mechanism-investigation). Minimum-delay discovery and DSH batching experiments are not release gates. The default compatibility workaround adds approximately 50 ms per unit plus platform scheduling overhead; it is not a universal guarantee about application processing. Dispatch remains unverified until a subsequent observation confirms the intended text. Successful results expose `inputDelayMs` so callers can identify the pacing actually used by the running helper. Default `auto` entry retains its existing quiet-edit/clipboard routing. Chords are mapped using the target thread's keyboard layout and use SendInput rather than an uninterruptible SendKeys call. The shared activity observer spans the whole call and distinguishes our tagged key events from intervening physical/foreign injected input; a later synthetic key cannot erase earlier intervention evidence. The helper rechecks target HWND/PID, foreground and cancellation before each character/key; text also checks focused control and user-held modifiers/buttons. Every held key/modifier is released in `finally`, even when its key-down result was uncertain. Partial text, timeout and cancellation never authorize automatic replay.
|
|
14
34
|
|
|
15
35
|
The independent stop event and retained-input acknowledgment from [pointer stopping](pointer.md) cover keyboard actions too. A timeout signals the operation to stop; missing release/clipboard acknowledgment retains ownership and blocks worker termination/replacement. `set_value` may escalate only an explicit background `WCU_NEEDS_INPUT`, after re-resolving the observed control, checking enabled/read-only state and focusing it; select-all and typing use the same guarded cleanup.
|
|
16
36
|
|
package/docs/tickets.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Implementation tickets
|
|
2
2
|
|
|
3
|
-
Requirements live in [spec](spec.md), delivery/coverage in [implementation-plan](implementation-plan.md), results in [validation](validation.md). This is the current Pi-first queue, reconciled on 2026-11-
|
|
3
|
+
Requirements live in [spec](spec.md), delivery/coverage in [implementation-plan](implementation-plan.md), results in [validation](validation.md). This is the current Pi-first queue, reconciled on 2026-11-10 under [ADR 0004](adr/0004-pi-first-acceptance.md). DSH manual/host/model comparisons, credentials and parity benchmarks are discarded.
|
|
4
4
|
|
|
5
5
|
## Current investigation and acceptance work
|
|
6
6
|
|
|
@@ -9,13 +9,13 @@ Requirements live in [spec](spec.md), delivery/coverage in [implementation-plan]
|
|
|
9
9
|
| Measure | Status after scope reconciliation |
|
|
10
10
|
| --- | --- |
|
|
11
11
|
| Tool implementation | All 25 target tools have implementations. Release acceptance concerns Pi correctness, preservation and usability. |
|
|
12
|
-
| Completed issues |
|
|
12
|
+
| Completed issues | 30: #1, #2, #3, #6, #7, #8, #14, #16, #17, #18, #19, #20, #21, #24, #25, #26, #27, #28, #29, #30, #31, #32, #33, #34, #35, #36, #40, #41, #42, #43. Historical comparison completions retain their original evidence scope. |
|
|
13
13
|
| Superseded issues | 10: #4, #5, #9, #10, #11, #12, #13, #15, #23, #37. Superseded is not a completed product delivery. |
|
|
14
|
-
| Open issues |
|
|
14
|
+
| Open issues | None in the first-release and accepted input-timing delivery queues. #40/#41/#42/#43 are complete with scoped evidence and merged through PR #44; 0.1.1 is being prepared, with npm publication pending. |
|
|
15
15
|
| Accepted Pi outcomes | Windows/WSL2 packaged launch, ownership, waits, settings/tool selection, Chromium and experimental Firefox workflows; six browser/mixed agent scenarios each passed three reset runs. Native creation/preservation, replacement/partial recovery and composed continuation (#35), #31 control interruption/cleanup and human-confirmed GitHub sign-in persistence/removal also passed. |
|
|
16
|
-
| Latest recorded checks |
|
|
16
|
+
| Latest recorded checks | Input-timing integration: **214 passed, one historical-reference skip**, scoped Windows keyboard/text/mouse acceptance and reviews passed; [receipt](validation.md#input-timing-and-modifier-tap-acceptance). First-release audit: 187 passed, one historical-reference skip; 91-entry 0.1.0 archive, MIT/notices/version/development-file exclusion and documentation links passed. [Audit](validation.md#final-release-audit). The final scoped/banner package check passed 92 entries. Public npm 0.1.0/latest and downloaded archive hash are verified; earlier Windows/model/physical receipts retain their scope. |
|
|
17
17
|
|
|
18
|
-
The
|
|
18
|
+
The first release is published as [@ycheung5/pi-computer-use@0.1.0](https://www.npmjs.com/package/@ycheung5/pi-computer-use). Its public archive matches the user-submitted build from 73616b5. An open issue does not reopen its accepted cases. Completion is scoped to the accepted first-release requirements and recorded test limitations. The earlier development baseline and historical comparison receipts remain available through commits/archive tags.
|
|
19
19
|
|
|
20
20
|
### Delivery outcomes and ownership
|
|
21
21
|
|
|
@@ -33,7 +33,7 @@ The package is implemented; remaining work is targeted Pi acceptance and repair.
|
|
|
33
33
|
| #31 — stop/control | Native Pi Esc, worker shortcut/suppression, native/WSL command stop, rejection/recovery and no-input host-exit cleanup passed; queued-settlement bug fixed. Four known-held Ctrl/button cancellation/target-error cases and actual WSL Pi Ctrl+Alt+F9 and human handback/resume also passed. | Completed: native availability-loss response and existing worker/held-input/host/handback results satisfy the narrowed scope; [acceptance](validation.md#control-interruption-acceptance). |
|
|
34
34
|
| #35 — native agent goals | Three preopened append/save tasks, three creation/preservation/save resets and three replacement/partial-observation recovery workflows have accepted independent evidence. [Creation](validation.md#native-document-workflow); [recovery and collector-failure disposition](validation.md#native-replacement-and-partial-observation-recovery-preparation). | Completed: three real-Pi continuation resets preserved the labelled stand-in edit after cancelled/retired setup. [Composed handback acceptance](validation.md#native-cancellation-and-handback-continuation) reuses #31 actual human/control proof; no new physical-control claim. |
|
|
35
35
|
| #36 — browser/mixed agent goals | Six named Pi scenarios passed three resets: form/delay; navigation/dialog/transfer; stale recovery; quiet mixed task; partial-observation recovery; held-navigation cancellation. | Completed for this scope. Shared physical control stays #31/#20; native creation/recovery stays #35. No DSH leg remains. |
|
|
36
|
-
| #21 under #1 — release | Capability mapping, existing checks and accepted scopes retained. |
|
|
36
|
+
| #21 under #1 — release | Capability mapping, existing checks and accepted scopes retained. | Completed: user-approved 0.1.0 publication, public version/latest and registry archive identity verified. [Release receipt](validation.md#npm-publication-receipt). |
|
|
37
37
|
|
|
38
38
|
### Execution queue
|
|
39
39
|
|
|
@@ -41,7 +41,8 @@ Work on one bounded missing outcome at a time. Reuse accepted evidence across is
|
|
|
41
41
|
|
|
42
42
|
| Order / owner | Next action | User involvement |
|
|
43
43
|
| --- | --- | --- |
|
|
44
|
-
|
|
|
44
|
+
| Input timing complete — #40/#41/#42/#43 | [Modifier taps, holds, pacing and timeout control](spec.md#input-timing-and-modifier-taps) merged into main through PR #44 for 0.1.1. [Composed acceptance](validation.md#input-timing-and-modifier-tap-acceptance) records portable, owned Windows and review results plus both fixture failures/corrections. | No further repeated tests needed for this accepted scope. Main merge is complete; see [0.1.1 release preparation](validation.md#011-release-preparation). npm publication remains pending; broader inputs remain deferred. |
|
|
45
|
+
| First release complete | 0.1.0 is public on npm; maintainer/latest and the downloaded archive checksum match. #21 and #1 are complete. | None required for this release. Do not reopen accepted cases or repeat tests without a demonstrated new issue. |
|
|
45
46
|
|
|
46
47
|
Standing scoped idle-test authorization remains. Preserve unrelated applications, documents, clipboard and profiles, serialize desktop activity and request human participation only for genuine physical/private/visual steps. Unknown input cleanup retains actual owners and blocks dependent activity. No DSH/OpenRouter credentials, DSH installation or side-by-side tests are needed.
|
|
47
48
|
|