deadbyte-mcp 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MANIFEST.SHA256 +101 -39
- package/README.txt +10 -9
- package/acceptance/r40/desktop-live-drift.mjs +106 -0
- package/acceptance/r40/desktop-live-ground.mjs +110 -0
- package/acceptance/r40/desktop-live-input.mjs +128 -0
- package/acceptance/r40/desktop-live-observe.mjs +100 -0
- package/acceptance/r40/desktop-live-uia.mjs +121 -0
- package/bin/deadbyte-desktop-input-r40.exe +0 -0
- package/bin/deadbyte-desktop-input-r40.obj +0 -0
- package/bin/deadbyte-desktop-observer-r40.exe +0 -0
- package/bin/deadbyte-desktop-observer-r40.obj +0 -0
- package/bin/deadbyte-desktop-uia-r40.exe +0 -0
- package/bin/deadbyte-desktop-uia-r40.obj +0 -0
- package/controller/README.TXT +7 -5
- package/controller/deadbyte-controller.config.psd1.example +3 -0
- package/controller/deadbyte-controller.ps1 +149 -4
- package/controller/deadbyte-desktop-policy.json +41 -0
- package/docs/ARCHITECTURE.md +6 -1
- package/docs/superpowers/specs/2026-09-19-r40-desktop-computer-use-design.md +214 -0
- package/native/deadbyte-desktop-input.cpp +455 -0
- package/native/deadbyte-desktop-observer.cpp +253 -0
- package/native/deadbyte-desktop-uia.cpp +307 -0
- package/package.json +10 -8
- package/scripts/build-desktop-r40.ps1 +106 -0
- package/scripts/deferred-slot-operation-r39.mjs +124 -0
- package/scripts/deferred-slot-operation-r40.mjs +126 -0
- package/scripts/final-closure-r39.mjs +113 -0
- package/scripts/final-closure-r40.mjs +114 -0
- package/scripts/final-closure-verify-r39.mjs +208 -0
- package/scripts/final-closure-verify-r40.mjs +212 -0
- package/scripts/gate-windows-r39.ps1 +42 -0
- package/scripts/gate-windows-r40.ps1 +43 -0
- package/scripts/init-desktop-policy-r40.ps1 +79 -0
- package/scripts/r40-compact-ledger-probe.mjs +4 -0
- package/scripts/release-parity-r39.mjs +200 -0
- package/scripts/release-parity-r40.mjs +212 -0
- package/scripts/release-parity-tests-r39.ps1 +9 -0
- package/scripts/release-parity-tests-r40.ps1 +9 -0
- package/scripts/windows-gate-evidence-r39.mjs +52 -0
- package/scripts/windows-gate-evidence-r40.mjs +52 -0
- package/src/autonomous-context-engine.mjs +2 -1
- package/src/autonomous-planner-contracts.mjs +19 -1
- package/src/autonomous-planner.mjs +22 -0
- package/src/autonomous-runtime.mjs +144 -24
- package/src/capability-ledger.mjs +6 -1
- package/src/compact-gateway-mcp-tools.mjs +14 -2
- package/src/deadbyte-cli.mjs +79 -47
- package/src/desktop-contract.mjs +263 -0
- package/src/desktop-input-r40.mjs +148 -0
- package/src/desktop-mcp-tools-r40.mjs +126 -0
- package/src/desktop-memory-r40.mjs +272 -0
- package/src/desktop-observer-r40.mjs +231 -0
- package/src/desktop-policy-r40.mjs +110 -0
- package/src/desktop-runtime-r40.mjs +318 -0
- package/src/desktop-uia-r40.mjs +123 -0
- package/src/mcp-server.mjs +74 -19
- package/src/r40-compact-disarmed-probe.mjs +71 -0
- package/src/r40-full-surface-probe.mjs +34 -0
- package/src/release-generation.mjs +38 -5
- package/src/remote-console.mjs +41 -0
- package/src/remote-live-log.mjs +215 -45
- package/src/remote-mcp-desktop-r40-smoke-client.mjs +70 -0
- package/src/remote-mcp-r40-parity-client.mjs +200 -0
- package/src/remote-mcp-smoke-client.mjs +20 -6
- package/src/remote-result-journal.mjs +10 -2
- package/src/runtime-update.mjs +55 -17
- package/src/tool-audit.mjs +29 -2
- package/src/version.mjs +1 -1
- package/test/autonomous-capability-r36.test.mjs +43 -1
- package/test/autonomous-desktop-r40.test.mjs +188 -0
- package/test/capability-ledger-r36.test.mjs +1 -0
- package/test/cli-entrypoint.test.mjs +1 -1
- package/test/compact-catalog-r36.test.mjs +2 -2
- package/test/compact-gateway-r36.test.mjs +56 -4
- package/test/compact-max-catalog-r40.test.mjs +95 -0
- package/test/controller-desktop-r40.test.mjs +85 -0
- package/test/core.test.mjs +1 -1
- package/test/desktop-adversarial-r40.test.mjs +62 -0
- package/test/desktop-contract-r40.test.mjs +200 -0
- package/test/desktop-input-r40.test.mjs +63 -0
- package/test/desktop-mcp-tools-r40.test.mjs +61 -0
- package/test/desktop-memory-r40.test.mjs +141 -0
- package/test/desktop-observer-r40.test.mjs +119 -0
- package/test/desktop-policy-r40.test.mjs +111 -0
- package/test/desktop-runtime-r40.test.mjs +278 -0
- package/test/desktop-uia-r40.test.mjs +42 -0
- package/test/full-max-catalog-r40.test.mjs +91 -0
- package/test/helpers/autonomous-r34-fixture.mjs +3 -3
- package/test/production-docs.test.mjs +3 -3
- package/test/r33-closeout-regression.test.mjs +8 -8
- package/test/r33-finalization.test.mjs +14 -14
- package/test/r37-remote-tool-call-timeline.test.mjs +7 -6
- package/test/r39-finalization.test.mjs +74 -0
- package/test/r40-finalization.test.mjs +118 -0
- package/test/release-generation.test.mjs +41 -9
- package/test/release-version.test.mjs +19 -58
- package/test/remote-console-r40.test.mjs +38 -0
- package/test/remote-live-log-r37.test.mjs +89 -32
- package/test/remote-r24-catalog.test.mjs +12 -3
- package/test/remote-result-journal.test.mjs +15 -0
- package/test/remote-session-cli.test.mjs +112 -55
- package/test/runtime-self-update.test.mjs +44 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { readFile } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { canonicalJson } from './canonical-json.mjs';
|
|
6
|
+
import { createDesktopFrame } from './desktop-contract.mjs';
|
|
7
|
+
|
|
8
|
+
const NATIVE_SCHEMA='deadbyte.desktop-native-frame.v1';
|
|
9
|
+
const BACKENDS=new Set(['gdi-bitblt-fallback','desktop-duplication','windows-graphics-capture']);
|
|
10
|
+
const PIXEL_FORMAT='BGRA8';
|
|
11
|
+
const HEX64=/^[0-9a-f]{64}$/;
|
|
12
|
+
const HWND=/^0x[0-9a-f]{1,16}$/;
|
|
13
|
+
const HEADER_MAX_BYTES=64*1024;
|
|
14
|
+
|
|
15
|
+
const sha256Bytes=value=>createHash('sha256').update(value).digest('hex');
|
|
16
|
+
const sha256Object=value=>sha256Bytes(Buffer.from(canonicalJson(value),'utf8'));
|
|
17
|
+
|
|
18
|
+
function insist(condition,message){ if(!condition) throw new Error(message); }
|
|
19
|
+
function safeInt(value,label,{min=Number.MIN_SAFE_INTEGER,max=Number.MAX_SAFE_INTEGER}={}){
|
|
20
|
+
insist(Number.isSafeInteger(value)&&value>=min&&value<=max,`${label} invalid`);
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
function boundedString(value,label,max=32768){
|
|
24
|
+
insist(typeof value==='string'&&Buffer.byteLength(value,'utf8')<=max,`${label} invalid`);
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
function normalizeMonitor(item,index){
|
|
28
|
+
insist(item&&typeof item==='object'&&!Array.isArray(item),`monitor[${index}] invalid`);
|
|
29
|
+
return Object.freeze({
|
|
30
|
+
left:safeInt(item.left,`monitor[${index}].left`,{min:-131072,max:131072}),
|
|
31
|
+
top:safeInt(item.top,`monitor[${index}].top`,{min:-131072,max:131072}),
|
|
32
|
+
width:safeInt(item.width,`monitor[${index}].width`,{min:1,max:131072}),
|
|
33
|
+
height:safeInt(item.height,`monitor[${index}].height`,{min:1,max:131072}),
|
|
34
|
+
primary:item.primary===true,
|
|
35
|
+
device:boundedString(item.device,`monitor[${index}].device`,512)
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
function monitorWithin(bounds,monitor){
|
|
39
|
+
return monitor.left>=bounds.left&&monitor.top>=bounds.top&&
|
|
40
|
+
monitor.left+monitor.width<=bounds.left+bounds.width&&
|
|
41
|
+
monitor.top+monitor.height<=bounds.top+bounds.height;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function parseNativeDesktopFrame(buffer,{maxFrameBytes=128*1024*1024}={}){
|
|
45
|
+
insist(Buffer.isBuffer(buffer),'native desktop frame must be Buffer');
|
|
46
|
+
safeInt(maxFrameBytes,'maxFrameBytes',{min:1024,max:256*1024*1024});
|
|
47
|
+
const newline=buffer.indexOf(0x0a);
|
|
48
|
+
insist(newline>0&&newline<=HEADER_MAX_BYTES,'native desktop frame header missing or oversized');
|
|
49
|
+
let header;
|
|
50
|
+
try { header=JSON.parse(buffer.subarray(0,newline).toString('utf8')); }
|
|
51
|
+
catch { throw new Error('native desktop frame header JSON invalid'); }
|
|
52
|
+
insist(header?.schema===NATIVE_SCHEMA,'native desktop frame schema mismatch');
|
|
53
|
+
insist(BACKENDS.has(header.backend),'native desktop frame backend invalid');
|
|
54
|
+
insist(header.pixel_format===PIXEL_FORMAT,'native desktop frame pixel format mismatch');
|
|
55
|
+
const left=safeInt(header.left,'native left',{min:-131072,max:131072});
|
|
56
|
+
const top=safeInt(header.top,'native top',{min:-131072,max:131072});
|
|
57
|
+
const width=safeInt(header.width,'native width',{min:1,max:32768});
|
|
58
|
+
const height=safeInt(header.height,'native height',{min:1,max:32768});
|
|
59
|
+
const stride=safeInt(header.stride,'native stride',{min:4,max:131072});
|
|
60
|
+
const byteLength=safeInt(header.byte_length,'native byte_length',{min:4,max:maxFrameBytes});
|
|
61
|
+
insist(stride===width*4,'native desktop stride mismatch');
|
|
62
|
+
insist(byteLength===stride*height,'native desktop byte_length mismatch');
|
|
63
|
+
insist(buffer.length===newline+1+byteLength,'native desktop frame payload length mismatch');
|
|
64
|
+
const systemDpi=safeInt(header.system_dpi,'system_dpi',{min:48,max:960});
|
|
65
|
+
const sessionId=safeInt(header.session_id,'session_id',{min:0,max:0xffffffff});
|
|
66
|
+
const inputDesktop=boundedString(header.input_desktop,'input_desktop',512);
|
|
67
|
+
const hwnd=String(header.foreground_hwnd??'').toLowerCase();
|
|
68
|
+
insist(HWND.test(hwnd),'foreground_hwnd invalid');
|
|
69
|
+
const foregroundPid=safeInt(header.foreground_pid,'foreground_pid',{min:0,max:0x7fffffff});
|
|
70
|
+
const foregroundTitle=boundedString(header.foreground_title??'','foreground_title',64*1024);
|
|
71
|
+
const foregroundImage=boundedString(header.foreground_image??'','foreground_image',64*1024);
|
|
72
|
+
insist(Array.isArray(header.monitors)&&header.monitors.length>=1&&header.monitors.length<=32,'native monitor list invalid');
|
|
73
|
+
const monitors=header.monitors.map(normalizeMonitor);
|
|
74
|
+
insist(monitors.filter(item=>item.primary).length===1,'native monitor list must have exactly one primary monitor');
|
|
75
|
+
const bounds=Object.freeze({left,top,width,height});
|
|
76
|
+
insist(monitors.every(item=>monitorWithin(bounds,item)),'native monitor lies outside virtual desktop bounds');
|
|
77
|
+
const pixels=buffer.subarray(newline+1);
|
|
78
|
+
const imageSha256=sha256Bytes(pixels);
|
|
79
|
+
const topology={
|
|
80
|
+
schema:'deadbyte.desktop-topology.v1',bounds,system_dpi:systemDpi,
|
|
81
|
+
monitors:[...monitors].sort((a,b)=>a.left-b.left||a.top-b.top||a.device.localeCompare(b.device))
|
|
82
|
+
};
|
|
83
|
+
const topologySha256=sha256Object(topology);
|
|
84
|
+
const activeWindow=foregroundPid>0&&hwnd!=='0x0'?Object.freeze({
|
|
85
|
+
hwnd,pid:foregroundPid,
|
|
86
|
+
...(foregroundImage?{process_image_sha256:sha256Bytes(Buffer.from(foregroundImage,'utf8'))}:{}),
|
|
87
|
+
...(foregroundTitle?{title_sha256:sha256Bytes(Buffer.from(foregroundTitle,'utf8'))}:{})
|
|
88
|
+
}):null;
|
|
89
|
+
return Object.freeze({
|
|
90
|
+
schema:NATIVE_SCHEMA,backend:header.backend,pixel_format:PIXEL_FORMAT,bounds,stride,byte_length:byteLength,
|
|
91
|
+
system_dpi:systemDpi,session_id:sessionId,input_desktop:inputDesktop,monitors,topology_sha256:topologySha256,
|
|
92
|
+
active_window:activeWindow,image_sha256:imageSha256,pixels
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export async function captureNativeDesktop({
|
|
97
|
+
observerPath,expectedObserverSha256,maxFrameBytes=128*1024*1024,timeoutMs=10000
|
|
98
|
+
}={}){
|
|
99
|
+
insist(typeof observerPath==='string'&&path.isAbsolute(observerPath),'observerPath must be absolute');
|
|
100
|
+
insist(HEX64.test(expectedObserverSha256??''),'expectedObserverSha256 invalid');
|
|
101
|
+
safeInt(maxFrameBytes,'maxFrameBytes',{min:1024,max:256*1024*1024});
|
|
102
|
+
safeInt(timeoutMs,'timeoutMs',{min:100,max:60000});
|
|
103
|
+
const exeBytes=await readFile(observerPath);
|
|
104
|
+
insist(exeBytes.length>0&&exeBytes.length<=4*1024*1024,'desktop observer binary size invalid');
|
|
105
|
+
insist(sha256Bytes(exeBytes)===expectedObserverSha256,'desktop observer executable hash mismatch');
|
|
106
|
+
return await new Promise((resolve,reject)=>{
|
|
107
|
+
const child=spawn(observerPath,[],{shell:false,windowsHide:true,stdio:['ignore','pipe','pipe']});
|
|
108
|
+
const chunks=[]; let stdoutBytes=0,stderr='',done=false;
|
|
109
|
+
const finish=(error,value)=>{
|
|
110
|
+
if(done)return;done=true;clearTimeout(timer);
|
|
111
|
+
error?reject(error):resolve(value);
|
|
112
|
+
};
|
|
113
|
+
child.stdout.on('data',chunk=>{
|
|
114
|
+
stdoutBytes+=chunk.length;
|
|
115
|
+
if(stdoutBytes>maxFrameBytes+HEADER_MAX_BYTES+1){
|
|
116
|
+
try{child.kill('SIGKILL');}catch{}
|
|
117
|
+
return finish(new Error('desktop observer stdout exceeds bound'));
|
|
118
|
+
}
|
|
119
|
+
chunks.push(Buffer.from(chunk));
|
|
120
|
+
});
|
|
121
|
+
child.stderr.on('data',chunk=>{
|
|
122
|
+
stderr+=chunk.toString('utf8');
|
|
123
|
+
if(Buffer.byteLength(stderr,'utf8')>64*1024){
|
|
124
|
+
try{child.kill('SIGKILL');}catch{}
|
|
125
|
+
finish(new Error('desktop observer stderr exceeds bound'));
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
child.once('error',error=>finish(error));
|
|
129
|
+
child.once('exit',(code,signal)=>{
|
|
130
|
+
if(code!==0) return finish(new Error(`desktop observer exit=${code} signal=${signal??'none'} stderr=${stderr.slice(-8192)}`));
|
|
131
|
+
try { finish(null,parseNativeDesktopFrame(Buffer.concat(chunks),{maxFrameBytes})); }
|
|
132
|
+
catch(error){ finish(error); }
|
|
133
|
+
});
|
|
134
|
+
const timer=setTimeout(()=>{
|
|
135
|
+
try{child.kill('SIGKILL');}catch{}
|
|
136
|
+
finish(new Error(`desktop observer timed out after ${timeoutMs}ms`));
|
|
137
|
+
},timeoutMs);
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function createDesktopPreview({pixels,width,height,stride,maxBytes=1024*1024}={}){
|
|
142
|
+
insist(Buffer.isBuffer(pixels),'desktop preview pixels invalid');
|
|
143
|
+
safeInt(width,'preview width',{min:1,max:32768}); safeInt(height,'preview height',{min:1,max:32768});
|
|
144
|
+
safeInt(stride,'preview stride',{min:4,max:131072}); insist(stride===width*4,'preview stride mismatch');
|
|
145
|
+
safeInt(maxBytes,'preview maxBytes',{min:64,max:4*1024*1024});
|
|
146
|
+
insist(pixels.length>=stride*height,'desktop preview pixel payload truncated');
|
|
147
|
+
let step=Math.max(1,Math.ceil(Math.sqrt((width*height*3)/Math.max(1,maxBytes-64))));
|
|
148
|
+
let outWidth,outHeight,header,total;
|
|
149
|
+
for(;;){
|
|
150
|
+
outWidth=Math.ceil(width/step);outHeight=Math.ceil(height/step);
|
|
151
|
+
header=Buffer.from('P6\n'+outWidth+' '+outHeight+'\n255\n','ascii');
|
|
152
|
+
total=header.length+outWidth*outHeight*3;
|
|
153
|
+
if(total<=maxBytes)break;
|
|
154
|
+
step++;
|
|
155
|
+
insist(step<=Math.max(width,height),'desktop preview cannot satisfy byte bound');
|
|
156
|
+
}
|
|
157
|
+
const data=Buffer.alloc(total);header.copy(data,0);let o=header.length;
|
|
158
|
+
for(let y=0;y<height;y+=step){
|
|
159
|
+
const row=y*stride;
|
|
160
|
+
for(let x=0;x<width;x+=step){
|
|
161
|
+
const i=row+x*4;
|
|
162
|
+
data[o++]=pixels[i+2];data[o++]=pixels[i+1];data[o++]=pixels[i];
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
insist(o===data.length,'desktop preview size mismatch');
|
|
166
|
+
return Object.freeze({
|
|
167
|
+
schema:'deadbyte.desktop-preview.v1',encoding:'ppm-base64',pixel_format:'RGB8',
|
|
168
|
+
width:outWidth,height:outHeight,source_width:width,source_height:height,sample_step:step,
|
|
169
|
+
byte_length:data.length,sha256:sha256Bytes(data),data_base64:data.toString('base64')
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function createDesktopObserverRuntime({
|
|
174
|
+
captureProvider=null,observerPath=null,expectedObserverSha256=null,maxFrameBytes=128*1024*1024,
|
|
175
|
+
timeoutMs=10000,maxCachedFrames=2,allowInputDesktops=['Default'],now=()=>new Date().toISOString(),
|
|
176
|
+
runtimeNonce=randomBytes(32).toString('hex')
|
|
177
|
+
}={}){
|
|
178
|
+
insist(Array.isArray(allowInputDesktops)&&allowInputDesktops.length>=1&&allowInputDesktops.length<=8,'allowInputDesktops invalid');
|
|
179
|
+
const allowed=new Set(allowInputDesktops.map(item=>boundedString(item,'allowed input desktop',512)));
|
|
180
|
+
safeInt(maxCachedFrames,'maxCachedFrames',{min:1,max:8});
|
|
181
|
+
insist(HEX64.test(sha256Bytes(Buffer.from(runtimeNonce,'utf8'))),'runtimeNonce invalid');
|
|
182
|
+
const provider=captureProvider??(()=>captureNativeDesktop({observerPath,expectedObserverSha256,maxFrameBytes,timeoutMs}));
|
|
183
|
+
const frames=new Map();
|
|
184
|
+
let generation=0;
|
|
185
|
+
|
|
186
|
+
async function capture(){
|
|
187
|
+
const native=await provider();
|
|
188
|
+
insist(native&&native.schema===NATIVE_SCHEMA,'desktop capture provider returned invalid native frame');
|
|
189
|
+
insist(allowed.has(native.input_desktop),`desktop input desktop '${native.input_desktop}' is not allowed`);
|
|
190
|
+
generation++;
|
|
191
|
+
const desktopEpoch=sha256Object({
|
|
192
|
+
schema:'deadbyte.desktop-epoch.v1',runtime_nonce:runtimeNonce,session_id:native.session_id,
|
|
193
|
+
input_desktop:native.input_desktop,observer_sha256:expectedObserverSha256??'provider',generation_family:1
|
|
194
|
+
});
|
|
195
|
+
const observedAtUtc=now();
|
|
196
|
+
const frame=createDesktopFrame({
|
|
197
|
+
desktopEpoch,topologySha256:native.topology_sha256,observedAtUtc,bounds:native.bounds,
|
|
198
|
+
activeWindow:native.active_window,imageSha256:native.image_sha256
|
|
199
|
+
});
|
|
200
|
+
const record=Object.freeze({
|
|
201
|
+
frame,backend:native.backend,pixel_format:native.pixel_format,stride:native.stride,byte_length:native.byte_length,
|
|
202
|
+
system_dpi:native.system_dpi,session_id:native.session_id,input_desktop:native.input_desktop,
|
|
203
|
+
monitors:native.monitors,pixels:native.pixels,generation
|
|
204
|
+
});
|
|
205
|
+
frames.set(frame.frame_id,record);
|
|
206
|
+
while(frames.size>maxCachedFrames) frames.delete(frames.keys().next().value);
|
|
207
|
+
return Object.freeze({
|
|
208
|
+
status:'ok',frame,backend:record.backend,pixel_format:record.pixel_format,stride:record.stride,byte_length:record.byte_length,
|
|
209
|
+
system_dpi:record.system_dpi,session_id:record.session_id,input_desktop:record.input_desktop,
|
|
210
|
+
monitors:record.monitors,generation
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
function getFrameRecord(frameId){
|
|
214
|
+
insist(HEX64.test(frameId??''),'frameId invalid');
|
|
215
|
+
const found=frames.get(frameId);
|
|
216
|
+
if(!found) throw new Error('desktop frame expired or unknown');
|
|
217
|
+
return found;
|
|
218
|
+
}
|
|
219
|
+
function frameStatus(frameId){
|
|
220
|
+
const r=getFrameRecord(frameId);
|
|
221
|
+
return Object.freeze({status:'ok',frame:r.frame,backend:r.backend,byte_length:r.byte_length,generation:r.generation});
|
|
222
|
+
}
|
|
223
|
+
function framePixels(frameId){
|
|
224
|
+
return getFrameRecord(frameId).pixels;
|
|
225
|
+
}
|
|
226
|
+
function framePreview(frameId,{maxBytes=1024*1024}={}){
|
|
227
|
+
const r=getFrameRecord(frameId);
|
|
228
|
+
return createDesktopPreview({pixels:r.pixels,width:r.frame.bounds.width,height:r.frame.bounds.height,stride:r.stride,maxBytes});
|
|
229
|
+
}
|
|
230
|
+
return Object.freeze({capture,frameStatus,framePixels,framePreview});
|
|
231
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdir, readFile } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { decodeUtf8Strict } from './text-policy.mjs';
|
|
5
|
+
|
|
6
|
+
export const DESKTOP_POLICY_SCHEMA='deadbyte.desktop-policy.v1';
|
|
7
|
+
export const DESKTOP_GRANT_SCHEMA='deadbyte.desktop-grant.v1';
|
|
8
|
+
export const DESKTOP_ACTION_KINDS=Object.freeze([
|
|
9
|
+
'pointer_move','click_primary','click_secondary','double_click_primary','scroll','key_chord','text_input'
|
|
10
|
+
]);
|
|
11
|
+
|
|
12
|
+
const HEX64=/^[0-9a-f]{64}$/;
|
|
13
|
+
const sha256Bytes=bytes=>createHash('sha256').update(bytes).digest('hex');
|
|
14
|
+
|
|
15
|
+
function insist(condition,message){if(!condition)throw new Error(message);}
|
|
16
|
+
function integer(value,name,min,max,fallback){
|
|
17
|
+
const n=value??fallback;
|
|
18
|
+
if(!Number.isInteger(n)||n<min||n>max)throw new Error(`${name} must be in [${min}, ${max}]`);
|
|
19
|
+
return n;
|
|
20
|
+
}
|
|
21
|
+
function absolute(value,label){
|
|
22
|
+
insist(typeof value==='string'&&path.isAbsolute(value),`${label} must be absolute`);
|
|
23
|
+
return path.resolve(value);
|
|
24
|
+
}
|
|
25
|
+
function hash(value,label){insist(typeof value==='string'&&HEX64.test(value),`${label} must be sha256`);return value;}
|
|
26
|
+
function actions(value,label='allowed_actions'){
|
|
27
|
+
insist(Array.isArray(value)&&value.length>=1&&value.length<=DESKTOP_ACTION_KINDS.length,`${label} invalid`);
|
|
28
|
+
const unique=[...new Set(value)];
|
|
29
|
+
insist(unique.length===value.length&&unique.every(item=>DESKTOP_ACTION_KINDS.includes(item)),`${label} invalid`);
|
|
30
|
+
return Object.freeze(unique);
|
|
31
|
+
}
|
|
32
|
+
function desktopNames(value){
|
|
33
|
+
insist(Array.isArray(value)&&value.length>=1&&value.length<=8,'allowed_input_desktops invalid');
|
|
34
|
+
const result=[...new Set(value)];
|
|
35
|
+
insist(result.length===value.length&&result.every(item=>typeof item==='string'&&item.length>=1&&item.length<=128&&!item.includes('\0')),'allowed_input_desktops invalid');
|
|
36
|
+
return Object.freeze(result);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function loadDesktopPolicy(policyPath){
|
|
40
|
+
if(!policyPath) throw new Error('missing DEADBYTE desktop policy path');
|
|
41
|
+
const absolutePath=path.resolve(policyPath);
|
|
42
|
+
const bytes=await readFile(absolutePath);
|
|
43
|
+
const raw=JSON.parse(decodeUtf8Strict(bytes));
|
|
44
|
+
insist(raw&&typeof raw==='object'&&!Array.isArray(raw),'desktop policy malformed');
|
|
45
|
+
insist(raw.schema===DESKTOP_POLICY_SCHEMA&&raw.enabled===true,'desktop policy schema/enabled mismatch');
|
|
46
|
+
const observer=raw.observer??{};
|
|
47
|
+
const input=raw.input??{};
|
|
48
|
+
const grounder=raw.grounder??{};
|
|
49
|
+
const limits=raw.limits??{};
|
|
50
|
+
const policy={
|
|
51
|
+
schema:DESKTOP_POLICY_SCHEMA,enabled:true,policy_path:absolutePath,
|
|
52
|
+
grant_file:absolute(raw.grant_file,'desktop grant_file'),
|
|
53
|
+
evidence_dir:absolute(raw.evidence_dir,'desktop evidence_dir'),
|
|
54
|
+
observer:Object.freeze({
|
|
55
|
+
executable:absolute(observer.executable,'desktop observer executable'),
|
|
56
|
+
sha256:hash(observer.sha256,'desktop observer sha256'),
|
|
57
|
+
backend:typeof observer.backend==='string'?observer.backend:'gdi-bitblt-fallback'
|
|
58
|
+
}),
|
|
59
|
+
input:Object.freeze({
|
|
60
|
+
executable:absolute(input.executable,'desktop input executable'),
|
|
61
|
+
sha256:hash(input.sha256,'desktop input sha256')
|
|
62
|
+
}),
|
|
63
|
+
grounder:Object.freeze({
|
|
64
|
+
executable:absolute(grounder.executable,'desktop grounder executable'),
|
|
65
|
+
sha256:hash(grounder.sha256,'desktop grounder sha256')
|
|
66
|
+
}),
|
|
67
|
+
allowed_input_desktops:desktopNames(raw.allowed_input_desktops??['Default']),
|
|
68
|
+
allowed_actions:actions(raw.allowed_actions??DESKTOP_ACTION_KINDS),
|
|
69
|
+
limits:Object.freeze({
|
|
70
|
+
max_frame_age_ms:integer(limits.max_frame_age_ms,'limits.max_frame_age_ms',100,10000,2000),
|
|
71
|
+
max_frame_bytes:integer(limits.max_frame_bytes,'limits.max_frame_bytes',1024*1024,256*1024*1024,128*1024*1024),
|
|
72
|
+
capture_timeout_ms:integer(limits.capture_timeout_ms,'limits.capture_timeout_ms',100,60000,10000),
|
|
73
|
+
input_timeout_ms:integer(limits.input_timeout_ms,'limits.input_timeout_ms',100,30000,5000),
|
|
74
|
+
ground_timeout_ms:integer(limits.ground_timeout_ms,'limits.ground_timeout_ms',100,30000,5000),
|
|
75
|
+
post_observe_timeout_ms:integer(limits.post_observe_timeout_ms,'limits.post_observe_timeout_ms',100,60000,10000),
|
|
76
|
+
max_preview_bytes:integer(limits.max_preview_bytes,'limits.max_preview_bytes',16*1024,4*1024*1024,1024*1024),
|
|
77
|
+
max_text_bytes:integer(limits.max_text_bytes,'limits.max_text_bytes',1,64*1024,4096)
|
|
78
|
+
})
|
|
79
|
+
};
|
|
80
|
+
policy.policy_sha256=sha256Bytes(bytes);
|
|
81
|
+
await mkdir(policy.evidence_dir,{recursive:true});
|
|
82
|
+
return Object.freeze(policy);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export async function desktopSessionStatus(policy){
|
|
86
|
+
if(!policy||policy.schema!==DESKTOP_POLICY_SCHEMA) throw new Error('desktop policy invalid');
|
|
87
|
+
let grant;
|
|
88
|
+
try{grant=JSON.parse(decodeUtf8Strict(await readFile(policy.grant_file)));}
|
|
89
|
+
catch(error){
|
|
90
|
+
return Object.freeze({configured:true,armed:false,reason:error?.code==='ENOENT'?'disarmed':'invalid-grant',standing:false,action_kinds:[]});
|
|
91
|
+
}
|
|
92
|
+
if(!grant||grant.schema!==DESKTOP_GRANT_SCHEMA) return Object.freeze({configured:true,armed:false,reason:'grant-schema',standing:false,action_kinds:[]});
|
|
93
|
+
if(grant.policy_sha256!==policy.policy_sha256) return Object.freeze({configured:true,armed:false,reason:'policy-digest',standing:false,action_kinds:[]});
|
|
94
|
+
if(typeof grant.instance_nonce!=='string'||!HEX64.test(grant.instance_nonce)) return Object.freeze({configured:true,armed:false,reason:'nonce-invalid',standing:false,action_kinds:[]});
|
|
95
|
+
let granted;
|
|
96
|
+
try{granted=actions(grant.action_kinds,'grant.action_kinds');}
|
|
97
|
+
catch{return Object.freeze({configured:true,armed:false,reason:'actions-invalid',standing:false,action_kinds:[]});}
|
|
98
|
+
if(granted.some(item=>!policy.allowed_actions.includes(item))) return Object.freeze({configured:true,armed:false,reason:'actions-outside-policy',standing:false,action_kinds:[]});
|
|
99
|
+
return Object.freeze({
|
|
100
|
+
configured:true,armed:true,reason:'standing-grant',standing:true,instance_nonce:grant.instance_nonce,action_kinds:granted
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export async function assertDesktopArmed(policy,actionKind){
|
|
105
|
+
insist(DESKTOP_ACTION_KINDS.includes(actionKind),'desktop action kind invalid');
|
|
106
|
+
const status=await desktopSessionStatus(policy);
|
|
107
|
+
if(!status.armed) throw new Error(`desktop plane is disarmed: ${status.reason}`);
|
|
108
|
+
if(!status.action_kinds.includes(actionKind)) throw new Error(`desktop action '${actionKind}' is outside active grant`);
|
|
109
|
+
return status;
|
|
110
|
+
}
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import { createHash, createPublicKey, randomBytes, sign as cryptoSign, verify as cryptoVerify } from 'node:crypto';
|
|
2
|
+
import { mkdir, open, readFile } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { canonicalJson } from './canonical-json.mjs';
|
|
5
|
+
import { createDesktopAction,createDesktopTarget,guardDesktopAction,bindDesktopPostObservation,bindDesktopActionChain } from './desktop-contract.mjs';
|
|
6
|
+
import { effectIdentity,bindEffectCompletion } from './effect-ledger.mjs';
|
|
7
|
+
import { DESKTOP_ACTION_KINDS,DESKTOP_POLICY_SCHEMA,assertDesktopArmed,desktopSessionStatus } from './desktop-policy-r40.mjs';
|
|
8
|
+
|
|
9
|
+
export const DESKTOP_RECEIPT_SCHEMA='deadbyte.desktop-receipt.v1';
|
|
10
|
+
const RECEIPT_DOMAIN=Buffer.from('DEADBYTE-DESKTOP-RECEIPT-V1\0','utf8');
|
|
11
|
+
const HEX64=/^[0-9a-f]{64}$/;
|
|
12
|
+
const RECEIPT_ID=/^[0-9a-f]{24}$/;
|
|
13
|
+
const sha256Bytes=bytes=>createHash('sha256').update(bytes).digest('hex');
|
|
14
|
+
const sha256Object=value=>sha256Bytes(Buffer.from(canonicalJson(value),'utf8'));
|
|
15
|
+
|
|
16
|
+
function insist(condition,message){if(!condition)throw new Error(message);}
|
|
17
|
+
function sameWindow(a,b){
|
|
18
|
+
if(a===null||b===null)return a===b;
|
|
19
|
+
return a.hwnd===b.hwnd&&a.pid===b.pid&&
|
|
20
|
+
(a.process_image_sha256??null)===(b.process_image_sha256??null)&&
|
|
21
|
+
(a.title_sha256??null)===(b.title_sha256??null);
|
|
22
|
+
}
|
|
23
|
+
function frameAgeMs(frame,nowUtc){
|
|
24
|
+
const a=Date.parse(frame.observed_at_utc),b=Date.parse(nowUtc);
|
|
25
|
+
insist(Number.isFinite(a)&&Number.isFinite(b),'desktop frame time invalid');
|
|
26
|
+
return b-a;
|
|
27
|
+
}
|
|
28
|
+
function targetCenter(rect){
|
|
29
|
+
return Object.freeze({
|
|
30
|
+
x:rect.left+Math.floor(rect.width/2),
|
|
31
|
+
y:rect.top+Math.floor(rect.height/2)
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
async function writeReceipt({policy,privateKey,keyId,evidenceDir,kind,payload,nowUtc}){
|
|
35
|
+
insist(policy?.schema===DESKTOP_POLICY_SCHEMA,'desktop receipt policy invalid');
|
|
36
|
+
insist(privateKey,'desktop receipt privateKey missing');
|
|
37
|
+
insist(typeof keyId==='string'&&keyId.length>=1&&keyId.length<=128,'desktop receipt keyId invalid');
|
|
38
|
+
const receiptId=randomBytes(12).toString('hex');
|
|
39
|
+
const body={
|
|
40
|
+
schema:DESKTOP_RECEIPT_SCHEMA,receipt_id:receiptId,kind,
|
|
41
|
+
policy_sha256:policy.policy_sha256,observed_at_utc:nowUtc,payload
|
|
42
|
+
};
|
|
43
|
+
const receiptSha256=sha256Object(body);
|
|
44
|
+
const signed={...body,receipt_sha256:receiptSha256};
|
|
45
|
+
const signature=cryptoSign(null,Buffer.concat([RECEIPT_DOMAIN,Buffer.from(canonicalJson(signed),'utf8')]),privateKey).toString('base64');
|
|
46
|
+
const receipt={...signed,attestation:{algorithm:'ed25519',key_id:keyId,signature_base64:signature}};
|
|
47
|
+
await mkdir(evidenceDir,{recursive:true});
|
|
48
|
+
const file=path.join(evidenceDir,`${receiptId}.json`);
|
|
49
|
+
const handle=await open(file,'wx',0o600);
|
|
50
|
+
try{await handle.writeFile(`${canonicalJson(receipt)}\n`);await handle.sync();}
|
|
51
|
+
finally{await handle.close();}
|
|
52
|
+
return Object.freeze({receipt_id:receiptId,receipt_sha256:receiptSha256,path:file,receipt});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function verifyDesktopReceiptObject(receipt,{policySha256,publicKey,keyId}={}){
|
|
56
|
+
try{
|
|
57
|
+
insist(receipt&&receipt.schema===DESKTOP_RECEIPT_SCHEMA,'receipt schema mismatch');
|
|
58
|
+
insist(RECEIPT_ID.test(receipt.receipt_id??''),'receipt id invalid');
|
|
59
|
+
insist(receipt.policy_sha256===policySha256,'receipt policy mismatch');
|
|
60
|
+
insist(HEX64.test(receipt.receipt_sha256??''),'receipt hash invalid');
|
|
61
|
+
const {attestation,receipt_sha256:claimed,...body}=receipt;
|
|
62
|
+
insist(sha256Object(body)===claimed,'receipt hash mismatch');
|
|
63
|
+
insist(attestation?.algorithm==='ed25519'&&attestation.key_id===keyId,'receipt attestation mismatch');
|
|
64
|
+
const sig=Buffer.from(attestation.signature_base64??'','base64');
|
|
65
|
+
insist(sig.length===64,'receipt signature length invalid');
|
|
66
|
+
const signed={...body,receipt_sha256:claimed};
|
|
67
|
+
insist(cryptoVerify(null,Buffer.concat([RECEIPT_DOMAIN,Buffer.from(canonicalJson(signed),'utf8')]),publicKey,sig),'receipt signature invalid');
|
|
68
|
+
return {ok:true,receipt_sha256:claimed,kind:receipt.kind,receipt_id:receipt.receipt_id};
|
|
69
|
+
}catch(error){return {ok:false,reason:String(error?.message??error)};}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function createDesktopRuntime({
|
|
73
|
+
policy,observerRuntime,inputProvider,grounderProvider=null,privateKey,publicKey=null,keyId,evidenceDir=policy?.evidence_dir,
|
|
74
|
+
now=()=>new Date().toISOString(),maxTargets=64
|
|
75
|
+
}={}){
|
|
76
|
+
insist(policy?.schema===DESKTOP_POLICY_SCHEMA,'desktop runtime policy invalid');
|
|
77
|
+
insist(observerRuntime?.capture&&observerRuntime?.frameStatus,'desktop runtime observer invalid');
|
|
78
|
+
insist(typeof inputProvider==='function','desktop runtime inputProvider required');
|
|
79
|
+
insist(privateKey,'desktop runtime privateKey required');
|
|
80
|
+
insist(typeof keyId==='string'&&keyId.length>=1,'desktop runtime keyId invalid');
|
|
81
|
+
insist(Number.isInteger(maxTargets)&&maxTargets>=1&&maxTargets<=1024,'maxTargets invalid');
|
|
82
|
+
const receiptPublicKey=publicKey??createPublicKey(privateKey);
|
|
83
|
+
const targets=new Map();
|
|
84
|
+
|
|
85
|
+
async function receipt(kind,payload,stamp=now()){
|
|
86
|
+
return writeReceipt({policy,privateKey,keyId,evidenceDir,kind,payload,nowUtc:stamp});
|
|
87
|
+
}
|
|
88
|
+
async function verifyReceipt({receiptId}={}){
|
|
89
|
+
insist(RECEIPT_ID.test(receiptId??''),'desktop receipt id invalid');
|
|
90
|
+
const bytes=await readFile(path.join(evidenceDir,receiptId+'.json'));
|
|
91
|
+
insist(bytes.length>=2&&bytes.length<=512*1024,'desktop receipt file size invalid');
|
|
92
|
+
let parsed;try{parsed=JSON.parse(bytes.toString('utf8'));}catch{throw new Error('desktop receipt JSON invalid');}
|
|
93
|
+
const proof=verifyDesktopReceiptObject(parsed,{policySha256:policy.policy_sha256,publicKey:receiptPublicKey,keyId});
|
|
94
|
+
insist(proof.ok,'desktop receipt verification failed: '+String(proof.reason??'unknown'));
|
|
95
|
+
return Object.freeze(proof);
|
|
96
|
+
}
|
|
97
|
+
async function status(){
|
|
98
|
+
const authority=await desktopSessionStatus(policy);
|
|
99
|
+
return Object.freeze({
|
|
100
|
+
status:'ok',configured:true,armed:authority.armed,reason:authority.reason,
|
|
101
|
+
action_kinds:authority.action_kinds,policy_sha256:policy.policy_sha256,
|
|
102
|
+
observer_backend:policy.observer.backend
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
async function observe({preview=false}={}){
|
|
106
|
+
insist(typeof preview==='boolean','desktop preview flag invalid');
|
|
107
|
+
const observed=await observerRuntime.capture();
|
|
108
|
+
const previewValue=preview
|
|
109
|
+
?observerRuntime.framePreview?.(observed.frame.frame_id,{maxBytes:policy.limits.max_preview_bytes})
|
|
110
|
+
:null;
|
|
111
|
+
if(preview) insist(previewValue?.schema==='deadbyte.desktop-preview.v1','desktop preview unavailable');
|
|
112
|
+
const evidence=await receipt('desktop_observation',{
|
|
113
|
+
frame_id:observed.frame.frame_id,desktop_epoch:observed.frame.desktop_epoch,
|
|
114
|
+
topology_sha256:observed.frame.topology_sha256,image_sha256:observed.frame.image_sha256,
|
|
115
|
+
observed_at_utc:observed.frame.observed_at_utc,active_window:observed.frame.active_window,
|
|
116
|
+
bounds:observed.frame.bounds,backend:observed.backend,byte_length:observed.byte_length,
|
|
117
|
+
preview:previewValue?{sha256:previewValue.sha256,byte_length:previewValue.byte_length,width:previewValue.width,height:previewValue.height,encoding:previewValue.encoding}:null
|
|
118
|
+
},observed.frame.observed_at_utc);
|
|
119
|
+
return Object.freeze({...observed,...(previewValue?{preview:previewValue}:{}),receipt_id:evidence.receipt_id,receipt_sha256:evidence.receipt_sha256});
|
|
120
|
+
}
|
|
121
|
+
function targetFor(targetId){
|
|
122
|
+
insist(HEX64.test(targetId??''),'targetId invalid');
|
|
123
|
+
const found=targets.get(targetId);if(!found)throw new Error('desktop target expired or unknown');return found;
|
|
124
|
+
}
|
|
125
|
+
async function target({frameId,rect,source,confidence,semanticLabelSha256=null,windowBinding=undefined}={}){
|
|
126
|
+
const record=observerRuntime.frameStatus(frameId);
|
|
127
|
+
const created=createDesktopTarget({
|
|
128
|
+
frame:record.frame,rect,source,confidence,semanticLabelSha256,windowBinding
|
|
129
|
+
});
|
|
130
|
+
targets.set(created.target_id,created);
|
|
131
|
+
while(targets.size>maxTargets)targets.delete(targets.keys().next().value);
|
|
132
|
+
return Object.freeze({status:'ok',target:created});
|
|
133
|
+
}
|
|
134
|
+
async function ground({frameId,query}={}){
|
|
135
|
+
insist(typeof grounderProvider==='function','desktop UIA grounder unavailable');
|
|
136
|
+
const expectedRecord=observerRuntime.frameStatus(frameId);
|
|
137
|
+
const expected=expectedRecord.frame;
|
|
138
|
+
const age=frameAgeMs(expected,now());
|
|
139
|
+
insist(age>=0&&age<=policy.limits.max_frame_age_ms,'desktop grounding frame is stale');
|
|
140
|
+
const fresh=await observerRuntime.capture();
|
|
141
|
+
insist(fresh.frame.desktop_epoch===expected.desktop_epoch,'desktop epoch changed before grounding');
|
|
142
|
+
insist(fresh.frame.topology_sha256===expected.topology_sha256,'desktop topology changed before grounding');
|
|
143
|
+
insist(sameWindow(fresh.frame.active_window,expected.active_window),'desktop foreground window changed before grounding');
|
|
144
|
+
const grounded=await grounderProvider({frame:fresh.frame,query});
|
|
145
|
+
insist(grounded&&grounded.status==='ok','desktop UIA grounder invalid result');
|
|
146
|
+
insist(grounded.ordinal<grounded.matches.length,'desktop UIA query produced no candidate at requested ordinal');
|
|
147
|
+
const candidate=grounded.matches[grounded.ordinal];
|
|
148
|
+
insist(candidate.enabled===true&&!candidate.offscreen,'desktop UIA candidate is not actionable');
|
|
149
|
+
const confidence=grounded.query_kind==='name_contains'?0.98:1;
|
|
150
|
+
const created=createDesktopTarget({
|
|
151
|
+
frame:fresh.frame,rect:candidate.rect,source:'uia',confidence,
|
|
152
|
+
semanticLabelSha256:grounded.query_sha256,windowBinding:fresh.frame.active_window
|
|
153
|
+
});
|
|
154
|
+
targets.set(created.target_id,created);
|
|
155
|
+
while(targets.size>maxTargets)targets.delete(targets.keys().next().value);
|
|
156
|
+
const evidence=await receipt('desktop_grounding',{
|
|
157
|
+
requested_frame_id:expected.frame_id,grounded_frame_id:fresh.frame.frame_id,
|
|
158
|
+
target_id:created.target_id,query_sha256:grounded.query_sha256,query_kind:grounded.query_kind,
|
|
159
|
+
ordinal:grounded.ordinal,match_count:grounded.matches.length,visited:grounded.visited,truncated:grounded.truncated,
|
|
160
|
+
provider_receipt_sha256:grounded.provider_receipt_sha256,control_type:candidate.control_type
|
|
161
|
+
},fresh.frame.observed_at_utc);
|
|
162
|
+
return Object.freeze({
|
|
163
|
+
status:'ok',frame:fresh.frame,target:created,
|
|
164
|
+
grounding:Object.freeze({
|
|
165
|
+
query_sha256:grounded.query_sha256,query_kind:grounded.query_kind,ordinal:grounded.ordinal,
|
|
166
|
+
match_count:grounded.matches.length,visited:grounded.visited,truncated:grounded.truncated,
|
|
167
|
+
control_type:candidate.control_type,provider_receipt_sha256:grounded.provider_receipt_sha256,
|
|
168
|
+
receipt_id:evidence.receipt_id,receipt_sha256:evidence.receipt_sha256
|
|
169
|
+
})
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
async function action({
|
|
173
|
+
intentId,kind,expectedFrameId,targetId=null,expectedWindow=undefined,point=null,
|
|
174
|
+
scrollDelta=null,keyChord=null,text=null,effectContext=null
|
|
175
|
+
}={}){
|
|
176
|
+
insist(DESKTOP_ACTION_KINDS.includes(kind),'desktop action kind invalid');
|
|
177
|
+
const authority=await assertDesktopArmed(policy,kind);
|
|
178
|
+
if(kind==='text_input'){
|
|
179
|
+
insist(typeof text==='string','desktop text input invalid');
|
|
180
|
+
insist(Buffer.byteLength(text,'utf8')<=policy.limits.max_text_bytes,'desktop text input exceeds policy byte limit');
|
|
181
|
+
}
|
|
182
|
+
const expectedRecord=observerRuntime.frameStatus(expectedFrameId);
|
|
183
|
+
const expected=expectedRecord.frame;
|
|
184
|
+
const requestAt=now();
|
|
185
|
+
const age=frameAgeMs(expected,requestAt);
|
|
186
|
+
insist(age>=0&&age<=policy.limits.max_frame_age_ms,'desktop expected frame is stale');
|
|
187
|
+
|
|
188
|
+
let chosenTarget=null;
|
|
189
|
+
if(targetId!==null){
|
|
190
|
+
chosenTarget=targetFor(targetId);
|
|
191
|
+
insist(chosenTarget.frame_id===expected.frame_id,'desktop target/frame mismatch');
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const pre=await observerRuntime.capture();
|
|
195
|
+
insist(pre.frame.desktop_epoch===expected.desktop_epoch,'desktop epoch changed before action');
|
|
196
|
+
insist(pre.frame.topology_sha256===expected.topology_sha256,'desktop topology changed before action');
|
|
197
|
+
insist(sameWindow(pre.frame.active_window,expected.active_window),'desktop foreground window changed before action');
|
|
198
|
+
// Pointer and text actions depend on visual/control state. A key chord is window-scoped:
|
|
199
|
+
// bind it to the same fresh desktop/topology/foreground identity and let the native helper
|
|
200
|
+
// revalidate HWND/PID immediately before SendInput instead of requiring unrelated pixels to freeze.
|
|
201
|
+
if(kind!=='key_chord') insist(pre.frame.image_sha256===expected.image_sha256,'desktop pixels changed before action');
|
|
202
|
+
|
|
203
|
+
let actionPoint=point;
|
|
204
|
+
if(chosenTarget&&['pointer_move','click_primary','click_secondary','double_click_primary','scroll'].includes(kind)){
|
|
205
|
+
actionPoint=targetCenter(chosenTarget.rect);
|
|
206
|
+
}
|
|
207
|
+
const expectedWindowValue=expectedWindow===undefined?pre.frame.active_window:expectedWindow;
|
|
208
|
+
const actionObject=createDesktopAction({
|
|
209
|
+
intentId,kind,frameId:pre.frame.frame_id,desktopEpoch:pre.frame.desktop_epoch,
|
|
210
|
+
topologySha256:pre.frame.topology_sha256,expectedWindow:expectedWindowValue,
|
|
211
|
+
point:actionPoint,scrollDelta,keyChord,text,targetId:chosenTarget?.target_id??null
|
|
212
|
+
});
|
|
213
|
+
const guarded=guardDesktopAction({
|
|
214
|
+
frame:pre.frame,action:actionObject,nowUtc:now(),maxFrameAgeMs:policy.limits.max_frame_age_ms
|
|
215
|
+
});
|
|
216
|
+
const effectAction=Object.freeze({
|
|
217
|
+
kind:`desktop_${kind}`,action_sha256:actionObject.action_sha256,frame_id:pre.frame.frame_id,
|
|
218
|
+
target_id:actionObject.target_id,point:actionObject.point,scroll_delta:actionObject.scroll_delta,
|
|
219
|
+
key_chord:actionObject.key_chord,text_sha256:actionObject.text_sha256
|
|
220
|
+
});
|
|
221
|
+
const ctx=effectContext??{};
|
|
222
|
+
const effectLedger=effectIdentity({
|
|
223
|
+
goalId:ctx.goalId??intentId,planId:ctx.planId??pre.frame.frame_id,
|
|
224
|
+
stepId:ctx.stepId??`desktop.${kind}`,mutationEpoch:ctx.mutationEpoch??0,
|
|
225
|
+
action:effectAction,observationHeadSha256:ctx.observationHeadSha256??pre.frame.frame_id
|
|
226
|
+
});
|
|
227
|
+
const started=await receipt('desktop_action_started',{
|
|
228
|
+
intent_id:intentId,kind,expected_frame_id:expected.frame_id,pre_frame_id:pre.frame.frame_id,
|
|
229
|
+
target_id:chosenTarget?.target_id??null,action_sha256:actionObject.action_sha256,
|
|
230
|
+
guard_sha256:guarded.guard_sha256,authority_instance_nonce:authority.instance_nonce,
|
|
231
|
+
effect_id:effectLedger.effect_id,effect_fingerprint:effectLedger.fingerprint
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
let effect;
|
|
235
|
+
try{
|
|
236
|
+
effect=await inputProvider(actionObject,{
|
|
237
|
+
timeoutMs:policy.limits.input_timeout_ms,guard:guarded,
|
|
238
|
+
inputDesktop:pre.input_desktop??null,expectedBounds:pre.frame.bounds,expectedWindow:pre.frame.active_window
|
|
239
|
+
});
|
|
240
|
+
insist(effect&&effect.injected===true,'desktop input provider did not confirm injection');
|
|
241
|
+
}catch(error){
|
|
242
|
+
const sideEffectState=error?.side_effect_state==='not_started'?'not_started':'unknown';
|
|
243
|
+
const evidenceKind=sideEffectState==='not_started'?'desktop_action_effect_failed':'desktop_action_outcome_unknown';
|
|
244
|
+
const failure=await receipt(evidenceKind,{
|
|
245
|
+
started_receipt_id:started.receipt_id,started_receipt_sha256:started.receipt_sha256,
|
|
246
|
+
action_sha256:actionObject.action_sha256,guard_sha256:guarded.guard_sha256,
|
|
247
|
+
effect_id:effectLedger.effect_id,effect_fingerprint:effectLedger.fingerprint,
|
|
248
|
+
side_effect_state:sideEffectState,error_sha256:sha256Bytes(Buffer.from(String(error?.message??error),'utf8'))
|
|
249
|
+
});
|
|
250
|
+
const wrapped=new Error(sideEffectState==='not_started'
|
|
251
|
+
?'desktop input effect did not start'
|
|
252
|
+
:'desktop action outcome unknown after input dispatch; inspect fresh desktop state before retry');
|
|
253
|
+
wrapped.side_effect_state=sideEffectState;
|
|
254
|
+
wrapped.receipt_id=failure.receipt_id;wrapped.receipt_sha256=failure.receipt_sha256;
|
|
255
|
+
wrapped.effect_id=effectLedger.effect_id;wrapped.effect_fingerprint=effectLedger.fingerprint;
|
|
256
|
+
throw wrapped;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
try{
|
|
260
|
+
const post=await observerRuntime.capture();
|
|
261
|
+
const bound=bindDesktopPostObservation({guard:guarded,beforeFrame:pre.frame,action:actionObject,afterFrame:post.frame});
|
|
262
|
+
const postEvidence=await receipt('desktop_post_observation',{
|
|
263
|
+
started_receipt_id:started.receipt_id,started_receipt_sha256:started.receipt_sha256,
|
|
264
|
+
effect_id:effectLedger.effect_id,effect_fingerprint:effectLedger.fingerprint,
|
|
265
|
+
frame_id:post.frame.frame_id,desktop_epoch:post.frame.desktop_epoch,
|
|
266
|
+
topology_sha256:post.frame.topology_sha256,image_sha256:post.frame.image_sha256,
|
|
267
|
+
observed_at_utc:post.frame.observed_at_utc,active_window:post.frame.active_window,
|
|
268
|
+
bounds:post.frame.bounds,backend:post.backend,byte_length:post.byte_length,
|
|
269
|
+
post_observation_sha256:bound.post_observation_sha256
|
|
270
|
+
},post.frame.observed_at_utc);
|
|
271
|
+
const chain=bindDesktopActionChain({
|
|
272
|
+
beforeFrameId:pre.frame.frame_id,actionSha256:actionObject.action_sha256,
|
|
273
|
+
afterFrameId:post.frame.frame_id,guardSha256:guarded.guard_sha256,
|
|
274
|
+
postObservationSha256:bound.post_observation_sha256,postReceiptSha256:postEvidence.receipt_sha256,
|
|
275
|
+
effectId:effectLedger.effect_id,effectFingerprint:effectLedger.fingerprint
|
|
276
|
+
});
|
|
277
|
+
const effectCompletion=bindEffectCompletion(effectLedger,{
|
|
278
|
+
side_effect_state:'confirmed_injected',provider_receipt_sha256:HEX64.test(effect.provider_receipt_sha256??'')?effect.provider_receipt_sha256:null,
|
|
279
|
+
post_observation_sha256:bound.post_observation_sha256,post_receipt_sha256:postEvidence.receipt_sha256,
|
|
280
|
+
chain_sha256:chain.chain_sha256
|
|
281
|
+
});
|
|
282
|
+
const completed=await receipt('desktop_action_completed',{
|
|
283
|
+
started_receipt_id:started.receipt_id,started_receipt_sha256:started.receipt_sha256,
|
|
284
|
+
post_receipt_id:postEvidence.receipt_id,post_receipt_sha256:postEvidence.receipt_sha256,
|
|
285
|
+
action_sha256:actionObject.action_sha256,guard_sha256:guarded.guard_sha256,
|
|
286
|
+
effect_id:effectLedger.effect_id,effect_fingerprint:effectLedger.fingerprint,effect_completion:effectCompletion,
|
|
287
|
+
effect:{injected:true,side_effect_state:'confirmed_injected',provider_receipt_sha256:HEX64.test(effect.provider_receipt_sha256??'')?effect.provider_receipt_sha256:null},
|
|
288
|
+
action_chain:chain,post_observation:bound
|
|
289
|
+
},post.frame.observed_at_utc);
|
|
290
|
+
return Object.freeze({
|
|
291
|
+
status:'ok',injected:true,task_success:null,side_effect_state:'confirmed_injected',
|
|
292
|
+
action:actionObject,guard:guarded,post_observation:bound,action_chain:chain,
|
|
293
|
+
effect_id:effectLedger.effect_id,effect_fingerprint:effectLedger.fingerprint,effect_completion:effectCompletion,
|
|
294
|
+
started_receipt_id:started.receipt_id,started_receipt_sha256:started.receipt_sha256,
|
|
295
|
+
post_receipt_id:postEvidence.receipt_id,post_receipt_sha256:postEvidence.receipt_sha256,
|
|
296
|
+
receipt_id:completed.receipt_id,receipt_sha256:completed.receipt_sha256
|
|
297
|
+
});
|
|
298
|
+
}catch(error){
|
|
299
|
+
if(error?.receipt_id&&error?.side_effect_state) throw error;
|
|
300
|
+
let unknown=null;
|
|
301
|
+
try{
|
|
302
|
+
unknown=await receipt('desktop_action_outcome_unknown',{
|
|
303
|
+
started_receipt_id:started.receipt_id,started_receipt_sha256:started.receipt_sha256,
|
|
304
|
+
action_sha256:actionObject.action_sha256,guard_sha256:guarded.guard_sha256,
|
|
305
|
+
effect_id:effectLedger.effect_id,effect_fingerprint:effectLedger.fingerprint,
|
|
306
|
+
side_effect_state:'unknown',injection_confirmed:true,
|
|
307
|
+
provider_receipt_sha256:HEX64.test(effect.provider_receipt_sha256??'')?effect.provider_receipt_sha256:null,
|
|
308
|
+
error_sha256:sha256Bytes(Buffer.from(String(error?.message??error),'utf8'))
|
|
309
|
+
});
|
|
310
|
+
}catch{}
|
|
311
|
+
const wrapped=new Error('desktop action outcome unknown after confirmed input; post-observation/evidence incomplete; inspect fresh desktop state before retry');
|
|
312
|
+
wrapped.side_effect_state='unknown';wrapped.effect_id=effectLedger.effect_id;wrapped.effect_fingerprint=effectLedger.fingerprint;
|
|
313
|
+
if(unknown){wrapped.receipt_id=unknown.receipt_id;wrapped.receipt_sha256=unknown.receipt_sha256;}
|
|
314
|
+
throw wrapped;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return Object.freeze({status,observe,target,ground,action,verifyReceipt});
|
|
318
|
+
}
|