dsh-world 0.7.6 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +541 -479
- package/CONTRIBUTING.md +174 -174
- package/INSTALL.md +314 -314
- package/NOTES.md +57 -3
- package/README.md +332 -259
- package/client/index.js +1126 -991
- package/lib/compaction.d.ts +1 -0
- package/lib/compaction.js +1 -1
- package/lib/console/routes.d.ts +11 -3
- package/lib/console/routes.js +12 -7
- package/lib/console/snapshot.d.ts +2 -0
- package/lib/console/snapshot.js +1 -0
- package/lib/ecosystem/computer/browser/browser.d.ts +34 -0
- package/lib/ecosystem/computer/browser/browser.js +101 -0
- package/lib/ecosystem/computer/browser/cdp.d.ts +22 -0
- package/lib/ecosystem/computer/browser/cdp.js +91 -0
- package/lib/ecosystem/computer/browser/edge.d.ts +15 -0
- package/lib/ecosystem/computer/browser/edge.js +80 -0
- package/lib/ecosystem/computer/browser/page.d.ts +41 -0
- package/lib/ecosystem/computer/browser/page.js +140 -0
- package/lib/ecosystem/computer/browser/pageLib.d.ts +7 -0
- package/lib/ecosystem/computer/browser/pageLib.js +106 -0
- package/lib/ecosystem/computer/desktop/desktop.d.ts +91 -0
- package/lib/ecosystem/computer/desktop/desktop.js +322 -0
- package/lib/ecosystem/computer/desktop/driver.d.ts +34 -0
- package/lib/ecosystem/computer/desktop/driver.js +197 -0
- package/lib/ecosystem/computer/desktop/processes.d.ts +5 -0
- package/lib/ecosystem/computer/desktop/processes.js +15 -0
- package/lib/ecosystem/computer/desktop/recording.d.ts +77 -0
- package/lib/ecosystem/computer/desktop/recording.js +187 -0
- package/lib/ecosystem/computer/desktop/screen.d.ts +33 -0
- package/lib/ecosystem/computer/desktop/screen.js +182 -0
- package/lib/ecosystem/computer/desktop/turns.d.ts +58 -0
- package/lib/ecosystem/computer/desktop/turns.js +94 -0
- package/lib/ecosystem/computer/desktop/watching.d.ts +25 -0
- package/lib/ecosystem/computer/desktop/watching.js +81 -0
- package/lib/ecosystem/computer/inbox/inbox.d.ts +32 -0
- package/lib/ecosystem/computer/inbox/inbox.js +142 -0
- package/lib/ecosystem/computer/index.d.ts +82 -0
- package/lib/ecosystem/computer/index.js +133 -0
- package/lib/ecosystem/computer/procedures/alarm.d.ts +13 -0
- package/lib/ecosystem/computer/procedures/alarm.js +46 -0
- package/lib/ecosystem/computer/procedures/archive.d.ts +32 -0
- package/lib/ecosystem/computer/procedures/archive.js +68 -0
- package/lib/ecosystem/computer/procedures/catalog.d.ts +7 -0
- package/lib/ecosystem/computer/procedures/catalog.js +75 -0
- package/lib/ecosystem/computer/procedures/format.d.ts +167 -0
- package/lib/ecosystem/computer/procedures/format.js +332 -0
- package/lib/ecosystem/computer/procedures/library.d.ts +53 -0
- package/lib/ecosystem/computer/procedures/library.js +178 -0
- package/lib/ecosystem/computer/procedures/run.d.ts +80 -0
- package/lib/ecosystem/computer/procedures/run.js +245 -0
- package/lib/ecosystem/computer/tools.d.ts +21 -0
- package/lib/ecosystem/computer/tools.js +197 -0
- package/lib/ecosystem/qq-native/images.d.ts +3 -0
- package/lib/ecosystem/qq-native/images.js +18 -6
- package/lib/ecosystem/qq-native/index.js +2 -2
- package/lib/ecosystem/shared/turn.d.ts +3 -0
- package/lib/ecosystem/shared/turn.js +13 -3
- package/lib/ecosystem/wechat/home.d.ts +1 -0
- package/lib/ecosystem/wechat/home.js +9 -3
- package/lib/ecosystem/wechat/images.js +3 -5
- package/lib/ecosystem/wechat/index.d.ts +1 -0
- package/lib/ecosystem/wechat/reading.d.ts +1 -1
- package/lib/ecosystem/wechat/reading.js +2 -2
- package/lib/history.js +2 -1
- package/lib/images.d.ts +30 -0
- package/lib/images.js +101 -0
- package/lib/index.d.ts +4 -0
- package/lib/index.js +90 -5
- package/lib/life/photoTools.js +4 -15
- package/lib/life/runtime.d.ts +12 -2
- package/lib/life/runtime.js +93 -12
- package/lib/life/toolRegistry.d.ts +9 -3
- package/lib/life/toolRegistry.js +46 -27
- package/lib/life/turnCost.d.ts +16 -0
- package/lib/life/turnCost.js +44 -0
- package/lib/life/types.d.ts +17 -0
- package/lib/memoryTools.d.ts +2 -1
- package/lib/memoryTools.js +16 -3
- package/lib/promptSection.d.ts +1 -0
- package/lib/promptSection.js +29 -42
- package/lib/runtime.js +1 -1
- package/lib/types.d.ts +4 -0
- package/lib/worldAdd.d.ts +9 -0
- package/lib/worldAdd.js +39 -0
- package/lib/worldPrompt.d.ts +3 -2
- package/lib/worldPrompt.js +3 -2
- package/lib/worldRoutes.d.ts +5 -2
- package/lib/worldRoutes.js +29 -4
- package/lib/worldSwitch.d.ts +8 -2
- package/lib/worldSwitch.js +11 -5
- package/package.json +132 -129
- package/src/ecosystem/README.md +2 -1
- package/src/ecosystem/computer/NOTICE.md +27 -0
- package/src/ecosystem/computer/README.md +244 -0
- package/src/ecosystem/computer/desktop/win32.ps1 +858 -0
- package/src/ecosystem/wechat/windows/README.md +4 -2
- package/src/ecosystem/wechat/windows/engine/NOTICE.md +6 -0
- package/src/ecosystem/wechat/windows/engine/profiles.json +28 -0
- package/templates/world-prompt-memory.md +10 -10
- package/templates/world-prompt.md +65 -65
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
// src/ecosystem/computer/desktop/driver.ts
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { randomBytes } from 'node:crypto';
|
|
4
|
+
import { createServer } from 'node:net';
|
|
5
|
+
import { createInterface } from 'node:readline';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
import { errorText } from "../../../faults.js";
|
|
8
|
+
/** A call that hangs this long has an unknown outcome; the helper is killed rather than trusted again. */
|
|
9
|
+
const CALL_MS = 30_000;
|
|
10
|
+
/** How long the partner has to answer the UAC prompt that starts foreground mode's helper. */
|
|
11
|
+
const ELEVATE_MS = 120_000;
|
|
12
|
+
/** Starts that end before any reply, in a row, after which it stays down until the switch is turned off and on. */
|
|
13
|
+
const MAX_FAILED_STARTS = 2;
|
|
14
|
+
// Shipped as source (package.json `files`), so the same path holds from `src/` and from the built `lib/`.
|
|
15
|
+
export const HELPER_SCRIPT = fileURLToPath(new URL('../../../../src/ecosystem/computer/desktop/win32.ps1', import.meta.url));
|
|
16
|
+
const POWERSHELL = ['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', HELPER_SCRIPT];
|
|
17
|
+
/** Words the elevated helper's launcher says when the partner declines the UAC prompt (English and Chinese Windows). */
|
|
18
|
+
const DECLINED = /canceled by the user|cancelled by the user|已被用户取消|操作已取消/i;
|
|
19
|
+
/**
|
|
20
|
+
* The Windows helper (`win32.ps1`) in a child process, started on first use and ended with the module. A crash or a
|
|
21
|
+
* hung call fails what is pending, is a fault, and the next call starts it again; starts that keep dying stop there,
|
|
22
|
+
* so a broken machine is said once instead of relaunched on every step. Her inbox runs a helper of its own (`as`), so
|
|
23
|
+
* a slow step of hers and a look at the notifications never wait on each other.
|
|
24
|
+
*
|
|
25
|
+
* `elevated` is foreground mode's helper: started as administrator, so the partner approves it once through UAC. A UAC
|
|
26
|
+
* prompt cannot share stdin and stdout, so it dials back to a pipe opened here and proves itself with a one-time key.
|
|
27
|
+
* It is never restarted behind the partner's back: once it is gone, foreground mode is over until it is turned on again.
|
|
28
|
+
*/
|
|
29
|
+
export function createDesktopDriver(deps) {
|
|
30
|
+
const [program, ...programArgs] = deps.command ?? POWERSHELL, callMs = deps.callMs ?? CALL_MS;
|
|
31
|
+
const { fault: FAULT, name, stopped } = deps.as ?? (deps.elevated
|
|
32
|
+
? { fault: 'computer-foreground', name: '前台模式的助手', stopped: '前台模式关掉了' }
|
|
33
|
+
: { fault: 'computer-desktop', name: '她用电脑的助手', stopped: '她用电脑的开关关掉了' });
|
|
34
|
+
const pending = new Map();
|
|
35
|
+
let link, starting, nextId = 1, failedStarts = 0, answered = false, halted;
|
|
36
|
+
function failAll(why) {
|
|
37
|
+
for (const [id, entry] of pending) {
|
|
38
|
+
clearTimeout(entry.timer);
|
|
39
|
+
entry.reject(new Error(why));
|
|
40
|
+
pending.delete(id);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function lost(gone, why) {
|
|
44
|
+
if (link !== gone)
|
|
45
|
+
return;
|
|
46
|
+
link = undefined;
|
|
47
|
+
if (deps.elevated)
|
|
48
|
+
halted = `${name}退出了:${why}。要再用前台模式,请伙伴把它关掉再打开(会再确认一次管理员权限)`;
|
|
49
|
+
else if (!answered && ++failedStarts >= MAX_FAILED_STARTS)
|
|
50
|
+
halted = `${name}起不来:${why}`;
|
|
51
|
+
deps.faults.set(FAULT, halted ?? `${name}退出了:${why},下一次用时重新启动`);
|
|
52
|
+
failAll(`${name}退出了:${why}`);
|
|
53
|
+
}
|
|
54
|
+
function settle(line) {
|
|
55
|
+
let message;
|
|
56
|
+
try {
|
|
57
|
+
message = JSON.parse(line);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
// Not an answer: something the helper saw happen, such as the partner stopping her.
|
|
63
|
+
if (typeof message.event === 'string') {
|
|
64
|
+
deps.events?.(message.event);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const entry = message.id === undefined ? undefined : pending.get(message.id);
|
|
68
|
+
if (!entry)
|
|
69
|
+
return;
|
|
70
|
+
pending.delete(message.id);
|
|
71
|
+
clearTimeout(entry.timer);
|
|
72
|
+
answered = true;
|
|
73
|
+
failedStarts = 0;
|
|
74
|
+
deps.faults.clear(FAULT);
|
|
75
|
+
if (message.error !== undefined)
|
|
76
|
+
entry.reject(new Error(message.error));
|
|
77
|
+
else
|
|
78
|
+
entry.resolve(message.data ?? {});
|
|
79
|
+
}
|
|
80
|
+
function startChild() {
|
|
81
|
+
answered = false;
|
|
82
|
+
const started = spawn(program, programArgs, { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
|
|
83
|
+
let stderr = '';
|
|
84
|
+
const made = { write: line => started.stdin.write(line), end: () => { started.stdin?.end(); started.kill(); } };
|
|
85
|
+
started.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString('utf8')).slice(-500); });
|
|
86
|
+
createInterface({ input: started.stdout }).on('line', settle);
|
|
87
|
+
started.stdin.on('error', () => { });
|
|
88
|
+
started.on('error', error => lost(made, errorText(error)));
|
|
89
|
+
started.on('exit', (code, signal) => lost(made, stderr.trim().split('\n').at(-1)?.trim() || `退出码 ${code ?? signal}`));
|
|
90
|
+
return made;
|
|
91
|
+
}
|
|
92
|
+
/** Opens a pipe only this run knows, starts the helper through UAC, and takes the first connection that says the key. */
|
|
93
|
+
function startElevated() {
|
|
94
|
+
answered = false;
|
|
95
|
+
const pipe = `dsh-world-${randomBytes(16).toString('hex')}`, key = randomBytes(24).toString('hex');
|
|
96
|
+
const elevateMs = deps.elevateMs ?? ELEVATE_MS;
|
|
97
|
+
return new Promise((resolve, reject) => {
|
|
98
|
+
let settled = false;
|
|
99
|
+
const server = createServer((socket) => {
|
|
100
|
+
if (settled) {
|
|
101
|
+
socket.destroy();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
let proven = false;
|
|
105
|
+
createInterface({ input: socket }).on('line', line => {
|
|
106
|
+
if (proven) {
|
|
107
|
+
settle(line);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
let hello;
|
|
111
|
+
try {
|
|
112
|
+
hello = JSON.parse(line).hello;
|
|
113
|
+
}
|
|
114
|
+
catch { /* not the helper */ }
|
|
115
|
+
// Anything else that found the pipe is not the helper this run started.
|
|
116
|
+
if (hello !== key || settled) {
|
|
117
|
+
socket.destroy();
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
proven = true;
|
|
121
|
+
settled = true;
|
|
122
|
+
clearTimeout(timer);
|
|
123
|
+
server.close();
|
|
124
|
+
const made = { write: text => socket.write(text), end: () => socket.destroy() };
|
|
125
|
+
socket.on('close', () => lost(made, '和 DSH 的连接断了'));
|
|
126
|
+
socket.on('error', () => { });
|
|
127
|
+
resolve(made);
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
const fail = (why) => { if (settled)
|
|
131
|
+
return; settled = true; clearTimeout(timer); server.close(); reject(new Error(why)); };
|
|
132
|
+
const timer = setTimeout(() => fail(`${Math.round(elevateMs / 1000)} 秒内没有确认管理员权限`), elevateMs);
|
|
133
|
+
server.on('error', error => fail(errorText(error)));
|
|
134
|
+
server.listen(`\\\\.\\pipe\\${pipe}`, () => {
|
|
135
|
+
// The launcher only asks Windows to start the helper as administrator; the helper itself dials the pipe.
|
|
136
|
+
const quoted = (text) => `'${text.replace(/'/g, "''")}'`;
|
|
137
|
+
const args = ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', `"${HELPER_SCRIPT}"`, '-Pipe', pipe, '-Key', key];
|
|
138
|
+
const launcher = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', `Start-Process -FilePath powershell.exe -Verb RunAs -WindowStyle Hidden -ArgumentList ${args.map(quoted).join(',')}`], { stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true });
|
|
139
|
+
let said = '';
|
|
140
|
+
launcher.stderr.on('data', (chunk) => { said += chunk.toString('utf8'); });
|
|
141
|
+
launcher.on('error', error => fail(errorText(error)));
|
|
142
|
+
launcher.on('exit', code => { if (code)
|
|
143
|
+
fail(DECLINED.test(said) ? '伙伴没有同意管理员权限' : said.trim().split('\n').find(Boolean)?.trim() || `启动失败(退出码 ${code})`); });
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
function send(target, op, args) {
|
|
148
|
+
const id = nextId++;
|
|
149
|
+
return new Promise((resolve, reject) => {
|
|
150
|
+
const timer = setTimeout(() => {
|
|
151
|
+
pending.delete(id);
|
|
152
|
+
reject(new Error(`助手 ${callMs / 1000} 秒没有回应,结果不明:先 desktop shot 看看现在的样子`));
|
|
153
|
+
if (link === target) {
|
|
154
|
+
target.end();
|
|
155
|
+
lost(target, '一次调用卡住了');
|
|
156
|
+
}
|
|
157
|
+
}, callMs);
|
|
158
|
+
pending.set(id, { resolve, reject, timer });
|
|
159
|
+
target.write(`${JSON.stringify({ id, op, args })}\n`);
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
function ready() {
|
|
163
|
+
if (halted)
|
|
164
|
+
return Promise.reject(new Error(halted));
|
|
165
|
+
if (link)
|
|
166
|
+
return Promise.resolve(link);
|
|
167
|
+
if (!deps.elevated)
|
|
168
|
+
return Promise.resolve(link = startChild());
|
|
169
|
+
return starting ??= startElevated().then(made => { link = made; deps.faults.clear(FAULT); return made; }, (error) => {
|
|
170
|
+
halted = `前台模式没有打开:${errorText(error)}`;
|
|
171
|
+
deps.faults.set(FAULT, halted);
|
|
172
|
+
throw new Error(halted);
|
|
173
|
+
}).finally(() => { starting = undefined; });
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
/** One helper operation by name, e.g. `windows` or `click`; a refusal rejects with the helper's reason. */
|
|
177
|
+
call(op, args = {}) {
|
|
178
|
+
// A helper already up takes the call at once, so stopping right after a call still fails it.
|
|
179
|
+
if (!link && !deps.elevated && !halted)
|
|
180
|
+
link = startChild();
|
|
181
|
+
return link ? send(link, op, args) : ready().then(target => send(target, op, args));
|
|
182
|
+
},
|
|
183
|
+
/** Starts it now rather than on first use: foreground mode asks for administrator rights when it is turned on. */
|
|
184
|
+
start: async () => { await ready(); },
|
|
185
|
+
running: () => link !== undefined,
|
|
186
|
+
/** Ends the helper and forgets a halted start, so turning the switch on again tries afresh. */
|
|
187
|
+
stop() {
|
|
188
|
+
const ending = link;
|
|
189
|
+
link = undefined;
|
|
190
|
+
halted = undefined;
|
|
191
|
+
failedStarts = 0;
|
|
192
|
+
failAll(stopped);
|
|
193
|
+
ending?.end();
|
|
194
|
+
deps.faults.clear(FAULT);
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// src/ecosystem/computer/desktop/processes.ts
|
|
2
|
+
import { execFile } from 'node:child_process';
|
|
3
|
+
/**
|
|
4
|
+
* Whether a program with this executable name is running, windows or not: WeChat closed to the tray has none, and
|
|
5
|
+
* launching it then would open a second copy.
|
|
6
|
+
*/
|
|
7
|
+
export function isRunning(executable) {
|
|
8
|
+
return new Promise((resolve, reject) => {
|
|
9
|
+
execFile('tasklist', ['/FI', `IMAGENAME eq ${executable}.exe`, '/FO', 'CSV', '/NH'], { windowsHide: true, timeout: 10_000 }, (error, stdout) => {
|
|
10
|
+
if (error)
|
|
11
|
+
return reject(new Error(`查不到 ${executable} 有没有在运行:${error.message}`));
|
|
12
|
+
resolve(stdout.toLowerCase().includes(`"${executable.toLowerCase()}.exe"`));
|
|
13
|
+
});
|
|
14
|
+
});
|
|
15
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
export type Recording = ReturnType<typeof createRecording>;
|
|
2
|
+
/**
|
|
3
|
+
* What is recorded: a window in background mode, the whole screen in foreground mode. `key` tells one target from
|
|
4
|
+
* another, `capture` pictures it into a file (FRAME_EDGE long) and gives its print, `space` presses the space bar on it.
|
|
5
|
+
*/
|
|
6
|
+
export interface RecordTarget {
|
|
7
|
+
key: string;
|
|
8
|
+
app: string;
|
|
9
|
+
capture(file: string, edge: number): Promise<string>;
|
|
10
|
+
space(): Promise<unknown>;
|
|
11
|
+
}
|
|
12
|
+
/** `print` is the helper's 16×16 grey thumbnail of the frame, as hex: what two frames are compared by. */
|
|
13
|
+
export interface Frame {
|
|
14
|
+
second: number;
|
|
15
|
+
file: string;
|
|
16
|
+
print: string;
|
|
17
|
+
}
|
|
18
|
+
/** What became of the video when the recording ended: stopped by the space bar, still running after it, or already not moving. */
|
|
19
|
+
export type Paused = 'paused' | 'running' | 'still';
|
|
20
|
+
/** A recording is at most this long and this many frames: ten minutes at one a second is some tens of megabytes. */
|
|
21
|
+
export declare const MAX_SECONDS = 600, MAX_FRAMES = 600;
|
|
22
|
+
/** Frames on one sheet, and a tile's width: twelve tiles read as one picture, more turn to mush. */
|
|
23
|
+
export declare const SHEET_FRAMES = 12;
|
|
24
|
+
/** `65s 1:05`: the second she passes back, and the clock reading a person would look for. */
|
|
25
|
+
export declare const frameLabel: (second: number) => string;
|
|
26
|
+
/** Up to `max` of the frames, spread evenly, first and last included. */
|
|
27
|
+
export declare function spread<T>(frames: readonly T[], max: number): T[];
|
|
28
|
+
/** Whether the picture changed between two frames; prints that cannot be compared count as changed, never as a false "still". */
|
|
29
|
+
export declare function moved(before: string, after: string): boolean;
|
|
30
|
+
/** A recording of something that was not playing: nearly every frame is the one before it. */
|
|
31
|
+
export declare function mostlyStill(frames: readonly Frame[]): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* How she watches something that moves: the window (or the screen) is pictured at a steady pace into a folder of frames, and what
|
|
34
|
+
* she looks at afterwards is always that folder, never the program again. One recording at a time; a new one, or the
|
|
35
|
+
* switch going off, deletes the last. The frames are pictures of the window only, with no sound.
|
|
36
|
+
*
|
|
37
|
+
* A video goes on playing while she reads what was recorded, and is somewhere else by the time she records again. So
|
|
38
|
+
* when a recording ends on a moving picture the space bar is pressed for her, which pauses nearly every player, and
|
|
39
|
+
* the next recording of the same window presses it again to go on. It is a guess: what came of it is looked at and told.
|
|
40
|
+
*/
|
|
41
|
+
export declare function createRecording(deps: {
|
|
42
|
+
call(op: string, args?: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
43
|
+
secondMs?: number;
|
|
44
|
+
}): {
|
|
45
|
+
/** Starts over a target; the first frame is taken before this returns, so one that cannot be pictured fails here. */
|
|
46
|
+
start(target: RecordTarget, seconds: unknown, every?: unknown, pauseAfter?: boolean): Promise<{
|
|
47
|
+
seconds: number;
|
|
48
|
+
every: number;
|
|
49
|
+
endsAt: Date;
|
|
50
|
+
resumed: boolean;
|
|
51
|
+
}>;
|
|
52
|
+
/** How it stands: what is recorded so far, whether it is over, and why it stopped early if it did. */
|
|
53
|
+
state(): {
|
|
54
|
+
app: string;
|
|
55
|
+
seconds: number;
|
|
56
|
+
every: number;
|
|
57
|
+
frames: Frame[];
|
|
58
|
+
ended: boolean;
|
|
59
|
+
stopping: boolean;
|
|
60
|
+
left: number;
|
|
61
|
+
resumed: boolean;
|
|
62
|
+
cut?: string;
|
|
63
|
+
paused?: Paused;
|
|
64
|
+
};
|
|
65
|
+
/** Resolves when the recording is over. */
|
|
66
|
+
finished: () => Promise<void>;
|
|
67
|
+
/** Ends it where it is and keeps what was recorded; over once the frame being taken, if any, is in. */
|
|
68
|
+
stop(quiet?: boolean): Promise<void>;
|
|
69
|
+
/** Whether a recording is still going. */
|
|
70
|
+
running: () => boolean;
|
|
71
|
+
/** She touched the window herself, so whether it is paused is no longer this module's to know. */
|
|
72
|
+
touched(): void;
|
|
73
|
+
/** Some frames as one picture, each with its time in the corner. */
|
|
74
|
+
sheet(frames: readonly Frame[]): Promise<Buffer>;
|
|
75
|
+
one: (target: Frame) => Buffer;
|
|
76
|
+
clear: () => void;
|
|
77
|
+
};
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// src/ecosystem/computer/desktop/recording.ts
|
|
2
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
/** A recording is at most this long and this many frames: ten minutes at one a second is some tens of megabytes. */
|
|
6
|
+
export const MAX_SECONDS = 600, MAX_FRAMES = 600;
|
|
7
|
+
const MIN_EVERY = 0.5, MAX_EVERY = 30;
|
|
8
|
+
/** A frame's long edge: enough to read subtitles in a single frame, small enough to keep hundreds. */
|
|
9
|
+
const FRAME_EDGE = 960;
|
|
10
|
+
/** Frames on one sheet, and a tile's width: twelve tiles read as one picture, more turn to mush. */
|
|
11
|
+
export const SHEET_FRAMES = 12;
|
|
12
|
+
const SHEET_COLUMNS = 4, TILE = 480;
|
|
13
|
+
/** Mean difference of two prints (0–255) above which the picture moved: a spinner or a blinking cursor stays under it. */
|
|
14
|
+
const MOVED = 1.5;
|
|
15
|
+
/** A recording in which fewer than this share of the frames differ from the one before is told as not moving. */
|
|
16
|
+
const STILL_SHARE = 0.1;
|
|
17
|
+
/** How long a player takes to act on the space bar, and how far apart the two looks are that tell whether it stopped, in seconds. */
|
|
18
|
+
const KEY_SETTLE = 0.6, PROBE_GAP = 0.8;
|
|
19
|
+
/** `65s 1:05`: the second she passes back, and the clock reading a person would look for. */
|
|
20
|
+
export const frameLabel = (second) => {
|
|
21
|
+
const whole = Math.round(second * 10) / 10;
|
|
22
|
+
return whole < 60 ? `${whole}s` : `${whole}s ${Math.floor(whole / 60)}:${String(Math.floor(whole % 60)).padStart(2, '0')}`;
|
|
23
|
+
};
|
|
24
|
+
/** Up to `max` of the frames, spread evenly, first and last included. */
|
|
25
|
+
export function spread(frames, max) {
|
|
26
|
+
if (frames.length <= max)
|
|
27
|
+
return [...frames];
|
|
28
|
+
return Array.from({ length: max }, (_unused, index) => frames[Math.round(index * (frames.length - 1) / (max - 1))]);
|
|
29
|
+
}
|
|
30
|
+
/** Whether the picture changed between two frames; prints that cannot be compared count as changed, never as a false "still". */
|
|
31
|
+
export function moved(before, after) {
|
|
32
|
+
if (!before || before.length !== after.length || before.length % 2)
|
|
33
|
+
return true;
|
|
34
|
+
let total = 0;
|
|
35
|
+
for (let at = 0; at < before.length; at += 2)
|
|
36
|
+
total += Math.abs(parseInt(before.slice(at, at + 2), 16) - parseInt(after.slice(at, at + 2), 16));
|
|
37
|
+
return total / (before.length / 2) > MOVED;
|
|
38
|
+
}
|
|
39
|
+
/** A recording of something that was not playing: nearly every frame is the one before it. */
|
|
40
|
+
export function mostlyStill(frames) {
|
|
41
|
+
if (frames.length < 2)
|
|
42
|
+
return false;
|
|
43
|
+
const changes = frames.slice(1).filter((each, index) => moved(frames[index].print, each.print)).length;
|
|
44
|
+
return changes / (frames.length - 1) < STILL_SHARE;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* How she watches something that moves: the window (or the screen) is pictured at a steady pace into a folder of frames, and what
|
|
48
|
+
* she looks at afterwards is always that folder, never the program again. One recording at a time; a new one, or the
|
|
49
|
+
* switch going off, deletes the last. The frames are pictures of the window only, with no sound.
|
|
50
|
+
*
|
|
51
|
+
* A video goes on playing while she reads what was recorded, and is somewhere else by the time she records again. So
|
|
52
|
+
* when a recording ends on a moving picture the space bar is pressed for her, which pauses nearly every player, and
|
|
53
|
+
* the next recording of the same window presses it again to go on. It is a guess: what came of it is looked at and told.
|
|
54
|
+
*/
|
|
55
|
+
export function createRecording(deps) {
|
|
56
|
+
// `call` is the helper, for laying frames out on a sheet; the frames themselves come from the target.
|
|
57
|
+
const secondMs = deps.secondMs ?? 1000;
|
|
58
|
+
const wait = (seconds) => new Promise(resolve => setTimeout(resolve, seconds * secondMs));
|
|
59
|
+
let take, files = 0;
|
|
60
|
+
/** The target this module paused and nothing has touched since: the next recording of it starts by un-pausing. */
|
|
61
|
+
let pausedTarget;
|
|
62
|
+
async function frame(into, index) {
|
|
63
|
+
const file = join(into.dir, `${String(index).padStart(5, '0')}.jpg`);
|
|
64
|
+
into.frames.push({ second: index * into.every, file, print: await into.target.capture(file, FRAME_EDGE) });
|
|
65
|
+
}
|
|
66
|
+
/** Two looks a moment apart, kept out of the recording. */
|
|
67
|
+
async function probe(into) {
|
|
68
|
+
const file = join(into.dir, `probe-${++files}.jpg`);
|
|
69
|
+
try {
|
|
70
|
+
const first = await into.target.capture(file, FRAME_EDGE);
|
|
71
|
+
await wait(PROBE_GAP);
|
|
72
|
+
return moved(first, await into.target.capture(file, FRAME_EDGE));
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
rmSync(file, { force: true });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Pauses what was playing when the recording ended; a picture that had already stopped is left alone, or the key would start it. */
|
|
79
|
+
async function pause(into) {
|
|
80
|
+
const last = into.frames.slice(-3);
|
|
81
|
+
if (last.length < 2 || !last.slice(1).some((each, index) => moved(last[index].print, each.print))) {
|
|
82
|
+
into.paused = 'still';
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
await into.target.space();
|
|
86
|
+
await wait(KEY_SETTLE);
|
|
87
|
+
into.paused = await probe(into) ? 'running' : 'paused';
|
|
88
|
+
if (into.paused === 'paused')
|
|
89
|
+
pausedTarget = into.target.key;
|
|
90
|
+
}
|
|
91
|
+
/** Each frame is aimed at its own moment from the start, so a slow one does not push the rest later. */
|
|
92
|
+
async function run(into, count, pauseAfter) {
|
|
93
|
+
for (let index = 1; index < count && !into.stopped && !into.discarded; index++) {
|
|
94
|
+
const left = into.startedAt + index * into.every * secondMs - Date.now();
|
|
95
|
+
if (left > 0)
|
|
96
|
+
await new Promise(resolve => setTimeout(resolve, left));
|
|
97
|
+
if (into.stopped || into.discarded)
|
|
98
|
+
break;
|
|
99
|
+
try {
|
|
100
|
+
await frame(into, index);
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
into.cut = error instanceof Error ? error.message : String(error);
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// The window may be gone, or the helper busy: the frames are what she asked for, the pause is a courtesy.
|
|
108
|
+
if (pauseAfter && !into.cut && !into.discarded && !into.quiet)
|
|
109
|
+
await pause(into).catch(() => { });
|
|
110
|
+
into.ended = true;
|
|
111
|
+
}
|
|
112
|
+
function clear() {
|
|
113
|
+
pausedTarget = undefined;
|
|
114
|
+
if (!take)
|
|
115
|
+
return;
|
|
116
|
+
take.discarded = true;
|
|
117
|
+
rmSync(take.dir, { recursive: true, force: true });
|
|
118
|
+
take = undefined;
|
|
119
|
+
}
|
|
120
|
+
function recorded() {
|
|
121
|
+
if (!take)
|
|
122
|
+
throw new Error('还没有录过:先 desktop record');
|
|
123
|
+
return take;
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
/** Starts over a target; the first frame is taken before this returns, so one that cannot be pictured fails here. */
|
|
127
|
+
async start(target, seconds, every = 1, pauseAfter = true) {
|
|
128
|
+
if (typeof seconds !== 'number' || !(seconds >= 1 && seconds <= MAX_SECONDS))
|
|
129
|
+
throw new Error(`seconds 要在 1 到 ${MAX_SECONDS} 之间(录多少秒)`);
|
|
130
|
+
if (typeof every !== 'number' || !(every >= MIN_EVERY && every <= MAX_EVERY))
|
|
131
|
+
throw new Error(`every 要在 ${MIN_EVERY} 到 ${MAX_EVERY} 之间(每几秒一帧)`);
|
|
132
|
+
const count = Math.floor(seconds / every) + 1;
|
|
133
|
+
if (count > MAX_FRAMES)
|
|
134
|
+
throw new Error(`这样要录 ${count} 帧,最多 ${MAX_FRAMES} 帧:把 every 调大`);
|
|
135
|
+
const resumed = pausedTarget === target.key;
|
|
136
|
+
clear();
|
|
137
|
+
if (resumed) {
|
|
138
|
+
await target.space();
|
|
139
|
+
await wait(KEY_SETTLE);
|
|
140
|
+
}
|
|
141
|
+
const fresh = { dir: mkdtempSync(join(tmpdir(), 'dsh-world-record-')), app: target.app, target, seconds, every, startedAt: Date.now(), frames: [], ended: false, resumed, stopped: false, discarded: false, done: Promise.resolve() };
|
|
142
|
+
try {
|
|
143
|
+
await frame(fresh, 0);
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
rmSync(fresh.dir, { recursive: true, force: true });
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
take = fresh;
|
|
150
|
+
fresh.done = run(fresh, count, pauseAfter);
|
|
151
|
+
return { seconds, every, endsAt: new Date(fresh.startedAt + seconds * secondMs), resumed };
|
|
152
|
+
},
|
|
153
|
+
/** How it stands: what is recorded so far, whether it is over, and why it stopped early if it did. */
|
|
154
|
+
state() {
|
|
155
|
+
const { app, seconds, every, frames, ended, cut, paused, resumed, startedAt, stopped } = recorded();
|
|
156
|
+
// Stopped but still taking its last frame: over in a moment, so it is waited for, not reported as still going.
|
|
157
|
+
return { app, seconds, every, frames: [...frames], ended, stopping: stopped && !ended, resumed, left: ended ? 0 : Math.max(0, seconds - (Date.now() - startedAt) / secondMs), ...(cut ? { cut } : {}), ...(paused ? { paused } : {}) };
|
|
158
|
+
},
|
|
159
|
+
/** Resolves when the recording is over. */
|
|
160
|
+
finished: () => recorded().done,
|
|
161
|
+
/** Ends it where it is and keeps what was recorded; over once the frame being taken, if any, is in. */
|
|
162
|
+
stop(quiet = false) {
|
|
163
|
+
const current = recorded();
|
|
164
|
+
current.stopped = true;
|
|
165
|
+
if (quiet)
|
|
166
|
+
current.quiet = true;
|
|
167
|
+
return current.done;
|
|
168
|
+
},
|
|
169
|
+
/** Whether a recording is still going. */
|
|
170
|
+
running: () => take !== undefined && !take.ended,
|
|
171
|
+
/** She touched the window herself, so whether it is paused is no longer this module's to know. */
|
|
172
|
+
touched() { pausedTarget = undefined; },
|
|
173
|
+
/** Some frames as one picture, each with its time in the corner. */
|
|
174
|
+
async sheet(frames) {
|
|
175
|
+
const file = join(recorded().dir, `sheet-${++files}.jpg`);
|
|
176
|
+
try {
|
|
177
|
+
await deps.call('sheet', { files: frames.map(each => each.file), labels: frames.map(each => frameLabel(each.second)), file, columns: Math.min(SHEET_COLUMNS, frames.length), tile: TILE });
|
|
178
|
+
return readFileSync(file);
|
|
179
|
+
}
|
|
180
|
+
finally {
|
|
181
|
+
rmSync(file, { force: true });
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
one: (target) => readFileSync(target.file),
|
|
185
|
+
clear,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type DesktopInput } from './desktop.ts';
|
|
2
|
+
import type { DesktopDriver } from './driver.ts';
|
|
3
|
+
import { type DesktopResult } from './turns.ts';
|
|
4
|
+
export type Screen = ReturnType<typeof createScreen>;
|
|
5
|
+
/**
|
|
6
|
+
* Foreground mode: the partner hands her the computer. She sees the whole screen and acts with the real mouse and
|
|
7
|
+
* keyboard, so every program answers her, games and administrators' windows too. `hands` is the helper the partner
|
|
8
|
+
* approved as administrator: pictures, input and the sign go through it, so its stop key works over any window.
|
|
9
|
+
* `eyes` is the ordinary helper: listing windows and starting programs stay out of the administrator's reach, so
|
|
10
|
+
* nothing she opens runs as administrator. One action at a time; the partner's stop works as in background mode.
|
|
11
|
+
*/
|
|
12
|
+
export declare function createScreen(deps: {
|
|
13
|
+
hands: Pick<DesktopDriver, 'call'>;
|
|
14
|
+
eyes: Pick<DesktopDriver, 'call'>;
|
|
15
|
+
companion?(): string;
|
|
16
|
+
signFault?(text: string | undefined): void;
|
|
17
|
+
settleMs?: number;
|
|
18
|
+
appearMs?: number;
|
|
19
|
+
secondMs?: number;
|
|
20
|
+
lingerMs?: number;
|
|
21
|
+
}): {
|
|
22
|
+
actions: string[];
|
|
23
|
+
act: (input: DesktopInput) => Promise<DesktopResult>;
|
|
24
|
+
recent: () => import("./turns.ts").DesktopAction[];
|
|
25
|
+
halt: () => {
|
|
26
|
+
cut: boolean;
|
|
27
|
+
recording: boolean;
|
|
28
|
+
};
|
|
29
|
+
release: () => void;
|
|
30
|
+
halted: () => boolean;
|
|
31
|
+
/** When foreground mode is turned off: what she saw and recorded goes with it. */
|
|
32
|
+
reset(): void;
|
|
33
|
+
};
|