dsh-world 0.7.5 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +541 -446
- package/CONTRIBUTING.md +174 -163
- package/INSTALL.md +314 -314
- package/NOTES.md +98 -5
- package/README.md +332 -258
- package/client/index.js +1126 -986
- package/lib/compaction.d.ts +7 -1
- package/lib/compaction.js +8 -3
- package/lib/console/routes.d.ts +11 -3
- package/lib/console/routes.js +12 -7
- package/lib/console/snapshot.d.ts +2 -0
- package/lib/console/snapshot.js +1 -0
- package/lib/ecosystem/computer/browser/browser.d.ts +34 -0
- package/lib/ecosystem/computer/browser/browser.js +101 -0
- package/lib/ecosystem/computer/browser/cdp.d.ts +22 -0
- package/lib/ecosystem/computer/browser/cdp.js +91 -0
- package/lib/ecosystem/computer/browser/edge.d.ts +15 -0
- package/lib/ecosystem/computer/browser/edge.js +80 -0
- package/lib/ecosystem/computer/browser/page.d.ts +41 -0
- package/lib/ecosystem/computer/browser/page.js +140 -0
- package/lib/ecosystem/computer/browser/pageLib.d.ts +7 -0
- package/lib/ecosystem/computer/browser/pageLib.js +106 -0
- package/lib/ecosystem/computer/desktop/desktop.d.ts +91 -0
- package/lib/ecosystem/computer/desktop/desktop.js +322 -0
- package/lib/ecosystem/computer/desktop/driver.d.ts +34 -0
- package/lib/ecosystem/computer/desktop/driver.js +197 -0
- package/lib/ecosystem/computer/desktop/processes.d.ts +5 -0
- package/lib/ecosystem/computer/desktop/processes.js +15 -0
- package/lib/ecosystem/computer/desktop/recording.d.ts +77 -0
- package/lib/ecosystem/computer/desktop/recording.js +187 -0
- package/lib/ecosystem/computer/desktop/screen.d.ts +33 -0
- package/lib/ecosystem/computer/desktop/screen.js +182 -0
- package/lib/ecosystem/computer/desktop/turns.d.ts +58 -0
- package/lib/ecosystem/computer/desktop/turns.js +94 -0
- package/lib/ecosystem/computer/desktop/watching.d.ts +25 -0
- package/lib/ecosystem/computer/desktop/watching.js +81 -0
- package/lib/ecosystem/computer/inbox/inbox.d.ts +32 -0
- package/lib/ecosystem/computer/inbox/inbox.js +142 -0
- package/lib/ecosystem/computer/index.d.ts +82 -0
- package/lib/ecosystem/computer/index.js +133 -0
- package/lib/ecosystem/computer/procedures/alarm.d.ts +13 -0
- package/lib/ecosystem/computer/procedures/alarm.js +46 -0
- package/lib/ecosystem/computer/procedures/archive.d.ts +32 -0
- package/lib/ecosystem/computer/procedures/archive.js +68 -0
- package/lib/ecosystem/computer/procedures/catalog.d.ts +7 -0
- package/lib/ecosystem/computer/procedures/catalog.js +75 -0
- package/lib/ecosystem/computer/procedures/format.d.ts +167 -0
- package/lib/ecosystem/computer/procedures/format.js +332 -0
- package/lib/ecosystem/computer/procedures/library.d.ts +53 -0
- package/lib/ecosystem/computer/procedures/library.js +178 -0
- package/lib/ecosystem/computer/procedures/run.d.ts +80 -0
- package/lib/ecosystem/computer/procedures/run.js +245 -0
- package/lib/ecosystem/computer/tools.d.ts +21 -0
- package/lib/ecosystem/computer/tools.js +197 -0
- package/lib/ecosystem/qq-native/images.d.ts +8 -1
- package/lib/ecosystem/qq-native/images.js +34 -15
- package/lib/ecosystem/qq-native/index.js +5 -5
- package/lib/ecosystem/shared/turn.d.ts +3 -0
- package/lib/ecosystem/shared/turn.js +13 -3
- package/lib/ecosystem/wechat/home.d.ts +1 -0
- package/lib/ecosystem/wechat/home.js +9 -3
- package/lib/ecosystem/wechat/images.js +3 -5
- package/lib/ecosystem/wechat/index.d.ts +6 -0
- package/lib/ecosystem/wechat/index.js +3 -1
- package/lib/ecosystem/wechat/presence.d.ts +5 -1
- package/lib/ecosystem/wechat/presence.js +22 -13
- package/lib/ecosystem/wechat/reading.d.ts +1 -1
- package/lib/ecosystem/wechat/reading.js +2 -2
- package/lib/elsewhere.d.ts +34 -0
- package/lib/elsewhere.js +40 -0
- package/lib/history.d.ts +5 -0
- package/lib/history.js +7 -1
- package/lib/hostContract.d.ts +6 -0
- package/lib/images.d.ts +30 -0
- package/lib/images.js +101 -0
- package/lib/index.d.ts +4 -0
- package/lib/index.js +98 -6
- package/lib/life/photoTools.js +4 -15
- package/lib/life/runtime.d.ts +12 -2
- package/lib/life/runtime.js +93 -12
- package/lib/life/toolRegistry.d.ts +9 -3
- package/lib/life/toolRegistry.js +46 -27
- package/lib/life/turnCost.d.ts +16 -0
- package/lib/life/turnCost.js +44 -0
- package/lib/life/types.d.ts +17 -0
- package/lib/memoryTools.d.ts +4 -2
- package/lib/memoryTools.js +25 -5
- package/lib/mood.d.ts +5 -1
- package/lib/mood.js +25 -8
- package/lib/promptSection.d.ts +3 -0
- package/lib/promptSection.js +29 -42
- package/lib/runtime.js +44 -17
- package/lib/sessionNews.d.ts +17 -0
- package/lib/sessionNews.js +55 -0
- package/lib/store.d.ts +6 -0
- package/lib/store.js +17 -3
- package/lib/types.d.ts +4 -0
- package/lib/worldAdd.d.ts +9 -0
- package/lib/worldAdd.js +39 -0
- package/lib/worldPrompt.d.ts +3 -2
- package/lib/worldPrompt.js +3 -2
- package/lib/worldRoutes.d.ts +5 -2
- package/lib/worldRoutes.js +29 -4
- package/lib/worldSwitch.d.ts +8 -2
- package/lib/worldSwitch.js +11 -5
- package/package.json +132 -129
- package/src/ecosystem/README.md +2 -1
- package/src/ecosystem/computer/NOTICE.md +27 -0
- package/src/ecosystem/computer/README.md +244 -0
- package/src/ecosystem/computer/desktop/win32.ps1 +858 -0
- package/src/ecosystem/wechat/windows/README.md +4 -2
- package/src/ecosystem/wechat/windows/engine/NOTICE.md +6 -0
- package/src/ecosystem/wechat/windows/engine/profiles.json +28 -0
- package/templates/world-prompt-memory.md +10 -10
- package/templates/world-prompt.md +65 -65
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// src/ecosystem/computer/desktop/recording.ts
|
|
2
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
/** A recording is at most this long and this many frames: ten minutes at one a second is some tens of megabytes. */
|
|
6
|
+
export const MAX_SECONDS = 600, MAX_FRAMES = 600;
|
|
7
|
+
const MIN_EVERY = 0.5, MAX_EVERY = 30;
|
|
8
|
+
/** A frame's long edge: enough to read subtitles in a single frame, small enough to keep hundreds. */
|
|
9
|
+
const FRAME_EDGE = 960;
|
|
10
|
+
/** Frames on one sheet, and a tile's width: twelve tiles read as one picture, more turn to mush. */
|
|
11
|
+
export const SHEET_FRAMES = 12;
|
|
12
|
+
const SHEET_COLUMNS = 4, TILE = 480;
|
|
13
|
+
/** Mean difference of two prints (0–255) above which the picture moved: a spinner or a blinking cursor stays under it. */
|
|
14
|
+
const MOVED = 1.5;
|
|
15
|
+
/** A recording in which fewer than this share of the frames differ from the one before is told as not moving. */
|
|
16
|
+
const STILL_SHARE = 0.1;
|
|
17
|
+
/** How long a player takes to act on the space bar, and how far apart the two looks are that tell whether it stopped, in seconds. */
|
|
18
|
+
const KEY_SETTLE = 0.6, PROBE_GAP = 0.8;
|
|
19
|
+
/** `65s 1:05`: the second she passes back, and the clock reading a person would look for. */
|
|
20
|
+
export const frameLabel = (second) => {
|
|
21
|
+
const whole = Math.round(second * 10) / 10;
|
|
22
|
+
return whole < 60 ? `${whole}s` : `${whole}s ${Math.floor(whole / 60)}:${String(Math.floor(whole % 60)).padStart(2, '0')}`;
|
|
23
|
+
};
|
|
24
|
+
/** Up to `max` of the frames, spread evenly, first and last included. */
|
|
25
|
+
export function spread(frames, max) {
|
|
26
|
+
if (frames.length <= max)
|
|
27
|
+
return [...frames];
|
|
28
|
+
return Array.from({ length: max }, (_unused, index) => frames[Math.round(index * (frames.length - 1) / (max - 1))]);
|
|
29
|
+
}
|
|
30
|
+
/** Whether the picture changed between two frames; prints that cannot be compared count as changed, never as a false "still". */
|
|
31
|
+
export function moved(before, after) {
|
|
32
|
+
if (!before || before.length !== after.length || before.length % 2)
|
|
33
|
+
return true;
|
|
34
|
+
let total = 0;
|
|
35
|
+
for (let at = 0; at < before.length; at += 2)
|
|
36
|
+
total += Math.abs(parseInt(before.slice(at, at + 2), 16) - parseInt(after.slice(at, at + 2), 16));
|
|
37
|
+
return total / (before.length / 2) > MOVED;
|
|
38
|
+
}
|
|
39
|
+
/** A recording of something that was not playing: nearly every frame is the one before it. */
|
|
40
|
+
export function mostlyStill(frames) {
|
|
41
|
+
if (frames.length < 2)
|
|
42
|
+
return false;
|
|
43
|
+
const changes = frames.slice(1).filter((each, index) => moved(frames[index].print, each.print)).length;
|
|
44
|
+
return changes / (frames.length - 1) < STILL_SHARE;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* How she watches something that moves: the window (or the screen) is pictured at a steady pace into a folder of frames, and what
|
|
48
|
+
* she looks at afterwards is always that folder, never the program again. One recording at a time; a new one, or the
|
|
49
|
+
* switch going off, deletes the last. The frames are pictures of the window only, with no sound.
|
|
50
|
+
*
|
|
51
|
+
* A video goes on playing while she reads what was recorded, and is somewhere else by the time she records again. So
|
|
52
|
+
* when a recording ends on a moving picture the space bar is pressed for her, which pauses nearly every player, and
|
|
53
|
+
* the next recording of the same window presses it again to go on. It is a guess: what came of it is looked at and told.
|
|
54
|
+
*/
|
|
55
|
+
export function createRecording(deps) {
|
|
56
|
+
// `call` is the helper, for laying frames out on a sheet; the frames themselves come from the target.
|
|
57
|
+
const secondMs = deps.secondMs ?? 1000;
|
|
58
|
+
const wait = (seconds) => new Promise(resolve => setTimeout(resolve, seconds * secondMs));
|
|
59
|
+
let take, files = 0;
|
|
60
|
+
/** The target this module paused and nothing has touched since: the next recording of it starts by un-pausing. */
|
|
61
|
+
let pausedTarget;
|
|
62
|
+
async function frame(into, index) {
|
|
63
|
+
const file = join(into.dir, `${String(index).padStart(5, '0')}.jpg`);
|
|
64
|
+
into.frames.push({ second: index * into.every, file, print: await into.target.capture(file, FRAME_EDGE) });
|
|
65
|
+
}
|
|
66
|
+
/** Two looks a moment apart, kept out of the recording. */
|
|
67
|
+
async function probe(into) {
|
|
68
|
+
const file = join(into.dir, `probe-${++files}.jpg`);
|
|
69
|
+
try {
|
|
70
|
+
const first = await into.target.capture(file, FRAME_EDGE);
|
|
71
|
+
await wait(PROBE_GAP);
|
|
72
|
+
return moved(first, await into.target.capture(file, FRAME_EDGE));
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
rmSync(file, { force: true });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Pauses what was playing when the recording ended; a picture that had already stopped is left alone, or the key would start it. */
|
|
79
|
+
async function pause(into) {
|
|
80
|
+
const last = into.frames.slice(-3);
|
|
81
|
+
if (last.length < 2 || !last.slice(1).some((each, index) => moved(last[index].print, each.print))) {
|
|
82
|
+
into.paused = 'still';
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
await into.target.space();
|
|
86
|
+
await wait(KEY_SETTLE);
|
|
87
|
+
into.paused = await probe(into) ? 'running' : 'paused';
|
|
88
|
+
if (into.paused === 'paused')
|
|
89
|
+
pausedTarget = into.target.key;
|
|
90
|
+
}
|
|
91
|
+
/** Each frame is aimed at its own moment from the start, so a slow one does not push the rest later. */
|
|
92
|
+
async function run(into, count, pauseAfter) {
|
|
93
|
+
for (let index = 1; index < count && !into.stopped && !into.discarded; index++) {
|
|
94
|
+
const left = into.startedAt + index * into.every * secondMs - Date.now();
|
|
95
|
+
if (left > 0)
|
|
96
|
+
await new Promise(resolve => setTimeout(resolve, left));
|
|
97
|
+
if (into.stopped || into.discarded)
|
|
98
|
+
break;
|
|
99
|
+
try {
|
|
100
|
+
await frame(into, index);
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
into.cut = error instanceof Error ? error.message : String(error);
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// The window may be gone, or the helper busy: the frames are what she asked for, the pause is a courtesy.
|
|
108
|
+
if (pauseAfter && !into.cut && !into.discarded && !into.quiet)
|
|
109
|
+
await pause(into).catch(() => { });
|
|
110
|
+
into.ended = true;
|
|
111
|
+
}
|
|
112
|
+
function clear() {
|
|
113
|
+
pausedTarget = undefined;
|
|
114
|
+
if (!take)
|
|
115
|
+
return;
|
|
116
|
+
take.discarded = true;
|
|
117
|
+
rmSync(take.dir, { recursive: true, force: true });
|
|
118
|
+
take = undefined;
|
|
119
|
+
}
|
|
120
|
+
function recorded() {
|
|
121
|
+
if (!take)
|
|
122
|
+
throw new Error('还没有录过:先 desktop record');
|
|
123
|
+
return take;
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
/** Starts over a target; the first frame is taken before this returns, so one that cannot be pictured fails here. */
|
|
127
|
+
async start(target, seconds, every = 1, pauseAfter = true) {
|
|
128
|
+
if (typeof seconds !== 'number' || !(seconds >= 1 && seconds <= MAX_SECONDS))
|
|
129
|
+
throw new Error(`seconds 要在 1 到 ${MAX_SECONDS} 之间(录多少秒)`);
|
|
130
|
+
if (typeof every !== 'number' || !(every >= MIN_EVERY && every <= MAX_EVERY))
|
|
131
|
+
throw new Error(`every 要在 ${MIN_EVERY} 到 ${MAX_EVERY} 之间(每几秒一帧)`);
|
|
132
|
+
const count = Math.floor(seconds / every) + 1;
|
|
133
|
+
if (count > MAX_FRAMES)
|
|
134
|
+
throw new Error(`这样要录 ${count} 帧,最多 ${MAX_FRAMES} 帧:把 every 调大`);
|
|
135
|
+
const resumed = pausedTarget === target.key;
|
|
136
|
+
clear();
|
|
137
|
+
if (resumed) {
|
|
138
|
+
await target.space();
|
|
139
|
+
await wait(KEY_SETTLE);
|
|
140
|
+
}
|
|
141
|
+
const fresh = { dir: mkdtempSync(join(tmpdir(), 'dsh-world-record-')), app: target.app, target, seconds, every, startedAt: Date.now(), frames: [], ended: false, resumed, stopped: false, discarded: false, done: Promise.resolve() };
|
|
142
|
+
try {
|
|
143
|
+
await frame(fresh, 0);
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
rmSync(fresh.dir, { recursive: true, force: true });
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
take = fresh;
|
|
150
|
+
fresh.done = run(fresh, count, pauseAfter);
|
|
151
|
+
return { seconds, every, endsAt: new Date(fresh.startedAt + seconds * secondMs), resumed };
|
|
152
|
+
},
|
|
153
|
+
/** How it stands: what is recorded so far, whether it is over, and why it stopped early if it did. */
|
|
154
|
+
state() {
|
|
155
|
+
const { app, seconds, every, frames, ended, cut, paused, resumed, startedAt, stopped } = recorded();
|
|
156
|
+
// Stopped but still taking its last frame: over in a moment, so it is waited for, not reported as still going.
|
|
157
|
+
return { app, seconds, every, frames: [...frames], ended, stopping: stopped && !ended, resumed, left: ended ? 0 : Math.max(0, seconds - (Date.now() - startedAt) / secondMs), ...(cut ? { cut } : {}), ...(paused ? { paused } : {}) };
|
|
158
|
+
},
|
|
159
|
+
/** Resolves when the recording is over. */
|
|
160
|
+
finished: () => recorded().done,
|
|
161
|
+
/** Ends it where it is and keeps what was recorded; over once the frame being taken, if any, is in. */
|
|
162
|
+
stop(quiet = false) {
|
|
163
|
+
const current = recorded();
|
|
164
|
+
current.stopped = true;
|
|
165
|
+
if (quiet)
|
|
166
|
+
current.quiet = true;
|
|
167
|
+
return current.done;
|
|
168
|
+
},
|
|
169
|
+
/** Whether a recording is still going. */
|
|
170
|
+
running: () => take !== undefined && !take.ended,
|
|
171
|
+
/** She touched the window herself, so whether it is paused is no longer this module's to know. */
|
|
172
|
+
touched() { pausedTarget = undefined; },
|
|
173
|
+
/** Some frames as one picture, each with its time in the corner. */
|
|
174
|
+
async sheet(frames) {
|
|
175
|
+
const file = join(recorded().dir, `sheet-${++files}.jpg`);
|
|
176
|
+
try {
|
|
177
|
+
await deps.call('sheet', { files: frames.map(each => each.file), labels: frames.map(each => frameLabel(each.second)), file, columns: Math.min(SHEET_COLUMNS, frames.length), tile: TILE });
|
|
178
|
+
return readFileSync(file);
|
|
179
|
+
}
|
|
180
|
+
finally {
|
|
181
|
+
rmSync(file, { force: true });
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
one: (target) => readFileSync(target.file),
|
|
185
|
+
clear,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type DesktopInput } from './desktop.ts';
|
|
2
|
+
import type { DesktopDriver } from './driver.ts';
|
|
3
|
+
import { type DesktopResult } from './turns.ts';
|
|
4
|
+
export type Screen = ReturnType<typeof createScreen>;
|
|
5
|
+
/**
|
|
6
|
+
* Foreground mode: the partner hands her the computer. She sees the whole screen and acts with the real mouse and
|
|
7
|
+
* keyboard, so every program answers her, games and administrators' windows too. `hands` is the helper the partner
|
|
8
|
+
* approved as administrator: pictures, input and the sign go through it, so its stop key works over any window.
|
|
9
|
+
* `eyes` is the ordinary helper: listing windows and starting programs stay out of the administrator's reach, so
|
|
10
|
+
* nothing she opens runs as administrator. One action at a time; the partner's stop works as in background mode.
|
|
11
|
+
*/
|
|
12
|
+
export declare function createScreen(deps: {
|
|
13
|
+
hands: Pick<DesktopDriver, 'call'>;
|
|
14
|
+
eyes: Pick<DesktopDriver, 'call'>;
|
|
15
|
+
companion?(): string;
|
|
16
|
+
signFault?(text: string | undefined): void;
|
|
17
|
+
settleMs?: number;
|
|
18
|
+
appearMs?: number;
|
|
19
|
+
secondMs?: number;
|
|
20
|
+
lingerMs?: number;
|
|
21
|
+
}): {
|
|
22
|
+
actions: string[];
|
|
23
|
+
act: (input: DesktopInput) => Promise<DesktopResult>;
|
|
24
|
+
recent: () => import("./turns.ts").DesktopAction[];
|
|
25
|
+
halt: () => {
|
|
26
|
+
cut: boolean;
|
|
27
|
+
recording: boolean;
|
|
28
|
+
};
|
|
29
|
+
release: () => void;
|
|
30
|
+
halted: () => boolean;
|
|
31
|
+
/** When foreground mode is turned off: what she saw and recorded goes with it. */
|
|
32
|
+
reset(): void;
|
|
33
|
+
};
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// src/ecosystem/computer/desktop/screen.ts
|
|
2
|
+
import { readFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { desktopKey, dragged, holdMs, IMAGE_EDGE, largest, NOTCHES, pressed, VK, windowsNamed, ZOOM_GROW } from "./desktop.js";
|
|
6
|
+
import { createRecording } from "./recording.js";
|
|
7
|
+
import { createTurns } from "./turns.js";
|
|
8
|
+
import { watching } from "./watching.js";
|
|
9
|
+
const SETTLE_MS = 400, APPEAR_MS = 10_000, WINDOWS_SHOWN = 30;
|
|
10
|
+
const pause = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
11
|
+
/** Here she has the whole computer: the sign stays up between her steps, which take her a while to think out. */
|
|
12
|
+
const SCREEN_LINGER_MS = 30_000;
|
|
13
|
+
/**
|
|
14
|
+
* Foreground mode: the partner hands her the computer. She sees the whole screen and acts with the real mouse and
|
|
15
|
+
* keyboard, so every program answers her, games and administrators' windows too. `hands` is the helper the partner
|
|
16
|
+
* approved as administrator: pictures, input and the sign go through it, so its stop key works over any window.
|
|
17
|
+
* `eyes` is the ordinary helper: listing windows and starting programs stay out of the administrator's reach, so
|
|
18
|
+
* nothing she opens runs as administrator. One action at a time; the partner's stop works as in background mode.
|
|
19
|
+
*/
|
|
20
|
+
export function createScreen(deps) {
|
|
21
|
+
const settleMs = deps.settleMs ?? SETTLE_MS, appearMs = deps.appearMs ?? APPEAR_MS;
|
|
22
|
+
let view, shots = 0, front;
|
|
23
|
+
const wrap = (driver) => (op, args = {}) => driver.call(op, args).catch((error) => {
|
|
24
|
+
throw new Error(`没做成:${error instanceof Error ? error.message : String(error)}`);
|
|
25
|
+
});
|
|
26
|
+
const hands = wrap(deps.hands), eyes = wrap(deps.eyes);
|
|
27
|
+
const recording = createRecording({ call: hands, ...(deps.secondMs === undefined ? {} : { secondMs: deps.secondMs }) });
|
|
28
|
+
const windows = async () => (await eyes('windows')).windows ?? [];
|
|
29
|
+
/** A picture pixel to a screen pixel; outside the picture is refused. */
|
|
30
|
+
function onScreen(x, y) {
|
|
31
|
+
if (!view)
|
|
32
|
+
throw new Error('还没看过屏幕:先 desktop shot');
|
|
33
|
+
if (typeof x !== 'number' || typeof y !== 'number')
|
|
34
|
+
throw new Error('要给 x、y(截图上的像素)');
|
|
35
|
+
if (x < 0 || y < 0 || x > view.width || y > view.height)
|
|
36
|
+
throw new Error(`x、y 要在截图里面(${view.width}×${view.height})`);
|
|
37
|
+
const local = { x: Math.round(x * view.scale), y: Math.round(y * view.scale) };
|
|
38
|
+
return { x: local.x + view.left, y: local.y + view.top, local };
|
|
39
|
+
}
|
|
40
|
+
async function picture(region, mark) {
|
|
41
|
+
const file = join(tmpdir(), `dsh-world-screen-${process.pid}-${++shots}.png`);
|
|
42
|
+
try {
|
|
43
|
+
const shot = await hands('screen', { file, edge: IMAGE_EDGE, grow: region ? ZOOM_GROW : 1, ...(region ?? { x: 0, y: 0, w: 0, h: 0 }), markX: mark?.x ?? -1, markY: mark?.y ?? -1 });
|
|
44
|
+
return { data: readFileSync(file), width: Number(shot.width), height: Number(shot.height), clientWidth: Number(shot.clientWidth), left: Number(shot.left ?? 0), top: Number(shot.top ?? 0) };
|
|
45
|
+
}
|
|
46
|
+
finally {
|
|
47
|
+
rmSync(file, { force: true });
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async function shot(said = '', mark) {
|
|
51
|
+
const taken = await picture(undefined, mark);
|
|
52
|
+
if (mark)
|
|
53
|
+
said += '图上的红圈是刚才点到的地方,没点中就看偏了多少,改了再点。';
|
|
54
|
+
view = { scale: taken.clientWidth / taken.width, width: taken.width, height: taken.height, left: taken.left, top: taken.top };
|
|
55
|
+
return { text: `${said}整个屏幕的截图附在下一条消息里(${taken.width}×${taken.height})。按图上的像素用 desktop click 带 x、y;字太小就 desktop zoom。`, image: { data: taken.data, mediaType: 'image/png' } };
|
|
56
|
+
}
|
|
57
|
+
async function after(done, clicked) {
|
|
58
|
+
await pause(settleMs);
|
|
59
|
+
return shot(done, clicked);
|
|
60
|
+
}
|
|
61
|
+
/** A program's window in front: an open one, then its tray icon, then the Start menu. */
|
|
62
|
+
async function open(app) {
|
|
63
|
+
const listed = await windows();
|
|
64
|
+
if (!app?.trim()) {
|
|
65
|
+
const titles = [...new Set(listed.map(entry => entry.title))].slice(0, WINDOWS_SHOWN);
|
|
66
|
+
return { text: `开着的窗口:${titles.join('、') || '(没有)'}。收在托盘里的、没开的,写它的名字也能打开。` };
|
|
67
|
+
}
|
|
68
|
+
const name = app.trim(), before = new Set(listed.map(entry => entry.hwnd));
|
|
69
|
+
let mine = windowsNamed(listed, name), told = '';
|
|
70
|
+
if (!mine.length && (await eyes('tray', { name })).result === 'opened') {
|
|
71
|
+
mine = await appear(name, before);
|
|
72
|
+
told = '它收在托盘里,已经从托盘图标点开。\n';
|
|
73
|
+
}
|
|
74
|
+
if (!mine.length) {
|
|
75
|
+
const apps = (await eyes('apps')).apps ?? [];
|
|
76
|
+
const wanted = name.toLowerCase();
|
|
77
|
+
const found = apps.find(entry => entry.name.toLowerCase() === wanted) ?? apps.find(entry => entry.name.toLowerCase().includes(wanted));
|
|
78
|
+
if (!found)
|
|
79
|
+
throw new Error(`找不到叫「${name}」的程序:开着的窗口、托盘图标和开始菜单里都没有。不带 app 调 desktop open 看看开着哪些`);
|
|
80
|
+
await eyes('launch', { appId: found.appId });
|
|
81
|
+
mine = await appear(found.name, before);
|
|
82
|
+
if (!mine.length)
|
|
83
|
+
throw new Error(`${found.name} 启动了,但 ${appearMs / 1000} 秒内没有出现窗口;过一会儿 desktop shot 看看`);
|
|
84
|
+
told = `${found.name} 没开,已经启动。\n`;
|
|
85
|
+
}
|
|
86
|
+
const chosen = mine.find(entry => entry.title.toLowerCase() === name.toLowerCase()) ?? largest(mine.filter(entry => !entry.owner)) ?? largest(mine);
|
|
87
|
+
if (chosen.minimized)
|
|
88
|
+
await hands('restore', { hwnd: chosen.hwnd });
|
|
89
|
+
if ((await hands('refocus', { hwnd: chosen.hwnd })).refocused !== true)
|
|
90
|
+
told += '没能把它放到最前面,看截图确认。\n';
|
|
91
|
+
front = name;
|
|
92
|
+
return after(told);
|
|
93
|
+
}
|
|
94
|
+
async function appear(name, before) {
|
|
95
|
+
for (const deadline = Date.now() + appearMs;;) {
|
|
96
|
+
const listed = await windows();
|
|
97
|
+
const mine = windowsNamed(listed, name);
|
|
98
|
+
if (mine.length)
|
|
99
|
+
return mine;
|
|
100
|
+
if (Date.now() >= deadline) {
|
|
101
|
+
const fresh = listed.filter(entry => !before.has(entry.hwnd));
|
|
102
|
+
return fresh.length === 1 ? fresh : [];
|
|
103
|
+
}
|
|
104
|
+
await pause(Math.min(500, appearMs));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// The sign covers the whole screen here, and lives in the administrator helper so its stop key is heard over any window.
|
|
108
|
+
async function sign(show) {
|
|
109
|
+
if (!deps.companion)
|
|
110
|
+
return;
|
|
111
|
+
try {
|
|
112
|
+
await hands('sign', show ? { show: true, hwnd: 0, text: `${deps.companion()}正在操作`, hint: '连按两下 Esc 停下', stop: '停下', stopped: '已停下' } : { show: false });
|
|
113
|
+
deps.signFault?.(undefined);
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
deps.signFault?.(`她操作时的提示没能显示:${error instanceof Error ? error.message : String(error)}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const watch = watching({
|
|
120
|
+
recording, what: () => '整个屏幕', keep: '',
|
|
121
|
+
target: async () => ({
|
|
122
|
+
key: 'screen', app: '屏幕',
|
|
123
|
+
capture: async (file, edge) => String((await hands('screen', { file, edge, grow: 1, x: 0, y: 0, w: 0, h: 0, markX: -1, markY: -1 })).print ?? ''),
|
|
124
|
+
space: () => hands('rkey', { vk: VK.space, modifiers: [] }),
|
|
125
|
+
}),
|
|
126
|
+
});
|
|
127
|
+
const run = {
|
|
128
|
+
record: input => watch.record(input),
|
|
129
|
+
stop: () => watch.stop(),
|
|
130
|
+
review: input => watch.review(input),
|
|
131
|
+
open: input => open(input.app),
|
|
132
|
+
shot: () => shot(),
|
|
133
|
+
click: async (input) => {
|
|
134
|
+
const at = onScreen(input.x, input.y);
|
|
135
|
+
await hands('rclick', { x: at.x, y: at.y, right: input.right === true, count: input.double ? 2 : 1, hold: holdMs(input.hold) });
|
|
136
|
+
return after(pressed('点击', input.hold), at.local);
|
|
137
|
+
},
|
|
138
|
+
drag: async (input) => {
|
|
139
|
+
const from = onScreen(input.x, input.y), to = onScreen(input.x2, input.y2);
|
|
140
|
+
await hands('rdrag', { x: from.x, y: from.y, x2: to.x, y2: to.y, right: input.right === true, hold: holdMs(input.hold) });
|
|
141
|
+
return after(dragged(input.hold), to.local);
|
|
142
|
+
},
|
|
143
|
+
type: async (input) => {
|
|
144
|
+
if (typeof input.text !== 'string' || !input.text)
|
|
145
|
+
throw new Error('text 不能是空的');
|
|
146
|
+
const at = input.x !== undefined || input.y !== undefined ? onScreen(input.x, input.y) : undefined;
|
|
147
|
+
if (at)
|
|
148
|
+
await hands('rclick', { x: at.x, y: at.y, right: false, count: 1 });
|
|
149
|
+
await hands('rtype', { text: input.text });
|
|
150
|
+
if (input.submit)
|
|
151
|
+
await hands('rkey', { vk: VK.return, modifiers: [] });
|
|
152
|
+
return after(input.submit ? '已输入并回车。' : '已输入。', at?.local);
|
|
153
|
+
},
|
|
154
|
+
press: async (input) => {
|
|
155
|
+
await hands('rkey', { ...desktopKey(String(input.key ?? '')), hold: holdMs(input.hold) });
|
|
156
|
+
return after(pressed(`按 ${input.key}`, input.hold));
|
|
157
|
+
},
|
|
158
|
+
scroll: async (input) => {
|
|
159
|
+
const step = { up: [NOTCHES, 0], down: [-NOTCHES, 0], left: [0, -NOTCHES], right: [0, NOTCHES] }[String(input.direction)];
|
|
160
|
+
if (!step)
|
|
161
|
+
throw new Error('direction 只能是 up、down、left 或 right');
|
|
162
|
+
if (!view)
|
|
163
|
+
throw new Error('还没看过屏幕:先 desktop shot');
|
|
164
|
+
const at = input.x === undefined && input.y === undefined ? onScreen(view.width / 2, view.height / 2) : onScreen(input.x, input.y);
|
|
165
|
+
await hands('rscroll', { x: at.x, y: at.y, up: step[0], right: step[1] });
|
|
166
|
+
return after('已滚动。');
|
|
167
|
+
},
|
|
168
|
+
zoom: async (input) => {
|
|
169
|
+
const from = onScreen(input.x, input.y).local, to = onScreen(input.x2, input.y2).local;
|
|
170
|
+
if (!(to.x > from.x && to.y > from.y))
|
|
171
|
+
throw new Error('zoom 要给左上角 x、y 和右下角 x2、y2(截图上的像素)');
|
|
172
|
+
const part = await picture({ x: from.x, y: from.y, w: to.x - from.x, h: to.y - from.y });
|
|
173
|
+
return { text: '放大的局部附在下一条消息里;在它上面量的坐标不能直接点,回到整张截图上点。', image: { data: part.data, mediaType: 'image/png' } };
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
const turns = createTurns({ run, sign, recording, app: () => front, lingerMs: deps.lingerMs ?? SCREEN_LINGER_MS });
|
|
177
|
+
return {
|
|
178
|
+
actions: turns.actions, act: turns.act, recent: turns.recent, halt: turns.halt, release: turns.release, halted: turns.halted,
|
|
179
|
+
/** When foreground mode is turned off: what she saw and recorded goes with it. */
|
|
180
|
+
reset() { view = undefined; front = undefined; recording.clear(); turns.reset(); },
|
|
181
|
+
};
|
|
182
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { ImageMediaType } from '@deepseek-ai/dsh-attachment';
|
|
2
|
+
import type { Recording } from './recording.ts';
|
|
3
|
+
/** What one desktop action hands back: short text, and the picture she acts on (`caption` when it is not a picture of now). */
|
|
4
|
+
export interface DesktopResult {
|
|
5
|
+
text: string;
|
|
6
|
+
image?: {
|
|
7
|
+
data: Buffer;
|
|
8
|
+
mediaType: ImageMediaType;
|
|
9
|
+
caption?: string;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export interface DesktopAction {
|
|
13
|
+
at: string;
|
|
14
|
+
action: string;
|
|
15
|
+
app?: string;
|
|
16
|
+
ok: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** What she is told once the partner has stopped her: in the step that was cut off, and in every later one. */
|
|
19
|
+
export declare const STOPPED = "\u4F19\u4F34\u6309\u4E86\u505C\u4E0B\u3002\u5728\u4F19\u4F34\u518D\u8DDF\u4F60\u8BF4\u8BDD\u4E4B\u524D\uFF0C\u4E0D\u80FD\u518D\u64CD\u4F5C\u4F19\u4F34\u7684\u7535\u8111\uFF1B\u521A\u624D\u90A3\u4E00\u6B65\u505A\u6CA1\u505A\u5B8C\u4E0D\u786E\u5B9A\uFF0C\u522B\u91CD\u8BD5\uFF0C\u5148\u95EE\u95EE\u4F19\u4F34\u3002";
|
|
20
|
+
/** The actions that send input, named as she is told about them. */
|
|
21
|
+
export declare const INPUTS: Record<string, string>;
|
|
22
|
+
/**
|
|
23
|
+
* Her turns on the partner's computer, in either mode: one action at a time, in order; the sign up while she acts and
|
|
24
|
+
* a little after; and the partner's stop, which cuts off the step under way and refuses every later one until they
|
|
25
|
+
* speak to her. Looking at what was recorded (`review`) touches nothing, so it is neither stopped nor signed.
|
|
26
|
+
*/
|
|
27
|
+
export declare function createTurns<I extends {
|
|
28
|
+
action: string;
|
|
29
|
+
}>(deps: {
|
|
30
|
+
run: Record<string, (input: I) => Promise<DesktopResult>>;
|
|
31
|
+
/** Shows the sign over what she is on (nothing when she is on nothing yet), or takes it away. */
|
|
32
|
+
sign(show: boolean): Promise<void>;
|
|
33
|
+
recording: Pick<Recording, 'running' | 'stop' | 'touched'>;
|
|
34
|
+
/** What she is on, for the console's list of her last actions. */
|
|
35
|
+
app(): string | undefined;
|
|
36
|
+
/** She sent input; `what` is how it is named to her. */
|
|
37
|
+
acted?(what: string): void;
|
|
38
|
+
lingerMs?: number;
|
|
39
|
+
}): {
|
|
40
|
+
actions: string[];
|
|
41
|
+
/** One action, after the ones before it; what she did is kept without what she typed. */
|
|
42
|
+
act(input: I): Promise<DesktopResult>;
|
|
43
|
+
recent: () => DesktopAction[];
|
|
44
|
+
/**
|
|
45
|
+
* The partner pressed stop. What she is doing ends here, a recording stops where it is with nothing more sent to
|
|
46
|
+
* the computer, and nothing reaches it until `release`. Says whether a step of hers was cut off (she hears it from
|
|
47
|
+
* that step) and whether a recording was.
|
|
48
|
+
*/
|
|
49
|
+
halt(): {
|
|
50
|
+
cut: boolean;
|
|
51
|
+
recording: boolean;
|
|
52
|
+
};
|
|
53
|
+
/** The partner spoke to her again: she may use the computer. */
|
|
54
|
+
release(): void;
|
|
55
|
+
halted: () => boolean;
|
|
56
|
+
/** When the switch goes off. */
|
|
57
|
+
reset(): void;
|
|
58
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/** What she is told once the partner has stopped her: in the step that was cut off, and in every later one. */
|
|
2
|
+
export const STOPPED = '伙伴按了停下。在伙伴再跟你说话之前,不能再操作伙伴的电脑;刚才那一步做没做完不确定,别重试,先问问伙伴。';
|
|
3
|
+
/** How long the sign stays after her last action: long enough to see, short enough not to linger over the partner's work. */
|
|
4
|
+
const SIGN_LINGER_MS = 5000;
|
|
5
|
+
const KEPT_ACTIONS = 20;
|
|
6
|
+
/** The actions that send input, named as she is told about them. */
|
|
7
|
+
export const INPUTS = { click: '点击', drag: '拖动', type: '输入', press: '按键', scroll: '滚动' };
|
|
8
|
+
/**
|
|
9
|
+
* Her turns on the partner's computer, in either mode: one action at a time, in order; the sign up while she acts and
|
|
10
|
+
* a little after; and the partner's stop, which cuts off the step under way and refuses every later one until they
|
|
11
|
+
* speak to her. Looking at what was recorded (`review`) touches nothing, so it is neither stopped nor signed.
|
|
12
|
+
*/
|
|
13
|
+
export function createTurns(deps) {
|
|
14
|
+
let queue = Promise.resolve(), halted = false, stops = 0, busy = 0, hideTimer;
|
|
15
|
+
const actions = [];
|
|
16
|
+
/** Gone a little after her last action; a recording keeps it up until the recording is over. */
|
|
17
|
+
function linger() {
|
|
18
|
+
if (hideTimer)
|
|
19
|
+
clearTimeout(hideTimer);
|
|
20
|
+
hideTimer = setTimeout(() => { hideTimer = undefined; if (deps.recording.running())
|
|
21
|
+
linger();
|
|
22
|
+
else
|
|
23
|
+
void deps.sign(false); }, deps.lingerMs ?? SIGN_LINGER_MS);
|
|
24
|
+
hideTimer.unref?.();
|
|
25
|
+
}
|
|
26
|
+
function quiet() { if (hideTimer) {
|
|
27
|
+
clearTimeout(hideTimer);
|
|
28
|
+
hideTimer = undefined;
|
|
29
|
+
} }
|
|
30
|
+
return {
|
|
31
|
+
actions: Object.keys(deps.run),
|
|
32
|
+
/** One action, after the ones before it; what she did is kept without what she typed. */
|
|
33
|
+
act(input) {
|
|
34
|
+
const work = deps.run[input.action];
|
|
35
|
+
if (!work)
|
|
36
|
+
return Promise.reject(new Error(`不认识的 action:${input.action},可用:${Object.keys(deps.run).join('、')}`));
|
|
37
|
+
const touches = input.action !== 'review';
|
|
38
|
+
const done = queue.then(async () => {
|
|
39
|
+
if (touches && halted)
|
|
40
|
+
throw new Error(STOPPED);
|
|
41
|
+
const what = INPUTS[input.action];
|
|
42
|
+
// Her own input may have paused or started it, so a pause made for her is no longer taken for granted.
|
|
43
|
+
if (what) {
|
|
44
|
+
deps.acted?.(what);
|
|
45
|
+
deps.recording.touched();
|
|
46
|
+
}
|
|
47
|
+
if (!touches)
|
|
48
|
+
return work(input);
|
|
49
|
+
const before = stops;
|
|
50
|
+
busy++;
|
|
51
|
+
try {
|
|
52
|
+
await deps.sign(true);
|
|
53
|
+
const result = await work(input);
|
|
54
|
+
if (stops !== before)
|
|
55
|
+
throw new Error(STOPPED);
|
|
56
|
+
await deps.sign(true);
|
|
57
|
+
return result;
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
throw stops !== before ? new Error(STOPPED) : error;
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
busy--;
|
|
64
|
+
if (!halted)
|
|
65
|
+
linger();
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
queue = done.catch(() => { });
|
|
69
|
+
const note = (ok) => { const app = deps.app(); actions.push({ at: new Date().toISOString(), action: input.action, ...(app ? { app } : {}), ok }); actions.splice(0, actions.length - KEPT_ACTIONS); };
|
|
70
|
+
return done.then(result => { note(true); return result; }, error => { note(false); throw error; });
|
|
71
|
+
},
|
|
72
|
+
recent: () => [...actions].reverse(),
|
|
73
|
+
/**
|
|
74
|
+
* The partner pressed stop. What she is doing ends here, a recording stops where it is with nothing more sent to
|
|
75
|
+
* the computer, and nothing reaches it until `release`. Says whether a step of hers was cut off (she hears it from
|
|
76
|
+
* that step) and whether a recording was.
|
|
77
|
+
*/
|
|
78
|
+
halt() {
|
|
79
|
+
const wasRecording = deps.recording.running();
|
|
80
|
+
halted = true;
|
|
81
|
+
stops++;
|
|
82
|
+
quiet();
|
|
83
|
+
deps.recording.touched();
|
|
84
|
+
if (wasRecording)
|
|
85
|
+
void deps.recording.stop(true);
|
|
86
|
+
return { cut: busy > 0, recording: wasRecording };
|
|
87
|
+
},
|
|
88
|
+
/** The partner spoke to her again: she may use the computer. */
|
|
89
|
+
release() { halted = false; },
|
|
90
|
+
halted: () => halted,
|
|
91
|
+
/** When the switch goes off. */
|
|
92
|
+
reset() { halted = false; quiet(); },
|
|
93
|
+
};
|
|
94
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type RecordTarget, type Recording } from './recording.ts';
|
|
2
|
+
import type { DesktopResult } from './turns.ts';
|
|
3
|
+
interface WatchInput {
|
|
4
|
+
seconds?: number;
|
|
5
|
+
every?: number;
|
|
6
|
+
from?: number;
|
|
7
|
+
to?: number;
|
|
8
|
+
at?: number;
|
|
9
|
+
pause?: boolean;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The three actions for watching something that moves, the same in both modes: record what she is on, stop it, and
|
|
13
|
+
* review it: first the whole recording on one sheet, then any stretch of it closer, then one frame.
|
|
14
|
+
*/
|
|
15
|
+
export declare function watching(deps: {
|
|
16
|
+
recording: Recording;
|
|
17
|
+
target(): Promise<RecordTarget>;
|
|
18
|
+
what(app: string): string;
|
|
19
|
+
keep: string;
|
|
20
|
+
}): {
|
|
21
|
+
record: (input: WatchInput) => Promise<DesktopResult>;
|
|
22
|
+
stop: () => Promise<DesktopResult>;
|
|
23
|
+
review: (input: WatchInput) => Promise<DesktopResult>;
|
|
24
|
+
};
|
|
25
|
+
export {};
|