linkshell-cli 0.8.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,14 @@
9
9
  // both two-finger tap or a long press: right click; two fingers moving:
10
10
  // scroll; a second tap that stays down and moves: drag; pinch: zoom;
11
11
  // three-finger tap: the keyboard
12
+ // Beside the gestures: a sheet of the computer's shortcuts, one tap each (and the user's own), and a
13
+ // box to write text in as on any phone, sent to the computer when it is ready.
12
14
  // With a mouse (a desktop browser) the pointer, buttons, wheel and keys go straight through.
15
+ //
16
+ // The picture comes one of two ways (docs/v2/screen-realtime.md). Where the computer's app and this
17
+ // device can reach each other, it is a video track from the app, with the hands and the pointer on
18
+ // data channels beside it; the page draws the pointer, which that picture does not have in it.
19
+ // Otherwise it is H.264 down the page's socket, decoded here onto a canvas, as it always was.
13
20
  const icon = (body) => `<svg viewBox="0 0 24 24" width="22" height="22" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round">${body}</svg>`;
14
21
  const ICONS = {
15
22
  view: icon('<path d="M2.5 12s3.5-6.5 9.5-6.5 9.5 6.5 9.5 6.5-3.5 6.5-9.5 6.5S2.5 12 2.5 12Z"/><circle cx="12" cy="12" r="2.8"/>'),
@@ -21,18 +28,24 @@ const ICONS = {
21
28
  expand: icon('<path d="M4 9V4h5M20 9V4h-5M4 15v5h5M20 15v5h-5"/>'),
22
29
  shrink: icon('<path d="M9 4v5H4M15 4v5h5M9 20v-5H4M15 20v-5h5"/>'),
23
30
  down: icon('<path d="m6 9.5 6 6 6-6"/>'),
31
+ quick: icon('<path d="M13 2.8 5.2 13.6H11l-1 7.6 7.8-10.8H12Z"/>'),
32
+ compose: icon('<path d="M4 5h16v10.5H9.5L5.5 19v-3.5H4Z"/><path d="M8 9h8M8 12h5"/>'),
33
+ close: icon('<path d="m6.5 6.5 11 11M17.5 6.5l-11 11"/>'),
24
34
  };
25
35
  const STYLE = String.raw `
26
36
  * { box-sizing: border-box; -webkit-tap-highlight-color: transparent; -webkit-touch-callout: none; -webkit-user-select: none; user-select: none; }
27
37
  html, body { margin: 0; height: 100%; background: #000; overflow: hidden; overscroll-behavior: none; }
28
38
  body { position: fixed; inset: 0; color: #fff; font: 15px/1.45 -apple-system, system-ui, "PingFang SC", "Noto Sans CJK SC", sans-serif; }
29
39
  #stage { position: absolute; inset: 0; overflow: hidden; touch-action: none; }
30
- canvas { position: absolute; left: 0; top: 0; transform-origin: 0 0; will-change: transform; }
40
+ canvas, video { position: absolute; left: 0; top: 0; transform-origin: 0 0; will-change: transform; }
41
+ video { object-fit: fill; pointer-events: none; }
31
42
  #pointer { position: absolute; left: -9px; top: -9px; width: 18px; height: 18px; border-radius: 50%; pointer-events: none;
32
43
  border: 2px solid rgba(255,255,255,0.95); box-shadow: 0 0 0 1px rgba(0,0,0,0.55), inset 0 0 0 1px rgba(0,0,0,0.35);
33
44
  opacity: 0; transition: opacity 0.3s; will-change: transform; }
34
45
  #pointer.on { opacity: 1; transition: none; }
35
46
  #pointer.pressed { background: rgba(255,255,255,0.4); }
47
+ #pointer.own { border: 0; border-radius: 0; box-shadow: none; background: transparent center / 100% 100% no-repeat; }
48
+ #stage.own { cursor: none; }
36
49
  #note { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; padding: 32px;
37
50
  color: rgba(255,255,255,0.72); text-align: center; pointer-events: none; }
38
51
  .glass { background: rgba(38,38,42,0.74); -webkit-backdrop-filter: blur(22px) saturate(1.6); backdrop-filter: blur(22px) saturate(1.6);
@@ -51,24 +64,227 @@ const STYLE = String.raw `
51
64
  .choice.on { background: rgba(255,255,255,0.16); }
52
65
  .choice b { display: block; font-weight: 600; font-size: 15px; }
53
66
  .choice span { display: block; font-size: 12.5px; color: rgba(255,255,255,0.62); }
67
+ #status { margin-top: 6px; padding: 8px 12px 5px; border-top: 0.5px solid rgba(255,255,255,0.12); font-size: 12px; line-height: 1.5;
68
+ color: rgba(255,255,255,0.5); white-space: pre-line; font-variant-numeric: tabular-nums; }
54
69
  #toast { position: absolute; left: 50%; max-width: min(86vw, 420px); width: max-content; transform: translateX(-50%); padding: 10px 16px; border-radius: 18px;
55
70
  font-size: 13.5px; text-align: center; color: rgba(255,255,255,0.94); pointer-events: none; opacity: 0; transition: opacity 0.3s; }
56
71
  #toast.on { opacity: 1; }
57
- #keys { position: absolute; left: 0; right: 0; height: 46px; display: flex; align-items: center; gap: 5px; padding: 0 6px;
72
+ #keys { position: absolute; left: 0; right: 0; height: 46px; display: flex; align-items: center; gap: 4px; padding: 0 5px;
58
73
  background: rgba(28,28,30,0.96); border-top: 0.5px solid rgba(255,255,255,0.14); }
59
74
  .key { flex: 1 1 0; min-width: 0; height: 34px; border-radius: 8px; display: flex; align-items: center; justify-content: center;
60
75
  background: rgba(255,255,255,0.13); color: #fff; font-size: 15px; font-variant-numeric: tabular-nums; }
61
76
  .key.word { font-size: 12.5px; }
77
+ .key svg { width: 20px; height: 20px; }
62
78
  .key:active { background: rgba(255,255,255,0.3); }
63
79
  .key.on { background: #fff; color: #111; }
80
+ #sheet { position: absolute; display: flex; flex-direction: column; border-radius: 24px; overflow: hidden; }
81
+ #sheethead { display: flex; align-items: center; gap: 2px; padding: 8px 6px 4px 10px; }
82
+ #compose { flex: 1; min-width: 0; height: 40px; margin-right: 4px; border-radius: 20px; display: flex; align-items: center; gap: 8px; padding: 0 14px;
83
+ background: rgba(255,255,255,0.1); color: rgba(255,255,255,0.62); font-size: 14.5px; white-space: nowrap; }
84
+ #compose:active { background: rgba(255,255,255,0.2); }
85
+ #compose svg { width: 19px; height: 19px; flex: none; }
86
+ #actions, #maker { flex: 1; min-height: 0; overflow-y: auto; overscroll-behavior: contain; -webkit-overflow-scrolling: touch; touch-action: pan-y; padding: 0 10px 12px; }
87
+ .group { display: flex; align-items: center; justify-content: space-between; padding: 12px 4px 7px; font-size: 12.5px; color: rgba(255,255,255,0.55); }
88
+ .group i { font-style: normal; padding: 4px 8px; margin: -4px -4px -4px 0; border-radius: 10px; color: rgba(255,255,255,0.86); }
89
+ .group i:active { background: rgba(255,255,255,0.16); }
90
+ .tiles { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 6px; }
91
+ .tiles.six { grid-template-columns: repeat(6, minmax(0, 1fr)); margin-bottom: 6px; }
92
+ #sheet.narrow .tiles { grid-template-columns: repeat(3, minmax(0, 1fr)); }
93
+ #sheet.narrow .tiles.six { grid-template-columns: repeat(6, minmax(0, 1fr)); }
94
+ .tile { position: relative; min-height: 52px; padding: 6px 3px; border-radius: 12px; display: flex; flex-direction: column; align-items: center; justify-content: center;
95
+ background: rgba(255,255,255,0.1); text-align: center; }
96
+ .tile b { max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 500; font-size: 13.5px; line-height: 1.3; }
97
+ .tile span { font-size: 11.5px; line-height: 1.3; color: rgba(255,255,255,0.55); font-variant-numeric: tabular-nums; }
98
+ .tiles.six .tile { min-height: 44px; }
99
+ .tile:active, .tile.hit { background: rgba(255,255,255,0.3); }
100
+ .tile.on { background: #fff; color: #111; }
101
+ .tile.on span { color: rgba(0,0,0,0.55); }
102
+ .tile.sure, .tile.sure:active { background: #d9392f; }
103
+ .tile.sure span { color: rgba(255,255,255,0.86); }
104
+ .tile.add { background: transparent; border: 1px dashed rgba(255,255,255,0.3); color: rgba(255,255,255,0.78); }
105
+ .tile.loose::after { content: "×"; position: absolute; top: -5px; right: -4px; width: 19px; height: 19px; border-radius: 10px; background: #d9392f;
106
+ font-size: 14px; line-height: 18px; text-align: center; }
107
+ .field { display: flex; align-items: center; gap: 12px; height: 46px; margin-top: 6px; padding: 0 12px; border-radius: 12px; background: rgba(255,255,255,0.1); }
108
+ .field span { flex: none; font-size: 14px; color: rgba(255,255,255,0.62); }
109
+ .field select, .field input { flex: 1; min-width: 0; height: 100%; padding: 0; border: 0; outline: 0; border-radius: 0; background: transparent; color: #fff;
110
+ font: inherit; font-size: 16px; text-align: right; text-align-last: right; -webkit-appearance: none; appearance: none; -webkit-user-select: text; user-select: text; }
111
+ .field input::placeholder { color: rgba(255,255,255,0.36); }
112
+ .ends { display: flex; justify-content: flex-end; gap: 8px; margin-top: 12px; }
113
+ .soft { flex: none; height: 38px; padding: 0 13px; border-radius: 19px; display: flex; align-items: center; justify-content: center; background: rgba(255,255,255,0.13);
114
+ font-size: 14px; white-space: nowrap; }
115
+ .soft:active { background: rgba(255,255,255,0.3); }
116
+ .soft.strong { padding: 0 17px; background: #fff; color: #111; font-weight: 600; }
117
+ .soft.strong:active { background: rgba(255,255,255,0.82); }
118
+ .soft svg { width: 20px; height: 20px; }
119
+ .spring { flex: 1; }
120
+ #composer { position: absolute; left: 0; right: 0; display: flex; flex-direction: column; gap: 8px; background: rgba(28,28,30,0.96); border-top: 0.5px solid rgba(255,255,255,0.14); }
121
+ #composer.wide { flex-direction: row; align-items: flex-end; }
122
+ #wordsbox { position: relative; flex: none; }
123
+ #composer.wide #wordsbox { flex: 1; min-width: 0; }
124
+ #words { display: block; width: 100%; height: 66px; margin: 0; padding: 9px 12px; border: 0; outline: 0; resize: none; border-radius: 12px; background: rgba(255,255,255,0.12);
125
+ color: #fff; font: inherit; font-size: 16px; line-height: 1.4; -webkit-user-select: text; user-select: text; }
126
+ #composer.wide #words { height: 44px; }
127
+ #wordshint { position: absolute; left: 12px; right: 12px; top: 9px; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; font-size: 16px; line-height: 1.4;
128
+ color: rgba(255,255,255,0.36); pointer-events: none; }
129
+ #sendrow { display: flex; align-items: center; gap: 6px; }
64
130
  #typing, #shortcut { position: absolute; left: 0; top: 0; width: 2px; height: 2px; padding: 0; border: 0; outline: 0; resize: none; opacity: 0;
65
131
  background: transparent; color: transparent; caret-color: transparent; font-size: 16px; -webkit-user-select: text; user-select: text; }
66
132
  `;
67
- const SCRIPT = String.raw `
68
- (() => {
133
+ /**
134
+ * The parts of the page's script that only work things out, with nothing of the page in them:
135
+ * exported so that they can be tried without a browser.
136
+ */
137
+ export const VIEWER_LOGIC = String.raw `
138
+ /** Undoes the Gray code the time on a clock strip is written in. */
139
+ function fromGray(gray) {
140
+ let n = gray;
141
+ for (let shift = 1; shift < 16; shift <<= 1) n ^= n >> shift;
142
+ return n;
143
+ }
144
+
145
+ /**
146
+ * The time on a clock strip, from one pixel (RGBA) at the middle of each of its 20 cells: white, black,
147
+ * the 16 bits, black, white. Undefined when what is there is not a strip.
148
+ */
149
+ function stripTime(pixels) {
150
+ const light = (cell) => (pixels[cell * 4] + pixels[cell * 4 + 1] + pixels[cell * 4 + 2]) / 3;
151
+ const white = (light(0) + light(19)) / 2, black = (light(1) + light(18)) / 2;
152
+ if (white - black < 80) return undefined;
153
+ let gray = 0;
154
+ for (let cell = 2; cell < 18; cell++) gray = (gray << 1) | (light(cell) > (white + black) / 2 ? 1 : 0);
155
+ return fromGray(gray);
156
+ }
157
+
158
+ /**
159
+ * How old a frame is when it is shown: shownAt is on the computer's clock, in milliseconds, and the time
160
+ * on the frame's strip is that clock's low 16 bits. More than half a minute is a misreading.
161
+ */
162
+ function lateness(shownAt, time) {
163
+ const late = (((shownAt - time) % 65536) + 65536) % 65536;
164
+ return late > 30000 ? undefined : Math.round(late * 10) / 10;
165
+ }
166
+
167
+ function summary(samples) {
168
+ const sorted = [...samples].sort((a, b) => a - b);
169
+ const rank = (p) => (sorted.length ? sorted[Math.max(0, Math.ceil(p * sorted.length) - 1)] : null);
170
+ return { n: sorted.length, min: rank(0), p50: rank(0.5), p95: rank(0.95), max: rank(1) };
171
+ }
172
+
173
+ /**
174
+ * How far the computer's clock is ahead of the page's, from a question sent at one time (the page's) and
175
+ * answered with the computer's. The answer that came back quickest is the one kept: it is off by half its trip at most.
176
+ */
177
+ function synced(clock, sent, back, now) {
178
+ const trip = back - sent;
179
+ return trip < clock.trip ? { trip, offset: now - (sent + trip / 2) } : clock;
180
+ }
181
+
182
+ /**
183
+ * Whether a position the computer reports for its pointer is one this page put it at within the last
184
+ * second (trail: what it sent, and when): its own move coming back, a round trip late.
185
+ */
186
+ function ownEcho(trail, x, y, now) {
187
+ return trail.some((point) => now - point.t < 1000 && Math.abs(point.x - x) < 0.003 && Math.abs(point.y - y) < 0.003);
188
+ }
189
+
190
+ /**
191
+ * Which of the app's two channels an event goes on. "pointer" (no order, sent once) has only what nothing
192
+ * waits for: a move or a turn of the wheel that is not placed (sent with what happens at it), not made
193
+ * with a button held, and not tied to a position that went on "input".
194
+ */
195
+ function laneOf(kind, placed, held, tied) {
196
+ return (kind === "move" || kind === "scroll") && !placed && !held && !tied ? "pointer" : "input";
197
+ }
198
+
199
+ /**
200
+ * Whether the phone's keyboard is up, from what is known. A field with the focus is not enough: a phone
201
+ * takes its keyboard down without a word to the page (the app was put aside and came back, something
202
+ * else of the app's took the keys), and the field goes on saying it has the focus.
203
+ * focused one of the page's fields has the focus
204
+ * asked milliseconds since the page asked for the keyboard: it takes a few hundred to come up, and
205
+ * nobody has seen it yet
206
+ * said what the app around the page says of the keyboard; undefined in a browser, which has
207
+ * nobody to ask, and in an app from before it said anything
208
+ * shrunk failing that, whether the window is shorter than it has been: undefined where it has never
209
+ * been seen to make room for a keyboard
210
+ * live the page is the one the keys go to. A field with the focus in a page that is not, is one
211
+ * the keyboard was taken from. In one that is, with no keyboard on show, the keys are a real
212
+ * keyboard's and typing works: the app says when a keyboard that was up has gone, and the
213
+ * field is let go of at that (see settle), so it is not that
214
+ */
215
+ function keyboardIsUp(known) {
216
+ if (!known.focused) return false;
217
+ if (known.asked < 1000) return true;
218
+ if (known.said !== undefined) return known.said || known.live;
219
+ return known.shrunk === undefined ? known.live : known.shrunk;
220
+ }
221
+
222
+ /** Text in pieces no longer than one event may carry, counted as the computer counts (UTF-16), and never cut through a character. */
223
+ function pieces(text, most) {
224
+ const out = [];
225
+ let piece = "";
226
+ for (const character of text) {
227
+ if (piece.length + character.length > most) {
228
+ out.push(piece);
229
+ piece = "";
230
+ }
231
+ piece += character;
232
+ }
233
+ if (piece) out.push(piece);
234
+ return out;
235
+ }
236
+
237
+ /** What typing a text on the computer is: its lines as text, the return key between them. */
238
+ function typingOf(text, most) {
239
+ const events = [];
240
+ text.replace(/\r\n?/g, "\n").split("\n").forEach((line, index) => {
241
+ if (index) events.push({ t: "key", k: "return" });
242
+ for (const s of pieces(line, most)) events.push({ t: "text", s });
243
+ });
244
+ return events;
245
+ }
246
+
247
+ // The keys the computer knows by name (apps/mac, Keys.swift), and how a shortcut is written.
248
+ const MOD_ORDER = ["ctrl", "alt", "shift", "cmd"];
249
+ const MOD_SIGNS = { ctrl: "⌃", alt: "⌥", shift: "⇧", cmd: "⌘" };
250
+ const KEY_SIGNS = { return: "↩\uFE0E", tab: "Tab", space: "Space", backspace: "⌫", escape: "Esc", left: "←", right: "→", down: "↓", up: "↑",
251
+ delete: "⌦", home: "Home", end: "End", pageup: "PgUp", pagedown: "PgDn" };
252
+ const KEY_CHARACTERS = "abcdefghijklmnopqrstuvwxyz1234567890-=[]\\;',./\x60";
253
+
254
+ function keyKnown(k) {
255
+ return typeof k === "string" && (Object.hasOwn(KEY_SIGNS, k) || /^f([1-9]|1[0-2])$/.test(k) || (k.length === 1 && KEY_CHARACTERS.includes(k)));
256
+ }
257
+
258
+ /** The modifiers among these names that are modifiers, each once, in the order they are written in. */
259
+ function modsOf(names) {
260
+ return MOD_ORDER.filter((mod) => Array.isArray(names) && names.includes(mod));
261
+ }
262
+
263
+ /** A shortcut as a Mac writes it: ⌃⌥⇧⌘ and the key. */
264
+ function comboSign(k, m) {
265
+ return modsOf(m).map((mod) => MOD_SIGNS[mod]).join("") + (KEY_SIGNS[k] || k.toUpperCase());
266
+ }
267
+
268
+ /**
269
+ * The user's own shortcuts, from wherever they were kept (the app, the browser): only those the computer
270
+ * would take, 24 at most, each with a name (the shortcut itself when it was given none).
271
+ */
272
+ function cleanShortcuts(list) {
273
+ const out = [];
274
+ for (const one of Array.isArray(list) ? list : []) {
275
+ if (!one || typeof one !== "object" || !keyKnown(one.k)) continue;
276
+ const m = modsOf(one.m), name = [...(typeof one.name === "string" ? one.name.trim() : "")].slice(0, 16).join("");
277
+ out.push({ name: name || comboSign(one.k, m), k: one.k, m });
278
+ if (out.length === 24) break;
279
+ }
280
+ return out;
281
+ }
282
+ `;
283
+ const BODY = String.raw `
69
284
  const $ = (id) => document.getElementById(id);
70
- const stage = $("stage"), canvas = $("screen"), ctx = canvas.getContext("2d"), pointer = $("pointer");
285
+ const stage = $("stage"), canvas = $("screen"), ctx = canvas.getContext("2d"), video = $("video"), pointer = $("pointer");
71
286
  const note = $("note"), bar = $("bar"), menu = $("menu"), toast = $("toast"), keys = $("keys"), typing = $("typing"), shortcut = $("shortcut");
287
+ const sheet = $("sheet"), actions = $("actions"), maker = $("maker"), naming = $("makername"), composer = $("composer"), words = $("words");
72
288
  const query = new URLSearchParams(location.search);
73
289
  const app = window.ReactNativeWebView;
74
290
  const tellApp = (message) => { if (app) app.postMessage(JSON.stringify(message)); };
@@ -95,19 +311,29 @@ const content = { w: 0, h: 0 };
95
311
  let zoom = 1, pan = { x: 0, y: 0 }, fit = 1;
96
312
  let area = { x: 0, y: 0, w: 1, h: 1 }, shown = { x: 0, y: 0, w: 1, h: 1 };
97
313
  const cursor = { x: 0.5, y: 0.5 };
98
- let keysOpen = false;
314
+ let keysOpen = false, sheetOpen = false, composerOpen = false;
99
315
  const mods = new Set();
100
316
 
101
317
  // ---- The picture ----------------------------------------------------------
102
318
 
103
319
  const say = (text) => { note.textContent = text; note.classList.toggle("gone", !text); };
320
+ const round = (value, digits) => (value == null || !isFinite(value) ? null : +value.toFixed(digits === undefined ? 1 : digits));
104
321
  const hex = (n) => n.toString(16).padStart(2, "0");
105
322
  function sps(unit) {
106
323
  for (let i = 0; i + 3 < unit.length; i++)
107
324
  if (unit[i] === 0 && unit[i + 1] === 0 && unit[i + 2] === 1 && (unit[i + 3] & 0x1f) === 7) return unit.subarray(i + 3);
108
325
  }
109
- if (!("VideoDecoder" in window)) say("这个系统版本的浏览器内核不支持视频解码,请升级系统后再试。");
110
- let decoder, skipping = false, arrived = 0;
326
+ // "low": the page is reached through a gateway. The picture is asked for as a video track where this end can
327
+ // show one: an app around the page that doesn't say it plays video in place would open the system's
328
+ // full-screen player for it. The app says so in the address as well (?video=1): on Android what it tells the
329
+ // page as it loads can arrive after this script has run. ?video=0 keeps to the socket, to try that way out.
330
+ const relayed = query.get("q") === "low";
331
+ const wantVideo = "RTCPeerConnection" in window && (!app || chrome.video === true || query.get("video") === "1") && query.get("video") !== "0";
332
+ const unseeable = () => say("这个系统版本的浏览器内核不支持视频解码,请升级系统后再试。");
333
+ if (!wantVideo && !("VideoDecoder" in window)) unseeable();
334
+ let decoder, skipping = false, skipped = 0, arrived = 0, keyAsked = -Infinity;
335
+ // Only a keyframe gets a decoder that lost its place going again, and the host makes them when asked rather than by the clock.
336
+ const askKey = () => { const now = performance.now(); if (now - keyAsked > 500) { keyAsked = now; send({ t: "keyframe" }); } };
111
337
  // The host is told which frame is on screen: that is how it knows when this end has fallen behind,
112
338
  // and stops sending rather than let the picture drift into the past.
113
339
  let lastShown = 0, telling = 0;
@@ -118,11 +344,17 @@ function shownUpTo(seq) {
118
344
  if (lag && performance.now() < lagUntil) { const late = lastShown; setTimeout(() => send({ t: "ack", n: late }), lag); return; }
119
345
  if (!telling) telling = setTimeout(() => { telling = 0; send({ t: "ack", n: lastShown }); }, 40);
120
346
  }
121
- let lighterAt = -Infinity;
122
- const ws = new WebSocket((location.protocol === "https:" ? "wss://" : "ws://") + location.host + "/stream" + location.search);
347
+ let lighterAt = -Infinity, failures = 0;
348
+ const asked = new URLSearchParams(location.search);
349
+ asked.delete("video");
350
+ if (wantVideo) asked.set("video", "1");
351
+ const ws = new WebSocket((location.protocol === "https:" ? "wss://" : "ws://") + location.host + "/stream?" + asked);
123
352
  ws.binaryType = "arraybuffer";
124
353
  const send = (message) => { if (ws.readyState === 1) ws.send(JSON.stringify(message)); };
125
- ws.onopen = () => { if (mode !== "view") askControl(); };
354
+ ws.onopen = () => { if (mode !== "view") askControl(); if (document.hidden) send({ t: "hidden" }); };
355
+ // Out of sight, a page answers late (its timers slow to one a second) and shows nothing: the host sends no
356
+ // picture down the socket meanwhile, and starts again at a keyframe.
357
+ document.addEventListener("visibilitychange", () => send({ t: document.hidden ? "hidden" : "shown" }));
126
358
  ws.onmessage = (event) => {
127
359
  if (typeof event.data === "string") return heard(JSON.parse(event.data));
128
360
  if (!("VideoDecoder" in window)) return;
@@ -133,33 +365,46 @@ ws.onmessage = (event) => {
133
365
  const unit = new Uint8Array(event.data, 5);
134
366
  arrived = seq;
135
367
  // A decoder that can't keep up skips to the next keyframe; what it skips has still arrived.
136
- if (decoder && !key && (skipping || decoder.decodeQueueSize > 8)) { skipping = true; return shownUpTo(seq); }
368
+ if (decoder && !key && (skipping || decoder.decodeQueueSize > 8)) { skipping = true; skipped += 1; askKey(); return shownUpTo(seq); }
137
369
  skipping = false;
138
370
  if (!decoder) {
139
371
  const params = key && sps(unit);
140
- if (!params) return;
372
+ if (!params) { askKey(); return shownUpTo(seq); }
141
373
  decoder = new VideoDecoder({
142
374
  output: (frame) => {
143
- if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
375
+ // The first frame down the socket takes the picture over from a video track that didn't last.
376
+ const taken = live;
377
+ if (taken) show(false);
378
+ if (content.w !== frame.displayWidth || content.h !== frame.displayHeight) {
144
379
  canvas.width = content.w = frame.displayWidth;
145
380
  canvas.height = content.h = frame.displayHeight;
146
381
  layout();
147
382
  }
383
+ if (taken && !menu.classList.contains("gone")) status();
148
384
  ctx.drawImage(frame, 0, 0);
149
385
  shownUpTo(frame.timestamp);
386
+ if (meter) meter.drawn(frame);
150
387
  frame.close();
151
388
  say("");
152
389
  },
153
- error: (error) => say("解码失败:" + error.message),
390
+ // A new decoder at the next keyframe, which is asked for: one bad frame is not the end of the picture.
391
+ error: (error) => { if (failures++ > 4) return say("解码失败:" + error.message); decoder = undefined; askKey(); },
154
392
  });
155
393
  decoder.configure({ codec: "avc1." + hex(params[1]) + hex(params[2]) + hex(params[3]), optimizeForLatency: true });
156
394
  }
157
395
  decoder.decode(new EncodedVideoChunk({ type: key ? "key" : "delta", timestamp: seq, data: unit }));
158
396
  };
159
- ws.onclose = () => { if (note.classList.contains("gone") || note.textContent.startsWith("正在")) say("屏幕连接已断开"); };
397
+ ws.onclose = () => {
398
+ // A video track is this socket's: without the one there is no finding the other again, and no hands.
399
+ drop();
400
+ if (blocked || note.classList.contains("gone") || note.textContent.startsWith("正在")) say("屏幕连接已断开");
401
+ blocked = false;
402
+ };
160
403
 
161
404
  function heard(message) {
162
405
  if (message.error) return say(message.error);
406
+ if (message.rtc) return signal(message.rtc);
407
+ if (message.pong) return meter && meter.pong(message.pong);
163
408
  if (message.restart) {
164
409
  // The host starts the stream again at another quality: a new decoder for it.
165
410
  try { if (decoder) decoder.close(); } catch {}
@@ -178,8 +423,229 @@ function heard(message) {
178
423
  if (mode !== "view") explain(was);
179
424
  refresh();
180
425
  }
181
- // The computer's own pointer moved: the next swipe starts from where it is now.
182
- if (message.cursor && !gesture) { cursor.x = message.cursor.x; cursor.y = message.cursor.y; place(); }
426
+ if (message.cursor) echo(message.cursor.x, message.cursor.y);
427
+ }
428
+
429
+ // ---- The picture as a video track -----------------------------------------
430
+
431
+ // The computer's app offers the track, and channels beside it for the hands and the pointer; this end only
432
+ // answers. The socket carries what the two need to find each other — and the picture itself once the host
433
+ // says so (rtc "off"): there is no app to send a track, or this end said the track didn't get across.
434
+ // rtc is the connection while there is one; live, that the picture on show is the track's. A socket that
435
+ // has given the track up does not try again.
436
+ let rtc = null, live = false, tried = false, gaveUp = false, blocked = false, patience = 0;
437
+ // For trying the way back out: ?failAfter=5 gives the track up 5 s after it was offered.
438
+ const failAfter = Number(query.get("failAfter")) || 0;
439
+
440
+ function signal(message) {
441
+ if (message.t === "off") return fallBack();
442
+ if (gaveUp) return;
443
+ if (message.t === "config") return connect(message.iceServers || []);
444
+ const mine = rtc;
445
+ if (!mine) return;
446
+ if (message.t === "offer") {
447
+ tried = true;
448
+ patience = setTimeout(() => giveUp("not connected within 8 s of the offer"), 8000);
449
+ if (failAfter) setTimeout(() => { if (rtc === mine) giveUp("asked to (failAfter)"); }, failAfter * 1000);
450
+ // The answer is the browser's own, as it made it: the sender's playout-delay header has to survive in it.
451
+ mine.described = mine.pc.setRemoteDescription({ type: "offer", sdp: message.sdp })
452
+ .then(() => mine.pc.createAnswer())
453
+ .then((answer) => mine.pc.setLocalDescription(answer))
454
+ .then(() => { if (rtc === mine) send({ t: "rtc.answer", sdp: mine.pc.localDescription.sdp }); });
455
+ mine.described.catch((error) => { if (rtc === mine) giveUp("the offer could not be answered: " + error.message); });
456
+ } else if (message.t === "ice") {
457
+ // A candidate that comes before the offer is in place waits for it.
458
+ mine.described.then(() => mine.pc.addIceCandidate({ candidate: message.candidate, sdpMid: message.sdpMid, sdpMLineIndex: message.sdpMLineIndex })).catch(() => {});
459
+ }
460
+ }
461
+
462
+ function connect(iceServers) {
463
+ drop();
464
+ const pc = new RTCPeerConnection({ iceServers });
465
+ const mine = (rtc = { pc, channels: {}, described: Promise.resolve(), up: false, away: 0 });
466
+ pc.onicecandidate = ({ candidate }) => {
467
+ // The empty one only says there are no more.
468
+ if (candidate && candidate.candidate) send({ t: "rtc.ice", candidate: candidate.candidate, sdpMid: candidate.sdpMid ?? null, sdpMLineIndex: candidate.sdpMLineIndex ?? null });
469
+ };
470
+ pc.ontrack = ({ receiver, track, streams }) => {
471
+ // No waiting beyond what decoding takes. Safari has neither knob: there the sender's playout-delay header does it.
472
+ if ("jitterBufferTarget" in receiver) receiver.jitterBufferTarget = 0;
473
+ if ("playoutDelayHint" in receiver) receiver.playoutDelayHint = 0;
474
+ video.srcObject = streams[0] || new MediaStream([track]);
475
+ play();
476
+ };
477
+ pc.ondatachannel = ({ channel }) => {
478
+ mine.channels[channel.label] = channel;
479
+ if (channel.label === "cursor" || channel.label === "shape") channel.onmessage = (event) => { if (rtc === mine) told(JSON.parse(event.data)); };
480
+ };
481
+ pc.oniceconnectionstatechange = pc.onconnectionstatechange = () => { if (rtc === mine) watch(mine); };
482
+ }
483
+
484
+ function watch(mine) {
485
+ // The whole connection's state where the browser has one: a path found is not yet a connection made.
486
+ const state = mine.pc.connectionState || mine.pc.iceConnectionState;
487
+ if (state === "failed" || mine.pc.iceConnectionState === "failed") return giveUp(mine.up ? "the connection failed" : "no path between the two (ICE failed)");
488
+ if (state === "disconnected") {
489
+ // A path that drops for a moment comes back by itself; one gone for longer is given up.
490
+ if (mine.up && !mine.away) mine.away = setTimeout(() => away(mine, performance.now()), 4000);
491
+ } else if (state === "connected" || state === "completed") {
492
+ clearTimeout(mine.away);
493
+ mine.away = 0;
494
+ if (mine.up) return;
495
+ mine.up = true;
496
+ clearTimeout(patience);
497
+ if (!live) patience = setTimeout(frameless, 4000);
498
+ }
499
+ }
500
+
501
+ function away(mine, since) {
502
+ // A page that was put aside (another app, another tab) could not hear the path come back: once it is
503
+ // looked at again it is given the time again.
504
+ const now = performance.now();
505
+ if (document.hidden || now - since > 6000) mine.away = setTimeout(() => away(mine, now), 4000);
506
+ else giveUp("disconnected for more than 4 s");
507
+ }
508
+
509
+ function frameless() {
510
+ // A page nobody is looking at is shown nothing, and one waiting for a tap has not begun: neither is the path's doing.
511
+ if (document.hidden || blocked) patience = setTimeout(frameless, 4000);
512
+ else giveUp("connected, but no frame within 4 s");
513
+ }
514
+
515
+ /** The track is not getting across: the host is asked for the picture down the socket instead. */
516
+ function giveUp(reason) {
517
+ if (gaveUp || !rtc) return;
518
+ gaveUp = true;
519
+ send({ t: "rtc.failed", reason });
520
+ drop();
521
+ }
522
+
523
+ function drop() {
524
+ clearTimeout(patience);
525
+ if (!rtc) return;
526
+ clearTimeout(rtc.away);
527
+ const pc = rtc.pc;
528
+ rtc = null;
529
+ pc.ontrack = pc.ondatachannel = pc.onicecandidate = pc.oniceconnectionstatechange = pc.onconnectionstatechange = null;
530
+ pc.close();
531
+ }
532
+
533
+ /** The picture comes down the socket from here on. What the track last showed stays up until its first frame. */
534
+ function fallBack() {
535
+ gaveUp = true;
536
+ drop();
537
+ if (blocked) say("正在连接电脑屏幕…");
538
+ blocked = false;
539
+ if (!("VideoDecoder" in window)) return unseeable();
540
+ if (tried) hint(live ? "直连断开了:已改用兼容方式传画面,延迟会高一些" : "没能和电脑直连:已改用兼容方式传画面,延迟会高一些", 5000);
541
+ }
542
+
543
+ function play() {
544
+ const started = video.play();
545
+ if (!started) return;
546
+ started.then(() => {
547
+ if (!blocked) return;
548
+ blocked = false;
549
+ say(live ? "" : "正在连接电脑屏幕…");
550
+ }, (error) => {
551
+ // A new stream replacing the one being started rejects the first call too; that is not a refusal.
552
+ if (error.name === "AbortError" || !rtc) return;
553
+ blocked = true;
554
+ say("轻点屏幕,开始显示画面");
555
+ });
556
+ }
557
+ // What a browser takes as someone's say-so to start a video.
558
+ if (wantVideo) for (const name of ["pointerup", "touchend", "click", "keydown"]) addEventListener(name, () => { if (blocked && rtc) play(); }, true);
559
+
560
+ /** The track's picture has a size: at its first frame, and whenever the sender changes its resolution. */
561
+ function sized() {
562
+ if (!rtc || !video.videoWidth) return;
563
+ content.w = video.videoWidth;
564
+ content.h = video.videoHeight;
565
+ if (!blocked) say("");
566
+ if (live) return layout();
567
+ clearTimeout(patience);
568
+ show(true);
569
+ }
570
+ video.addEventListener("resize", sized);
571
+ video.addEventListener("loadedmetadata", sized);
572
+
573
+ /** Which of the two the picture is on: the video while its track runs, the canvas otherwise. */
574
+ function show(track) {
575
+ live = track;
576
+ video.classList.toggle("gone", !track);
577
+ canvas.classList.toggle("gone", track);
578
+ if (!track) {
579
+ video.srcObject = null;
580
+ content.w = content.h = 0;
581
+ pointer.classList.remove("on");
582
+ }
583
+ refresh();
584
+ }
585
+
586
+ // ---- The pointer in a video ----------------------------------------------
587
+
588
+ // A video track has no pointer in its picture. The app says where the pointer is (every position, on
589
+ // "cursor") and what it looks like ("shape": the picture the first time, its id after that), and the page
590
+ // draws it — an arrow of its own until it has been told.
591
+ const ARROW = { w: 14.5, h: 20, hotX: 1.5, hotY: 1.5, url: "data:image/svg+xml," + encodeURIComponent("<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 14.5 20'><path d='M1.5 1.5v15l4-3.4 2.6 5.4 2.5-1.2-2.6-5.3 5-.6Z' fill='#000' stroke='#fff' stroke-width='1.3' stroke-linejoin='round'/></svg>") };
592
+ const shapes = new Map();
593
+ // worn: the shape's id. span: the display's width in the points the shapes are measured in.
594
+ // pointed: the pointer has been somewhere to draw it at (it may be on another display, and never be).
595
+ let worn = "", span = 0, drawn = 0, pointed = false;
596
+
597
+ function told(message) {
598
+ if (message.t === "cursor") {
599
+ // The channel keeps no order: a position older than the one drawn is not drawn after it.
600
+ if (message.i <= drawn) return;
601
+ if (message.i) drawn = message.i;
602
+ return echo(message.x, message.y);
603
+ }
604
+ if (message.t !== "shape") return;
605
+ if (message.png) shapes.set(message.id, { url: "data:image/png;base64," + message.png, w: message.w, h: message.h, hotX: message.hotX, hotY: message.hotY });
606
+ if (message.displayW) span = message.displayW;
607
+ worn = message.id;
608
+ dress();
609
+ }
610
+
611
+ /** The computer says where its pointer is. */
612
+ function echo(x, y) {
613
+ if (live) {
614
+ // Every position comes back, this page's own moves among them, a round trip late; and the hand here is
615
+ // ahead of that. So what is drawn stays where the hand has it while the hand is moving it, and for any
616
+ // position the hand put it at within the last second. Anything else is news — someone at the computer,
617
+ // a program — and is drawn at once.
618
+ const now = performance.now();
619
+ if (now - movedAt < 250 || ownEcho(trail, x, y, now)) return;
620
+ }
621
+ // In the socket's picture the pointer is drawn already. It moved by itself: the next swipe starts from where it is now.
622
+ else if (gesture) return;
623
+ cursor.x = x;
624
+ cursor.y = y;
625
+ if (rtc && !pointed) point();
626
+ else place();
627
+ }
628
+
629
+ /** The pointer is somewhere, for the first time. */
630
+ function point() {
631
+ pointed = true;
632
+ dress();
633
+ }
634
+
635
+ /** What the pointer looks like: the computer's own over a video; otherwise the ring that follows a finger on the trackpad. */
636
+ function dress() {
637
+ const look = live ? shapes.get(worn) || ARROW : null, style = pointer.style;
638
+ pointer.classList.toggle("own", !!look);
639
+ if (look) pointer.classList.toggle("on", pointed);
640
+ // A mouse here would be a second pointer on top of the one drawn: while it controls, the drawn one is it.
641
+ stage.classList.toggle("own", !!look && controlling());
642
+ style.width = look ? look.w + "px" : "";
643
+ style.height = look ? look.h + "px" : "";
644
+ style.left = look ? -look.hotX + "px" : "";
645
+ style.top = look ? -look.hotY + "px" : "";
646
+ style.transformOrigin = look ? look.hotX + "px " + look.hotY + "px" : "";
647
+ style.backgroundImage = look ? 'url("' + look.url + '")' : "";
648
+ place();
183
649
  }
184
650
 
185
651
  // ---- Layout ---------------------------------------------------------------
@@ -189,11 +655,57 @@ function viewport() {
189
655
  return v ? { w: v.width, h: v.height, x: v.offsetLeft, y: v.offsetTop } : { w: innerWidth, h: innerHeight, x: 0, y: 0 };
190
656
  }
191
657
 
658
+ // What a browser's window has been seen to do: a keyboard coming up makes it shorter, on a phone (see keyboardIsUp).
659
+ const seen = { w: 0, h: 0, follows: false };
660
+
192
661
  function layout() {
193
662
  const v = viewport(), inset = chrome.insets;
194
- // Above the keyboard, or above the phone's bottom edge when the keys are a real keyboard's.
195
- const keysHeight = keysOpen ? 46 + inset.bottom : 0;
196
- area = { x: v.x + inset.left, y: v.y + inset.top, w: Math.max(1, v.w - inset.left - inset.right), h: Math.max(1, v.h - inset.top - keysHeight) };
663
+ if (v.w !== seen.w) Object.assign(seen, { w: v.w, h: v.h });
664
+ else if (v.h > seen.h) seen.h = v.h;
665
+ if (seen.h - v.h > 120 && fieldFocused()) seen.follows = true;
666
+ // A phone lying down has its room beside the picture, not under it.
667
+ const wide = v.w > v.h && v.w >= 560;
668
+ // Beside the picture, things take the side without the camera.
669
+ const onLeft = chrome.clear ? chrome.clear === "left" : inset.left < inset.right;
670
+ // Under the picture: the box text is written in, or the key bar (above the keyboard, or above the phone's
671
+ // bottom edge when the keys are a real keyboard's), or the shortcuts. Each has the picture make room for it.
672
+ let under = 0, aside = 0;
673
+ composer.classList.toggle("gone", !composerOpen);
674
+ composer.classList.toggle("wide", wide);
675
+ keys.classList.toggle("gone", !keysOpen || composerOpen);
676
+ sheet.classList.toggle("gone", !sheetOpen);
677
+ sheet.classList.toggle("narrow", wide);
678
+ if (composerOpen) {
679
+ composer.style.padding = "8px " + (8 + inset.right) + "px " + (8 + inset.bottom) + "px " + (8 + inset.left) + "px";
680
+ under = composer.offsetHeight;
681
+ composer.style.top = v.y + v.h - under + "px";
682
+ } else if (keysOpen) {
683
+ under = 46 + inset.bottom;
684
+ keys.style.top = v.y + v.h - under + "px";
685
+ keys.style.height = under + "px";
686
+ // Clear of the camera, with the phone lying down.
687
+ keys.style.padding = "0 " + (5 + inset.right) + "px " + inset.bottom + "px " + (5 + inset.left) + "px";
688
+ } else if (sheetOpen && wide) {
689
+ // Clear of the screen's round corners; on a side that may have something of the system's in it, clear of that too.
690
+ const edge = chrome.clear ? 12 : Math.max(12, onLeft ? inset.left : inset.right), top = Math.max(inset.top, 12);
691
+ const width = Math.min(320, Math.round(v.w * 0.42));
692
+ sheet.style.width = width + "px";
693
+ sheet.style.height = Math.max(96, v.h - top - 12) + "px";
694
+ sheet.style.left = (onLeft ? v.x + edge : v.x + v.w - edge - width) + "px";
695
+ sheet.style.top = v.y + top + "px";
696
+ aside = edge + width + 8;
697
+ } else if (sheetOpen) {
698
+ const edge = Math.max(inset.bottom, 8), side = Math.max(inset.left, inset.right, 8), most = Math.max(96, v.h - inset.top - edge - 8);
699
+ // All the room there is while a name is typed into it: the keyboard has the rest.
700
+ const height = document.activeElement === naming ? most : Math.min(most, clamp(Math.round(v.h * 0.5), 260, 400));
701
+ sheet.style.width = v.w - side * 2 + "px";
702
+ sheet.style.height = height + "px";
703
+ sheet.style.left = v.x + side + "px";
704
+ sheet.style.top = v.y + v.h - edge - height + "px";
705
+ under = height + edge + 8;
706
+ }
707
+ const left = Math.max(inset.left, onLeft ? aside : 0), right = Math.max(inset.right, onLeft ? 0 : aside);
708
+ area = { x: v.x + left, y: v.y + inset.top, w: Math.max(1, v.w - left - right), h: Math.max(1, v.h - inset.top - under) };
197
709
  if (content.w) {
198
710
  fit = Math.min(area.w / content.w, area.h / content.h);
199
711
  const w = content.w * fit * zoom, h = content.h * fit * zoom;
@@ -201,19 +713,20 @@ function layout() {
201
713
  pan.x = w <= area.w ? area.x + (area.w - w) / 2 : clamp(pan.x, area.x + area.w - w, area.x);
202
714
  pan.y = h <= area.h ? area.y + (area.h - h) / 2 : clamp(pan.y, area.y + area.h - h, area.y);
203
715
  shown = { x: pan.x, y: pan.y, w, h };
204
- canvas.style.transform = "translate(" + pan.x + "px," + pan.y + "px) scale(" + fit * zoom + ")";
716
+ if (live) {
717
+ // The video's box is the picture's place, not its pixels: the sender changes the resolution as the
718
+ // network changes, and the picture stays where it is and as large, only sharper or softer.
719
+ video.style.width = content.w * fit + "px";
720
+ video.style.height = content.h * fit + "px";
721
+ video.style.transform = "translate(" + pan.x + "px," + pan.y + "px) scale(" + zoom + ")";
722
+ } else canvas.style.transform = "translate(" + pan.x + "px," + pan.y + "px) scale(" + fit * zoom + ")";
205
723
  }
206
- keys.classList.toggle("gone", !keysOpen);
207
- keys.style.top = v.y + v.h - keysHeight + "px";
208
- keys.style.height = keysHeight + "px";
209
- keys.style.paddingBottom = inset.bottom + "px";
210
- // The toolbar stands in the black beside the picture when there is some, under it otherwise; the keyboard takes its place.
724
+ // The toolbar stands in the black beside the picture when there is some, under it otherwise; whatever opens takes its place.
211
725
  const beside = content.w ? area.w - content.w * fit > area.h - content.h * fit + inset.bottom : v.w > v.h;
212
726
  bar.classList.toggle("side", beside);
213
- bar.classList.toggle("hidden", keysOpen);
727
+ bar.classList.toggle("hidden", keysOpen || sheetOpen || composerOpen);
214
728
  const size = { w: bar.offsetWidth, h: bar.offsetHeight };
215
- // Beside the picture it takes the side without the camera, and stands in the middle of the black there.
216
- const onLeft = beside && (chrome.clear ? chrome.clear === "left" : inset.left < inset.right);
729
+ // Beside the picture it stands in the middle of the black there.
217
730
  if (beside) {
218
731
  const edge = onLeft ? inset.left : inset.right;
219
732
  const black = edge + (content.w ? (area.w - content.w * fit) / 2 : 0);
@@ -234,7 +747,10 @@ function layout() {
234
747
  }
235
748
 
236
749
  function place() {
237
- pointer.style.transform = "translate(" + (shown.x + cursor.x * shown.w) + "px," + (shown.y + cursor.y * shown.h) + "px)";
750
+ // Over a video the pointer is as large as it is in the picture, and never smaller than it is on the computer:
751
+ // on a phone the whole screen is a few fingers wide, and a pointer to scale would be lost in it.
752
+ const size = live ? " scale(" + (span ? Math.max(1, shown.w / span) : 1) + ")" : "";
753
+ pointer.style.transform = "translate(" + (shown.x + cursor.x * shown.w) + "px," + (shown.y + cursor.y * shown.h) + "px)" + size;
238
754
  }
239
755
 
240
756
  addEventListener("resize", layout);
@@ -269,27 +785,72 @@ function askControl() {
269
785
  send({ t: "control" });
270
786
  }
271
787
 
788
+ // Where an event travels: down the socket, in order, until a video's two channels to the app are open; then
789
+ // input in order, nothing lost buttons, keys, text
790
+ // pointer no order, sent once the pointer on its way somewhere, the wheel
791
+ // The two keep no time with each other, so nothing goes on "pointer" that something on "input" has to come
792
+ // after, or before. The position a button, a key or the wheel acts at goes on "input" just ahead of it,
793
+ // however it was sent before. With a button down every position does: a drag is a path, and ends where it
794
+ // is let go. The wheel stays on "input" for the rest of a gesture whose position went there. And a pointer
795
+ // that comes to rest says where once more on "input", in case its last move was the one lost. Each event on
796
+ // the channels has a number (one count for both): by it the app drops a move from "pointer" that arrives
797
+ // after something sent later.
798
+ let wired = false, held = 0, sure = true, sent = 0, rest = 0, movedAt = -Infinity;
799
+ const trail = [];
800
+
801
+ function lanes() {
802
+ const channels = rtc && rtc.channels;
803
+ const open = !!channels && !!channels.input && channels.input.readyState === "open" && !!channels.pointer && channels.pointer.readyState === "open";
804
+ // Not taken up with a button down: its release goes the way it went down.
805
+ if (open !== wired && (!open || !held)) wired = open;
806
+ }
807
+
808
+ function post(message, lane) {
809
+ if (!wired) return send(message);
810
+ message.i = ++sent;
811
+ rtc.channels[lane].send(JSON.stringify(message));
812
+ }
813
+
272
814
  let moveWaiting = false;
273
- function flush() {
274
- if (!moveWaiting) return;
815
+ /** Says where the pointer is: in order, when something is about to happen there. */
816
+ function tell(placed) {
817
+ lanes();
275
818
  moveWaiting = false;
276
- send({ t: "move", x: +cursor.x.toFixed(5), y: +cursor.y.toFixed(5) });
819
+ const now = performance.now(), lane = laneOf("move", placed, held > 0, false);
820
+ if (rtc) {
821
+ // What was sent, to know it by when it comes back (see echo).
822
+ trail.push({ x: cursor.x, y: cursor.y, t: now });
823
+ while (now - trail[0].t > 1000) trail.shift();
824
+ }
825
+ post({ t: "move", x: +cursor.x.toFixed(5), y: +cursor.y.toFixed(5) }, lane);
826
+ sure = !wired || lane === "input";
827
+ clearTimeout(rest);
828
+ if (!sure) rest = setTimeout(() => { if (!sure) tell(true); }, 80);
277
829
  }
830
+ const flush = () => { if (moveWaiting) tell(false); };
278
831
  // A pointer position a frame: as often as the picture could show it.
279
832
  function moved() {
280
833
  if (!moveWaiting) requestAnimationFrame(flush);
281
834
  moveWaiting = true;
835
+ movedAt = performance.now();
282
836
  glow();
283
837
  place();
284
838
  }
285
839
  // Everything else goes at once, after the position it happens at.
286
840
  function act(message) {
287
- flush();
288
- send(message);
841
+ lanes();
842
+ const placed = moveWaiting || !sure;
843
+ if (placed) tell(true);
844
+ if (message.t === "down") held += 1;
845
+ if (message.t === "scroll" && placed && gesture) gesture.tied = true;
846
+ post(message, laneOf(message.t, placed, held > 0, !!gesture && !!gesture.tied));
847
+ if (message.t === "up") held = Math.max(0, held - 1);
289
848
  }
290
849
 
291
850
  let fade = 0;
292
851
  function glow() {
852
+ // Over a video the pointer is on show for good once it has been somewhere.
853
+ if (live) return pointed || point();
293
854
  if (mode !== "trackpad") return;
294
855
  pointer.classList.add("on");
295
856
  clearTimeout(fade);
@@ -308,7 +869,7 @@ function takeMods() {
308
869
  function click(buttonName, count) {
309
870
  const m = takeMods();
310
871
  act({ t: "down", b: buttonName, n: count, m });
311
- send({ t: "up", b: buttonName, n: count, m });
872
+ act({ t: "up", b: buttonName, n: count, m });
312
873
  }
313
874
 
314
875
  /** Puts the pointer under a point of the page; false when that is outside the picture. */
@@ -360,8 +921,8 @@ function endOne() {
360
921
 
361
922
  stage.addEventListener("pointerdown", (event) => {
362
923
  wake();
363
- // A touch that closes the menu does nothing else.
364
- const closing = closeMenu();
924
+ // A touch that closes the menu, or the shortcuts, does nothing else.
925
+ const closing = [closeMenu(), closeSheet()].some(Boolean);
365
926
  if (event.pointerType === "mouse") return closing ? undefined : mouseDown(event);
366
927
  event.preventDefault();
367
928
  try { stage.setPointerCapture(event.pointerId); } catch {}
@@ -501,7 +1062,8 @@ function tap(g, now) {
501
1062
  let mouseHeld = null, mouseClicks = { t: 0, n: 0, x: 0, y: 0, b: 0 };
502
1063
  function mouseDown(event) {
503
1064
  if (!controlling()) { mouseHeld = "pan"; return; }
504
- typing.focus({ preventScroll: true });
1065
+ // Keys go to the computer as they are pressed, unless text is being written here to be sent whole.
1066
+ if (!composerOpen) typing.focus({ preventScroll: true });
505
1067
  event.preventDefault();
506
1068
  if (!pointAt(event.clientX, event.clientY)) return;
507
1069
  const now = performance.now();
@@ -556,7 +1118,7 @@ function typed(text) {
556
1118
  if (mods.size && [...text].length === 1) return press(KEY_OF[text] || text.toLowerCase());
557
1119
  const lines = text.split("\n");
558
1120
  lines.forEach((line, index) => {
559
- if (line) act({ t: "text", s: line });
1121
+ for (const s of pieces(line, 4000)) act({ t: "text", s });
560
1122
  if (index < lines.length - 1) press("return");
561
1123
  });
562
1124
  }
@@ -626,9 +1188,9 @@ function isLeftover(text) {
626
1188
  leftover = null;
627
1189
  return mine;
628
1190
  }
629
- const keyboardUp = () => document.activeElement === typing || document.activeElement === shortcut;
1191
+ const fieldFocused = () => document.activeElement === typing || document.activeElement === shortcut;
630
1192
  function syncField() {
631
- if (!keyboardUp()) return;
1193
+ if (!fieldFocused()) return;
632
1194
  const field = mods.size ? shortcut : typing;
633
1195
  if (document.activeElement === field) return;
634
1196
  switching = true;
@@ -640,10 +1202,83 @@ for (const field of [typing, shortcut]) {
640
1202
  field.addEventListener("blur", () => { if (switching) return; keysOpen = false; mods.clear(); drawKeys(); layout(); refresh(); });
641
1203
  }
642
1204
 
1205
+ // A field with the focus is not a keyboard on the screen. A phone takes its keyboard down without the field
1206
+ // hearing of it — the app was put aside and came back, something else of the app's took the keys, the
1207
+ // system's Back on Android — and the field still says it has the focus: asked for the keyboard then, it
1208
+ // would only be let go of, and nothing would come up. So the app, which hears of the keyboard from the
1209
+ // system, says whether it is up (chrome.keyboard), and a keyboard is asked for in a way that works from
1210
+ // any state.
1211
+ let askedAt = -Infinity, wanted = null, wentDown = false, settling = 0;
1212
+
1213
+ const keyboardShown = () => keyboardIsUp({
1214
+ focused: fieldFocused(),
1215
+ asked: performance.now() - askedAt,
1216
+ said: app && typeof chrome.keyboard === "boolean" ? chrome.keyboard : undefined,
1217
+ live: document.hasFocus(),
1218
+ shrunk: coarse && seen.follows ? seen.h - viewport().h > 120 : undefined,
1219
+ });
1220
+
1221
+ /** Brings the phone's keyboard up for a field. Within a touch: a browser gives a keyboard to nothing else. */
1222
+ function ask(field, again) {
1223
+ // A keyboard goes to the view the keys go to, and inside an app that is not always the page: the app
1224
+ // sees to it, and has the page ask once more (linkshellKeyboard) if no keyboard has come.
1225
+ if (!again) tellApp({ type: "keyboard" });
1226
+ askedAt = performance.now();
1227
+ wanted = field;
1228
+ // Focusing the field that has the focus does nothing: one the keyboard was taken from is let go of first,
1229
+ // without the page taking that for the keyboard going.
1230
+ if (document.activeElement === field) {
1231
+ switching = true;
1232
+ field.blur();
1233
+ switching = false;
1234
+ }
1235
+ field.focus({ preventScroll: true });
1236
+ }
1237
+ window.linkshellKeyboard = () => {
1238
+ if (chrome.keyboard === true || !wanted || performance.now() - askedAt > 3000) return;
1239
+ ask(wanted, true);
1240
+ };
1241
+
1242
+ /** The keyboard goes, and the key bar with it: said here too, for a field that hears nothing because the page is not the one the keys go to. */
1243
+ function dropKeys() {
1244
+ wanted = null;
1245
+ if (fieldFocused()) document.activeElement.blur();
1246
+ if (!keysOpen && !mods.size) return;
1247
+ keysOpen = false;
1248
+ mods.clear();
1249
+ drawKeys();
1250
+ refresh();
1251
+ }
1252
+
1253
+ function raiseKeys() {
1254
+ closeSheet();
1255
+ closeComposer();
1256
+ ask(mods.size ? shortcut : typing);
1257
+ }
1258
+
643
1259
  function toggleKeys() {
644
- if (keyboardUp()) document.activeElement.blur();
645
- else typing.focus({ preventScroll: true });
1260
+ if (keyboardShown()) dropKeys();
1261
+ else raiseKeys();
1262
+ }
1263
+
1264
+ /** The app said the keyboard that was up has gone: the page lets go of the field it was up for. */
1265
+ function settle() {
1266
+ // Turning the phone takes the keyboard down and brings it straight back, and so does putting the app
1267
+ // aside and coming back to it: only a keyboard that stays down is gone. A page nobody sees waits to be seen.
1268
+ if (!wentDown || chrome.keyboard !== false || document.hidden) return;
1269
+ // One asked for a moment ago may be on its way up still.
1270
+ if (performance.now() - askedAt < 1000) return settleIn(1000);
1271
+ wentDown = false;
1272
+ const field = document.activeElement;
1273
+ wanted = null;
1274
+ if (field === words || field === naming) field.blur();
1275
+ else dropKeys();
646
1276
  }
1277
+ function settleIn(ms) {
1278
+ clearTimeout(settling);
1279
+ settling = setTimeout(settle, ms);
1280
+ }
1281
+ document.addEventListener("visibilitychange", () => { if (!document.hidden) settleIn(1200); });
647
1282
 
648
1283
  // ---- Toolbar, menu, key bar ----------------------------------------------
649
1284
 
@@ -660,6 +1295,22 @@ function tappable(element, action) {
660
1295
  });
661
1296
  }
662
1297
 
1298
+ /** The same inside something that scrolls: the touch is left to the browser, and counts if it stayed where it landed. */
1299
+ function pressable(element, action) {
1300
+ let from = null;
1301
+ element.addEventListener("pointerdown", (event) => { event.stopPropagation(); wake(); from = { id: event.pointerId, x: event.clientX, y: event.clientY }; });
1302
+ element.addEventListener("pointercancel", () => { from = null; });
1303
+ // The mouse's events a browser makes up after a touch would land on whatever the action has left under the
1304
+ // finger, and take the focus from the field it gave it to. A touch that scrolled has none to stop.
1305
+ element.addEventListener("touchend", (event) => { if (event.cancelable) event.preventDefault(); });
1306
+ element.addEventListener("mousedown", (event) => event.preventDefault());
1307
+ element.addEventListener("pointerup", (event) => {
1308
+ const was = from;
1309
+ from = null;
1310
+ if (was && was.id === event.pointerId && Math.hypot(event.clientX - was.x, event.clientY - was.y) < SLOP) action();
1311
+ });
1312
+ }
1313
+
663
1314
  let idle = 0;
664
1315
  function wake() {
665
1316
  bar.classList.remove("idle");
@@ -693,9 +1344,11 @@ function setMode(next) {
693
1344
  mode = next;
694
1345
  try { localStorage.setItem("linkshell.screen.mode", mode); } catch {}
695
1346
  tellApp({ type: "mode", mode });
696
- pointer.classList.remove("on");
1347
+ if (!live) pointer.classList.remove("on");
697
1348
  if (mode === "view") {
698
- if (keyboardUp()) document.activeElement.blur();
1349
+ closeSheet();
1350
+ closeComposer();
1351
+ dropKeys();
699
1352
  hint(HINTS.view);
700
1353
  } else {
701
1354
  askControl();
@@ -707,6 +1360,7 @@ function setMode(next) {
707
1360
  function closeMenu() {
708
1361
  if (menu.classList.contains("gone")) return false;
709
1362
  menu.classList.add("gone");
1363
+ clearInterval(statusTimer);
710
1364
  wake();
711
1365
  return true;
712
1366
  }
@@ -715,13 +1369,15 @@ function refresh() {
715
1369
  $("mode").innerHTML = ICON[mode];
716
1370
  $("mode").classList.toggle("on", mode !== "view");
717
1371
  $("keyboard").classList.toggle("gone", mode === "view");
1372
+ $("quick").classList.toggle("gone", mode === "view");
718
1373
  $("fit").classList.toggle("gone", zoom < 1.05);
719
1374
  $("rotate").classList.toggle("gone", !app || !chrome.canRotate);
720
1375
  $("rotate").classList.toggle("on", chrome.landscape);
721
1376
  const full = app ? chrome.fullscreen : !!document.fullscreenElement;
722
1377
  $("full").classList.toggle("gone", !app && !document.fullscreenEnabled);
723
1378
  $("full").innerHTML = full ? ICON.shrink : ICON.expand;
724
- for (const choice of menu.children) choice.classList.toggle("on", choice.dataset.mode === mode);
1379
+ for (const choice of choices) choice.classList.toggle("on", choice.dataset.mode === mode);
1380
+ dress();
725
1381
  layout();
726
1382
  }
727
1383
 
@@ -730,9 +1386,74 @@ function drawKeys() {
730
1386
  syncField();
731
1387
  }
732
1388
 
1389
+ // ---- The way the picture comes ---------------------------------------------
1390
+
1391
+ const RANK = ["host", "prflx", "srflx", "relay"];
1392
+ /** What the connection says of the track: the picture's size and pace, the round trip, the waits. */
1393
+ async function vitals(pc, before) {
1394
+ const pairs = [], ends = new Map();
1395
+ let track = {}, transport = {};
1396
+ (await pc.getStats()).forEach((entry) => {
1397
+ if (entry.type === "inbound-rtp" && (entry.kind || entry.mediaType) === "video") track = entry;
1398
+ else if (entry.type === "transport") transport = entry;
1399
+ else if (entry.type === "candidate-pair") pairs.push(entry);
1400
+ else if (entry.type === "local-candidate" || entry.type === "remote-candidate") ends.set(entry.id, entry);
1401
+ });
1402
+ const pair = pairs.find((one) => one.id === transport.selectedCandidatePairId) || pairs.find((one) => one.selected) || pairs.find((one) => one.nominated && one.state === "succeeded") || {};
1403
+ const near = ends.get(pair.localCandidateId), far = ends.get(pair.remoteCandidateId);
1404
+ const was = before || {}, since = (field) => (track[field] || 0) - (was[field] || 0);
1405
+ const each = (total, count) => (since(count) > 0 ? (since(total) / since(count)) * 1000 : null);
1406
+ return {
1407
+ raw: track,
1408
+ width: track.frameWidth || video.videoWidth,
1409
+ height: track.frameHeight || video.videoHeight,
1410
+ fps: round(before ? (since("framesDecoded") * 1000) / (track.timestamp - was.timestamp) : track.framesPerSecond),
1411
+ rttMs: round(pair.currentRoundTripTime == null ? null : pair.currentRoundTripTime * 1000),
1412
+ jitterBufferMs: round(each("jitterBufferDelay", "jitterBufferEmittedCount")),
1413
+ decodeMs: round(each("totalDecodeTime", "framesDecoded"), 2),
1414
+ dropped: track.framesDropped ?? null,
1415
+ freezes: track.freezeCount ?? null,
1416
+ // The less direct of the two ends.
1417
+ path: near && far ? RANK[Math.max(RANK.indexOf(near.candidateType), RANK.indexOf(far.candidateType))] : null,
1418
+ };
1419
+ }
1420
+
1421
+ /** The line at the foot of the menu: which way the picture comes and, for a video track, how it is doing. */
1422
+ let statusTimer = 0, statusWas = null;
1423
+ async function status() {
1424
+ const line = $("status");
1425
+ const put = (text) => {
1426
+ if (line.textContent === text) return;
1427
+ line.textContent = text;
1428
+ layout();
1429
+ };
1430
+ const way = !content.w ? "正在连接…" : live ? "直连 · 视频" : (relayed ? "中继" : "直连") + " · 兼容\n" + content.w + "×" + content.h;
1431
+ const pc = live && rtc && rtc.pc;
1432
+ if (!pc || !line.textContent.startsWith(way)) put(way);
1433
+ const now = pc && (await vitals(pc, statusWas).catch(() => null));
1434
+ if (!now || rtc === null || rtc.pc !== pc) return;
1435
+ statusWas = now.raw;
1436
+ put(way + "\n" + [now.width + "×" + now.height, now.fps === null ? "" : Math.round(now.fps) + " 帧/秒", now.rttMs === null ? "" : "往返 " + Math.round(now.rttMs) + " 毫秒"].filter(Boolean).join(" · "));
1437
+ }
1438
+
733
1439
  const ICON = JSON.parse($("icons").textContent);
734
- tappable($("mode"), () => { menu.classList.toggle("gone"); layout(); });
735
- tappable($("keyboard"), toggleKeys);
1440
+ const choices = menu.querySelectorAll(".choice");
1441
+ tappable($("mode"), () => {
1442
+ closeSheet();
1443
+ menu.classList.toggle("gone");
1444
+ clearInterval(statusTimer);
1445
+ // Kept up to date while it can be seen.
1446
+ if (!menu.classList.contains("gone")) {
1447
+ statusWas = null;
1448
+ status();
1449
+ statusTimer = setInterval(status, 1500);
1450
+ }
1451
+ layout();
1452
+ });
1453
+ // On a phone the button is there only while the key bar is not, which is while the page knows of no keyboard:
1454
+ // there it only ever raises one, whatever state the field was left in. With a mouse it gives the keys and takes them.
1455
+ tappable($("keyboard"), () => (coarse ? raiseKeys() : toggleKeys()));
1456
+ tappable($("quick"), openSheet);
736
1457
  tappable($("fit"), () => zoomTo(1, { x: area.x + area.w / 2, y: area.y + area.h / 2 }));
737
1458
  tappable($("rotate"), () => tellApp({ type: "landscape", on: !chrome.landscape }));
738
1459
  tappable($("full"), () => {
@@ -741,7 +1462,7 @@ tappable($("full"), () => {
741
1462
  else document.documentElement.requestFullscreen().catch(() => {});
742
1463
  });
743
1464
  document.addEventListener("fullscreenchange", refresh);
744
- for (const choice of menu.children) {
1465
+ for (const choice of choices) {
745
1466
  choice.querySelector("i").innerHTML = ICON[choice.dataset.mode];
746
1467
  tappable(choice, () => { closeMenu(); setMode(choice.dataset.mode); });
747
1468
  }
@@ -753,23 +1474,444 @@ for (const key of keys.children) {
753
1474
  return drawKeys();
754
1475
  }
755
1476
  if (key.dataset.key) return press(key.dataset.key);
756
- if (keyboardUp()) document.activeElement.blur();
1477
+ if (key.id === "quickkey") return openSheet();
1478
+ if (key.id === "composekey") return openComposer();
1479
+ dropKeys();
757
1480
  });
758
1481
  }
759
1482
  $("hide").innerHTML = ICON.down;
1483
+ $("quickkey").innerHTML = ICON.quick;
1484
+ $("composekey").innerHTML = ICON.compose;
1485
+
1486
+ // ---- Shortcuts --------------------------------------------------------------
1487
+
1488
+ // Keys for the computer that a phone's keyboard has not got, one tap each, by what they are for: name, key,
1489
+ // modifiers, and how the sheet takes the tap. "again": it stays, for another (the rest close it: what comes
1490
+ // next is on the picture). "sure": asked twice, being a slip of the finger away from losing work.
1491
+ const QUICK = [
1492
+ ["常用", [["复制", "c", "cmd"], ["粘贴", "v", "cmd"], ["剪切", "x", "cmd"], ["全选", "a", "cmd"], ["撤销", "z", "cmd", "again"], ["重做", "z", "shift cmd", "again"],
1493
+ ["保存", "s", "cmd"], ["查找", "f", "cmd"]]],
1494
+ ["窗口与应用", [["切换应用", "tab", "cmd", "again"], ["同应用窗口", "\x60", "cmd", "again"], ["新标签", "t", "cmd"], ["关闭窗口", "w", "cmd"], ["最小化", "m", "cmd"],
1495
+ ["全屏", "f", "ctrl cmd"], ["退出应用", "q", "cmd", "sure"]]],
1496
+ ["系统", [["聚焦搜索", "space", "cmd"], ["调度中心", "up", "ctrl"], ["应用窗口", "down", "ctrl"], ["左边桌面", "left", "ctrl", "again"], ["右边桌面", "right", "ctrl", "again"],
1497
+ ["切换输入法", "space", "ctrl", "again"], ["截图", "4", "shift cmd"], ["截图工具", "5", "shift cmd"], ["强制退出", "escape", "alt cmd", "sure"], ["锁定屏幕", "q", "ctrl cmd", "sure"]]],
1498
+ ];
1499
+ const F_KEYS = Array.from({ length: 12 }, (_, n) => "f" + (n + 1));
1500
+ const PLAIN_KEYS = { home: "Home", end: "End", pageup: "PgUp", pagedown: "PgDn", delete: "⌦", escape: "Esc", tab: "Tab", return: "回车" };
1501
+ const SHORTCUTS_KEPT = "linkshell.screen.shortcuts";
1502
+
1503
+ // The user's own. The app keeps them (a page's own storage goes with its address, and this page's changes
1504
+ // with every visit) and hands them back with the rest of what it tells the page; a browser has only its own.
1505
+ // An app from before there were any does neither, and is known by saying nothing of them.
1506
+ const helped = () => !!app && Array.isArray(chrome.shortcuts);
1507
+ let mine = [];
1508
+ if (helped()) mine = cleanShortcuts(chrome.shortcuts);
1509
+ else try { mine = cleanShortcuts(JSON.parse(localStorage.getItem(SHORTCUTS_KEPT))); } catch {}
1510
+ function keepMine() {
1511
+ if (helped()) return tellApp({ type: "shortcuts", list: mine });
1512
+ try { localStorage.setItem(SHORTCUTS_KEPT, JSON.stringify(mine)); } catch {}
1513
+ }
1514
+
1515
+ // back: the sheet took the keyboard's place, and gives it back once a key has done what it was opened for.
1516
+ let back = false, loosening = false, armed = null, disarming = 0;
1517
+
1518
+ function disarm() {
1519
+ if (!armed) return;
1520
+ clearTimeout(disarming);
1521
+ armed.classList.remove("sure");
1522
+ armed.firstChild.textContent = armed.dataset.name;
1523
+ armed = null;
1524
+ }
1525
+
1526
+ function fire(item, element) {
1527
+ if (!controlling()) return explain(false);
1528
+ if (item.how === "sure" && armed !== element) {
1529
+ disarm();
1530
+ armed = element;
1531
+ element.dataset.name = item.name;
1532
+ element.classList.add("sure");
1533
+ element.firstChild.textContent = "再点一次";
1534
+ disarming = setTimeout(disarm, 3000);
1535
+ return;
1536
+ }
1537
+ disarm();
1538
+ act({ t: "key", k: item.k, m: item.m.length ? item.m : undefined });
1539
+ tellApp({ type: "haptic", kind: "light" });
1540
+ element.classList.add("hit");
1541
+ setTimeout(() => element.classList.remove("hit"), 160);
1542
+ const sign = comboSign(item.k, item.m);
1543
+ hint(sign === item.name ? sign : item.name + " " + sign, 1300);
1544
+ if (item.how !== "again") closeSheet(back);
1545
+ }
1546
+
1547
+ function tile(name, k, m, how, action) {
1548
+ const item = { name, k, m: modsOf(m), how }, element = document.createElement("div"), label = document.createElement("b"), sign = document.createElement("span");
1549
+ element.className = "tile";
1550
+ label.textContent = name;
1551
+ sign.textContent = comboSign(k, item.m);
1552
+ element.append(label);
1553
+ if (sign.textContent !== name) element.append(sign);
1554
+ pressable(element, () => (action ? action(element) : fire(item, element)));
1555
+ return element;
1556
+ }
1557
+
1558
+ function group(into, title, tiles, six) {
1559
+ const head = document.createElement("div"), grid = document.createElement("div");
1560
+ head.className = "group";
1561
+ head.append(Object.assign(document.createElement("span"), { textContent: title }));
1562
+ grid.className = six ? "tiles six" : "tiles";
1563
+ grid.append(...tiles);
1564
+ if (title) into.append(head);
1565
+ into.append(grid);
1566
+ return head;
1567
+ }
1568
+
1569
+ for (const [title, items] of QUICK) group(actions, title, items.map(([name, k, m, how]) => tile(name, k, m.split(" "), how)));
1570
+ group(actions, "按键", F_KEYS.map((k) => tile(k.toUpperCase(), k, [], "again")), true);
1571
+ group(actions, "", Object.keys(PLAIN_KEYS).map((k) => tile(PLAIN_KEYS[k], k, [], "again")));
1572
+ const own = actions.appendChild(document.createElement("div"));
1573
+
1574
+ function drawMine() {
1575
+ own.textContent = "";
1576
+ if (!mine.length) loosening = false;
1577
+ const tiles = mine.map((one, index) => {
1578
+ const made = tile(one.name, one.k, one.m, "", loosening ? () => { mine.splice(index, 1); keepMine(); drawMine(); } : null);
1579
+ made.classList.toggle("loose", loosening);
1580
+ return made;
1581
+ });
1582
+ const add = document.createElement("div");
1583
+ add.className = "tile add";
1584
+ add.append(Object.assign(document.createElement("b"), { textContent: "+ 添加" }));
1585
+ pressable(add, () => (mine.length < 24 ? showMaker(true) : hint("最多 24 个:先删掉用不到的", 2500)));
1586
+ const head = group(own, "我的", [...tiles, add]);
1587
+ if (!mine.length) return;
1588
+ const turn = head.appendChild(Object.assign(document.createElement("i"), { textContent: loosening ? "完成" : "删除" }));
1589
+ pressable(turn, () => { loosening = !loosening; drawMine(); });
1590
+ }
1591
+
1592
+ // A shortcut of the user's own making: the modifiers, one key of those the computer knows by name, a name.
1593
+ const making = new Set(), makerKey = $("makerkey");
1594
+ const KEY_NAMES = { return: "回车 ↩\uFE0E", tab: "Tab", space: "空格", backspace: "删除 ⌫", delete: "向后删除 ⌦", escape: "Esc", left: "← 左", right: "→ 右", up: "↑ 上", down: "↓ 下",
1595
+ home: "Home", end: "End", pageup: "Page Up", pagedown: "Page Down" };
1596
+ for (const [title, list] of [["字母", [..."abcdefghijklmnopqrstuvwxyz"]], ["数字", [..."1234567890"]], ["符号", [..."-=[]\\;',./\x60"]], ["功能键", F_KEYS], ["其他", Object.keys(KEY_NAMES)]]) {
1597
+ const set = makerKey.appendChild(Object.assign(document.createElement("optgroup"), { label: title }));
1598
+ for (const k of list) set.append(Object.assign(document.createElement("option"), { value: k, textContent: KEY_NAMES[k] || k.toUpperCase() }));
1599
+ }
1600
+ function drawMaker() {
1601
+ for (const one of $("makermods").children) one.classList.toggle("on", making.has(one.dataset.mod));
1602
+ $("makersign").textContent = comboSign(makerKey.value, [...making]);
1603
+ }
1604
+ function showMaker(on) {
1605
+ if (on) {
1606
+ making.clear();
1607
+ making.add("cmd");
1608
+ makerKey.value = "a";
1609
+ naming.value = "";
1610
+ drawMaker();
1611
+ } else if (document.activeElement === naming) naming.blur();
1612
+ actions.classList.toggle("gone", on);
1613
+ maker.classList.toggle("gone", !on);
1614
+ layout();
1615
+ }
1616
+ function saveMaker() {
1617
+ mine = cleanShortcuts([...mine, { name: naming.value, k: makerKey.value, m: [...making] }]);
1618
+ keepMine();
1619
+ loosening = false;
1620
+ drawMine();
1621
+ showMaker(false);
1622
+ // The new one is the last thing in the sheet.
1623
+ actions.scrollTop = actions.scrollHeight;
1624
+ }
1625
+ for (const one of $("makermods").children) tappable(one, () => { if (!making.delete(one.dataset.mod)) making.add(one.dataset.mod); drawMaker(); });
1626
+ makerKey.addEventListener("change", drawMaker);
1627
+ naming.addEventListener("focus", layout);
1628
+ naming.addEventListener("blur", layout);
1629
+ naming.addEventListener("keydown", (event) => { if (event.key === "Enter" && !event.isComposing) { event.preventDefault(); saveMaker(); } });
1630
+ tappable($("makercancel"), () => showMaker(false));
1631
+ tappable($("makersave"), saveMaker);
1632
+
1633
+ function openSheet() {
1634
+ if (mode === "view" || sheetOpen) return;
1635
+ closeMenu();
1636
+ closeComposer();
1637
+ back = keysOpen && fieldFocused();
1638
+ dropKeys();
1639
+ sheetOpen = true;
1640
+ loosening = false;
1641
+ drawMine();
1642
+ showMaker(false);
1643
+ refresh();
1644
+ }
1645
+
1646
+ /** True when there was a sheet to close. toKeys: the keyboard it took the place of comes back. */
1647
+ function closeSheet(toKeys) {
1648
+ if (!sheetOpen) return false;
1649
+ sheetOpen = false;
1650
+ disarm();
1651
+ if (document.activeElement === naming) naming.blur();
1652
+ refresh();
1653
+ if (toKeys) raiseKeys();
1654
+ return true;
1655
+ }
1656
+ $("compose").insertAdjacentHTML("afterbegin", ICON.compose);
1657
+ $("sheetkeys").innerHTML = ICON.keyboard;
1658
+ $("sheetclose").innerHTML = ICON.close;
1659
+ tappable($("compose"), openComposer);
1660
+ tappable($("sheetkeys"), () => closeSheet(true));
1661
+ tappable($("sheetclose"), () => closeSheet(false));
1662
+
1663
+ // ---- Text, written here and sent whole --------------------------------------
1664
+
1665
+ // Typing into the computer a key at a time suits a word or two. Anything longer is written in a box on the
1666
+ // phone, where its keyboard, dictation and paste work as they do everywhere, and sent when it is ready.
1667
+ // writing: an input method (pinyin, say) is in the middle of a word in the box. What it holds is not text yet,
1668
+ // and the box is not emptied under it: on a phone it would go on composing with what is no longer there.
1669
+ let writing = false;
1670
+ words.addEventListener("compositionstart", () => { writing = true; });
1671
+ words.addEventListener("compositionend", () => { writing = false; });
1672
+ const unfinished = () => writing && (hint("先在键盘上选好字,再发送", 2500), true);
1673
+
1674
+ function grow() {
1675
+ const wide = composer.classList.contains("wide");
1676
+ // What to write there, said by the page: a phone's browser has been seen to draw a text box's own
1677
+ // placeholder cut to the width of what an input method last wrote in it.
1678
+ $("wordshint").classList.toggle("gone", words.value !== "");
1679
+ words.style.height = "";
1680
+ words.style.height = clamp(words.scrollHeight, wide ? 44 : 66, wide ? 66 : 132) + "px";
1681
+ layout();
1682
+ }
1683
+
1684
+ function openComposer() {
1685
+ if (mode === "view") return;
1686
+ closeMenu();
1687
+ closeSheet();
1688
+ if (!composerOpen) {
1689
+ composerOpen = true;
1690
+ // The app reads the clipboard for the page (one that knows how: see helped); a browser may let the page.
1691
+ $("pasteclip").classList.toggle("gone", app ? !helped() : !(navigator.clipboard && navigator.clipboard.readText));
1692
+ refresh();
1693
+ grow();
1694
+ }
1695
+ ask(words);
1696
+ }
1697
+
1698
+ function closeComposer() {
1699
+ if (!composerOpen) return;
1700
+ composerOpen = false;
1701
+ wanted = null;
1702
+ writing = false;
1703
+ if (document.activeElement === words) words.blur();
1704
+ refresh();
1705
+ }
1706
+
1707
+ function sendWords(andReturn) {
1708
+ if (!controlling()) return explain(false);
1709
+ if (unfinished()) return;
1710
+ const text = words.value;
1711
+ if (!text && !andReturn) return;
1712
+ for (const event of typingOf(text, 4000)) act(event);
1713
+ if (andReturn) act({ t: "key", k: "return" });
1714
+ words.value = "";
1715
+ grow();
1716
+ tellApp({ type: "haptic", kind: "light" });
1717
+ hint(text ? "已发送 " + [...text].length + " 个字" + (andReturn ? ",并回车" : "") : "已回车", 1500);
1718
+ }
1719
+
1720
+ /** The phone's clipboard, put where the caret is: to be looked over, and sent by the user. */
1721
+ function pasted(text) {
1722
+ if (!composerOpen) return;
1723
+ if (typeof text !== "string" || !text) return hint("手机剪贴板里没有文字", 2500);
1724
+ if (writing) return hint("先在键盘上选好字,再粘贴", 2500);
1725
+ words.setRangeText(text.slice(0, 20000), words.selectionStart, words.selectionEnd, "end");
1726
+ grow();
1727
+ words.scrollTop = words.scrollHeight;
1728
+ }
1729
+ window.linkshellClipboard = pasted;
1730
+
1731
+ words.addEventListener("input", grow);
1732
+ tappable($("composeclose"), closeComposer);
1733
+ tappable($("pasteclip"), () => {
1734
+ if (app) return tellApp({ type: "clipboard" });
1735
+ navigator.clipboard.readText().then(pasted, () => hint("浏览器没有允许读取剪贴板", 2500));
1736
+ });
1737
+ tappable($("sendwords"), () => sendWords(false));
1738
+ tappable($("sendreturn"), () => sendWords(true));
1739
+ $("composeclose").innerHTML = ICON.down;
760
1740
 
761
- /** The app tells the page what it did with the phone's screen. */
1741
+ /** The app tells the page what it did with the phone's screen, whether the keyboard is on it, and what shortcuts it keeps for the user. */
762
1742
  window.linkshellChrome = (next) => {
1743
+ const was = chrome.keyboard;
763
1744
  Object.assign(chrome, next);
1745
+ if (chrome.keyboard === true) wentDown = false;
1746
+ else if (was === true && chrome.keyboard === false) {
1747
+ wentDown = true;
1748
+ settleIn(300);
1749
+ }
1750
+ if (next.shortcuts && JSON.stringify(cleanShortcuts(next.shortcuts)) !== JSON.stringify(mine)) {
1751
+ mine = cleanShortcuts(next.shortcuts);
1752
+ drawMine();
1753
+ }
764
1754
  refresh();
765
1755
  };
766
1756
 
1757
+ // ---- Measuring (?measure=1) ------------------------------------------------
1758
+
1759
+ // How late the picture is, by a clock the computer's app draws on its screen when the host is started with
1760
+ // LINKSHELL_SCREEN_CLOCK=1: a strip of 20 cells across x 0.05–0.45 of the display at y 0.12, holding the time
1761
+ // in milliseconds. Each frame's strip is read as the frame is shown, and once a second the host is told what
1762
+ // was seen, whichever way the picture comes. The page's clock is taken to be the computer's (a browser on
1763
+ // it, a simulator); with ?measure=sync the host is asked the time, for a device with a clock of its own.
1764
+ let meter = null;
1765
+ if (query.get("measure")) meter = (() => {
1766
+ const pageTime = () => performance.timeOrigin + performance.now();
1767
+ const sync = query.get("measure") === "sync", pings = new Map();
1768
+ let clock = { offset: 0, trip: Infinity }, pinged = 0;
1769
+ let late = [], unread = 0, frames = 0, presented = 0;
1770
+ let before = { at: performance.now(), presented: 0, raw: null };
1771
+
1772
+ const small = () => Object.assign(document.createElement("canvas"), { width: 20, height: 1 });
1773
+ // One pixel a cell, at its middle. Through WebGL the frame stays on the GPU and only those 20 come back.
1774
+ function throughGL() {
1775
+ const gl = small().getContext("webgl", { antialias: false, depth: false, alpha: false });
1776
+ if (!gl) return null;
1777
+ const shader = (type, source) => {
1778
+ const made = gl.createShader(type);
1779
+ gl.shaderSource(made, source);
1780
+ gl.compileShader(made);
1781
+ return made;
1782
+ };
1783
+ const program = gl.createProgram();
1784
+ gl.attachShader(program, shader(gl.VERTEX_SHADER, "attribute vec2 corner; varying vec2 at; void main() { at = vec2(0.05 + 0.4 * (corner.x * 0.5 + 0.5), 0.12); gl_Position = vec4(corner, 0.0, 1.0); }"));
1785
+ gl.attachShader(program, shader(gl.FRAGMENT_SHADER, "precision mediump float; uniform sampler2D frame; varying vec2 at; void main() { gl_FragColor = texture2D(frame, at); }"));
1786
+ gl.bindAttribLocation(program, 0, "corner");
1787
+ gl.linkProgram(program);
1788
+ if (!gl.getProgramParameter(program, gl.LINK_STATUS)) return null;
1789
+ gl.useProgram(program);
1790
+ gl.bindBuffer(gl.ARRAY_BUFFER, gl.createBuffer());
1791
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), gl.STATIC_DRAW);
1792
+ gl.enableVertexAttribArray(0);
1793
+ gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
1794
+ gl.bindTexture(gl.TEXTURE_2D, gl.createTexture());
1795
+ // Not averaged with its neighbours; and a frame is no power of two, which leaves only these settings.
1796
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
1797
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
1798
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
1799
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
1800
+ const pixels = new Uint8Array(80);
1801
+ return (source) => {
1802
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
1803
+ gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
1804
+ gl.readPixels(0, 0, 20, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
1805
+ return pixels;
1806
+ };
1807
+ }
1808
+ // The same through a canvas in memory, where WebGL can't be had or won't take the frame.
1809
+ function throughCanvas() {
1810
+ const context = small().getContext("2d", { willReadFrequently: true });
1811
+ context.imageSmoothingEnabled = false;
1812
+ return (source, w, h) => {
1813
+ context.drawImage(source, 0.05 * w, 0.12 * h - 0.5, 0.4 * w, 1, 0, 0, 20, 1);
1814
+ return context.getImageData(0, 0, 20, 1).data;
1815
+ };
1816
+ }
1817
+ let reader = throughGL();
1818
+ let plain = reader ? null : throughCanvas();
1819
+
1820
+ /** One frame, w by h, shown at a time on the page's clock. Returns the time on its strip. */
1821
+ function seen(source, w, h, at) {
1822
+ let pixels;
1823
+ try {
1824
+ pixels = (reader || plain)(source, w, h);
1825
+ } catch {
1826
+ reader = null;
1827
+ pixels = (plain = plain || throughCanvas())(source, w, h);
1828
+ }
1829
+ const time = stripTime(pixels), ms = time === undefined ? undefined : lateness(at + clock.offset, time);
1830
+ frames += 1;
1831
+ if (ms === undefined) unread += 1;
1832
+ else late.push(ms);
1833
+ return time;
1834
+ }
1835
+
1836
+ if ("requestVideoFrameCallback" in video) {
1837
+ const each = (now, metadata) => {
1838
+ video.requestVideoFrameCallback(each);
1839
+ presented = metadata.presentedFrames;
1840
+ seen(video, video.videoWidth, video.videoHeight, performance.timeOrigin + metadata.expectedDisplayTime);
1841
+ };
1842
+ video.requestVideoFrameCallback(each);
1843
+ } else {
1844
+ // Without frame callbacks a frame is seen on several refreshes of the screen: only its first sighting counts.
1845
+ let last;
1846
+ const each = () => {
1847
+ requestAnimationFrame(each);
1848
+ if (!live || video.readyState < 2) return;
1849
+ const count = late.length, time = seen(video, video.videoWidth, video.videoHeight, pageTime());
1850
+ if (time === last) {
1851
+ frames -= 1;
1852
+ if (late.length > count) late.pop();
1853
+ else unread -= 1;
1854
+ }
1855
+ last = time;
1856
+ };
1857
+ requestAnimationFrame(each);
1858
+ }
1859
+
1860
+ if (sync) {
1861
+ const ping = () => {
1862
+ pings.set(++pinged, pageTime());
1863
+ send({ t: "ping", n: pinged, at: Date.now() });
1864
+ };
1865
+ ws.addEventListener("open", () => {
1866
+ for (let i = 0; i < 10; i++) setTimeout(ping, i * 100);
1867
+ setInterval(ping, 5000);
1868
+ });
1869
+ }
1870
+
1871
+ setInterval(async () => {
1872
+ const counted = { late, unread, frames }, now = { at: performance.now(), presented, raw: null };
1873
+ late = [];
1874
+ unread = frames = 0;
1875
+ const track = live, pc = track && rtc && rtc.pc, seconds = (now.at - before.at) / 1000;
1876
+ const said = pc ? await vitals(pc, before.raw).catch(() => null) : null;
1877
+ if (said) now.raw = said.raw;
1878
+ send({
1879
+ t: "measure",
1880
+ mode: track ? "video" : "legacy",
1881
+ latency: summary(counted.late),
1882
+ // What was put on the screen: the browser's own count for a video, the frames drawn for the socket's picture.
1883
+ fps: round((track && now.presented > before.presented ? now.presented - before.presented : counted.frames) / seconds),
1884
+ width: content.w,
1885
+ height: content.h,
1886
+ jitterBufferMs: said && said.jitterBufferMs,
1887
+ decodeMs: said && said.decodeMs,
1888
+ rttMs: said && said.rttMs,
1889
+ path: said ? said.path : relayed ? "relay" : "socket",
1890
+ dropped: said ? said.dropped : skipped,
1891
+ freezes: said && said.freezes,
1892
+ unread: counted.unread,
1893
+ clock: sync ? { offsetMs: round(clock.offset), tripMs: round(clock.trip) } : undefined,
1894
+ });
1895
+ before = now;
1896
+ }, 1000);
1897
+
1898
+ return {
1899
+ /** A frame of the socket's picture, just drawn. */
1900
+ drawn: (frame) => seen(frame, frame.displayWidth, frame.displayHeight, pageTime()),
1901
+ pong: (answer) => {
1902
+ const sent = pings.get(answer.n);
1903
+ pings.delete(answer.n);
1904
+ if (sent !== undefined) clock = synced(clock, sent, pageTime(), answer.now);
1905
+ },
1906
+ };
1907
+ })();
1908
+
767
1909
  resetTyping();
768
1910
  refresh();
769
1911
  wake();
770
1912
  tellApp({ type: "ready", mode });
771
- })();
772
1913
  `;
1914
+ const SCRIPT = `(() => {${VIEWER_LOGIC}${BODY}})();`;
773
1915
  export function viewerPage() {
774
1916
  return `<!doctype html>
775
1917
  <html><head><meta charset="utf-8">
@@ -777,22 +1919,56 @@ export function viewerPage() {
777
1919
  <title>屏幕</title>
778
1920
  <style>${STYLE}</style></head>
779
1921
  <body>
780
- <div id="stage"><canvas id="screen"></canvas><div id="pointer"></div></div>
1922
+ <div id="stage"><canvas id="screen"></canvas><video id="video" class="gone" autoplay playsinline muted></video><div id="pointer"></div></div>
781
1923
  <div id="note">正在连接电脑屏幕…</div>
782
1924
  <div id="toast" class="glass"></div>
783
1925
  <div id="menu" class="glass gone">
784
1926
  <div class="choice" data-mode="view"><i></i><div><b>只看</b><span>不会碰到电脑上的任何东西</span></div></div>
785
1927
  <div class="choice" data-mode="trackpad"><i></i><div><b>触控板</b><span>滑动移动指针,轻点点击</span></div></div>
786
1928
  <div class="choice" data-mode="touch"><i></i><div><b>点按</b><span>点哪里,就点击哪里</span></div></div>
1929
+ <div id="status"></div>
787
1930
  </div>
788
1931
  <div id="bar" class="glass">
789
1932
  <div class="tool" id="mode" role="button" aria-label="控制方式"></div>
790
1933
  <div class="tool" id="keyboard" role="button" aria-label="键盘">${ICONS.keyboard}</div>
1934
+ <div class="tool" id="quick" role="button" aria-label="快捷操作">${ICONS.quick}</div>
791
1935
  <div class="tool" id="fit" role="button" aria-label="还原缩放">${ICONS.fit}</div>
792
1936
  <div class="tool" id="rotate" role="button" aria-label="横屏">${ICONS.rotate}</div>
793
1937
  <div class="tool" id="full" role="button" aria-label="全屏"></div>
794
1938
  </div>
1939
+ <div id="sheet" class="glass gone">
1940
+ <div id="sheethead">
1941
+ <div id="compose" role="button">发送文字…</div>
1942
+ <div class="tool" id="sheetkeys" role="button" aria-label="键盘"></div>
1943
+ <div class="tool" id="sheetclose" role="button" aria-label="关闭"></div>
1944
+ </div>
1945
+ <div id="actions"></div>
1946
+ <div id="maker" class="gone">
1947
+ <div class="group"><span>添加快捷键</span><span id="makersign"></span></div>
1948
+ <div class="tiles" id="makermods">
1949
+ <div class="tile" data-mod="ctrl"><b>⌃</b><span>control</span></div>
1950
+ <div class="tile" data-mod="alt"><b>⌥</b><span>option</span></div>
1951
+ <div class="tile" data-mod="shift"><b>⇧</b><span>shift</span></div>
1952
+ <div class="tile" data-mod="cmd"><b>⌘</b><span>command</span></div>
1953
+ </div>
1954
+ <label class="field"><span>按键</span><select id="makerkey" aria-label="按键"></select></label>
1955
+ <label class="field"><span>名称</span><input id="makername" type="text" maxlength="16" placeholder="可不填" autocapitalize="off" autocomplete="off" autocorrect="off" spellcheck="false" enterkeyhint="done" aria-label="名称"></label>
1956
+ <div class="ends"><div class="soft" id="makercancel" role="button">取消</div><div class="soft strong" id="makersave" role="button">添加</div></div>
1957
+ </div>
1958
+ </div>
1959
+ <div id="composer" class="gone">
1960
+ <div id="wordsbox"><textarea id="words" maxlength="20000" autocapitalize="off" autocorrect="off" spellcheck="false" aria-label="要发送的文字"></textarea><div id="wordshint">在这里写好,再发到电脑上</div></div>
1961
+ <div id="sendrow">
1962
+ <div class="soft" id="composeclose" role="button" aria-label="收起"></div>
1963
+ <div class="soft" id="pasteclip" role="button">粘贴手机剪贴板</div>
1964
+ <div class="spring"></div>
1965
+ <div class="soft" id="sendreturn" role="button">发送并回车</div>
1966
+ <div class="soft strong" id="sendwords" role="button">发送</div>
1967
+ </div>
1968
+ </div>
795
1969
  <div id="keys" class="gone">
1970
+ <div class="key" id="quickkey" role="button" aria-label="快捷操作"></div>
1971
+ <div class="key" id="composekey" role="button" aria-label="发送文字"></div>
796
1972
  <div class="key word" data-key="escape">esc</div>
797
1973
  <div class="key word" data-key="tab">tab</div>
798
1974
  <div class="key" data-mod="ctrl">⌃</div>