npm - @kln-mcp/ctrl-mobile-cn - Versions diffs - 0.0.6 - Mend

@kln-mcp/ctrl-mobile-cn 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (13) hide show

package/bin/install.js +314 -0
package/cli/setup.js +304 -0
package/config/index.d.ts +19 -0
package/config/index.js +78 -0
package/config/pro.json +12 -0
package/core/index.d.ts +57 -0
package/core/index.js +315 -0
package/index.js +23 -0
package/package.json +31 -0
package/skills/kln-mobile-ctrl/SKILL.md +122 -0
package/wrapper/index.d.ts +107 -0
package/wrapper/index.js +2553 -0
package/wrapper/live-view.html +438 -0

package/wrapper/live-view.html ADDED Viewed

@@ -0,0 +1,438 @@
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>KLN Live View</title>
+  <style>
+    html, body { margin: 0; height: 100%; background: #101114; color: #e8eaed; font-family: system-ui, sans-serif; }
+    body { display: grid; grid-template-rows: auto 1fr; }
+    header { display: flex; gap: 16px; align-items: center; padding: 10px 14px; background: #191b20; border-bottom: 1px solid #2b2f38; }
+    #status { color: #8ab4f8; }
+    #stage { display: grid; place-items: center; overflow: hidden; }
+    canvas { max-width: 100vw; max-height: calc(100vh - 44px); background: #000; touch-action: none; }
+  </style>
+</head>
+<body>
+  <header><strong>KLN Live View</strong><span id="status">connecting...</span><span id="stats"></span></header>
+  <main id="stage"><canvas id="canvas"></canvas></main>
+  <script>
+    const statusEl = document.getElementById('status');
+    const statsEl = document.getElementById('stats');
+    const canvas = document.getElementById('canvas');
+    const ctx = canvas.getContext('2d');
+    let decoder;
+    let configData;
+    let frames = 0;
+    let bytes = 0;
+    let configured = false;
+    let pointerStart = null;
+    let deviceContext = null;
+    function log(...args) {
+      console.log('[KLN Live View]', ...args);
+    }
+    function setStatus(text) {
+      log('status:', text);
+      statusEl.textContent = text;
+    }
+    log('page loaded', {
+      location: location.href,
+      userAgent: navigator.userAgent,
+      hasVideoDecoder: 'VideoDecoder' in window
+    });
+    function configureDecoder() {
+      if (configured) {
+        log('decoder already configured');
+        return;
+      }
+      if (!('VideoDecoder' in window)) {
+        setStatus('WebCodecs VideoDecoder is not available in this browser');
+        return;
+      }
+      log('configuring decoder');
+      decoder = new VideoDecoder({
+        output(frame) {
+          if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+            log('resize canvas from decoded frame', {
+              width: frame.displayWidth,
+              height: frame.displayHeight
+            });
+            canvas.width = frame.displayWidth;
+            canvas.height = frame.displayHeight;
+          }
+          ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+          frame.close();
+          frames += 1;
+          statsEl.textContent = frames + ' frames, ' + Math.round(bytes / 1024) + ' KB';
+        },
+        error(error) {
+          setStatus('decoder error: ' + error.message);
+          console.error('[KLN Live View] decoder error', error);
+        }
+      });
+      decoder.configure({ codec: 'avc1.42E01E', optimizeForLatency: true });
+      configured = true;
+      setStatus('streaming');
+    }
+    function readU32LE(view, offset) { return view.getUint32(offset, true); }
+    function readI64LE(view, offset) { return Number(view.getBigInt64(offset, true)); }
+    function concatBytes(a, b) {
+      const out = new Uint8Array(a.byteLength + b.byteLength);
+      out.set(new Uint8Array(a), 0);
+      out.set(new Uint8Array(b), a.byteLength);
+      return out;
+    }
+    function handlePacket(buffer) {
+      const view = new DataView(buffer);
+      const kind = view.getUint8(0);
+      const flags = readU32LE(view, 1);
+      const width = readU32LE(view, 5);
+      const height = readU32LE(view, 9);
+      const timestamp = readI64LE(view, 13);
+      const len = readU32LE(view, 29);
+      const data = buffer.slice(33, 33 + len);
+      bytes += len;
+      if (width && height && (canvas.width === 0 || canvas.height === 0)) {
+        log('initialize canvas size from packet', { width, height });
+        canvas.width = width;
+        canvas.height = height;
+      }
+      if (kind === 1 || (flags & 2) === 2) {
+        log('decoder config packet received', { len });
+        configData = data;
+        configureDecoder();
+        return;
+      }
+      if (kind !== 2 || !configured) {
+        return;
+      }
+      const isKey = (flags & 1) === 1;
+      const chunkData = isKey && configData ? concatBytes(configData, data) : new Uint8Array(data);
+      try {
+        decoder.decode(new EncodedVideoChunk({
+          type: isKey ? 'key' : 'delta',
+          timestamp: timestamp || performance.now() * 1000,
+          data: chunkData
+        }));
+      } catch (error) {
+        setStatus('decode enqueue error: ' + error.message);
+        console.error('[KLN Live View] decode enqueue error', { kind, flags, len, isKey, error });
+      }
+    }
+    function canvasPoint(event) {
+      const rect = canvas.getBoundingClientRect();
+      if (!rect.width || !rect.height || !canvas.width || !canvas.height) {
+        log('cannot map pointer to canvas point', {
+          rectWidth: rect.width,
+          rectHeight: rect.height,
+          canvasWidth: canvas.width,
+          canvasHeight: canvas.height
+        });
+        return null;
+      }
+      const point = {
+        x: Math.round((event.clientX - rect.left) * canvas.width / rect.width),
+        y: Math.round((event.clientY - rect.top) * canvas.height / rect.height)
+      };
+      log('pointer mapped to canvas point', {
+        clientX: event.clientX,
+        clientY: event.clientY,
+        rectLeft: rect.left,
+        rectTop: rect.top,
+        rectWidth: rect.width,
+        rectHeight: rect.height,
+        canvasWidth: canvas.width,
+        canvasHeight: canvas.height,
+        point
+      });
+      return point;
+    }
+    function getBoundsSize(bounds) {
+      if (!bounds || typeof bounds !== 'object') return null;
+      const width = Number(bounds.width ?? (bounds.right - bounds.left));
+      const height = Number(bounds.height ?? (bounds.bottom - bounds.top));
+      if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
+        return null;
+      }
+      return { width, height };
+    }
+    function toDevicePoint(videoPoint) {
+      const screen = getBoundsSize(deviceContext && (deviceContext.screenBounds || deviceContext.screen_bounds));
+      if (!screen || !canvas.width || !canvas.height) {
+        log('device context unavailable; skip input to avoid sending video coordinates as device coordinates', {
+          videoPoint,
+          deviceContext,
+          canvasWidth: canvas.width,
+          canvasHeight: canvas.height
+        });
+        setStatus('waiting for device context');
+        return null;
+      }
+      const x = Math.round(videoPoint.x * screen.width / canvas.width);
+      const y = Math.round(videoPoint.y * screen.height / canvas.height);
+      const point = {
+        x: Math.max(0, Math.min(screen.width - 1, x)),
+        y: Math.max(0, Math.min(screen.height - 1, y)),
+        videoX: videoPoint.x,
+        videoY: videoPoint.y,
+        videoWidth: canvas.width,
+        videoHeight: canvas.height,
+        deviceWidth: screen.width,
+        deviceHeight: screen.height,
+        coordinateSpace: 'android-device'
+      };
+      log('video point mapped to device point', point);
+      return point;
+    }
+    function sendInput(payload) {
+      const message = {
+        type: 'input',
+        profile: 'human',
+        clientSentAtMs: Date.now(),
+        clientPerfNowMs: performance.now(),
+        ...payload
+      };
+      log('send input requested', { readyState: ws.readyState, message });
+      if (ws.readyState !== WebSocket.OPEN) {
+        log('skip input send because websocket is not open', { readyState: ws.readyState, message });
+        return;
+      }
+      ws.send(JSON.stringify(message));
+      log('input sent', message);
+    }
+    const ws = new WebSocket('ws://' + location.host + '/ws');
+    log('websocket created', { url: ws.url });
+    ws.binaryType = 'arraybuffer';
+    ws.onopen = () => {
+      log('websocket open');
+      setStatus('view connected');
+    };
+    ws.onmessage = (event) => {
+      if (typeof event.data === 'string') {
+        const msg = JSON.parse(event.data);
+        log('websocket json message', msg);
+        if (msg.type === 'hello') setStatus('device ' + (msg.connectionState || 'unknown'));
+        if (msg.type === 'connection') setStatus('device ' + msg.state);
+        if (msg.type === 'end') setStatus('ended: ' + msg.reason);
+        if (msg.type === 'error') setStatus('error: ' + msg.message);
+        if (msg.type === 'input-error') setStatus('input error: ' + msg.message);
+        if (msg.type === 'device-context') {
+          deviceContext = msg.context;
+          log('device context updated', deviceContext);
+        }
+        if (msg.type === 'device-context-error') {
+          log('device context query failed', msg);
+        }
+        return;
+      }
+      handlePacket(event.data);
+    };
+    ws.onclose = (event) => {
+      log('websocket closed', {
+        code: event.code,
+        reason: event.reason,
+        wasClean: event.wasClean
+      });
+      setStatus('closed');
+    };
+    ws.onerror = (event) => {
+      console.error('[KLN Live View] websocket error', event);
+      setStatus('websocket error');
+    };
+    const PATH_MIN_STEP_PX = 2;
+    const PATH_MAX_SAMPLES = 96;
+    const TAP_THRESHOLD_PX = 8;
+    // Android's default long-press timeout is 500ms. A press-hold that stays
+    // within tap distance for at least this long is dispatched as a long-press
+    // (instead of an instantaneous tap).
+    const LONG_PRESS_THRESHOLD_MS = 500;
+    // Samples within this radius of the press point (leading) or release point
+    // (trailing) are treated as idle "hold" wiggle and trimmed. Set above the
+    // 2px push threshold so several jittery micro-steps during a press-hold get
+    // absorbed rather than dispatched to Android as long-duration tiny-motion
+    // strokes (which the OS interprets as a long-press).
+    const IDLE_TRIM_RADIUS_PX = 8;
+    function pushPathSample(samples, vx, vy, t) {
+      const last = samples[samples.length - 1];
+      if (last && Math.hypot(vx - last.vx, vy - last.vy) < PATH_MIN_STEP_PX) {
+        return;
+      }
+      samples.push({ vx, vy, t });
+    }
+    function trimIdleEnds(samples) {
+      if (samples.length < 3) return samples;
+      const origin = samples[0];
+      const endPoint = samples[samples.length - 1];
+      let lead = 0;
+      for (let i = 1; i < samples.length - 1; i += 1) {
+        if (Math.hypot(samples[i].vx - origin.vx, samples[i].vy - origin.vy) >= IDLE_TRIM_RADIUS_PX) break;
+        lead = i;
+      }
+      let trail = samples.length - 1;
+      for (let i = samples.length - 2; i > lead; i -= 1) {
+        if (Math.hypot(samples[i].vx - endPoint.vx, samples[i].vy - endPoint.vy) >= IDLE_TRIM_RADIUS_PX) break;
+        trail = i;
+      }
+      return samples.slice(lead, trail + 1);
+    }
+    function downsamplePath(samples, max) {
+      if (samples.length <= max) return samples;
+      const stride = (samples.length - 1) / (max - 1);
+      const out = new Array(max);
+      for (let i = 0; i < max; i += 1) {
+        out[i] = samples[Math.min(samples.length - 1, Math.round(i * stride))];
+      }
+      return out;
+    }
+    canvas.addEventListener('pointerdown', (event) => {
+      const point = canvasPoint(event);
+      if (!point) return;
+      const now = performance.now();
+      pointerStart = {
+        ...point,
+        pointerId: event.pointerId,
+        startedAt: now,
+        samples: [{ vx: point.x, vy: point.y, t: 0 }]
+      };
+      log('pointer down', pointerStart);
+      canvas.setPointerCapture(event.pointerId);
+      event.preventDefault();
+    });
+    canvas.addEventListener('pointermove', (event) => {
+      if (!pointerStart || pointerStart.pointerId !== event.pointerId) return;
+      const point = canvasPoint(event);
+      if (!point) return;
+      const t = Math.round(performance.now() - pointerStart.startedAt);
+      pushPathSample(pointerStart.samples, point.x, point.y, t);
+    });
+    canvas.addEventListener('pointerup', (event) => {
+      if (!pointerStart || pointerStart.pointerId !== event.pointerId) {
+        log('ignore pointer up without matching pointer start', {
+          pointerId: event.pointerId,
+          pointerStart
+        });
+        return;
+      }
+      const point = canvasPoint(event);
+      if (!point) return;
+      const wallDurationMs = Math.max(1, Math.round(performance.now() - pointerStart.startedAt));
+      pushPathSample(pointerStart.samples, point.x, point.y, wallDurationMs);
+      const rawSamples = pointerStart.samples;
+      const samples = trimIdleEnds(rawSamples);
+      const startSample = samples[0];
+      const endSample = samples[samples.length - 1];
+      const dx = endSample.vx - startSample.vx;
+      const dy = endSample.vy - startSample.vy;
+      // Effective duration covers only the active motion window — leading/trailing
+      // idle is trimmed so the dispatched gesture doesn't look like a long-press.
+      const effectiveDurationMs = Math.max(1, endSample.t - startSample.t);
+      log('pointer up', {
+        point,
+        dx,
+        dy,
+        wallDurationMs,
+        effectiveDurationMs,
+        rawSamples: rawSamples.length,
+        trimmedSamples: samples.length,
+        trimmedLeading: rawSamples.length - samples.length > 0
+      });
+      let totalDistance = 0;
+      for (let i = 1; i < samples.length; i += 1) {
+        totalDistance += Math.hypot(samples[i].vx - samples[i - 1].vx, samples[i].vy - samples[i - 1].vy);
+      }
+      if (Math.hypot(dx, dy) < TAP_THRESHOLD_PX && totalDistance < TAP_THRESHOLD_PX * 2) {
+        const devicePoint = toDevicePoint({ x: endSample.vx, y: endSample.vy });
+        if (!devicePoint) return;
+        if (wallDurationMs >= LONG_PRESS_THRESHOLD_MS) {
+          log('recognize long-press', { ...devicePoint, wallDurationMs });
+          sendInput({ action: 'longPress', durationMs: wallDurationMs, ...devicePoint });
+        } else {
+          log('recognize tap', devicePoint);
+          sendInput({ action: 'tap', ...devicePoint });
+        }
+      } else if (samples.length <= 2) {
+        const deviceStart = toDevicePoint({ x: startSample.vx, y: startSample.vy });
+        const deviceEnd = toDevicePoint({ x: endSample.vx, y: endSample.vy });
+        if (!deviceStart || !deviceEnd) return;
+        log('recognize swipe (2-point)', { deviceStart, deviceEnd, effectiveDurationMs });
+        sendInput({
+          action: 'swipe',
+          startX: deviceStart.x,
+          startY: deviceStart.y,
+          endX: deviceEnd.x,
+          endY: deviceEnd.y,
+          videoStartX: startSample.vx,
+          videoStartY: startSample.vy,
+          videoEndX: endSample.vx,
+          videoEndY: endSample.vy,
+          videoWidth: canvas.width,
+          videoHeight: canvas.height,
+          deviceWidth: deviceStart.deviceWidth,
+          deviceHeight: deviceStart.deviceHeight,
+          coordinateSpace: deviceStart.coordinateSpace,
+          durationMs: effectiveDurationMs
+        });
+      } else {
+        const sampled = downsamplePath(samples, PATH_MAX_SAMPLES);
+        const baseT = sampled[0].t;
+        const devicePoints = [];
+        let deviceWidth = 0;
+        let deviceHeight = 0;
+        for (const s of sampled) {
+          const mapped = toDevicePoint({ x: s.vx, y: s.vy });
+          if (!mapped) return;
+          // Re-zero timestamps so the Android side sees t starting at 0; this
+          // keeps duration math consistent after leading-idle trim.
+          devicePoints.push({ x: mapped.x, y: mapped.y, tMs: s.t - baseT });
+          deviceWidth = mapped.deviceWidth;
+          deviceHeight = mapped.deviceHeight;
+        }
+        log('recognize gesture', {
+          rawSamples: rawSamples.length,
+          trimmedSamples: samples.length,
+          sentPoints: devicePoints.length,
+          effectiveDurationMs,
+          totalDistance: Math.round(totalDistance)
+        });
+        sendInput({
+          action: 'gesture',
+          points: devicePoints,
+          durationMs: effectiveDurationMs,
+          videoWidth: canvas.width,
+          videoHeight: canvas.height,
+          deviceWidth,
+          deviceHeight,
+          coordinateSpace: 'android-device'
+        });
+      }
+      pointerStart = null;
+      event.preventDefault();
+    });
+    canvas.addEventListener('pointercancel', (event) => {
+      log('pointer cancel', { pointerId: event.pointerId, pointerStart });
+      pointerStart = null;
+    });
+  </script>
+</body>
+</html>