@mazees/mark 5.0.0

package/src/server/tools/media-tools.js

@@ -0,0 +1,244 @@
+import { MsEdgeTTS, OUTPUT_FORMAT } from 'msedge-tts'
+import { Readable } from 'stream'
+import ytSearch from 'yt-search'
+import { YoutubeTranscript } from 'youtube-transcript-plus'
+import path from 'path'
+import os from 'os'
+import fs from 'fs'
+
+// Patch MsEdgeTTS prototype to prevent unhandled crashes when streams close prematurely,
+// when connections are rejected, or when WebSocket events throw without handler.
+if (!MsEdgeTTS.__mark_patched) {
+  MsEdgeTTS.__mark_patched = true
+
+  MsEdgeTTS.prototype._pushAudioData = function (data, requestId) {
+    if (this._streams && this._streams[requestId] && this._streams[requestId].audio) {
+      try {
+        if (!this._streams[requestId].audio.destroyed) {
+          this._streams[requestId].audio.push(data)
+        }
+      } catch (_) {}
+    }
+  }
+
+  MsEdgeTTS.prototype._pushMetadata = function (data, requestId) {
+    if (this._streams && this._streams[requestId] && this._streams[requestId].metadata) {
+      try {
+        if (!this._streams[requestId].metadata.destroyed) {
+          this._streams[requestId].metadata.push(data)
+        }
+      } catch (_) {}
+    }
+  }
+
+  // Override _rawSSMLRequest with explicit catch handler on _send() to eliminate UnhandledPromiseRejection
+  MsEdgeTTS.prototype._rawSSMLRequest = function (requestSSML) {
+    this._metadataCheck()
+    const requestId = MsEdgeTTS.randomHex(16)
+    const request =
+      `X-RequestId:${requestId}\r\nContent-Type:application/ssml+xml\r\nPath:ssml\r\n\r\n` +
+      requestSSML.trim()
+    const self = this
+    const audioStream = new Readable({
+      read() {},
+      destroy(error, callback) {
+        delete self._streams[requestId]
+        if (typeof callback === 'function') callback(error)
+      }
+    })
+    const metadataStream = this._hasMetadataBoundaries()
+      ? new Readable({
+          read() {},
+          destroy(error, callback) {
+            delete self._streams[requestId]
+            if (typeof callback === 'function') callback(error)
+          }
+        })
+      : null
+
+    audioStream.on('error', () => {
+      audioStream.destroy()
+      metadataStream?.destroy()
+    })
+    audioStream.once('close', () => {
+      audioStream.destroy()
+      metadataStream?.destroy()
+    })
+
+    this._streams[requestId] = {
+      audio: audioStream,
+      metadata: metadataStream
+    }
+
+    // Attach catch handler directly to eliminate UnhandledPromiseRejection and pipe error to audioStream
+    this._send(request).catch((err) => {
+      const errMsg = typeof err === 'string' ? err : err?.message || 'TTS connection failed'
+      if (!audioStream.destroyed) {
+        audioStream.destroy(new Error(errMsg))
+      }
+    })
+
+    return { audioStream, metadataStream, requestId }
+  }
+
+  const originalInitClient = MsEdgeTTS.prototype._initClient
+  MsEdgeTTS.prototype._initClient = async function () {
+    try {
+      const res = await originalInitClient.apply(this, arguments)
+      if (this._ws) {
+        const origOnMessage = this._ws.onmessage
+        this._ws.onmessage = (m) => {
+          try {
+            const buffer = Buffer.from(m.data)
+            const message = buffer.toString()
+            const match = /X-RequestId:(.*?)\r\n/gm.exec(message)
+            const requestId = match ? match[1] : null
+
+            if (message.includes('Path:turn.end') && requestId) {
+              if (this._streams && this._streams[requestId] && this._streams[requestId].audio) {
+                try {
+                  if (!this._streams[requestId].audio.destroyed) {
+                    this._streams[requestId].audio.push(null)
+                  }
+                } catch (_) {}
+              }
+              return
+            }
+
+            if (origOnMessage) {
+              origOnMessage.call(this._ws, m)
+            }
+          } catch (_) {
+            // Ignore unparseable or orphaned socket packets
+          }
+        }
+      }
+      return res
+    } catch (connectErr) {
+      const msg =
+        typeof connectErr === 'string' ? connectErr : connectErr?.message || 'TTS connection failed'
+      throw new Error(msg)
+    }
+  }
+}
+
+const TEMP_DIR = path.join(os.tmpdir(), 'mark-audio')
+if (!fs.existsSync(TEMP_DIR)) {
+  fs.mkdirSync(TEMP_DIR, { recursive: true })
+}
+
+export const CUTE_ROBOT_PITCH = 55
+export const CUTE_ROBOT_RATE = 10
+
+/**
+ * Menghasilkan readable stream audio langsung untuk HTTP response streaming.
+ * Menggunakan instance terisolasi per permintaan dengan penutupan socket otomatis
+ * agar tahan terhadap idle timeout dan permintaan kalimat paralel.
+ * Default pitch +55Hz dan rate +10% menghasilkan nada suara robot kecil yang imut & ekspresif.
+ * @param {string} text
+ * @param {string} [voice='id-ID-ArdiNeural']
+ * @param {number|string} [rate=10]
+ * @param {number|string} [pitch=55]
+ */
+export async function streamTTS(text, voice = 'id-ID-ArdiNeural', rate = 10, pitch = 55) {
+  if (!text || !text.trim()) throw new Error('Teks kosong untuk TTS')
+
+  let selectedVoice = 'id-ID-ArdiNeural'
+  if (typeof voice === 'string' && voice.trim() && !/^-?\d+$/.test(voice.trim())) {
+    selectedVoice = voice.trim()
+  }
+
+  const numRate = typeof rate === 'number' ? rate : (rate !== undefined && rate !== '' ? parseInt(rate, 10) : CUTE_ROBOT_RATE) || CUTE_ROBOT_RATE
+  const numPitch = typeof pitch === 'number' ? pitch : (pitch !== undefined && pitch !== '' ? parseInt(pitch, 10) : CUTE_ROBOT_PITCH) || CUTE_ROBOT_PITCH
+  const rateStr = numRate >= 0 ? `+${numRate}%` : `${numRate}%`
+  const pitchStr = numPitch >= 0 ? `+${numPitch}Hz` : `${numPitch}Hz`
+
+  const tts = new MsEdgeTTS()
+  await tts.setMetadata(selectedVoice, OUTPUT_FORMAT.AUDIO_24KHZ_48KBITRATE_MONO_MP3)
+  const streamObj = tts.toStream(text, { rate: rateStr, pitch: pitchStr })
+
+  const cleanup = () => {
+    try {
+      tts.close()
+    } catch (_) {}
+  }
+
+  streamObj.audioStream.once('end', cleanup)
+  streamObj.audioStream.once('close', cleanup)
+  streamObj.audioStream.once('error', cleanup)
+
+  return streamObj.audioStream
+}
+
+/**
+ * Sintesis suara menggunakan Microsoft Edge TTS (Buffer RAM langsung tanpa disk write)
+ * @param {string} text
+ * @param {string} [voice='id-ID-ArdiNeural']
+ * @param {number|string} [rate=10]
+ * @param {number|string} [pitch=55]
+ * @returns {Promise<{ filePath: string, audioBase64: string }>}
+ */
+export async function synthesizeTTS(text, voice = 'id-ID-ArdiNeural', rate = 10, pitch = 55) {
+  if (!text || !text.trim()) throw new Error('Teks kosong untuk TTS')
+
+  let selectedVoice = 'id-ID-ArdiNeural'
+  if (typeof voice === 'string' && voice.trim() && !/^-?\d+$/.test(voice.trim())) {
+    selectedVoice = voice.trim()
+  }
+
+  const audioStream = await streamTTS(text, selectedVoice, rate, pitch)
+  const chunks = []
+
+  await new Promise((resolve, reject) => {
+    audioStream.on('data', (chunk) => chunks.push(chunk))
+    audioStream.on('end', resolve)
+    audioStream.on('error', reject)
+  })
+
+  const buffer = Buffer.concat(chunks)
+  const audioBase64 = `data:audio/mp3;base64,${buffer.toString('base64')}`
+
+  return { filePath: '', audioBase64 }
+}
+
+/**
+ * Cari video atau musik YouTube
+ * @param {string} query
+ */
+export async function searchYoutube(query) {
+  if (!query) return []
+  const result = await ytSearch(query)
+  return result.videos.slice(0, 10).map((v) => ({
+    id: v.videoId,
+    videoId: v.videoId,
+    title: v.title,
+    url: v.url,
+    timestamp: v.timestamp,
+    seconds: v.seconds,
+    views: v.views,
+    author: v.author?.name || 'Unknown Artist',
+    artist: v.author?.name || 'Unknown Artist',
+    thumbnail: v.thumbnail || v.image || `https://img.youtube.com/vi/${v.videoId}/hqdefault.jpg`
+  }))
+}
+
+/**
+ * Mengambil transkrip dari video YouTube
+ * @param {string} url
+ * @param {string} [lang='id']
+ */
+export async function getTranscript(url, lang = 'id') {
+  if (!url || typeof url !== 'string') return ''
+  try {
+    const transcriptList = await YoutubeTranscript.fetchTranscript(url, { lang })
+    if (transcriptList && transcriptList.length > 0) {
+      return transcriptList.map((t) => t.text).join(' ')
+    }
+    // Fallback tanpa filter bahasa jika bahasa target tidak tersedia
+    const fallbackList = await YoutubeTranscript.fetchTranscript(url)
+    return fallbackList.map((t) => t.text).join(' ')
+  } catch (err) {
+    console.warn('[YouTube Transcript] Gagal mengambil transkrip:', err?.message || err)
+    return `Gagal mengambil transkrip video: ${err?.message || 'Video tidak memiliki transkrip atau dibatasi'}`
+  }
+}

package/src/server/tools/pc-agent.js

@@ -0,0 +1,327 @@
+import { spawn } from 'child_process'
+import path from 'path'
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+import { wsHub } from '../ws-hub.js'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+let daemonProcess = null
+let daemonReady = false
+let pendingResolve = null
+let daemonBuffer = ''
+let lastReadResult = null
+let lastReadTimestamp = 0
+let stateChanged = false
+const CACHE_TTL = 10000
+
+let overlayProcess = null
+let overlayReady = false
+
+function getScriptPath(scriptName) {
+  const candidates = [
+    path.resolve(__dirname, `../../../src/main/pc-agent-scripts/${scriptName}`),
+    path.resolve(__dirname, `../../main/pc-agent-scripts/${scriptName}`),
+    path.resolve(__dirname, `../../../resources/scripts/${scriptName}`),
+    path.resolve(__dirname, `../../resources/scripts/${scriptName}`)
+  ]
+  for (const c of candidates) {
+    if (fs.existsSync(c)) return c
+  }
+  return candidates[0]
+}
+
+function getDaemonScriptPath() {
+  return getScriptPath('pc-daemon.ps1')
+}
+
+function getOverlayScriptPath() {
+  return getScriptPath('pc-overlay.ps1')
+}
+
+export function isOverlayAlive() {
+  return overlayProcess && !overlayProcess.killed
+}
+
+export function startOverlay() {
+  if (isOverlayAlive()) return
+
+  const scriptPath = getOverlayScriptPath()
+  if (!fs.existsSync(scriptPath)) {
+    console.warn('[PC-Agent] Skrip overlay tidak ditemukan:', scriptPath)
+    return
+  }
+
+  try {
+    overlayProcess = spawn('powershell.exe', [
+      '-NoProfile',
+      '-STA',
+      '-ExecutionPolicy',
+      'Bypass',
+      '-File',
+      scriptPath
+    ])
+
+    overlayReady = false
+
+    overlayProcess.stdout.on('data', (chunk) => {
+      const text = chunk.toString().trim()
+      if (text.includes('READY')) {
+        overlayReady = true
+      }
+      if (text.includes('"event":"abort"') || text.includes('{"event":"abort"}')) {
+        console.warn('[PC-Agent] Sinyal ABORT diterima dari Overlay (Ctrl+Shift+S / Tombol Batal)!')
+        // Hentikan sesi dan picu abort di WebSocket hub
+        stopDaemon()
+        stopOverlay()
+        wsHub.broadcast('ai:abort', { source: 'pc-overlay', reason: 'User pressed Ctrl+Shift+S or clicked Cancel' })
+      }
+    })
+
+    overlayProcess.stderr.on('data', (chunk) => {
+      console.warn('[PC-Agent Overlay Error]:', chunk.toString())
+    })
+
+    overlayProcess.on('close', () => {
+      overlayProcess = null
+      overlayReady = false
+    })
+
+    overlayProcess.on('error', (err) => {
+      console.error('[PC-Agent] Gagal spawn overlay:', err)
+      overlayProcess = null
+      overlayReady = false
+    })
+  } catch (err) {
+    console.error('[PC-Agent] Error menjalankan overlay:', err)
+  }
+}
+
+export function stopOverlay() {
+  if (overlayProcess && !overlayProcess.killed) {
+    try {
+      overlayProcess.kill()
+    } catch (_) {}
+    overlayProcess = null
+    overlayReady = false
+  }
+}
+
+export function isDaemonAlive() {
+  return daemonProcess && !daemonProcess.killed && daemonReady
+}
+
+export function startDaemon() {
+  return new Promise((resolve, reject) => {
+    if (isDaemonAlive()) {
+      resolve()
+      return
+    }
+
+    const scriptPath = getDaemonScriptPath()
+
+    daemonProcess = spawn('powershell.exe', [
+      '-NoProfile',
+      '-WindowStyle',
+      'Hidden',
+      '-ExecutionPolicy',
+      'Bypass',
+      '-File',
+      scriptPath
+    ])
+
+    daemonBuffer = ''
+    daemonReady = false
+
+    daemonProcess.stdout.on('data', (chunk) => {
+      daemonBuffer += chunk.toString()
+
+      const delimiterIndex = daemonBuffer.indexOf('---MARK_DONE---')
+      if (delimiterIndex !== -1) {
+        const response = daemonBuffer.substring(0, delimiterIndex).trim()
+        daemonBuffer = daemonBuffer.substring(delimiterIndex + '---MARK_DONE---'.length).trimStart()
+
+        if (!daemonReady) {
+          daemonReady = true
+          resolve()
+          return
+        }
+
+        if (pendingResolve) {
+          const resolveFn = pendingResolve
+          pendingResolve = null
+          resolveFn(response)
+        }
+      }
+    })
+
+    daemonProcess.stderr.on('data', () => {})
+
+    daemonProcess.on('close', () => {
+      daemonProcess = null
+      daemonReady = false
+      if (pendingResolve) {
+        const resolveFn = pendingResolve
+        pendingResolve = null
+        resolveFn('')
+      }
+    })
+
+    daemonProcess.on('error', (err) => {
+      console.error('[PC-Agent] Gagal spawn daemon:', err)
+      daemonProcess = null
+      reject(err)
+    })
+
+    setTimeout(() => {
+      if (!daemonReady) {
+        console.warn('[PC-Agent] Timeout menunggu startup daemon (15s)')
+        resolve()
+      }
+    }, 15000)
+  })
+}
+
+export function stopDaemon() {
+  if (daemonProcess && !daemonProcess.killed) {
+    try {
+      daemonProcess.stdin.write(JSON.stringify({ cmd: 'exit' }) + '\n')
+    } catch (_) {}
+    setTimeout(() => {
+      if (daemonProcess && !daemonProcess.killed) {
+        try {
+          daemonProcess.kill()
+        } catch (_) {}
+      }
+      daemonProcess = null
+      daemonReady = false
+    }, 1000)
+  }
+}
+
+export function sendCommand(cmd) {
+  return new Promise((resolve) => {
+    if (!isDaemonAlive()) {
+      resolve(JSON.stringify({ status: 'error', message: 'Daemon Win32 belum berjalan' }))
+      return
+    }
+
+    pendingResolve = resolve
+    try {
+      daemonProcess.stdin.write(JSON.stringify(cmd) + '\n')
+    } catch (err) {
+      pendingResolve = null
+      resolve(JSON.stringify({ status: 'error', message: 'Gagal kirim ke daemon: ' + err.message }))
+    }
+
+    setTimeout(() => {
+      if (pendingResolve === resolve) {
+        pendingResolve = null
+        resolve(JSON.stringify({ status: 'error', message: 'Command timeout (30s)' }))
+      }
+    }, 30000)
+  })
+}
+
+export async function readDesktop(options = {}) {
+  const maxElements = options.maxElements || 60
+  if (!stateChanged && lastReadResult && Date.now() - lastReadTimestamp < CACHE_TTL) {
+    return { ...lastReadResult, method: 'cached' }
+  }
+
+  if (!isDaemonAlive()) {
+    await startDaemon()
+  }
+
+  const raw = await sendCommand({ cmd: 'read-ui', maxElements, roles: options.roles })
+  try {
+    const parsed = JSON.parse(raw)
+    lastReadResult = parsed
+    lastReadTimestamp = Date.now()
+    stateChanged = false
+    return parsed
+  } catch (_) {
+    return { status: 'error', message: raw }
+  }
+}
+
+export async function executeClick(x, y) {
+  startOverlay()
+  if (!isDaemonAlive()) await startDaemon()
+  stateChanged = true
+  const raw = await sendCommand({ cmd: 'click', x, y })
+  return raw
+}
+
+export async function executeType(text) {
+  startOverlay()
+  if (!isDaemonAlive()) await startDaemon()
+  stateChanged = true
+  const raw = await sendCommand({ cmd: 'type', text })
+  return raw
+}
+
+export async function executeKey(combo) {
+  startOverlay()
+  if (!isDaemonAlive()) await startDaemon()
+  stateChanged = true
+  const raw = await sendCommand({ cmd: 'key', combo })
+  return raw
+}
+
+export async function executeScroll(direction = 'down', amount = 3) {
+  startOverlay()
+  if (!isDaemonAlive()) await startDaemon()
+  stateChanged = true
+  const raw = await sendCommand({ cmd: 'scroll', direction, amount })
+  return raw
+}
+
+export async function openApp(target) {
+  startOverlay()
+  if (!isDaemonAlive()) await startDaemon()
+  stateChanged = true
+  const raw = await sendCommand({ cmd: 'open', target })
+  return raw
+}
+
+export async function listWindows() {
+  if (!isDaemonAlive()) await startDaemon()
+  const raw = await sendCommand({ cmd: 'list-windows' })
+  try {
+    return JSON.parse(raw)
+  } catch (_) {
+    return []
+  }
+}
+
+export async function focusWindow(title) {
+  startOverlay()
+  if (!isDaemonAlive()) await startDaemon()
+  stateChanged = true
+  const raw = await sendCommand({ cmd: 'focus-window', title })
+  return raw
+}
+
+export async function executeDoubleClick(x, y) {
+  await executeClick(x, y)
+  await new Promise((r) => setTimeout(r, 60))
+  return await executeClick(x, y)
+}
+
+export async function openPCSession() {
+  startOverlay()
+  await startDaemon()
+  return { success: true }
+}
+
+export async function closePCSession() {
+  stopOverlay()
+  stopDaemon()
+  return { success: true }
+}
+
+export function isPCSessionOpen() {
+  return isDaemonAlive()
+}

package/src/server/tools/screen-service.js

@@ -0,0 +1,51 @@
+import { exec } from 'child_process'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import util from 'util'
+import fs from 'fs'
+
+const execPromise = util.promisify(exec)
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+export async function captureDesktopScreenshotsBase64() {
+  if (process.platform !== 'win32') {
+    throw new Error('Screen capture hanya didukung di lingkungan Windows.')
+  }
+
+  try {
+    const scriptPath = path.resolve(__dirname, '../../main/pc-agent-scripts/take-screenshot.ps1')
+    if (!fs.existsSync(scriptPath)) {
+      throw new Error(`Skrip screenshot tidak ditemukan di ${scriptPath}`)
+    }
+
+    const { stdout, stderr } = await execPromise(
+      `powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "${scriptPath}"`,
+      { maxBuffer: 50 * 1024 * 1024 }
+    )
+
+    const rawOutput = (stdout || '').trim()
+    if (!rawOutput) {
+      throw new Error(stderr || 'Output screenshot kosong.')
+    }
+
+    const base64List = rawOutput
+      .split('|||')
+      .map((str) => str.trim())
+      .filter(Boolean)
+
+    if (base64List.length === 0) {
+      throw new Error('Tidak ada gambar layar yang berhasil diambil.')
+    }
+
+    return base64List.map((b64) => `data:image/jpeg;base64,${b64}`)
+  } catch (err) {
+    console.error('[Screen Capture Error]:', err)
+    throw new Error(`Gagal mengambil screenshot: ${err.message}`)
+  }
+}
+
+export async function captureDesktopScreenshotBase64() {
+  const all = await captureDesktopScreenshotsBase64()
+  return all[0]
+}