huaweicloud-devkit 1.0.2-next.8 → 1.0.2

package/plugins/huaweicloud-core/src/sandbox/sandbox-file-server.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+import hashlib, http.server, json, os, sys
+class UploadHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        if self.path != '/upload':
+            self.send_error(404)
+            return
+        target_path = self.headers.get('X-Target-Path', '/workspace/upload.tar.gz')
+        content_length = int(self.headers.get('Content-Length', 0))
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+        md5 = hashlib.md5()
+        bytes_written = 0
+        with open(target_path, 'wb') as f:
+            remaining = content_length
+            while remaining > 0:
+                chunk = self.rfile.read(min(remaining, 65536))
+                if not chunk:
+                    break
+                f.write(chunk)
+                md5.update(chunk)
+                bytes_written += len(chunk)
+                remaining -= len(chunk)
+        result = json.dumps({'ok': True, 'path': target_path, 'bytes': bytes_written, 'md5': md5.hexdigest()})
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(result)))
+        self.end_headers()
+        self.wfile.write(result.encode())
+    def do_GET(self):
+        if self.path == '/health':
+            body = b'ok'
+            self.send_response(200)
+            self.send_header('Content-Length', str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        else:
+            self.send_error(404)
+    def log_message(self, format, *args):
+        pass
+if __name__ == '__main__':
+    port = int(sys.argv[1]) if len(sys.argv) > 1 else 8888
+    token = sys.argv[2] if len(sys.argv) > 2 else ''
+    server = http.server.HTTPServer(('127.0.0.1', port), UploadHandler)
+    server.upload_token = token
+    server.serve_forever()

package/plugins/huaweicloud-core/src/sandbox/session-manager.mjs

@@ -1,15 +1,25 @@
-import { spawn } from 'node:child_process';
-import { existsSync, readFileSync, statSync } from 'node:fs';
-import { createHash } from 'node:crypto';
-import { join, dirname } from 'node:path';
+import { spawn, execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { createConnection as netConnect } from 'node:net';
+import { existsSync, readFileSync, statSync, mkdirSync, rmSync, createReadStream, appendFileSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { join, dirname, basename } from 'node:path';
+import { tmpdir } from 'node:os';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { createConnection, getCredentials } from './hwlink-api.mjs';
 import { getWebSocketImpl } from '../proxy/proxy-agent.mjs';
 
+const execFileAsync = promisify(execFile);
+
 const __dirname = dirname(fileURLToPath(import.meta.url));
 export const WS_EXEC_INDEX_URL = pathToFileURL(join(__dirname, '..', 'ws-exec', 'index.js')).href;
 
-const DEFAULT_WORKSPACE_ID = process.env.HW_WORKSPACE_ID || '0107bd9997aa4287bd2b4890b49af07d';
+let currentWorkspaceId = process.env.HW_WORKSPACE_ID || null;
+
+function setWorkspaceId(id) {
+  currentWorkspaceId = id;
+  process.env.HW_WORKSPACE_ID = id;
+}
 
 function resolveEnv() {
   const env = { ...process.env };
@@ -77,6 +87,45 @@ async function getSession(workspaceId, username, timeoutMs) {
   return session;
 }
 
+async function createTunnelSession(workspaceId, username, timeoutMs = 30000) {
+  const { ak, sk, securitytoken } = getCredentials();
+  const { wsUrl, source } = await createConnection(workspaceId, ak, sk, securitytoken);
+  const WebSocketImpl = await getWebSocketImpl(wsUrl);
+
+  const { HwlinkWebSocketMultiplexer } = await import(WS_EXEC_INDEX_URL);
+  const mux = new HwlinkWebSocketMultiplexer(wsUrl, source, { WebSocketImpl, protocol: 'devenv' });
+
+  await new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      clearInterval(interval);
+      reject(new Error('tunnel session WebSocket open timeout'));
+    }, timeoutMs);
+    const interval = setInterval(() => {
+      if (mux.readyState === 1) {
+        clearTimeout(timer);
+        clearInterval(interval);
+        resolve();
+      } else if (mux.readyState === 3) {
+        clearTimeout(timer);
+        clearInterval(interval);
+        reject(new Error('tunnel session WebSocket closed'));
+      }
+    }, 100);
+    mux.onClose = () => {
+      clearTimeout(timer);
+      clearInterval(interval);
+      reject(new Error('tunnel session WebSocket closed'));
+    };
+    mux.onError = (err) => {
+      clearTimeout(timer);
+      clearInterval(interval);
+      reject(err);
+    };
+  });
+
+  return { mux, close: () => mux.close() };
+}
+
 export async function execOneShot(workspaceId, command, username, timeoutMs) {
   const { ak, sk, securitytoken } = getCredentials();
   const { wsUrl, source } = await createConnection(workspaceId, ak, sk, securitytoken);
@@ -99,7 +148,9 @@ export async function execWithSession(workspaceId, command, username, timeoutMs)
   return await session.exec(command, { timeoutMs });
 }
 
-export const UPLOAD_CHUNK_SIZE = 3072;
+export const UPLOAD_CHUNK_SIZE = 30000;
+
+export const UPLOAD_BATCH_SIZE = 5;
 
 export function splitBase64Chunks(base64, chunkSize = UPLOAD_CHUNK_SIZE) {
   const chunks = [];
@@ -127,13 +178,21 @@ export async function uploadFileWithSession(workspaceId, localPath, remotePath,
     throw new Error(`sandbox upload: failed to reset temp file: ${reset.stdout || reset.error || reset.exitCode}`);
   }
 
-  for (const [i, chunk] of chunks.entries()) {
-    const res = await execWithSession(workspaceId, `printf '%s' '${chunk}' >> "${tmp}"`, username, timeoutMs);
+  for (let batchStart = 0; batchStart < chunks.length; batchStart += UPLOAD_BATCH_SIZE) {
+    const batch = chunks.slice(batchStart, batchStart + UPLOAD_BATCH_SIZE);
+    const combinedChunk = batch.join('');
+    const batchNum = Math.floor(batchStart / UPLOAD_BATCH_SIZE) + 1;
+    const totalBatches = Math.ceil(chunks.length / UPLOAD_BATCH_SIZE);
+    const cmd = `printf '%s' '${combinedChunk}' >> "${tmp}"`;
+    const res = await execWithSession(workspaceId, cmd, username, timeoutMs);
     if (res.exitCode !== 0) {
       throw new Error(
-        `sandbox upload: failed writing chunk ${i + 1}/${chunks.length}: ${res.stdout || res.error || res.exitCode}`,
+        `sandbox upload: failed writing batch ${batchNum}/${totalBatches}: ${res.stdout || res.error || res.exitCode}`,
       );
     }
+    if (batchNum % 10 === 0 || batchNum === totalBatches) {
+      console.error(`  upload progress: batch ${batchNum}/${totalBatches}`);
+    }
   }
 
   const decode = await execWithSession(
@@ -152,6 +211,10 @@ export async function uploadFileWithSession(workspaceId, localPath, remotePath,
   let md5Verified = false;
   if (verify.exitCode === 0) {
     const remoteMd5 = String(verify.stdout || '')
+      // eslint-disable-next-line no-control-regex
+      .replace(/\x1b\[[0-9;]*[A-Za-z]/g, '')
+      // eslint-disable-next-line no-control-regex
+      .replace(/\x1b\][^\x07]*\x07/g, '')
       .trim()
       .split(/\s+/)[0];
     md5Verified = remoteMd5 === expectedMd5;
@@ -173,6 +236,365 @@ export async function uploadFileWithSession(workspaceId, localPath, remotePath,
   };
 }
 
+const SANDBOX_FILE_SERVER_SCRIPT = readFileSync(join(__dirname, 'sandbox-file-server.py'), 'utf8');
+
+const TUNNEL_READY_TIMEOUT_MS = 30000;
+const SERVER_HEALTH_MAX_RETRIES = 30;
+const SERVER_HEALTH_INTERVAL_MS = 1000;
+const UPLOAD_LOG_PATH = join(tmpdir(), 'sandbox-upload.log');
+
+function uploadLog(message) {
+  const ts = new Date().toISOString();
+  const line = `[${ts}] ${message}\n`;
+  console.error(line.trimEnd());
+  try {
+    appendFileSync(UPLOAD_LOG_PATH, line);
+  } catch {}
+}
+
+function generateUploadToken() {
+  return randomBytes(16).toString('hex');
+}
+
+async function createTarGz(localDir, exclude = []) {
+  const archiveName = `${basename(localDir)}.tar.gz`;
+  const archiveDir = join(tmpdir(), `sandbox-upload-${Date.now()}`);
+  mkdirSync(archiveDir, { recursive: true });
+  const archivePath = join(archiveDir, archiveName);
+
+  const args = [
+    ...exclude.flatMap((p) => ['--exclude', p]),
+    '-czf',
+    archivePath,
+    '-C',
+    dirname(localDir),
+    basename(localDir),
+  ];
+  await execFileAsync('tar', args);
+  return archivePath;
+}
+
+async function computeMd5(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash('md5');
+    createReadStream(filePath)
+      .on('data', (chunk) => hash.update(chunk))
+      .on('end', () => resolve(hash.digest('hex')))
+      .on('error', reject);
+  });
+}
+
+function cleanupLocalArchive(archivePath) {
+  try {
+    rmSync(dirname(archivePath), { recursive: true, force: true });
+  } catch {}
+}
+
+async function deployFileServer(workspaceId, username, port = 8888, token = '') {
+  const scriptPath = '/tmp/sandbox-file-server.py';
+  const pidFile = '/tmp/sandbox-file-server.pid';
+  uploadLog(`deployFileServer: killing old server (pidFile=${pidFile})`);
+  await execWithSession(workspaceId, `kill $(cat ${pidFile} 2>/dev/null) 2>/dev/null; rm -f ${pidFile}`, username);
+  const b64 = Buffer.from(SANDBOX_FILE_SERVER_SCRIPT).toString('base64');
+  uploadLog(`deployFileServer: writing script (${b64.length} b64 chars)`);
+  await execWithSession(workspaceId, `echo '${b64}' | base64 -d > ${scriptPath}`, username);
+  const cmd = token
+    ? `python3 ${scriptPath} ${port} ${token} & echo $! > ${pidFile}`
+    : `python3 ${scriptPath} ${port} & echo $! > ${pidFile}`;
+  uploadLog(`deployFileServer: starting server on port ${port}`);
+  const startResult = await execWithSession(workspaceId, cmd, username);
+  uploadLog(
+    `deployFileServer: start result exitCode=${startResult.exitCode} stdout=${JSON.stringify(startResult.stdout?.slice(0, 200))}`,
+  );
+  return startResult;
+}
+
+async function uploadViaTunnel(localPort, archivePath, archiveSize, archiveRemotePath, uploadToken, timeoutMs) {
+  const archiveBuffer = readFileSync(archivePath);
+  const headers = [
+    'POST /upload HTTP/1.1',
+    'Host: localhost',
+    'Content-Type: application/octet-stream',
+    `Content-Length: ${archiveSize}`,
+    `X-Target-Path: ${archiveRemotePath}`,
+    `X-Upload-Token: ${uploadToken}`,
+    'Connection: close',
+    '',
+    '',
+  ].join('\r\n');
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const done = (fn) => {
+      if (!settled) {
+        settled = true;
+        clearTimeout(timer);
+        sock.destroy();
+        fn();
+      }
+    };
+    const timer = setTimeout(() => done(() => reject(new Error(`upload timeout after ${timeoutMs}ms`))), timeoutMs);
+    const sock = netConnect({ host: '127.0.0.1', port: localPort }, () => {
+      sock.write(headers);
+      sock.write(archiveBuffer);
+    });
+    let respBuf = Buffer.alloc(0);
+    let contentLength = -1;
+    let headerEnd = -1;
+    sock.on('data', (chunk) => {
+      respBuf = Buffer.concat([respBuf, chunk]);
+      if (contentLength < 0) {
+        const resp = respBuf.toString();
+        headerEnd = resp.indexOf('\r\n\r\n');
+        if (headerEnd > 0) {
+          const headerBlock = resp.slice(0, headerEnd);
+          const clMatch = headerBlock.match(/Content-Length:\s*(\d+)/i);
+          if (clMatch) contentLength = parseInt(clMatch[1], 10);
+        }
+      }
+      if (contentLength >= 0 && headerEnd > 0) {
+        const bodyReceived = respBuf.length - (headerEnd + 4);
+        if (bodyReceived >= contentLength) {
+          done(() => {
+            const resp = respBuf.toString();
+            const statusLine = resp.split('\r\n')[0] || '';
+            const statusCode = parseInt(statusLine.split(' ')[1], 10);
+            const body = resp.slice(headerEnd + 4, headerEnd + 4 + contentLength);
+            if (!statusCode || statusCode < 200 || statusCode >= 300) {
+              uploadLog(`uploadViaTunnel: POST failed with HTTP ${statusCode}: ${body.slice(0, 200)}`);
+              reject(new Error(`upload HTTP ${statusCode}: ${body}`));
+              return;
+            }
+            try {
+              resolve(JSON.parse(body));
+            } catch (e) {
+              reject(new Error(`invalid JSON response: ${body.slice(0, 200)}`));
+            }
+          });
+        }
+      }
+    });
+    sock.on('close', () => {
+      done(() => {
+        const resp = respBuf.toString();
+        const statusLine = resp.split('\r\n')[0] || '';
+        const statusCode = parseInt(statusLine.split(' ')[1], 10);
+        const he = resp.indexOf('\r\n\r\n');
+        const body = he > 0 ? resp.slice(he + 4) : '';
+        if (!statusCode || statusCode < 200 || statusCode >= 300) {
+          uploadLog(`uploadViaTunnel: POST failed with HTTP ${statusCode}: ${body.slice(0, 200)}`);
+          reject(new Error(`upload HTTP ${statusCode}: ${body}`));
+          return;
+        }
+        try {
+          resolve(JSON.parse(body));
+        } catch (e) {
+          reject(new Error(`invalid JSON response: ${body.slice(0, 200)}`));
+        }
+      });
+    });
+    sock.on('error', (err) => {
+      done(() => {
+        uploadLog(`uploadViaTunnel: socket error: ${err.message}`);
+        reject(err);
+      });
+    });
+  });
+}
+
+async function waitForServerReady(localPort) {
+  for (let i = 0; i < SERVER_HEALTH_MAX_RETRIES; i++) {
+    try {
+      uploadLog(`waitForServerReady: attempt ${i + 1}, checking http://localhost:${localPort}/health`);
+      const ok = await new Promise((resolve) => {
+        let settled = false;
+        const done = (val) => {
+          if (!settled) {
+            settled = true;
+            sock.destroy();
+            resolve(val);
+          }
+        };
+        const sock = netConnect({ host: '127.0.0.1', port: localPort }, () => {
+          sock.write('GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n');
+        });
+        let resp = '';
+        sock.on('data', (c) => {
+          resp += c.toString();
+          if (resp.includes('200 OK')) done(true);
+        });
+        sock.on('close', () => done(resp.includes('200 OK')));
+        sock.on('error', () => done(false));
+        setTimeout(() => done(false), 3000);
+      });
+      if (ok) {
+        uploadLog(`waitForServerReady: server ready on port ${localPort} after ${i} retries`);
+        return;
+      }
+      uploadLog(`waitForServerReady: health check returned non-200 (retry ${i + 1})`);
+    } catch (err) {
+      uploadLog(`waitForServerReady: health check failed: ${err.message} (retry ${i + 1})`);
+    }
+    await new Promise((r) => setTimeout(r, SERVER_HEALTH_INTERVAL_MS));
+  }
+  throw new Error(
+    `sandbox file server not ready after ${SERVER_HEALTH_MAX_RETRIES * SERVER_HEALTH_INTERVAL_MS}ms (log: ${UPLOAD_LOG_PATH})`,
+  );
+}
+
+async function cleanupFileServer(workspaceId, username) {
+  const pidFile = '/tmp/sandbox-file-server.pid';
+  const scriptPath = '/tmp/sandbox-file-server.py';
+  await execWithSession(workspaceId, `kill $(cat ${pidFile}) 2>/dev/null; rm -f ${pidFile} ${scriptPath}`, username);
+}
+
+async function uploadViaHttpTunnel(workspaceId, archivePath, archiveRemotePath, username, timeoutMs, options) {
+  const sandboxPort = options.sandboxPort || 8888;
+  const uploadToken = generateUploadToken();
+  const archiveSize = statSync(archivePath).size;
+
+  uploadLog(
+    `uploadViaHttpTunnel: start (archive=${archivePath}, size=${archiveSize}, remotePath=${archiveRemotePath})`,
+  );
+
+  uploadLog(`uploadViaHttpTunnel: deploying file server on sandbox port ${sandboxPort}`);
+  await deployFileServer(workspaceId, username, sandboxPort, uploadToken);
+
+  uploadLog(`uploadViaHttpTunnel: creating dedicated tunnel session`);
+  const tunnelSession = await createTunnelSession(workspaceId, username);
+
+  uploadLog(`uploadViaHttpTunnel: creating tunnel channel (localPort=0, remotePort=${sandboxPort})`);
+  const { HwlinkTunnelChannel } = await import(WS_EXEC_INDEX_URL);
+  const tunnel = new HwlinkTunnelChannel({
+    localPort: 0,
+    remotePort: sandboxPort,
+  });
+  tunnel.attach(tunnelSession.mux);
+
+  uploadLog(`uploadViaHttpTunnel: waiting for tunnel ready (timeout=${TUNNEL_READY_TIMEOUT_MS}ms)`);
+  try {
+    await Promise.race([
+      tunnel.ready,
+      new Promise((_, reject) =>
+        setTimeout(
+          () => reject(new Error(`tunnel ready timeout after ${TUNNEL_READY_TIMEOUT_MS}ms`)),
+          TUNNEL_READY_TIMEOUT_MS,
+        ),
+      ),
+    ]);
+    uploadLog(`uploadViaHttpTunnel: tunnel ready, localPort=${tunnel.localPort}`);
+  } catch (tunnelReadyError) {
+    uploadLog(`uploadViaHttpTunnel: TUNNEL READY FAILED: ${tunnelReadyError.message}`);
+    tunnel.close();
+    throw new Error(
+      `HTTP tunnel failed to establish: ${tunnelReadyError.message}. ` +
+        `This means the WebSocket port-forwarding channel to sandbox port ${sandboxPort} could not be opened. ` +
+        `Common causes: (1) Python file server not running on sandbox, (2) sandbox port ${sandboxPort} blocked, ` +
+        `(3) hwlink multiplexer channel rejected. ` +
+        `Diagnostic log: ${UPLOAD_LOG_PATH}`,
+      { cause: tunnelReadyError },
+    );
+  }
+
+  try {
+    uploadLog(`uploadViaHttpTunnel: waiting for server health on localhost:${tunnel.localPort}`);
+    await waitForServerReady(tunnel.localPort);
+
+    uploadLog(`uploadViaHttpTunnel: sending POST with ${archiveSize} bytes`);
+    const result = await uploadViaTunnel(
+      tunnel.localPort,
+      archivePath,
+      archiveSize,
+      archiveRemotePath,
+      uploadToken,
+      timeoutMs,
+    );
+    uploadLog(`uploadViaHttpTunnel: upload complete (bytes=${result.bytes}, md5=${result.md5})`);
+    return result;
+  } catch (uploadError) {
+    uploadLog(`uploadViaHttpTunnel: UPLOAD FAILED: ${uploadError.message}`);
+    throw uploadError;
+  } finally {
+    tunnel.close();
+    tunnelSession.close();
+  }
+}
+
+export async function uploadProjectWithSession(
+  workspaceId,
+  localDir,
+  remoteDir,
+  username = 'root',
+  timeoutMs = 300000,
+  options = {},
+) {
+  if (!workspaceId) {
+    throw new Error(
+      'sandbox upload project: workspace_id is required. ' +
+        'Set HW_WORKSPACE_ID env var or ensure huaweicloud_sandbox_connect was called first.',
+    );
+  }
+  if (!existsSync(localDir)) {
+    throw new Error(`sandbox upload project: local directory not found: ${localDir}`);
+  }
+  if (!statSync(localDir).isDirectory()) {
+    throw new Error(`sandbox upload project: path is not a directory: ${localDir}`);
+  }
+
+  const projectName = basename(localDir);
+  const targetParentDir = remoteDir || '/workspace';
+  const archiveRemotePath = `${targetParentDir}/${projectName}.tar.gz`;
+
+  const archivePath = await createTarGz(localDir, options.exclude);
+  const archiveSize = statSync(archivePath).size;
+  const expectedMd5 = await computeMd5(archivePath);
+
+  uploadLog(`uploadProject: ${localDir} -> ${archiveRemotePath} (archive=${archiveSize} bytes, md5=${expectedMd5})`);
+
+  let result;
+  try {
+    result = await uploadViaHttpTunnel(workspaceId, archivePath, archiveRemotePath, username, timeoutMs, options);
+  } catch (tunnelError) {
+    uploadLog(`uploadProject: HTTP tunnel upload failed: ${tunnelError.message}`);
+    uploadLog(`uploadProject: NOT falling back to base64 (removed). Rethrowing with diagnostics.`);
+    cleanupLocalArchive(archivePath);
+    throw new Error(
+      `sandbox upload failed: HTTP tunnel could not transfer the project archive. ` +
+        `Archive size: ${(archiveSize / 1024).toFixed(1)}KB. ` +
+        `Root cause: ${tunnelError.message}. ` +
+        `Diagnostic log: ${UPLOAD_LOG_PATH}`,
+      { cause: tunnelError },
+    );
+  }
+
+  if (options.verify !== false && result.md5 && result.md5 !== expectedMd5) {
+    throw new Error(`md5 mismatch: expected ${expectedMd5}, got ${result.md5}`);
+  }
+
+  if (options.extract !== false) {
+    await execWithSession(
+      workspaceId,
+      `mkdir -p "${targetParentDir}" && tar -xzf "${archiveRemotePath}" -C "${targetParentDir}" && rm -f "${archiveRemotePath}"`,
+      username,
+      timeoutMs,
+    );
+  }
+
+  try {
+    await cleanupFileServer(workspaceId, username);
+  } catch {}
+  cleanupLocalArchive(archivePath);
+
+  return {
+    ok: true,
+    localDir,
+    remotePath: options.extract !== false ? `${targetParentDir}/${projectName}` : archiveRemotePath,
+    bytes: result.bytes || 0,
+    md5: result.md5 || expectedMd5,
+    md5Verified: result.md5 ? result.md5 === expectedMd5 : true,
+    extracted: options.extract !== false,
+  };
+}
+
 export async function closeSession(workspaceId, username) {
   const key = `${workspaceId}:${username}`;
   const session = sessions.get(key);
@@ -193,4 +615,4 @@ export async function closeAllSessions() {
   }
 }
 
-export { DEFAULT_WORKSPACE_ID, runNodeExec };
+export { currentWorkspaceId, setWorkspaceId, runNodeExec };