@myagentroam/marctl 0.9.120

package/src/scp-worker.cjs

@@ -0,0 +1,254 @@
+const { createHash, randomUUID } = require('node:crypto');
+const { constants } = require('node:fs');
+const { link, lstat, open, rename, unlink } = require('node:fs/promises');
+const path = require('node:path');
+
+const CHUNK = 32 * 1024;
+const [mode, encodedPath, sizeArgument, encodedMetadata, preserveArgument, encodedSourceBasename] =
+  process.argv.slice(1);
+const target = Buffer.from(encodedPath ?? '', 'base64url').toString('utf8');
+const sourceBasename = Buffer.from(encodedSourceBasename ?? '', 'base64url').toString('utf8');
+const input = process.stdin[Symbol.asyncIterator]();
+let pending = Buffer.alloc(0);
+
+async function readExact(length) {
+  while (pending.length < length) {
+    const next = await input.next();
+    if (next.done) throw new Error('SCP_STREAM_TRUNCATED');
+    pending = Buffer.concat([pending, next.value]);
+    if (pending.length > CHUNK * 4) throw new Error('SCP_INPUT_OVERFLOW');
+  }
+  const data = pending.subarray(0, length);
+  pending = pending.subarray(length);
+  return data;
+}
+
+async function send(data) {
+  await new Promise((resolve, reject) => {
+    process.stdout.write(data, (error) => (error ? reject(error) : resolve()));
+  });
+}
+
+function frameLength(size) {
+  const buffer = Buffer.alloc(4);
+  buffer.writeUInt32BE(size);
+  return buffer;
+}
+
+function fileMetadata(stat) {
+  const bytes = Buffer.alloc(20);
+  bytes.writeUInt32BE(stat.mode & 0o777, 0);
+  bytes.writeDoubleBE(stat.atimeMs, 4);
+  bytes.writeDoubleBE(stat.mtimeMs, 12);
+  return bytes;
+}
+
+function appendFilename(directory, filename) {
+  const separator =
+    directory.endsWith(path.sep) || (process.platform === 'win32' && directory.endsWith('/'))
+      ? ''
+      : path.sep;
+  return `${directory}${separator}${filename}`;
+}
+
+async function destinationPath() {
+  const targetPath = process.platform === 'win32' ? path.win32 : path.posix;
+  const directoryIntent = (
+    process.platform === 'win32' ? /(?:^|[\\/])\.{0,2}$/u : /(?:^|\/)\.{0,2}$/u
+  ).test(target);
+  // A trailing slash or dot must not disguise the final directory symlink.
+  let inspected = target;
+  const trailingDirectorySyntax = process.platform === 'win32' ? /[\\/]+\.?$/u : /\/+\.?$/u;
+  for (;;) {
+    const trimmed = inspected.replace(trailingDirectorySyntax, '');
+    if (trimmed === inspected || trimmed.length < targetPath.parse(inspected).root.length) break;
+    inspected = trimmed;
+  }
+  let previous;
+  try {
+    previous = await lstat(inspected);
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error;
+    if (directoryIntent) throw new Error('SCP_TARGET_INVALID');
+    return target;
+  }
+  if (previous.isDirectory() && !previous.isSymbolicLink()) {
+    if (
+      !sourceBasename ||
+      sourceBasename === '.' ||
+      sourceBasename === '..' ||
+      sourceBasename.includes('\0') ||
+      targetPath.basename(sourceBasename) !== sourceBasename ||
+      (process.platform === 'win32' &&
+        (/[<>:"/\\|?*\x00-\x1f]/u.test(sourceBasename) ||
+          /[. ]$/u.test(sourceBasename) ||
+          /^(?:CON|PRN|AUX|NUL|COM[1-9¹²³]|LPT[1-9¹²³])(?:\.|$)/iu.test(sourceBasename)))
+    )
+      throw new Error('SCP_SOURCE_FILENAME_INVALID');
+    return appendFilename(target, sourceBasename);
+  }
+  if (directoryIntent) throw new Error('SCP_TARGET_INVALID');
+  return target;
+}
+
+async function source() {
+  const before = await lstat(target);
+  if (!before.isFile() || before.isSymbolicLink() || !Number.isSafeInteger(before.size))
+    throw new Error('SCP_SOURCE_INVALID');
+  const handle = await open(target, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    const current = await handle.stat();
+    if (
+      (before.dev !== current.dev && !(process.platform === 'win32' && before.dev === 0)) ||
+      before.ino !== current.ino ||
+      before.size !== current.size
+    )
+      throw new Error('SCP_SOURCE_CHANGED');
+    const size = Buffer.alloc(8);
+    size.writeBigUInt64BE(BigInt(current.size));
+    await send(size);
+    await send(fileMetadata(current));
+    const hash = createHash('sha256');
+    const buffer = Buffer.alloc(CHUNK);
+    let offset = 0;
+    while (offset < current.size) {
+      const { bytesRead } = await handle.read(
+        buffer,
+        0,
+        Math.min(CHUNK, current.size - offset),
+        offset
+      );
+      if (bytesRead === 0) throw new Error('SCP_SOURCE_CHANGED');
+      hash.update(buffer.subarray(0, bytesRead));
+      await send(Buffer.concat([frameLength(bytesRead), buffer.subarray(0, bytesRead)]));
+      if ((await readExact(1))[0] !== 1) throw new Error('SCP_ACK_INVALID');
+      offset += bytesRead;
+    }
+    const after = await handle.stat();
+    if (
+      after.size !== current.size ||
+      after.mtimeMs !== current.mtimeMs ||
+      after.ctimeMs !== current.ctimeMs ||
+      after.ino !== current.ino
+    )
+      throw new Error('SCP_SOURCE_CHANGED');
+    await send(Buffer.concat([frameLength(0), hash.digest()]));
+  } finally {
+    await handle.close();
+  }
+}
+
+async function destination() {
+  const destinationTarget = await destinationPath();
+  const size = Number(sizeArgument);
+  if (!Number.isSafeInteger(size) || size < 0) throw new Error('SCP_SIZE_INVALID');
+  const metadata = Buffer.from(encodedMetadata ?? '', 'base64url');
+  if (metadata.length !== 20) throw new Error('SCP_METADATA_INVALID');
+  const sourceMode = metadata.readUInt32BE(0);
+  const atimeMs = metadata.readDoubleBE(4);
+  const mtimeMs = metadata.readDoubleBE(12);
+  if (sourceMode > 0o777 || !Number.isFinite(atimeMs) || !Number.isFinite(mtimeMs))
+    throw new Error('SCP_METADATA_INVALID');
+  const filename = path.basename(destinationTarget);
+  const temporary = appendFilename(
+    path.dirname(destinationTarget),
+    `.${filename}.marctl-${randomUUID()}.tmp`
+  );
+  const handle = await open(temporary, 'wx', 0o600);
+  let published = false;
+  try {
+    let previous;
+    try {
+      previous = await lstat(destinationTarget);
+      if (!previous.isFile() || previous.isSymbolicLink()) throw new Error('SCP_TARGET_INVALID');
+      const existing = await open(
+        destinationTarget,
+        constants.O_WRONLY | (constants.O_NOFOLLOW ?? 0)
+      );
+      try {
+        const accessible = await existing.stat();
+        if (
+          (!(process.platform === 'win32' && previous.dev === 0) &&
+            previous.dev !== accessible.dev) ||
+          previous.ino !== accessible.ino
+        )
+          throw new Error('SCP_TARGET_CHANGED');
+      } finally {
+        await existing.close();
+      }
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error;
+    }
+    await send(Buffer.from([1]));
+    const hash = createHash('sha256');
+    let received = 0;
+    for (;;) {
+      const length = (await readExact(4)).readUInt32BE();
+      if (length === 0) break;
+      if (length > CHUNK || received + length > size) throw new Error('SCP_SIZE_INVALID');
+      const bytes = await readExact(length);
+      hash.update(bytes);
+      let written = 0;
+      while (written < bytes.length) {
+        const result = await handle.write(
+          bytes,
+          written,
+          bytes.length - written,
+          received + written
+        );
+        written += result.bytesWritten;
+      }
+      received += length;
+      await send(Buffer.from([1]));
+    }
+    const expectedHash = await readExact(32);
+    if (received !== size || !hash.digest().equals(expectedHash))
+      throw new Error('SCP_CHECKSUM_MISMATCH');
+    if (preserveArgument === '1') await handle.utimes(new Date(atimeMs), new Date(mtimeMs));
+    await handle.chmod(
+      preserveArgument === '1'
+        ? sourceMode
+        : previous === undefined
+          ? sourceMode & ~process.umask()
+          : previous.mode & 0o777
+    );
+    await handle.sync();
+    await handle.close();
+    if (previous === undefined) {
+      await link(temporary, destinationTarget);
+    } else {
+      const current = await lstat(destinationTarget);
+      if (
+        !current.isFile() ||
+        current.isSymbolicLink() ||
+        previous.dev !== current.dev ||
+        previous.ino !== current.ino ||
+        previous.size !== current.size ||
+        previous.mtimeMs !== current.mtimeMs ||
+        previous.ctimeMs !== current.ctimeMs
+      )
+        throw new Error('SCP_TARGET_CHANGED');
+      await rename(temporary, destinationTarget);
+    }
+    published = true;
+    await send(Buffer.from([2]));
+  } finally {
+    await handle.close().catch(() => undefined);
+    await unlink(temporary).catch(() => undefined);
+    if (!published) process.exitCode = 1;
+  }
+}
+
+async function main() {
+  if (!path.isAbsolute(target) || target.includes('\0')) throw new Error('SCP_PATH_INVALID');
+  if (mode === 'read') await source();
+  else if (mode === 'write') await destination();
+  else throw new Error('SCP_MODE_INVALID');
+}
+
+void main()
+  .catch((error) => {
+    process.stderr.write(`${error instanceof Error ? error.message : 'SCP_FAILED'}\n`);
+    process.exitCode = 1;
+  })
+  .finally(() => process.stdin.destroy());

package/src/scp.mjs

@@ -0,0 +1,321 @@
+import { spawn } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import { gzipSync } from 'node:zlib';
+import { MarctlError } from './config.mjs';
+import { parseFrame, runTerminalSession } from './terminal.mjs';
+
+const CHUNK = 32 * 1024;
+const script = gzipSync(readFileSync(new URL('./scp-worker.cjs', import.meta.url))).toString(
+  'base64'
+);
+const expression = `eval(require('zlib').gunzipSync(Buffer.from('${script}','base64')).toString())`;
+
+export function parseScpEndpoint(value) {
+  if (typeof value !== 'string' || !value || value.includes('\0'))
+    throw new MarctlError('Invalid copy endpoint.');
+  const explicit = value.indexOf('://');
+  const separator = explicit >= 0 ? explicit : value.indexOf(':');
+  if (separator < 0 || (separator === 1 && /^[A-Za-z]:[\\/]/u.test(value))) {
+    // Let the filesystem resolve dot components and symlink parents.
+    return { path: path.isAbsolute(value) ? value : `${process.cwd()}${path.sep}${value}` };
+  }
+  const host = value.slice(0, separator);
+  const file = value.slice(separator + (explicit >= 0 ? 3 : 1));
+  if (!host || !file || (!file.startsWith('/') && !/^[A-Za-z]:[\\/]/u.test(file)))
+    throw new MarctlError('Invalid Node file path.');
+  return { host, path: file };
+}
+
+export function scpSourceBasename(value) {
+  const windowsPath = /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith('\\\\');
+  if (value.endsWith('/') || (windowsPath && value.endsWith('\\')))
+    throw new MarctlError('Invalid source filename.');
+  const sourcePath = windowsPath ? path.win32 : path.posix;
+  const basename = sourcePath.basename(value);
+  if (!basename || basename === '.' || basename === '..')
+    throw new MarctlError('Invalid source filename.');
+  return basename;
+}
+
+function workerArguments(mode, file, size, metadata, preserve, sourceBasename) {
+  const encoded = Buffer.from(file, 'utf8').toString('base64url');
+  return [
+    mode,
+    encoded,
+    ...(size === undefined
+      ? []
+      : [
+          String(size),
+          metadata.toString('base64url'),
+          preserve ? '1' : '0',
+          Buffer.from(sourceBasename, 'utf8').toString('base64url')
+        ])
+  ];
+}
+
+function workerCommand(mode, file, size, metadata, preserve, sourceBasename) {
+  return `node -e "${expression}" ${workerArguments(mode, file, size, metadata, preserve, sourceBasename).join(' ')}`;
+}
+
+class ByteChannel {
+  #buffer = Buffer.alloc(0);
+  #waiting = [];
+  #ended;
+  #failure;
+
+  constructor(send, signal) {
+    this.send = (bytes) => {
+      if (this.#failure) throw this.#failure;
+      return send(bytes);
+    };
+    if (signal) {
+      if (signal.aborted) this.fail(new MarctlError('Copy was interrupted.'));
+      else
+        signal.addEventListener(
+          'abort',
+          () => this.fail(new MarctlError('Copy was interrupted.')),
+          {
+            once: true
+          }
+        );
+    }
+  }
+
+  feed(bytes) {
+    if (this.#failure) return;
+    this.#buffer = Buffer.concat([this.#buffer, bytes]);
+    if (this.#buffer.length > CHUNK * 4)
+      return this.fail(new MarctlError('Copy output exceeded the bounded window.'));
+    this.#wake();
+  }
+
+  finish(code) {
+    this.#ended = code;
+    this.#wake();
+  }
+
+  fail(error) {
+    this.#failure ??= error;
+    this.#wake();
+  }
+
+  #wake() {
+    for (const resolve of this.#waiting.splice(0)) resolve();
+  }
+
+  async read(length) {
+    if (this.#failure) throw this.#failure;
+    while (this.#buffer.length < length) {
+      if (this.#failure) throw this.#failure;
+      if (this.#ended !== undefined) throw new MarctlError('Copy stream ended unexpectedly.');
+      await new Promise((resolve) => this.#waiting.push(resolve));
+    }
+    const bytes = this.#buffer.subarray(0, length);
+    this.#buffer = this.#buffer.subarray(length);
+    return bytes;
+  }
+
+  async done() {
+    if (this.#failure) throw this.#failure;
+    while (this.#ended === undefined && this.#failure === undefined)
+      await new Promise((resolve) => this.#waiting.push(resolve));
+    if (this.#failure) throw this.#failure;
+    if (this.#ended !== 0) throw new MarctlError('Remote copy command failed.');
+  }
+}
+
+async function copyBytes(source, destination) {
+  const sizeBytes = await source.read(8);
+  const size = sizeBytes.readBigUInt64BE();
+  if (size > BigInt(Number.MAX_SAFE_INTEGER)) throw new MarctlError('Copy source is too large.');
+  const metadata = await source.read(20);
+  await destination.open(Number(size), metadata, async (target) => {
+    if ((await target.read(1))[0] !== 1)
+      throw new MarctlError('Copy target did not accept the file.');
+    let remaining = Number(size);
+    for (;;) {
+      const header = await source.read(4);
+      const length = header.readUInt32BE();
+      if (length > CHUNK || length > remaining)
+        throw new MarctlError('Copy source sent an invalid chunk.');
+      await target.send(header);
+      if (length === 0) {
+        await target.send(await source.read(32));
+        if (remaining !== 0 || (await target.read(1))[0] !== 2)
+          throw new MarctlError('Copy target did not verify the file.');
+        break;
+      }
+      await target.send(await source.read(length));
+      if ((await target.read(1))[0] !== 1) throw new MarctlError('Copy target rejected the chunk.');
+      await source.send(Buffer.from([1]));
+      remaining -= length;
+    }
+    await target.done();
+  });
+  await source.done();
+}
+
+async function withLocalWorker(mode, file, size, metadata, preserve, sourceBasename, use, signal) {
+  const child = spawn(
+    process.execPath,
+    ['-e', expression, ...workerArguments(mode, file, size, metadata, preserve, sourceBasename)],
+    {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      windowsHide: true
+    }
+  );
+  child.stdin.on('error', () => undefined);
+  child.stderr.resume();
+  const channel = new ByteChannel((bytes) => child.stdin.write(bytes), signal);
+  child.stdout.on('data', (bytes) => channel.feed(bytes));
+  child.once('error', (error) => channel.fail(error));
+  child.once('close', (code) => channel.finish(code ?? 1));
+  try {
+    return await use(channel);
+  } finally {
+    if (child.exitCode === null) child.kill();
+  }
+}
+
+async function withRemoteWorker(
+  {
+    client,
+    WebSocketClass,
+    nodeId,
+    file,
+    mode,
+    size,
+    metadata,
+    preserve,
+    sourceBasename,
+    translate,
+    signal
+  },
+  use
+) {
+  return runTerminalSession({
+    client,
+    WebSocketClass,
+    nodeId,
+    command: workerCommand(mode, file, size, metadata, preserve, sourceBasename),
+    binary: true,
+    raceConnections: true,
+    translate,
+    attached: async ({ transport, terminalId }) => {
+      const channel = new ByteChannel(
+        (bytes) =>
+          transport.send(
+            JSON.stringify({ type: 'input', terminalId, data: bytes.toString('base64') })
+          ),
+        signal
+      );
+      let sequence = 0;
+      const onMessage = (raw, isBinary) => {
+        try {
+          const frame = parseFrame(raw, isBinary, terminalId, translate);
+          if (frame.type === 'output') {
+            if (frame.sequence !== sequence + 1)
+              throw new MarctlError('Copy output sequence is invalid.');
+            sequence = frame.sequence;
+            const bytes = Buffer.from(frame.data, 'base64');
+            if (bytes.toString('base64') !== frame.data)
+              throw new MarctlError('Copy output is not valid base64.');
+            channel.feed(bytes);
+          } else if (frame.type === 'exit') channel.finish(frame.exitCode ?? 1);
+          else if (frame.type === 'error')
+            channel.fail(new MarctlError(translate('errors.terminalRemoteError')));
+          else throw new MarctlError(translate('errors.terminalUnexpectedFrame'));
+        } catch (error) {
+          channel.fail(error);
+        }
+      };
+      const onError = () =>
+        channel.fail(new MarctlError(translate('errors.terminalConnectionFailed')));
+      const onClose = () =>
+        channel.fail(new MarctlError(translate('errors.terminalConnectionLost')));
+      transport.on('message', onMessage);
+      transport.on('error', onError);
+      transport.on('close', onClose);
+      try {
+        transport.activate?.();
+        return await use(channel);
+      } finally {
+        transport.removeListener?.('message', onMessage);
+        transport.removeListener?.('error', onError);
+        transport.removeListener?.('close', onClose);
+      }
+    }
+  });
+}
+
+async function withWorker(endpoint, mode, size, metadata, context, use) {
+  if (endpoint.nodeId === undefined)
+    return withLocalWorker(
+      mode,
+      endpoint.path,
+      size,
+      metadata,
+      context.preserve,
+      context.sourceBasename,
+      use,
+      context.signal
+    );
+  return withRemoteWorker(
+    {
+      ...context,
+      nodeId: endpoint.nodeId,
+      file: endpoint.path,
+      mode,
+      size,
+      metadata
+    },
+    use
+  );
+}
+
+export async function runScp({
+  source,
+  destination,
+  preserve = false,
+  client,
+  WebSocketClass,
+  resolveNodeId,
+  translate,
+  signals = process
+}) {
+  const from = parseScpEndpoint(source);
+  const to = parseScpEndpoint(destination);
+  const sourceBasename = scpSourceBasename(from.path);
+  if (!from.host && !to.host) throw new MarctlError('At least one copy endpoint must be a Node.');
+  const references = [...new Set([from.host, to.host].filter(Boolean))];
+  const ids = await Promise.all(references.map((host) => resolveNodeId(host)));
+  const resolved = (endpoint) => ({
+    path: endpoint.path,
+    ...(endpoint.host ? { nodeId: ids[references.indexOf(endpoint.host)] } : {})
+  });
+  const controller = new AbortController();
+  const interrupt = () => controller.abort();
+  signals.on?.('SIGINT', interrupt);
+  signals.on?.('SIGTERM', interrupt);
+  try {
+    const context = {
+      client,
+      WebSocketClass,
+      translate,
+      signal: controller.signal,
+      preserve,
+      sourceBasename
+    };
+    await withWorker(resolved(from), 'read', undefined, undefined, context, (channel) =>
+      copyBytes(channel, {
+        open: (size, metadata, use) =>
+          withWorker(resolved(to), 'write', size, metadata, context, use)
+      })
+    );
+    return 0;
+  } finally {
+    signals.removeListener?.('SIGINT', interrupt);
+    signals.removeListener?.('SIGTERM', interrupt);
+  }
+}