@vantran-se/openchamber-web 2.0.4 → 2.0.6

package/server/lib/spaces/code-transfer.js

@@ -0,0 +1,173 @@
+// What code in and code out share: the host git session that starts nothing of the user's, the one
+// git version floor of the feature, the work tree check, the `ext::` URL, and the time limits of a
+// transfer. Both directions follow docs/isolated-spaces/stage-0/e4-git-over-exec.md.
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { SpaceError } from './errors.js';
+import { isGitVersionAtLeast, parseGitVersion } from './host-git.js';
+import { requireSpaceId } from './labels.js';
+
+// The oldest host git the feature works with. `fetch --no-write-fetch-head` first appears in the
+// git-fetch manual of 2.29.0 (the 2.28.0 page does not have it), and without it code out would write
+// `FETCH_HEAD` into the user's `.git`. Code in needs 2.27.0: `git init --object-format`. Every other
+// option either direction uses is older. See DOCUMENTATION.md, "Host git".
+const GIT_VERSION_FLOOR = Object.freeze({ major: 2, minor: 29, patch: 0 });
+const GIT_VERSION_FLOOR_TEXT = '2.29';
+
+// Commits the host or the fixed scripts write carry a fixed identity, so a user without `user.name`
+// can still create a space and bring its work out.
+export const SNAPSHOT_IDENTITY = Object.freeze({
+  GIT_AUTHOR_NAME: 'OpenChamber',
+  GIT_AUTHOR_EMAIL: 'spaces@openchamber.invalid',
+  GIT_COMMITTER_NAME: 'OpenChamber',
+  GIT_COMMITTER_EMAIL: 'spaces@openchamber.invalid',
+});
+
+// A git `ext::` URL splits on spaces itself and cannot carry these at all.
+export const EXT_UNCARRIABLE = /[\x00-\x1f\x7f]/;
+
+// The limit inside is the host's limit plus this margin, so it never ends a transfer the host still
+// waits for, and a transfer the host killed leaves nothing running inside once it passes.
+export const INNER_MARGIN_SECONDS = 5;
+const MAX_INNER_MARGIN_SECONDS = 3600;
+const MIN_TIMEOUT_MS = 1_000;
+const MAX_TIMEOUT_MS = 24 * 60 * 60_000;
+
+export const spaceRefPrefix = (spaceId) => `refs/openchamber/spaces/${requireSpaceId(spaceId)}/`;
+export const startRef = (spaceId) => `${spaceRefPrefix(spaceId)}start`;
+
+/** One line of git output, without its line end. */
+export const line = (text) => text.replace(/\r?\n$/, '');
+export const zeroObjectId = (objectFormat) => '0'.repeat(objectFormat === 'sha256' ? 64 : 40);
+export const objectIdPattern = (objectFormat) => (objectFormat === 'sha256' ? /^[0-9a-f]{64}$/ : /^[0-9a-f]{40}$/);
+export const innerSeconds = (timeoutMs, marginSeconds) => Math.ceil(timeoutMs / 1000) + marginSeconds;
+
+/** The host's limit on a transfer, in whole milliseconds from a second to a day. */
+export const requireTimeout = (value) => {
+  if (!Number.isInteger(value) || value < MIN_TIMEOUT_MS || value > MAX_TIMEOUT_MS) {
+    throw new SpaceError('invalid_timeout', `The time limit of a transfer is a whole number of milliseconds from ${MIN_TIMEOUT_MS} to ${MAX_TIMEOUT_MS}`);
+  }
+  return value;
+};
+
+/** The margin between the host's limit and the one inside, in whole seconds: a length, and nothing else. */
+export const requireInnerMargin = (value) => {
+  if (!Number.isInteger(value) || value < 1 || value > MAX_INNER_MARGIN_SECONDS) {
+    throw new SpaceError('invalid_inner_margin', `The margin of the limit inside the space is a whole number of seconds from 1 to ${MAX_INNER_MARGIN_SECONDS}`);
+  }
+  return value;
+};
+
+/**
+ * A git `ext::` URL for an argv. git-remote-ext splits the text after `ext::` on spaces itself, so a
+ * space inside an argument is written `% ` and a percent `%%`, as its documentation says. The docker
+ * CLI on Windows can live under a path with spaces. An empty argument or one with a control
+ * character cannot be carried, and is refused. The URL goes to git as one argv element, never
+ * through a shell.
+ */
+export function buildExtUrl(argv) {
+  if (!Array.isArray(argv) || argv.length === 0) {
+    throw new SpaceError('invalid_ext_argument', 'A command for git is a non-empty list of arguments');
+  }
+  const escaped = argv.map((argument) => {
+    const text = String(argument);
+    if (text === '' || EXT_UNCARRIABLE.test(text)) {
+      throw new SpaceError('invalid_ext_argument', 'A command for git cannot carry an empty argument or a control character');
+    }
+    return text.replaceAll('%', '%%').replaceAll(' ', '% ');
+  });
+  return `ext::${escaped.join(' ')}`;
+}
+
+const removeWithRetries = (directory) => fs.rm(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
+
+/**
+ * The host side of one direction. `git` is a host git from `createHostGit`, `temporaryDirectory` where
+ * a call's temporary folder lives, `removeDirectory` how it goes, injectable so a test can make it
+ * fail. `name` is how messages call the direction, `code in` or `code out`.
+ */
+export function createTransferSession({ git, temporaryDirectory, removeDirectory = removeWithRetries, name }) {
+  const Name = `${name[0].toUpperCase()}${name.slice(1)}`;
+
+  /**
+   * Runs `work(g, directory)` with a temporary directory and a host git `g` that starts nothing of
+   * the user's. `core.hooksPath` points at an empty folder of ours: without it the user's
+   * `post-index-change` hook ran three times per snapshot, with `GIT_INDEX_FILE` pointing at our
+   * copy, and their `reference-transaction` hook ran on every write of our ref. `core.fsmonitor` is
+   * empty: a user's fsmonitor program ran ten times per snapshot, and `core.fsmonitor=true` left a
+   * `git fsmonitor--daemon` running on the host with its files in the user's `.git`. Empty, and not
+   * `false`, because before git 2.36 the value is a command path, where `false` names a program.
+   *
+   * A failure that is not a SpaceError becomes one with `failureCode`. Removing the folder is retried,
+   * and it never replaces the error of the work: when it still fails, the error says which folder was
+   * left in `details.temporaryDirectoryLeft`. After a successful call a leftover folder is let go.
+   */
+  const withHostGit = async (failureCode, work) => {
+    let directory;
+    try {
+      directory = await fs.mkdtemp(path.join(temporaryDirectory, `openchamber-${name.replace(' ', '-')}-`));
+    } catch (error) {
+      throw new SpaceError(failureCode, `Could not make a temporary folder for ${name}: ${error.message}`, { step: 'make a temporary folder', cause: error.code ?? null });
+    }
+    let outcome;
+    let failed = false;
+    try {
+      const hooks = path.join(directory, 'no-hooks');
+      await fs.mkdir(hooks);
+      const isolated = ['-c', `core.hooksPath=${hooks.replaceAll('\\', '/')}`, '-c', 'core.fsmonitor='];
+      const g = {
+        run: (where, args, options) => git.run(where, [...isolated, ...args], options),
+        output: (where, args, options) => git.output(where, [...isolated, ...args], options),
+      };
+      outcome = await work(g, directory);
+    } catch (error) {
+      failed = true;
+      outcome = error instanceof SpaceError
+        ? error
+        : new SpaceError(failureCode, `${Name} failed on the host: ${error.message}`, { step: 'host side', cause: error.code ?? null });
+    }
+    const left = await removeDirectory(directory).then(() => false, () => true);
+    if (failed) {
+      if (left) outcome.details = { ...outcome.details, temporaryDirectoryLeft: directory };
+      throw outcome;
+    }
+    return outcome;
+  };
+
+  /** Refuses a host git older than the floor, with the version it needs. */
+  const requireGitVersion = async (g, directory) => {
+    const unreadable = (cause) => new SpaceError('git_version_unreadable', `Could not tell which git this computer has. ${Name} needs git to be installed and working.`, { cause });
+    let result;
+    try {
+      result = await g.run(directory, ['version']);
+    } catch (error) {
+      // A git that cannot be started at all, missing among them.
+      throw unreadable(error.code ?? null);
+    }
+    const version = result.code === 0 ? parseGitVersion(result.stdout) : null;
+    if (!version) {
+      throw unreadable(null);
+    }
+    if (!isGitVersionAtLeast(version, GIT_VERSION_FLOOR)) {
+      throw new SpaceError('git_too_old', `${Name} needs git ${GIT_VERSION_FLOOR_TEXT} or newer, and this computer has git ${version.major}.${version.minor}.${version.patch}. Update git, then try again.`, { version });
+    }
+  };
+
+  /** The top level of the work tree that `repository` is in, as a native path, or a refusal. */
+  const requireWorkTree = async (g, repository) => {
+    const found = await fs.stat(repository).then((stats) => stats.isDirectory(), () => false);
+    if (!found) {
+      throw new SpaceError('project_folder_missing', `${repository} does not exist, or is not a folder.`);
+    }
+    const inside = await g.run(repository, ['rev-parse', '--is-inside-work-tree']);
+    if (inside.code !== 0 || line(inside.stdout) !== 'true') {
+      throw new SpaceError('not_a_git_work_tree', `${repository} is not inside a git working tree. Code can travel between a space and this computer only for a git repository.`);
+    }
+    // git answers with forward slashes on Windows too.
+    return path.resolve(line(await g.output(repository, ['rev-parse', '--show-toplevel'])));
+  };
+
+  return { withHostGit, requireGitVersion, requireWorkTree };
+}

package/server/lib/spaces/command-stream.test.js

@@ -0,0 +1,181 @@
+// `openCommandStream` of run-command.js, and the bridge of `connect` from layout.js, run on the
+// host with this Node: the stream that the dispatcher's agent uses as a socket.
+
+import crypto from 'node:crypto';
+import http from 'node:http';
+import net from 'node:net';
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { CONNECT_BRIDGE_PROGRAM } from './layout.js';
+import { openCommandStream } from './run-command.js';
+
+const node = process.execPath;
+const ECHO = 'process.stdin.pipe(process.stdout)';
+const servers = [];
+const streams = [];
+
+const listen = (server) => new Promise((resolve) => {
+  server.listen(0, '127.0.0.1', () => { servers.push(server); resolve(server.address().port); });
+});
+const open = (file, args) => {
+  const stream = openCommandStream(file, args);
+  streams.push(stream);
+  return stream;
+};
+const bridgeTo = (port) => open(node, ['-e', CONNECT_BRIDGE_PROGRAM, '127.0.0.1', String(port)]);
+const closed = (stream) => new Promise((resolve) => { if (stream.closed) resolve(); else stream.once('close', resolve); });
+const failure = (stream) => new Promise((resolve) => stream.once('error', resolve));
+const collect = (stream) => new Promise((resolve) => {
+  const chunks = [];
+  stream.on('data', (chunk) => chunks.push(chunk));
+  stream.on('end', () => resolve(Buffer.concat(chunks)));
+});
+const alive = (pid) => {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+afterEach(async () => {
+  for (const stream of streams.splice(0)) {
+    stream.destroy();
+    await closed(stream);
+  }
+  for (const server of servers.splice(0)) await new Promise((resolve) => server.close(() => resolve()));
+});
+
+describe('openCommandStream', () => {
+  it('carries bytes both ways and ends when the command ends', async () => {
+    const stream = open(node, ['-e', ECHO]);
+    const sent = crypto.randomBytes(1024 * 1024);
+    const received = collect(stream);
+    stream.end(sent);
+    expect((await received).equals(sent)).toBe(true);
+    await closed(stream);
+    expect(stream.errored).toBeNull();
+  });
+
+  it('ends with an error that carries the command\'s stderr when it fails before it answers', async () => {
+    const stream = open(node, ['-e', 'process.stderr.write("no route to the server"); process.exit(3)']);
+    stream.resume();
+    const error = await failure(stream);
+    expect(error).toMatchObject({ code: 'command_stream_failed', message: expect.stringMatching(/exited with code 3: no route to the server/) });
+  });
+
+  it('ends with an error when the command answers nothing and leaves quietly', async () => {
+    const stream = open(node, ['-e', 'process.exit(0)']);
+    stream.resume();
+    expect(await failure(stream)).toMatchObject({ code: 'command_stream_failed' });
+  });
+
+  it('kills the command when it is destroyed', async () => {
+    const stream = open(node, ['-e', 'setInterval(() => {}, 1000)']);
+    const { pid } = stream;
+    stream.destroy();
+    await closed(stream);
+    for (let attempt = 0; attempt < 50 && alive(pid); attempt += 1) await new Promise((resolve) => { setTimeout(resolve, 20); });
+    expect(alive(pid)).toBe(false);
+  });
+
+  it('emits timeout after the given silence, and not while bytes flow', async () => {
+    const stream = open(node, ['-e', ECHO]);
+    stream.resume();
+    let timeouts = 0;
+    stream.setTimeout(150, () => { timeouts += 1; });
+    for (let round = 0; round < 4; round += 1) {
+      stream.write('tick');
+      await new Promise((resolve) => { setTimeout(resolve, 60); });
+    }
+    expect(timeouts).toBe(0);
+    await new Promise((resolve) => { setTimeout(resolve, 300); });
+    expect(timeouts).toBe(1);
+    expect(stream.timeout).toBe(150);
+  });
+
+  it('answers a command that cannot start with an error, not an exception', async () => {
+    const stream = open('/nonexistent/program/of-ours', []);
+    stream.resume();
+    expect(await failure(stream)).toMatchObject({ code: 'command_spawn_failed' });
+  });
+});
+
+describe('the connect bridge', () => {
+  it('carries HTTP requests over a keep-alive agent, many on one command', async () => {
+    let requests = 0;
+    const port = await listen(http.createServer((request, response) => {
+      requests += 1;
+      let bytes = 0;
+      request.on('data', (chunk) => { bytes += chunk.length; });
+      request.on('end', () => response.end(JSON.stringify({ url: request.url, bytes, big: 'y'.repeat(300_000) })));
+    }));
+    let opened = 0;
+    const agent = new http.Agent({ keepAlive: true, maxSockets: 1 });
+    agent.createConnection = (_options, callback) => { opened += 1; callback(null, bridgeTo(port)); };
+    const call = (path, body = null) => new Promise((resolve, reject) => {
+      const request = http.request({ agent, host: '127.0.0.1', port: 9, path, method: body ? 'POST' : 'GET' }, (response) => {
+        const chunks = [];
+        response.on('data', (chunk) => chunks.push(chunk));
+        response.on('end', () => resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))));
+      });
+      request.on('error', reject);
+      request.end(body);
+    });
+
+    expect(await call('/one')).toMatchObject({ url: '/one', bytes: 0 });
+    expect(await call('/two', Buffer.alloc(2 * 1024 * 1024, 1))).toMatchObject({ url: '/two', bytes: 2 * 1024 * 1024 });
+    const parallel = await Promise.all(['/a', '/b', '/c'].map((path) => call(path)));
+    expect(parallel.map((answer) => answer.url)).toEqual(['/a', '/b', '/c']);
+    expect(requests).toBe(5);
+    expect(opened).toBe(1);
+    agent.destroy();
+  });
+
+  it('carries a large answer whole while the other side is still sending', async () => {
+    const outgoing = crypto.randomBytes(4 * 1024 * 1024);
+    const incoming = crypto.randomBytes(4 * 1024 * 1024);
+    let seenAtServer = null;
+    const port = await listen(net.createServer((socket) => {
+      const chunks = [];
+      socket.on('data', (chunk) => chunks.push(chunk));
+      socket.on('end', () => { seenAtServer = Buffer.concat(chunks); socket.end(); });
+      socket.write(incoming);
+    }));
+    const stream = bridgeTo(port);
+    const received = collect(stream);
+    stream.end(outgoing);
+    const answer = await received;
+    expect(crypto.createHash('sha256').update(answer).digest('hex')).toBe(crypto.createHash('sha256').update(incoming).digest('hex'));
+    await closed(stream);
+    expect(crypto.createHash('sha256').update(seenAtServer).digest('hex')).toBe(crypto.createHash('sha256').update(outgoing).digest('hex'));
+  });
+
+  it('fails with the connection error when nothing listens on the port', async () => {
+    const stream = bridgeTo(1);
+    stream.resume();
+    stream.write('GET / HTTP/1.1\r\n\r\n');
+    const error = await failure(stream);
+    expect(error).toMatchObject({ code: 'command_stream_failed', message: expect.stringMatching(/exited with code 1: ECONNREFUSED/) });
+  });
+
+  it('ends its command when the host lets go, and when the server closes', async () => {
+    const sockets = [];
+    const port = await listen(net.createServer((socket) => { sockets.push(socket); socket.write('hello'); }));
+    const first = bridgeTo(port);
+    await new Promise((resolve) => first.once('data', resolve));
+    const firstPid = first.pid;
+    first.destroy();
+    await closed(first);
+    for (let attempt = 0; attempt < 50 && alive(firstPid); attempt += 1) await new Promise((resolve) => { setTimeout(resolve, 20); });
+    expect(alive(firstPid)).toBe(false);
+
+    const second = bridgeTo(port);
+    await new Promise((resolve) => second.once('data', resolve));
+    sockets[1].end();
+    await closed(second);
+    expect(second.errored).toBeNull();
+  });
+});