@procedures/archive 0.0.0-stage → 0.2.3

package/src/tar.ts

@@ -0,0 +1,198 @@
+/**
+ * Tar and tar.gz archives with `tar` (node-tar), the tar library of npm.
+ *
+ * - The list reads the headers with a `Parser`. It stops at the limit of entries or of bytes, so a
+ *   compressed archive with a very large content stops early.
+ * - The extract runs after the check of `planExtract()`. It keeps the protections of node-tar:
+ *   no absolute path, no `..`, and no write through a symbolic link. Its filter lets only the
+ *   checked entries through, and `strict` makes each warning an error.
+ */
+
+import { createReadStream } from "node:fs";
+import { createGunzip, gzipSync } from "node:zlib";
+import { open, writeFile } from "node:fs/promises";
+import { create, extract, Parser, type ReadEntry } from "tar";
+import { ArchiveError } from "./errors.js";
+import type { ArchiveEntryKind } from "./types.js";
+
+/** One header of a tar archive. */
+export interface TarEntryInfo {
+  readonly name: string;
+  readonly kind: ArchiveEntryKind;
+  readonly size: number;
+  readonly mode: number | undefined;
+  readonly mtime: Date | undefined;
+  readonly linkTarget: string | undefined;
+}
+
+/** The kind of a tar entry type. */
+function kindOf(type: string): ArchiveEntryKind {
+  switch (type) {
+    case "File":
+    case "OldFile":
+    case "ContiguousFile":
+      return "file";
+    case "Directory":
+      return "directory";
+    case "SymbolicLink":
+      return "symlink";
+    case "Link":
+      return "hardlink";
+    default:
+      return "other";
+  }
+}
+
+/** An error of node-tar as an `ArchiveError`. */
+function archiveErrorOf(error: unknown): ArchiveError {
+  if (error instanceof ArchiveError) return error;
+  const message = error instanceof Error ? error.message : String(error);
+  return new ArchiveError("BAD_ARCHIVE", `The tar archive is damaged: ${message}`);
+}
+
+/** The most zero bytes of an empty tar: an empty tar of GNU tar has 10240. */
+const MAX_EMPTY_TAR_BYTES = 1024 * 1024;
+
+/**
+ * True when the content of a tar or tar.gz archive is only zero bytes: an archive without
+ * entries. node-tar calls such an archive "unrecognized". The check reads at most 1 MiB.
+ */
+async function isEmptyTar(path: string): Promise<boolean> {
+  const head = Buffer.alloc(2);
+  const handle = await open(path, "r");
+  try {
+    await handle.read(head, 0, 2, 0);
+  } finally {
+    await handle.close();
+  }
+  const input = createReadStream(path);
+  const stream = head[0] === 0x1f && head[1] === 0x8b ? input.pipe(createGunzip()) : input;
+  let bytes = 0;
+  try {
+    for await (const chunk of stream as AsyncIterable<Buffer>) {
+      bytes += chunk.length;
+      if (bytes > MAX_EMPTY_TAR_BYTES || chunk.some((byte) => byte !== 0)) return false;
+    }
+    return bytes > 0;
+  } catch {
+    return false;
+  } finally {
+    input.destroy();
+  }
+}
+
+/**
+ * This function reads the headers of a tar or tar.gz archive. It throws `TOO_LARGE` at more than
+ * `maxEntries` entries or more than `maxBytes` bytes of files.
+ */
+export async function readTarEntries(path: string, limits: { maxEntries: number; maxBytes: number }): Promise<TarEntryInfo[]> {
+  try {
+    return await parseTarEntries(path, limits);
+  } catch (error) {
+    if (error instanceof ArchiveError && error.code === "BAD_ARCHIVE" && error.message.includes("TAR_BAD_ARCHIVE") && (await isEmptyTar(path))) return [];
+    throw error;
+  }
+}
+
+async function parseTarEntries(path: string, limits: { maxEntries: number; maxBytes: number }): Promise<TarEntryInfo[]> {
+  const entries: TarEntryInfo[] = [];
+  let bytes = 0;
+  await new Promise<void>((resolve, reject) => {
+    let settled = false;
+    const input = createReadStream(path);
+    const finish = (error?: unknown): void => {
+      if (settled) return;
+      settled = true;
+      if (error === undefined) {
+        resolve();
+        return;
+      }
+      input.destroy();
+      reject(archiveErrorOf(error));
+    };
+    const parser = new Parser({
+      strict: true,
+      onReadEntry: (entry: ReadEntry) => {
+        entry.resume();
+        if (settled) return;
+        const kind = kindOf(entry.type);
+        if (kind === "file") bytes += entry.size;
+        if (entries.length >= limits.maxEntries) {
+          finish(new ArchiveError("TOO_LARGE", `The archive has more entries than the limit of ${limits.maxEntries}`));
+          return;
+        }
+        if (bytes > limits.maxBytes) {
+          finish(new ArchiveError("TOO_LARGE", `The files of the archive have more than ${limits.maxBytes} bytes`, entry.path));
+          return;
+        }
+        entries.push({
+          name: entry.path,
+          kind,
+          size: entry.size,
+          mode: entry.mode,
+          mtime: entry.mtime,
+          linkTarget: entry.linkpath === undefined || entry.linkpath === "" ? undefined : entry.linkpath,
+        });
+      },
+    });
+    parser.on("warn", (code: string, message: string | Error) => finish(new ArchiveError("BAD_ARCHIVE", `The tar archive is damaged: ${code} ${message instanceof Error ? message.message : message}`)));
+    parser.on("error", (error: unknown) => finish(error));
+    parser.on("end", () => finish());
+    input.on("error", (error) => finish(error));
+    input.on("data", (chunk) => {
+      if (settled) return;
+      if (!parser.write(chunk as Buffer)) {
+        input.pause();
+        parser.once("drain", () => input.resume());
+      }
+    });
+    input.on("end", () => {
+      if (!settled) parser.end();
+    });
+  });
+  return entries;
+}
+
+/**
+ * This function extracts a tar or tar.gz archive into `target`, after the check. Only the names
+ * of `allowed` pass the filter. A name that the check did not see is an error at the end.
+ */
+export async function extractTar(path: string, target: string, allowed: ReadonlySet<string>): Promise<void> {
+  const refused: string[] = [];
+  try {
+    await extract({
+      file: path,
+      cwd: target,
+      strict: true,
+      preservePaths: false,
+      preserveOwner: false,
+      // Remove an old file before the new one: the write never goes through a link or into a hard link
+      unlink: true,
+      filter: (name: string) => {
+        if (allowed.has(name)) return true;
+        refused.push(name);
+        return false;
+      },
+    });
+  } catch (error) {
+    throw archiveErrorOf(error);
+  }
+  const unexpected = refused.filter((name) => name.split(/[/\\]+/).some((part) => part !== "" && part !== "."));
+  if (unexpected.length > 0) {
+    throw new ArchiveError("BAD_ARCHIVE", `The archive changed during the extract: ${unexpected[0]} was not in the check`, unexpected[0]);
+  }
+}
+
+/**
+ * This function writes a tar or tar.gz archive of the `names` (relative to `cwd`) to `file`. The
+ * names are the full list: the walk of `archive.create` gives each file and folder.
+ */
+export async function createTar(file: string, cwd: string, names: readonly string[], gzip: boolean): Promise<void> {
+  if (names.length === 0) {
+    // An empty tar: two blocks of zeros
+    const empty = Buffer.alloc(1024);
+    await writeFile(file, gzip ? gzipSync(empty) : empty, { flag: "wx" });
+    return;
+  }
+  await create({ file, cwd, gzip, portable: true, follow: false, noDirRecurse: true }, [...names]);
+}

package/src/types.ts

@@ -0,0 +1,101 @@
+/**
+ * Type definitions of the procedures of this package
+ *
+ * mark procedure new adds the types of each new procedure here. The zod schemas are in
+ * `module.ts`: an exported schema needs a written type (isolatedDeclarations).
+ */
+
+/** The formats of an archive. `tar` is a tar file without compression. */
+export type ArchiveFormat = "tar.gz" | "tar" | "zip";
+
+/** The kind of an entry of an archive. `other` is a device, a FIFO or another special file. */
+export type ArchiveEntryKind = "file" | "directory" | "symlink" | "hardlink" | "other";
+
+// =============================================================================
+// archive.create
+// =============================================================================
+
+export interface ArchiveCreateInput {
+  /** The archive to make. */
+  output: string;
+  /** A folder: the archive gets its content, with paths relative to the folder. */
+  source?: string | undefined;
+  /** Files and folders, relative to `cwd`. A folder goes in with its content. */
+  files?: string[] | undefined;
+  /** The base folder of `files`. The default is the working folder of the process. */
+  cwd?: string | undefined;
+  /** The format. The default comes from the name of `output`: `.tar.gz`, `.tgz`, `.tar` or `.zip`. */
+  format?: ArchiveFormat | undefined;
+  /** True: replace an existing archive. The default is false: an existing archive is an error. */
+  overwrite?: boolean | undefined;
+}
+
+export interface ArchiveCreateOutput {
+  /** The absolute path of the archive. */
+  path: string;
+  format: ArchiveFormat;
+  /** The number of entries: files, folders and links. */
+  entries: number;
+  /** The size of the archive, in bytes. */
+  bytes: number;
+}
+
+// =============================================================================
+// archive.extract
+// =============================================================================
+
+export interface ArchiveExtractInput {
+  /** The archive. */
+  archive: string;
+  /** The folder that gets the content. The procedure makes it when it does not exist. */
+  target: string;
+  /** The format. The default comes from the first bytes of the archive. */
+  format?: ArchiveFormat | undefined;
+  /** True: replace an existing file. The default is false: an existing file is an error, and nothing changes. */
+  overwrite?: boolean | undefined;
+}
+
+export interface ArchiveExtractOutput {
+  /** The absolute path of the target folder. */
+  target: string;
+  format: ArchiveFormat;
+  files: number;
+  directories: number;
+  /** The symbolic links and the hard links. */
+  links: number;
+  /** The bytes of the files. */
+  bytes: number;
+}
+
+// =============================================================================
+// archive.list
+// =============================================================================
+
+export interface ArchiveListInput {
+  /** The archive. */
+  archive: string;
+  /** The format. The default comes from the first bytes of the archive. */
+  format?: ArchiveFormat | undefined;
+}
+
+/** One entry of an archive. */
+export interface ArchiveEntry {
+  /** The path in the archive, as the archive has it. */
+  path: string;
+  kind: ArchiveEntryKind;
+  /** The size of the content, in bytes. */
+  size: number;
+  /** The mode bits (`0o644`), when the archive has them. */
+  mode?: number | undefined;
+  /** The time of the last change, as an ISO text, when the archive has it. */
+  mtime?: string | undefined;
+  /** The target of a link. */
+  linkTarget?: string | undefined;
+}
+
+export interface ArchiveListOutput {
+  format: ArchiveFormat;
+  entries: ArchiveEntry[];
+  /** The sum of the sizes of the entries, in bytes. */
+  bytes: number;
+}

package/src/walk.ts

@@ -0,0 +1,85 @@
+/**
+ * The walk of `archive.create`: each file, folder and symbolic link under the inputs, in a fixed
+ * order. The walk does not follow a link: the archive gets the link itself.
+ */
+
+import { lstat, readdir, readlink } from "node:fs/promises";
+import { join, resolve, sep } from "node:path";
+import { ArchiveError } from "./errors.js";
+
+/** One path for the archive. */
+export interface WalkEntry {
+  /** The name in the archive: relative, with `/`. */
+  readonly name: string;
+  /** The absolute path on the disk. */
+  readonly path: string;
+  readonly kind: "file" | "directory" | "symlink";
+  readonly size: number;
+  readonly mode: number;
+  readonly mtime: Date;
+  readonly linkTarget?: string | undefined;
+}
+
+/** The limits and the paths to leave out (the new archive and its temporary file). */
+export interface WalkOptions {
+  readonly maxEntries: number;
+  readonly maxBytes: number;
+  readonly exclude: ReadonlySet<string>;
+}
+
+const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
+/**
+ * This function gives the target of a link with `/`, as an archive keeps it. Windows gives `..\a.txt`
+ * for a link that Node made from `../a.txt`. On the other systems, a `\` is a part of a name.
+ */
+export function portableLinkTarget(target: string, separator: string = sep): string {
+  return separator === "\\" ? target.replaceAll("\\", "/") : target;
+}
+
+/** This function walks the names (relative to `base`) and gives each entry for the archive. */
+export async function walk(base: string, names: readonly string[], options: WalkOptions): Promise<WalkEntry[]> {
+  const entries: WalkEntry[] = [];
+  const seen = new Set<string>();
+  let bytes = 0;
+
+  const visit = async (name: string): Promise<void> => {
+    const path = resolve(base, ...name.split("/"));
+    if (options.exclude.has(path) || seen.has(name)) return;
+    seen.add(name);
+    let stats;
+    try {
+      stats = await lstat(path);
+    } catch {
+      throw new ArchiveError("UNSAFE_PATH", `${path} does not exist`, name);
+    }
+    if (entries.length >= options.maxEntries) throw new ArchiveError("TOO_LARGE", `The archive gets more entries than the limit of ${options.maxEntries}`, name);
+    const common = { name, path, size: 0, mode: stats.mode & 0o7777, mtime: stats.mtime };
+    if (stats.isSymbolicLink()) {
+      entries.push({ ...common, kind: "symlink", linkTarget: portableLinkTarget(await readlink(path)) });
+    } else if (stats.isDirectory()) {
+      entries.push({ ...common, kind: "directory" });
+      const children = (await readdir(path)).sort(byName);
+      for (const child of children) await visit(`${name}/${child}`);
+    } else if (stats.isFile()) {
+      bytes += stats.size;
+      if (bytes > options.maxBytes) throw new ArchiveError("TOO_LARGE", `The files have more than ${options.maxBytes} bytes`, name);
+      entries.push({ ...common, kind: "file", size: stats.size });
+    } else {
+      throw new ArchiveError("UNSUPPORTED_ENTRY", `${path} is a special file (a socket, a device or a FIFO)`, name);
+    }
+  };
+
+  for (const name of names) await visit(name);
+  return entries;
+}
+
+/** The names of the content of a folder, sorted. */
+export async function folderNames(folder: string): Promise<string[]> {
+  return (await readdir(folder)).sort(byName);
+}
+
+/** The absolute path of a name under a base. */
+export function under(base: string, name: string): string {
+  return join(base, ...name.split("/"));
+}