@push.rocks/smartlinuxrepo 0.0.0-stage → 1.0.0

package/ts/readers/decompress.ts

@@ -0,0 +1,332 @@
+import * as plugins from '../plugins.js';
+import { LinuxRepoError, limitExceeded, truncated } from '../shared/errors.js';
+import type { ILinuxPackageLimits } from '../shared/limits.js';
+import { READ_CHUNK_BYTES, takeFront, type IByteReader } from './bytes.js';
+
+/** The stream compressions the readers decode. */
+export type TLinuxCompression = 'none' | 'gzip' | 'xz' | 'zstd';
+
+export interface IDecompressOptions {
+  /** Bound on the decompressed bytes delivered. */
+  maxOutputBytes: number;
+  outputLimit: keyof ILinuxPackageLimits;
+}
+
+/**
+ * A reader over the decompressed bytes of an inner reader. Pulling is lazy
+ * and bounded: decompressed output past `maxOutputBytes` fails with
+ * `limit-exceeded` and stops the decoder. `close()` releases the decoder and
+ * must be called (see `withDecompressingReader`).
+ *
+ * - gzip (multi-member) and zstd come from `node:zlib`, xz from
+ *   `@napi-rs/lzma` `xz.decompressStream` (64 KiB chunks, cancellable).
+ * - An error from the inner reader (the caller's source, or a codec limit on
+ *   it) propagates unchanged.
+ * - A compressed stream that ends early is `truncated`; corrupt data is
+ *   `malformed-archive` with the decoder error as `cause`.
+ * - The decoder reads the inner reader ahead and never hands bytes back, so
+ *   the compressed stream must run to the end of the inner reader (a whole
+ *   package, or a `LimitedReader` over one archive member).
+ * - Opening a zstd decoder on a Node.js without nodejs/node#64593 throws a
+ *   plain `Error` (see `isZstdDecodingSafe`): a server fault, not a verdict
+ *   on the package.
+ */
+export interface IDecompressingReader extends IByteReader {
+  readonly outputBytes: number;
+  close(): Promise<void>;
+}
+
+interface IChunkSource {
+  next(): Promise<Uint8Array | null>;
+  close(): Promise<void>;
+}
+
+export function openDecompressingReader(
+  inputArg: IByteReader,
+  compressionArg: TLinuxCompression,
+  optionsArg: IDecompressOptions,
+): IDecompressingReader {
+  let source: IChunkSource;
+  switch (compressionArg) {
+    case 'none':
+      source = new PassThroughSource(inputArg);
+      break;
+    case 'gzip':
+      source = new ZlibSource(inputArg, 'gzip');
+      break;
+    case 'zstd':
+      if (!isZstdDecodingSafe(process.versions.node)) {
+        throw new Error(
+          `zstd decoding needs Node.js ${ZSTD_NODE_FLOOR} (nodejs/node#64593 reports a zstd frame that ends early); ` +
+            `this process runs Node.js ${process.versions.node}`,
+        );
+      }
+      source = new ZlibSource(inputArg, 'zstd');
+      break;
+    case 'xz':
+      source = new XzSource(inputArg);
+      break;
+    default:
+      throw new LinuxRepoError('unsupported-compression', `unknown compression "${String(compressionArg)}"`, {
+        field: 'compression',
+      });
+  }
+  return new BoundedOutputReader(source, optionsArg);
+}
+
+/** The Node.js releases whose node:zlib reports a truncated zstd frame. */
+const ZSTD_NODE_FLOOR = '^24.20.0 || >=26.6.0';
+
+/**
+ * True when Node.js `versionArg` (a `process.versions.node` string) carries
+ * nodejs/node#64593, so node:zlib reports a zstd frame that ends early as
+ * `Z_BUF_ERROR` instead of returning a prefix: 24.x from 24.20.0, 26.x from
+ * 26.6.0, and every major from 27 on. Anything else, including all of 22.x,
+ * 23.x and 25.x, is false.
+ */
+export function isZstdDecodingSafe(versionArg: string): boolean {
+  const match = /^(\d+)\.(\d+)\.(\d+)(?:$|[-+])/.exec(versionArg);
+  if (match === null) {
+    return false;
+  }
+  const major = Number(match[1]);
+  const minor = Number(match[2]);
+  if (major === 24) {
+    return minor >= 20;
+  }
+  if (major === 26) {
+    return minor >= 6;
+  }
+  return major >= 27;
+}
+
+/** Runs `bodyArg` with a decompressing reader and always closes it. */
+export async function withDecompressingReader<T>(
+  inputArg: IByteReader,
+  compressionArg: TLinuxCompression,
+  optionsArg: IDecompressOptions,
+  bodyArg: (readerArg: IDecompressingReader) => Promise<T>,
+): Promise<T> {
+  const reader = openDecompressingReader(inputArg, compressionArg, optionsArg);
+  try {
+    return await bodyArg(reader);
+  } finally {
+    await reader.close();
+  }
+}
+
+class BoundedOutputReader implements IDecompressingReader {
+  private delivered = 0;
+  private readonly pushedBack: Uint8Array[] = [];
+  private closed = false;
+
+  constructor(
+    private readonly source: IChunkSource,
+    private readonly options: IDecompressOptions,
+  ) {}
+
+  public get outputBytes(): number {
+    return this.delivered;
+  }
+
+  public async read(maxArg: number): Promise<Uint8Array | null> {
+    const pushed = this.pushedBack.pop();
+    if (pushed !== undefined) {
+      return takeFront(pushed, maxArg, this.pushedBack);
+    }
+    if (this.closed) {
+      return null;
+    }
+    const chunk = await this.source.next();
+    if (chunk === null) {
+      return null;
+    }
+    this.delivered += chunk.length;
+    if (this.delivered > this.options.maxOutputBytes) {
+      await this.close();
+      throw limitExceeded(this.options.outputLimit, this.options.maxOutputBytes, 'decompressed data');
+    }
+    return takeFront(chunk, maxArg, this.pushedBack);
+  }
+
+  public unread(bytesArg: Uint8Array): void {
+    if (bytesArg.length > 0) {
+      this.pushedBack.push(bytesArg);
+    }
+  }
+
+  public async close(): Promise<void> {
+    this.pushedBack.length = 0;
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    await this.source.close();
+  }
+}
+
+class PassThroughSource implements IChunkSource {
+  constructor(private readonly input: IByteReader) {}
+
+  public next(): Promise<Uint8Array | null> {
+    return this.input.read(READ_CHUNK_BYTES);
+  }
+
+  public async close(): Promise<void> {}
+}
+
+/** gzip (multi-member, like Go's default) and zstd through `node:zlib`, with stream backpressure. */
+class ZlibSource implements IChunkSource {
+  private readonly kind: 'gzip' | 'zstd';
+  private readonly feeder: plugins.stream.Readable;
+  private readonly decoder: plugins.zlib.Gunzip | plugins.zlib.ZstdDecompress;
+  private readonly iterator: AsyncIterator<Buffer>;
+  private inputFailed = false;
+  private inputError: unknown = undefined;
+  private finished = false;
+
+  constructor(inputArg: IByteReader, kindArg: 'gzip' | 'zstd') {
+    this.kind = kindArg;
+    const pump = async function* (this: ZlibSource): AsyncGenerator<Uint8Array> {
+      for (;;) {
+        let chunk: Uint8Array | null;
+        try {
+          chunk = await inputArg.read(READ_CHUNK_BYTES);
+        } catch (error) {
+          this.inputFailed = true;
+          this.inputError = error;
+          throw error;
+        }
+        if (chunk === null) {
+          return;
+        }
+        yield chunk;
+      }
+    };
+    this.feeder = plugins.stream.Readable.from(pump.call(this), { objectMode: false });
+    this.decoder = kindArg === 'gzip' ? plugins.zlib.createGunzip() : plugins.zlib.createZstdDecompress();
+    // errors surface through the iterator below; the callback only completes the pipeline
+    plugins.stream.pipeline(this.feeder, this.decoder, () => {});
+    this.iterator = this.decoder[Symbol.asyncIterator]();
+  }
+
+  public async next(): Promise<Uint8Array | null> {
+    if (this.finished) {
+      return null;
+    }
+    try {
+      const result = await this.iterator.next();
+      if (result.done) {
+        this.finished = true;
+        return null;
+      }
+      return result.value;
+    } catch (error) {
+      this.finished = true;
+      if (this.inputFailed) {
+        throw this.inputError;
+      }
+      throw mapZlibError(error, this.kind);
+    }
+  }
+
+  public async close(): Promise<void> {
+    this.finished = true;
+    this.feeder.destroy();
+    this.decoder.destroy();
+  }
+}
+
+function mapZlibError(errorArg: unknown, kindArg: 'gzip' | 'zstd'): LinuxRepoError {
+  const code = (errorArg as { code?: unknown } | null)?.code;
+  // gunzip, and zstd on the Node.js floor, report an input that ended inside the stream as Z_BUF_ERROR
+  if (code === 'Z_BUF_ERROR') {
+    const error = truncated(`${kindArg} stream`);
+    return new LinuxRepoError(error.code, error.message, error.detail, { cause: errorArg });
+  }
+  return new LinuxRepoError('malformed-archive', `corrupt ${kindArg} data`, { field: `${kindArg} stream` }, {
+    cause: errorArg,
+  });
+}
+
+/**
+ * xz through `@napi-rs/lzma` `xz.decompressStream`, the one bounded decoder of
+ * that package (its class API buffers whole outputs). The input is a
+ * pull-based web stream that never reads ahead; cancelling the output stops it.
+ */
+class XzSource implements IChunkSource {
+  private readonly reader: ReadableStreamDefaultReader<Uint8Array>;
+  private inputFailed = false;
+  private inputError: unknown = undefined;
+  private inputEnded = false;
+  private finished = false;
+  private stopped = false;
+
+  constructor(inputArg: IByteReader) {
+    const input = new ReadableStream<Uint8Array>(
+      {
+        pull: async (controller) => {
+          if (this.stopped) {
+            controller.close();
+            return;
+          }
+          let chunk: Uint8Array | null;
+          try {
+            chunk = await inputArg.read(READ_CHUNK_BYTES);
+          } catch (error) {
+            this.inputFailed = true;
+            this.inputError = error;
+            controller.error(error);
+            return;
+          }
+          if (chunk === null) {
+            this.inputEnded = true;
+            controller.close();
+          } else {
+            controller.enqueue(chunk);
+          }
+        },
+        cancel: () => {
+          this.stopped = true;
+        },
+      },
+      { highWaterMark: 0 },
+    );
+    this.reader = plugins.lzma.xz.decompressStream(input).getReader();
+  }
+
+  public async next(): Promise<Uint8Array | null> {
+    if (this.finished) {
+      return null;
+    }
+    try {
+      const result = await this.reader.read();
+      if (result.done) {
+        this.finished = true;
+        return null;
+      }
+      return result.value;
+    } catch (error) {
+      this.finished = true;
+      if (this.inputFailed) {
+        throw this.inputError;
+      }
+      // @napi-rs/lzma 1.5.3 reports a decoder that ran out of input as Rust's
+      // read_exact error; with the input at its end that is a truncated stream
+      const message = error instanceof Error ? error.message : '';
+      if (this.inputEnded && message.includes('failed to fill whole buffer')) {
+        const base = truncated('xz stream');
+        throw new LinuxRepoError(base.code, base.message, base.detail, { cause: error });
+      }
+      throw new LinuxRepoError('malformed-archive', 'corrupt xz data', { field: 'xz stream' }, { cause: error });
+    }
+  }
+
+  public async close(): Promise<void> {
+    this.stopped = true;
+    if (!this.finished) {
+      this.finished = true;
+      await this.reader.cancel();
+    }
+  }
+}

package/ts/readers/gzipmember.ts

@@ -0,0 +1,209 @@
+import * as plugins from '../plugins.js';
+import { LinuxRepoError, limitExceeded, truncated } from '../shared/errors.js';
+import type { ILinuxPackageLimits } from '../shared/limits.js';
+import { concatBytes, READ_CHUNK_BYTES, readExact, readUpTo, type IByteReader } from './bytes.js';
+
+const FLAG_HCRC = 0x02;
+const FLAG_EXTRA = 0x04;
+const FLAG_NAME = 0x08;
+const FLAG_COMMENT = 0x10;
+
+export interface IGzipMemberOptions {
+  /** Bound on the member's own bytes (header, deflate data and trailer). */
+  maxMemberBytes: number;
+  memberLimit: keyof ILinuxPackageLimits;
+  /** Bound on the member's decompressed bytes. */
+  maxOutputBytes: number;
+  outputLimit: keyof ILinuxPackageLimits;
+  /** Bound on the header's FNAME and FCOMMENT strings. */
+  maxHeaderBytes: number;
+}
+
+export interface IGzipMember {
+  /** The decompressed member. */
+  output: Uint8Array;
+  /** The member's own byte count: header, deflate data and trailer. */
+  memberBytes: number;
+  /** Lowercase hex SHA-1 of exactly the member's bytes. */
+  memberSha1: string;
+}
+
+/**
+ * Reads exactly one gzip member (RFC 1952) from `inputArg` and leaves the
+ * reader positioned at the first byte after it, so concatenated members (an
+ * apk v2 package) can be read one at a time with their exact boundaries.
+ *
+ * The header is parsed here; the deflate data is inflated with
+ * `zlib.createInflateRaw()`, whose `bytesWritten` counts only the bytes the
+ * deflate stream consumed, and the bytes past the stream's end are pushed back
+ * with `unread`. The trailer's CRC-32 and ISIZE are checked. Returns `null`
+ * when the input ends exactly where a member would start.
+ */
+export async function readGzipMember(
+  inputArg: IByteReader,
+  optionsArg: IGzipMemberOptions,
+): Promise<IGzipMember | null> {
+  const sha1 = plugins.crypto.createHash('sha1');
+  let memberBytes = 0;
+  const account = (bytesArg: Uint8Array) => {
+    memberBytes += bytesArg.length;
+    if (memberBytes > optionsArg.maxMemberBytes) {
+      throw limitExceeded(optionsArg.memberLimit, optionsArg.maxMemberBytes, 'gzip member');
+    }
+    sha1.update(bytesArg);
+  };
+
+  // header
+  const fixed = await readUpTo(inputArg, 10);
+  if (fixed.length === 0) {
+    return null;
+  }
+  if (fixed[0] !== 0x1f || (fixed.length > 1 && fixed[1] !== 0x8b) || (fixed.length > 2 && fixed[2] !== 0x08)) {
+    throw new LinuxRepoError('malformed-archive', 'input is not a gzip member', { field: 'gzip header' });
+  }
+  if (fixed.length < 10) {
+    throw truncated('gzip header');
+  }
+  account(fixed);
+  let headerCrc = plugins.zlib.crc32(fixed);
+  const flags = fixed[3];
+  if ((flags & FLAG_EXTRA) !== 0) {
+    const lengthBytes = await readExact(inputArg, 2, 'gzip header');
+    account(lengthBytes);
+    headerCrc = plugins.zlib.crc32(lengthBytes, headerCrc);
+    const extra = await readExact(inputArg, lengthBytes[0] | (lengthBytes[1] << 8), 'gzip header');
+    account(extra);
+    headerCrc = plugins.zlib.crc32(extra, headerCrc);
+  }
+  for (const flag of [FLAG_NAME, FLAG_COMMENT]) {
+    if ((flags & flag) !== 0) {
+      const text = await readZeroTerminated(inputArg, optionsArg.maxHeaderBytes);
+      account(text);
+      headerCrc = plugins.zlib.crc32(text, headerCrc);
+    }
+  }
+  if ((flags & FLAG_HCRC) !== 0) {
+    const stored = await readExact(inputArg, 2, 'gzip header');
+    account(stored);
+    if ((stored[0] | (stored[1] << 8)) !== (headerCrc & 0xffff)) {
+      throw new LinuxRepoError('malformed-archive', 'gzip header CRC mismatch', { field: 'gzip header' });
+    }
+  }
+
+  // deflate data
+  const output = await inflateRawMember(inputArg, optionsArg, account);
+
+  // trailer
+  const trailer = await readExact(inputArg, 8, 'gzip trailer');
+  account(trailer);
+  const view = new DataView(trailer.buffer, trailer.byteOffset, trailer.byteLength);
+  if (view.getUint32(0, true) !== output.crc || view.getUint32(4, true) !== output.bytes.length % 0x1_0000_0000) {
+    throw new LinuxRepoError('malformed-archive', 'gzip trailer does not match the data', { field: 'gzip trailer' });
+  }
+  return { output: output.bytes, memberBytes, memberSha1: sha1.digest('hex') };
+}
+
+async function inflateRawMember(
+  inputArg: IByteReader,
+  optionsArg: IGzipMemberOptions,
+  accountArg: (bytesArg: Uint8Array) => void,
+): Promise<{ bytes: Uint8Array; crc: number }> {
+  const inflate = plugins.zlib.createInflateRaw();
+  const chunks: Uint8Array[] = [];
+  let outputBytes = 0;
+  let crc = 0;
+  let failure: LinuxRepoError | null = null;
+  let streamError: Error | null = null;
+  inflate.on('data', (chunk: Buffer) => {
+    if (failure !== null) {
+      return;
+    }
+    outputBytes += chunk.length;
+    if (outputBytes > optionsArg.maxOutputBytes) {
+      failure = limitExceeded(optionsArg.outputLimit, optionsArg.maxOutputBytes, 'decompressed gzip member');
+      inflate.destroy();
+      return;
+    }
+    crc = plugins.zlib.crc32(chunk, crc);
+    chunks.push(chunk);
+  });
+  inflate.on('error', (error: Error) => {
+    streamError ??= error;
+  });
+  try {
+    for (;;) {
+      const chunk = await inputArg.read(READ_CHUNK_BYTES);
+      if (chunk === null) {
+        throw truncated('gzip member');
+      }
+      const before = inflate.bytesWritten;
+      try {
+        await writeChunk(inflate, chunk);
+      } catch (error) {
+        throw failure ?? mapInflateError(streamError ?? error);
+      }
+      if (failure !== null) {
+        throw failure;
+      }
+      const consumed = inflate.bytesWritten - before;
+      accountArg(chunk.subarray(0, consumed));
+      if (consumed < chunk.length) {
+        inputArg.unread(chunk.subarray(consumed));
+        break;
+      }
+    }
+    if (!inflate.readableEnded) {
+      await new Promise<void>((resolve) => inflate.once('end', resolve));
+    }
+    if (failure !== null) {
+      throw failure;
+    }
+    return { bytes: concatBytes(chunks, outputBytes), crc };
+  } finally {
+    inflate.destroy();
+  }
+}
+
+/** Writes one chunk and waits until the stream processed it, failing if the stream closes first. */
+function writeChunk(streamArg: plugins.zlib.InflateRaw, chunkArg: Uint8Array): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const onClose = () => reject(new Error('inflate stream closed'));
+    streamArg.once('close', onClose);
+    streamArg.write(chunkArg, (error) => {
+      streamArg.off('close', onClose);
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function mapInflateError(errorArg: unknown): LinuxRepoError {
+  return new LinuxRepoError('malformed-archive', 'gzip member has corrupt deflate data', { field: 'gzip member' }, {
+    cause: errorArg,
+  });
+}
+
+async function readZeroTerminated(inputArg: IByteReader, maxBytesArg: number): Promise<Uint8Array> {
+  const parts: Uint8Array[] = [];
+  let total = 0;
+  for (;;) {
+    const chunk = await inputArg.read(READ_CHUNK_BYTES);
+    if (chunk === null) {
+      throw truncated('gzip header');
+    }
+    const nul = chunk.indexOf(0);
+    const take = nul < 0 ? chunk : chunk.subarray(0, nul + 1);
+    total += take.length;
+    if (total > maxBytesArg) {
+      throw limitExceeded('maxHeaderBytes', maxBytesArg, 'gzip header string');
+    }
+    parts.push(take);
+    if (nul >= 0) {
+      inputArg.unread(chunk.subarray(nul + 1));
+      return concatBytes(parts, total);
+    }
+  }
+}