@push.rocks/smartlinuxrepo 1.1.0 → 1.1.2

package/ts/readers/decompress.ts

@@ -10,6 +10,11 @@ export interface IDecompressOptions {
   /** Bound on the decompressed bytes delivered. */
   maxOutputBytes: number;
   outputLimit: keyof ILinuxPackageLimits;
+  /**
+   * Bound on the Zstandard frames of a zstd stream (skippable frames do not
+   * count): one more is `limit-exceeded` on `maxEntries`.
+   */
+  maxZstdFrames: number;
 }
 
 /**
@@ -18,15 +23,20 @@ export interface IDecompressOptions {
  * `limit-exceeded` and stops the decoder. `close()` releases the decoder and
  * must be called (see `withDecompressingReader`).
  *
- * - gzip (multi-member) and zstd come from `node:zlib`, xz from
- *   `@napi-rs/lzma` `xz.decompressStream` (64 KiB chunks, cancellable).
+ * - gzip (multi-member) comes from `node:zlib` `createGunzip`; zstd from one
+ *   `node:zlib` `createZstdDecompress` fed by the codec's RFC 8878 frame walk,
+ *   at most `maxZstdFrames` Zstandard frames; xz from `@napi-rs/lzma`
+ *   `xz.decompressStream` (64 KiB chunks, cancellable).
  * - An error from the inner reader (the caller's source, or a codec limit on
  *   it) propagates unchanged.
  * - A compressed stream that ends early is `truncated`; corrupt data is
  *   `malformed-archive` with the decoder error as `cause`.
  * - The decoder reads the inner reader ahead and never hands bytes back, so
  *   the compressed stream must run to the end of the inner reader (a whole
- *   package, or a `LimitedReader` over one archive member).
+ *   package, or a `LimitedReader` over one archive member). Bytes after the
+ *   last gzip member or zstd frame are `malformed-archive` (zstd skippable
+ *   frames are skipped). A reader closed before the end skips the end check;
+ *   the zstd walk still refuses such bytes when it has already read them.
  * - Opening a zstd decoder on a Node.js without nodejs/node#64593 throws a
  *   plain `Error` (see `isZstdDecodingSafe`): a server fault, not a verdict
  *   on the package.
@@ -52,7 +62,7 @@ export function openDecompressingReader(
       source = new PassThroughSource(inputArg);
       break;
     case 'gzip':
-      source = new ZlibSource(inputArg, 'gzip');
+      source = new GzipSource(inputArg);
       break;
     case 'zstd':
       if (!isZstdDecodingSafe(process.versions.node)) {
@@ -61,7 +71,7 @@ export function openDecompressingReader(
             `this process runs Node.js ${process.versions.node}`,
         );
       }
-      source = new ZlibSource(inputArg, 'zstd');
+      source = new ZstdSource(inputArg, optionsArg.maxZstdFrames);
       break;
     case 'xz':
       source = new XzSource(inputArg);
@@ -175,28 +185,36 @@ class PassThroughSource implements IChunkSource {
   public async close(): Promise<void> {}
 }
 
-/** gzip (multi-member, like Go's default) and zstd through `node:zlib`, with stream backpressure. */
-class ZlibSource implements IChunkSource {
-  private readonly kind: 'gzip' | 'zstd';
+/**
+ * gzip through `node:zlib` `createGunzip`, with stream backpressure. Gunzip
+ * decodes concatenated members, as Go's default reader does, but it stops at a
+ * NUL byte after a member and ignores the rest of the input, a valid member
+ * included; its `bytesWritten` counts only the bytes it consumed. So when the
+ * decoder ends by itself it must have consumed every byte read from the inner
+ * reader, and the inner reader must be at its end; anything else is
+ * `malformed-archive`. A source closed before that skips the check.
+ */
+class GzipSource implements IChunkSource {
+  private readonly input: IByteReader;
   private readonly feeder: plugins.stream.Readable;
-  private readonly decoder: plugins.zlib.Gunzip | plugins.zlib.ZstdDecompress;
+  private readonly decoder: plugins.zlib.Gunzip;
   private readonly iterator: AsyncIterator<Buffer>;
+  /** Bytes read from the inner reader. */
+  private inputBytes = 0;
+  private inputEnded = false;
   private inputFailed = false;
   private inputError: unknown = undefined;
+  /** The read in flight on the inner reader; there is never more than one. */
+  private pendingRead: Promise<Uint8Array | null> | null = null;
+  /** Set when the decoder has ended: the feeder starts no further read. */
+  private stopped = false;
   private finished = false;
 
-  constructor(inputArg: IByteReader, kindArg: 'gzip' | 'zstd') {
-    this.kind = kindArg;
-    const pump = async function* (this: ZlibSource): AsyncGenerator<Uint8Array> {
-      for (;;) {
-        let chunk: Uint8Array | null;
-        try {
-          chunk = await inputArg.read(READ_CHUNK_BYTES);
-        } catch (error) {
-          this.inputFailed = true;
-          this.inputError = error;
-          throw error;
-        }
+  constructor(inputArg: IByteReader) {
+    this.input = inputArg;
+    const pump = async function* (this: GzipSource): AsyncGenerator<Uint8Array> {
+      while (!this.stopped) {
+        const chunk = await this.readInput(READ_CHUNK_BYTES);
         if (chunk === null) {
           return;
         }
@@ -204,7 +222,7 @@ class ZlibSource implements IChunkSource {
       }
     };
     this.feeder = plugins.stream.Readable.from(pump.call(this), { objectMode: false });
-    this.decoder = kindArg === 'gzip' ? plugins.zlib.createGunzip() : plugins.zlib.createZstdDecompress();
+    this.decoder = plugins.zlib.createGunzip();
     // errors surface through the iterator below; the callback only completes the pipeline
     plugins.stream.pipeline(this.feeder, this.decoder, () => {});
     this.iterator = this.decoder[Symbol.asyncIterator]();
@@ -214,27 +232,421 @@ class ZlibSource implements IChunkSource {
     if (this.finished) {
       return null;
     }
+    let result: IteratorResult<Buffer>;
     try {
-      const result = await this.iterator.next();
-      if (result.done) {
-        this.finished = true;
-        return null;
-      }
-      return result.value;
+      result = await this.iterator.next();
     } catch (error) {
       this.finished = true;
       if (this.inputFailed) {
         throw this.inputError;
       }
-      throw mapZlibError(error, this.kind);
+      throw mapZlibError(error, 'gzip');
+    }
+    if (!result.done) {
+      return result.value;
+    }
+    this.finished = true;
+    await this.checkEnd();
+    return null;
+  }
+
+  public async close(): Promise<void> {
+    this.finished = true;
+    this.stopped = true;
+    this.feeder.destroy();
+    this.decoder.destroy();
+  }
+
+  /** Reads the inner reader and counts what it returns. */
+  private readInput(maxArg: number): Promise<Uint8Array | null> {
+    const pending = this.input.read(maxArg).then(
+      (chunk) => {
+        this.pendingRead = null;
+        if (chunk === null) {
+          this.inputEnded = true;
+        } else {
+          this.inputBytes += chunk.length;
+        }
+        return chunk;
+      },
+      (error: unknown) => {
+        this.pendingRead = null;
+        this.inputFailed = true;
+        this.inputError = error;
+        throw error;
+      },
+    );
+    this.pendingRead = pending;
+    return pending;
+  }
+
+  /**
+   * The decoder ended by itself. It must have consumed every byte read from
+   * the inner reader, and one more read must report the end of input.
+   */
+  private async checkEnd(): Promise<void> {
+    this.stopped = true;
+    if (this.pendingRead !== null && this.decoder.bytesWritten === this.inputBytes) {
+      // a read the feeder started settles first: its bytes count, and no two reads overlap
+      await this.pendingRead;
+    }
+    if (this.inputFailed) {
+      throw this.inputError;
+    }
+    const consumedAll =
+      this.decoder.bytesWritten === this.inputBytes && (this.inputEnded || (await this.readInput(1)) === null);
+    if (!consumedAll) {
+      throw new LinuxRepoError('malformed-archive', 'bytes follow the last gzip member', { field: 'gzip stream' });
+    }
+  }
+}
+
+/** RFC 8878 §3.1.1: the Zstandard frame magic number 0xFD2FB528, little-endian. */
+const ZSTD_FRAME_MAGIC = Uint8Array.of(0x28, 0xb5, 0x2f, 0xfd);
+/** RFC 8878 §3.1.2: a skippable frame magic number 0x184D2A5?, little-endian, after its first byte 0x5?. */
+const ZSTD_SKIPPABLE_MAGIC = Uint8Array.of(0x50, 0x2a, 0x4d, 0x18);
+/** Dictionary_ID field bytes by Dictionary_ID_Flag (RFC 8878 §3.1.1.1.1). */
+const ZSTD_DICTIONARY_ID_BYTES = [0, 1, 2, 4];
+/** Frame_Content_Size field bytes by Frame_Content_Size_Flag; flag 0 means 1 byte with Single_Segment_Flag, else 0. */
+const ZSTD_CONTENT_SIZE_BYTES = [0, 2, 4, 8];
+const ZSTD_BLOCK_TYPE_RLE = 1;
+const ZSTD_BLOCK_TYPE_RESERVED = 3;
+/** Place values of the bytes of a little-endian integer. */
+const LITTLE_ENDIAN_PLACE = [1, 0x100, 0x10000, 0x1000000];
+
+/**
+ * Where the frame walk stands: the magic number at a frame start, a skippable
+ * frame's size and content, or a Zstandard frame's header descriptor, header
+ * fields, block header, block content and checksum.
+ */
+type TZstdWalkState =
+  | 'magic'
+  | 'skippable-size'
+  | 'skippable-body'
+  | 'frame-header'
+  | 'header-fields'
+  | 'block-header'
+  | 'block-body'
+  | 'checksum';
+
+/**
+ * zstd through one `node:zlib` `createZstdDecompress` for the whole stream,
+ * fed by the codec's own RFC 8878 frame walk. node:zlib's zstd decoder before
+ * Node.js 26 drops the rest of a write after a frame (a second frame in the
+ * same write is lost, one in the next write is decoded), and every release
+ * ignores non-frame bytes after a frame. So the walk decides the frame
+ * boundaries, and no write to the decoder holds the end of one frame and any
+ * byte after it: every frame therefore costs at least one write (about 35 µs),
+ * which is why `maxZstdFrames` caps the Zstandard frames of a stream.
+ * - The walk reads the inner reader in chunks of `READ_CHUNK_BYTES` and parses
+ *   each chunk synchronously; it writes the longest runs of Zstandard-frame
+ *   bytes as views into the chunk, each ending at the end of the chunk or at a
+ *   frame end, so a frame costs one write per chunk it spans, never one per
+ *   header or block. Magic-number bytes from an earlier read are written as a
+ *   copy before the run. It keeps no bytes across reads, only the state below.
+ * - Skippable frames are skipped, never written; they do not count against
+ *   the cap. Anything else at a frame start, a set reserved header bit and
+ *   block type 3 are `malformed-archive`, raised as soon as the byte is read.
+ * - Input that ends inside a frame, a magic-number prefix included, or is
+ *   empty, is `truncated`; at least one Zstandard frame is required.
+ * - When the decoder ends by itself it must have consumed every byte written:
+ *   libzstd and the walk read the same header fields, so one check at the end
+ *   fails closed. A walk error is rethrown unchanged; a source closed before
+ *   the end skips the check.
+ */
+class ZstdSource implements IChunkSource {
+  private readonly input: IByteReader;
+  private readonly maxFrames: number;
+  private readonly feeder: plugins.stream.Readable;
+  private readonly decoder: plugins.zlib.ZstdDecompress;
+  private readonly iterator: AsyncIterator<Buffer>;
+  /** Frame bytes the walk has written to the decoder. */
+  private written = 0;
+  private walkFailed = false;
+  private walkError: unknown = undefined;
+  /** Set when the source closes: the walk starts no further read. */
+  private stopped = false;
+  private finished = false;
+
+  private state: TZstdWalkState = 'magic';
+  /** Bytes of the current magic number, skippable frame size or block header read so far. */
+  private fieldBytes = 0;
+  /** Whether the magic number being read is a skippable frame's. */
+  private skippable = false;
+  /** A skippable frame size or block header as read so far, little-endian. */
+  private fieldValue = 0;
+  /** Bytes left in the current header fields, block content, checksum or skippable content. */
+  private remaining = 0;
+  private descriptor = 0;
+  private lastBlock = false;
+  private zstandardFrames = 0;
+  private sawInput = false;
+
+  constructor(inputArg: IByteReader, maxFramesArg: number) {
+    this.input = inputArg;
+    this.maxFrames = maxFramesArg;
+    this.feeder = plugins.stream.Readable.from(this.walk(), { objectMode: false });
+    this.decoder = plugins.zlib.createZstdDecompress();
+    // errors surface through the iterator below; the callback only completes the pipeline
+    plugins.stream.pipeline(this.feeder, this.decoder, () => {});
+    this.iterator = this.decoder[Symbol.asyncIterator]();
+  }
+
+  public async next(): Promise<Uint8Array | null> {
+    if (this.finished) {
+      return null;
+    }
+    let result: IteratorResult<Buffer>;
+    try {
+      result = await this.iterator.next();
+    } catch (error) {
+      await this.close();
+      if (this.walkFailed) {
+        throw this.walkError;
+      }
+      throw mapZlibError(error, 'zstd');
+    }
+    if (!result.done) {
+      return result.value;
+    }
+    const consumed = this.decoder.bytesWritten;
+    await this.close();
+    if (this.walkFailed) {
+      throw this.walkError;
+    }
+    if (consumed !== this.written) {
+      throw zstdMalformed(`the zstd decoder ended after ${consumed} of the ${this.written} frame bytes`);
     }
+    return null;
   }
 
   public async close(): Promise<void> {
     this.finished = true;
+    this.stopped = true;
     this.feeder.destroy();
     this.decoder.destroy();
   }
+
+  /** Reads and parses the inner reader chunk by chunk and yields the runs of Zstandard-frame bytes to write. */
+  private async *walk(): AsyncGenerator<Uint8Array> {
+    const runs: Uint8Array[] = [];
+    for (;;) {
+      // only errors the walk raises or reads count as walk errors, not ones a destroyed feeder injects at a yield
+      try {
+        const chunk = await this.input.read(READ_CHUNK_BYTES);
+        if (this.stopped) {
+          return;
+        }
+        if (chunk === null) {
+          this.endOfInput();
+          return;
+        }
+        this.parse(chunk, runs);
+      } catch (error) {
+        this.walkFailed = true;
+        this.walkError = error;
+        throw error;
+      }
+      for (const run of runs) {
+        this.written += run.length;
+        yield run;
+      }
+      runs.length = 0;
+    }
+  }
+
+  /** Advances the state over `chunkArg` and appends the runs of Zstandard-frame bytes it holds to `runsArg`. */
+  private parse(chunkArg: Uint8Array, runsArg: Uint8Array[]): void {
+    const length = chunkArg.length;
+    if (length > 0) {
+      this.sawInput = true;
+    }
+    const outsideFrame = this.state === 'magic' || this.state === 'skippable-size' || this.state === 'skippable-body';
+    /** Start of the current run in this chunk, or -1 outside a Zstandard frame. */
+    let runStart = outsideFrame ? -1 : 0;
+    /** Magic-number bytes read before this chunk. */
+    let earlier = this.state === 'magic' ? this.fieldBytes : 0;
+    let index = 0;
+    while (index < length) {
+      switch (this.state) {
+        case 'magic': {
+          const byte = chunkArg[index];
+          if (this.fieldBytes === 0) {
+            this.skippable = (byte & 0xf0) === ZSTD_SKIPPABLE_MAGIC[0];
+            if (!this.skippable && byte !== ZSTD_FRAME_MAGIC[0]) {
+              throw zstdMalformed('bytes at a zstd frame start are not a zstd frame');
+            }
+          } else if (byte !== (this.skippable ? ZSTD_SKIPPABLE_MAGIC : ZSTD_FRAME_MAGIC)[this.fieldBytes]) {
+            throw zstdMalformed('bytes at a zstd frame start are not a zstd frame');
+          }
+          index += 1;
+          this.fieldBytes += 1;
+          if (this.fieldBytes < 4) {
+            break;
+          }
+          this.fieldBytes = 0;
+          if (this.skippable) {
+            this.fieldValue = 0;
+            this.state = 'skippable-size';
+          } else {
+            this.zstandardFrames += 1;
+            if (this.zstandardFrames > this.maxFrames) {
+              throw limitExceeded('maxEntries', this.maxFrames, 'zstd frames');
+            }
+            if (earlier > 0) {
+              runsArg.push(ZSTD_FRAME_MAGIC.slice(0, earlier));
+              runStart = 0;
+            } else {
+              runStart = index - 4;
+            }
+            this.state = 'frame-header';
+          }
+          earlier = 0;
+          break;
+        }
+        case 'skippable-size':
+          this.fieldValue += chunkArg[index] * LITTLE_ENDIAN_PLACE[this.fieldBytes];
+          index += 1;
+          this.fieldBytes += 1;
+          if (this.fieldBytes === 4) {
+            this.fieldBytes = 0;
+            this.remaining = this.fieldValue;
+            this.state = this.remaining === 0 ? 'magic' : 'skippable-body';
+          }
+          break;
+        case 'skippable-body': {
+          const take = Math.min(this.remaining, length - index);
+          index += take;
+          this.remaining -= take;
+          if (this.remaining === 0) {
+            this.state = 'magic';
+          }
+          break;
+        }
+        case 'frame-header': {
+          const flags = chunkArg[index];
+          index += 1;
+          if ((flags & 0x08) !== 0) {
+            throw zstdMalformed('a zstd frame header sets the reserved bit');
+          }
+          const contentSizeFlag = flags >> 6;
+          const singleSegment = (flags >> 5) & 1;
+          this.descriptor = flags;
+          this.remaining =
+            (singleSegment === 1 ? 0 : 1) +
+            ZSTD_DICTIONARY_ID_BYTES[flags & 3] +
+            (contentSizeFlag === 0 ? singleSegment : ZSTD_CONTENT_SIZE_BYTES[contentSizeFlag]);
+          this.fieldValue = 0;
+          this.state = this.remaining === 0 ? 'block-header' : 'header-fields';
+          break;
+        }
+        case 'header-fields': {
+          const take = Math.min(this.remaining, length - index);
+          index += take;
+          this.remaining -= take;
+          if (this.remaining === 0) {
+            this.state = 'block-header';
+          }
+          break;
+        }
+        case 'block-header': {
+          this.fieldValue += chunkArg[index] * LITTLE_ENDIAN_PLACE[this.fieldBytes];
+          index += 1;
+          this.fieldBytes += 1;
+          if (this.fieldBytes < 3) {
+            break;
+          }
+          const header = this.fieldValue;
+          this.fieldBytes = 0;
+          this.fieldValue = 0;
+          const blockType = (header >> 1) & 3;
+          if (blockType === ZSTD_BLOCK_TYPE_RESERVED) {
+            throw zstdMalformed('a zstd block has the reserved block type');
+          }
+          this.lastBlock = (header & 1) === 1;
+          this.remaining = blockType === ZSTD_BLOCK_TYPE_RLE ? 1 : header >>> 3;
+          if (this.remaining > 0) {
+            this.state = 'block-body';
+          } else if (this.endBlock()) {
+            runsArg.push(chunkArg.subarray(runStart, index));
+            runStart = -1;
+          }
+          break;
+        }
+        case 'block-body':
+        case 'checksum': {
+          const take = Math.min(this.remaining, length - index);
+          index += take;
+          this.remaining -= take;
+          if (this.remaining > 0) {
+            break;
+          }
+          if (this.state === 'checksum' || this.endBlock()) {
+            this.state = 'magic';
+            runsArg.push(chunkArg.subarray(runStart, index));
+            runStart = -1;
+          }
+          break;
+        }
+      }
+    }
+    if (runStart >= 0 && runStart < length) {
+      runsArg.push(chunkArg.subarray(runStart, length));
+    }
+  }
+
+  /** A block's content has been read: returns true when that ends the frame, else moves to the next field. */
+  private endBlock(): boolean {
+    if (!this.lastBlock) {
+      this.state = 'block-header';
+      return false;
+    }
+    if ((this.descriptor & 0x04) !== 0) {
+      this.remaining = 4;
+      this.state = 'checksum';
+      return false;
+    }
+    this.state = 'magic';
+    return true;
+  }
+
+  /** The inner reader has ended: only a frame start after at least one Zstandard frame is a clean end. */
+  private endOfInput(): void {
+    switch (this.state) {
+      case 'magic':
+        if (this.fieldBytes > 0) {
+          throw zstdTruncated('a zstd frame magic number');
+        }
+        if (this.zstandardFrames > 0) {
+          return;
+        }
+        if (!this.sawInput) {
+          throw zstdTruncated('the zstd stream');
+        }
+        throw zstdMalformed('the zstd stream holds no Zstandard frame');
+      case 'skippable-size':
+        throw zstdTruncated('a zstd skippable frame header');
+      case 'skippable-body':
+        throw zstdTruncated('a zstd skippable frame');
+      case 'frame-header':
+      case 'header-fields':
+        throw zstdTruncated('a zstd frame header');
+      case 'block-header':
+        throw zstdTruncated('a zstd block header');
+      case 'block-body':
+        throw zstdTruncated('a zstd block');
+      case 'checksum':
+        throw zstdTruncated('a zstd frame checksum');
+    }
+  }
+}
+
+function zstdTruncated(whereArg: string): LinuxRepoError {
+  return new LinuxRepoError('truncated', `input ended inside ${whereArg}`, { field: 'zstd stream' });
+}
+
+function zstdMalformed(messageArg: string): LinuxRepoError {
+  return new LinuxRepoError('malformed-archive', messageArg, { field: 'zstd stream' });
 }
 
 function mapZlibError(errorArg: unknown, kindArg: 'gzip' | 'zstd'): LinuxRepoError {

package/ts/shared/architecture.ts

@@ -2,7 +2,11 @@
  * Whether an architecture can name an index path segment. Readers record any
  * architecture a package states (Gitea accepts them); builders refuse an entry
  * whose architecture fails this rule with `invalid-input`, and never skip it
- * silently. Refused: the empty string, `.` and `..`, and any value containing
+ * silently. The exception is `readDebianPackage`: the record's file name
+ * (`<name>_<version>_<architecture>.deb`) embeds the architecture, so it
+ * refuses one that fails this rule or the pool file name rule with
+ * `invalid-field` (`Architecture`), and never returns a record
+ * `buildDebianIndex` refuses. Refused: the empty string, `.` and `..`, and any value containing
  * `/`, whitespace or a control character.
  */
 export function isIndexableArchitecture(architectureArg: string): boolean {

package/ts/shared/limits.ts

@@ -19,7 +19,7 @@ export interface ILinuxPackageLimits {
   maxDecompressedMetadataBytes: number;
   /** The decompressed Arch payload walked for the file list, in bytes. */
   maxPayloadBytes: number;
-  /** Tar or ar entries per archive, and values per multi-valued metadata field. */
+  /** Tar or ar entries per archive, Zstandard frames per zstd stream, and values per multi-valued metadata field. */
   maxEntries: number;
   /** Total bytes of Arch file names. */
   maxFileListBytes: number;