@push.rocks/smartlinuxrepo 0.0.0-stage → 1.0.0

package/ts/debian/reader.ts

@@ -0,0 +1,126 @@
+import { LinuxRepoError, limitExceeded } from '../shared/errors.js';
+import { resolveLinuxPackageLimits, type ILinuxPackageLimits } from '../shared/limits.js';
+import { goPathBase } from '../shared/gopath.js';
+import type { IDebianPackageRecord } from '../shared/types.js';
+import { ArReader } from '../readers/ar.js';
+import { skipToEnd } from '../readers/bytes.js';
+import { PackageByteSource } from '../readers/bytesource.js';
+import { withDecompressingReader, type TLinuxCompression } from '../readers/decompress.js';
+import { readTarEntryData, TarReader } from '../readers/tar.js';
+import { parseDebianControlFile, type IDebianControl } from './control.js';
+
+const CONTROL_TAR = 'control.tar';
+
+/** The control archive forms Gitea 28.2.0 reads (`metadata.go:108-136`). */
+const CONTROL_COMPRESSIONS: Record<string, TLinuxCompression> = {
+  '': 'none',
+  '.gz': 'gzip',
+  '.xz': 'xz',
+  '.zst': 'zstd',
+};
+
+/**
+ * Reads a `.deb` package in one pass into Gitea 28.2.0's record
+ * (`modules/packages/debian/metadata.go:92-163`).
+ *
+ * - The `ar` members are walked in order; `debian-binary` is not required.
+ *   The first member whose name starts with `control.tar` (a trailing `/`
+ *   stripped from the suffix) is decompressed: `control.tar`, `.gz`, `.xz` or
+ *   `.zst`; any other suffix is `unsupported-compression`.
+ * - In it, the first regular entry whose base name is `control` is parsed. A
+ *   control archive without one moves on to the next `ar` member, as Gitea
+ *   does; none at all is `missing-field` (`control`).
+ * - The control member is decoded to the decoder's end on both paths, after
+ *   the control file is parsed and when it holds none: a cut or corrupt
+ *   control member is `truncated` or `malformed-archive` in every form, where
+ *   Gitea, which stops at the control entry, accepts it. For a plain
+ *   `control.tar` the end is the end of its `ar` member; the bytes after the
+ *   tar end marker are read, not parsed.
+ * - Once the control file is parsed the `ar` archive is not walked further:
+ *   the rest of the input is only hashed (`metadata.go:155-156`).
+ * - Bounds: the control member's compressed size by `maxMetadataBytes`, its
+ *   decompressed bytes by `maxDecompressedMetadataBytes` (Gitea's 32 MiB),
+ *   tar entries and `ar` members by `maxEntries`, tar header chains and
+ *   control lines by `maxHeaderBytes`, the whole input by `maxPackageBytes`.
+ *   A control member that decompresses past `maxDecompressedMetadataBytes` is
+ *   `limit-exceeded` even when `control` comes first, where Gitea cuts the
+ *   stream at 32 MiB and keeps a `control` found before the cut.
+ */
+export async function readDebianPackage(
+  source: AsyncIterable<Uint8Array>,
+  limits?: Partial<ILinuxPackageLimits>,
+): Promise<IDebianPackageRecord> {
+  const resolved = resolveLinuxPackageLimits(limits);
+  const input = new PackageByteSource(source, resolved.maxPackageBytes);
+  const control = await readControl(input, resolved);
+  await input.drain();
+  return {
+    format: 'debian',
+    name: control.name,
+    version: control.version,
+    architecture: control.architecture,
+    fileName: `${control.name}_${control.version}_${control.architecture}.deb`,
+    digests: input.digests(),
+    metadata: control.metadata,
+  };
+}
+
+async function readControl(inputArg: PackageByteSource, limitsArg: Readonly<ILinuxPackageLimits>): Promise<IDebianControl> {
+  const archive = new ArReader(inputArg, limitsArg.maxEntries);
+  for (;;) {
+    const member = await archive.next();
+    if (member === null) {
+      throw new LinuxRepoError('missing-field', 'the package has no control file', { field: 'control' });
+    }
+    if (!member.name.startsWith(CONTROL_TAR)) {
+      continue;
+    }
+    const suffix = member.name.slice(CONTROL_TAR.length).replace(/\/$/, '');
+    const compression = Object.hasOwn(CONTROL_COMPRESSIONS, suffix) ? CONTROL_COMPRESSIONS[suffix] : undefined;
+    if (compression === undefined) {
+      throw new LinuxRepoError('unsupported-compression', `unsupported control archive "${member.name}"`, {
+        field: 'control.tar',
+        value: member.name,
+      });
+    }
+    if (member.size > limitsArg.maxMetadataBytes) {
+      throw limitExceeded('maxMetadataBytes', limitsArg.maxMetadataBytes, `control member ${member.name}`);
+    }
+    const control = await withDecompressingReader(
+      member.body,
+      compression,
+      { maxOutputBytes: limitsArg.maxDecompressedMetadataBytes, outputLimit: 'maxDecompressedMetadataBytes' },
+      async (decompressed) => {
+        const tar = new TarReader(decompressed, {
+          maxEntries: limitsArg.maxEntries,
+          maxHeaderBytes: limitsArg.maxHeaderBytes,
+        });
+        let parsed: IDebianControl | null = null;
+        for (;;) {
+          const entry = await tar.next();
+          if (entry === null) {
+            break;
+          }
+          if (entry.typeflag !== '0' || goPathBase(entry.name) !== 'control') {
+            continue;
+          }
+          const bytes = await readTarEntryData(
+            entry,
+            limitsArg.maxDecompressedMetadataBytes,
+            'maxDecompressedMetadataBytes',
+          );
+          parsed = parseDebianControlFile(bytes, limitsArg.maxHeaderBytes);
+          break;
+        }
+        // Gitea stops at the control entry; the codec decodes the member to
+        // the decoder's end, so a cut or corrupt control member is refused in
+        // every form (a plain control.tar ends with its ar member)
+        await skipToEnd(decompressed);
+        return parsed;
+      },
+    );
+    if (control !== null) {
+      return control;
+    }
+  }
+}

package/ts/index.ts

@@ -0,0 +1,46 @@
+export type {
+  TLinuxPackageFormat,
+  ILinuxPackageDigests,
+  IAlpinePackageDigests,
+  ILinuxPackageRecordBase,
+  IAlpinePackageMetadata,
+  IAlpinePackageRecord,
+  IDebianPackageMetadata,
+  IDebianPackageRecord,
+  IRpmDependency,
+  IRpmFile,
+  IRpmChangelog,
+  IRpmPackageMetadata,
+  IRpmPackageRecord,
+  IRpmUpdate,
+  IRpmUpdateReference,
+  IRpmUpdateCollection,
+  IRpmUpdatePackage,
+  IArchPackageMetadata,
+  IArchPackageRecord,
+  TLinuxPackageRecord,
+  TLinuxIndexFileStage,
+  ILinuxIndexFile,
+} from './shared/types.js';
+export { DEFAULT_LINUX_PACKAGE_LIMITS } from './shared/limits.js';
+export type { ILinuxPackageLimits } from './shared/limits.js';
+export { LinuxRepoError } from './shared/errors.js';
+export type { TLinuxRepoErrorCode, ILinuxRepoErrorDetail } from './shared/errors.js';
+export { isIndexableArchitecture } from './shared/architecture.js';
+export { createAlpineSigner, createOpenPgpSigner } from './signing/signers.js';
+export type { IAlpineSigner, IOpenPgpSigner } from './signing/signers.js';
+export { generateAlpineKeyPair, generateOpenPgpKeyPair, giteaAlpineKeyName } from './signing/keys.js';
+export type { IAlpineKeyPair, IOpenPgpKeyPair } from './signing/keys.js';
+export { readDebianPackage } from './debian/reader.js';
+export { buildDebianIndex } from './debian/indexbuilder.js';
+export { readRpmPackage } from './rpm/reader.js';
+export { signRpmPackage } from './rpm/sign.js';
+export { buildRpmIndex } from './rpm/builder.js';
+export { mergeRpmUpdates } from './rpm/updates.js';
+export { renderRpmRepoConfig } from './rpm/repoconfig.js';
+export { readArchPackage } from './arch/reader.js';
+export { buildArchIndex } from './arch/builder.js';
+export type { IArchIndexEntry } from './arch/builder.js';
+export { signArchPackage } from './arch/signer.js';
+export { readAlpinePackage } from './alpine/reader.js';
+export { buildAlpineIndex } from './alpine/apkindex.js';

package/ts/plugins.ts

@@ -0,0 +1,13 @@
+// node native scope
+import * as crypto from 'node:crypto';
+import * as stream from 'node:stream';
+import * as streamWeb from 'node:stream/web';
+import * as zlib from 'node:zlib';
+
+export { crypto, stream, streamWeb, zlib };
+
+// third party scope
+import * as lzma from '@napi-rs/lzma';
+import * as openpgp from 'openpgp';
+
+export { lzma, openpgp };

package/ts/readers/ar.ts

@@ -0,0 +1,102 @@
+import { LinuxRepoError, limitExceeded, truncated } from '../shared/errors.js';
+import { LimitedReader, readUpTo, type IByteReader } from './bytes.js';
+import { decodeGoUtf8, decodeLatin1 } from './text.js';
+
+const AR_MAGIC = '!<arch>\n';
+const AR_HEADER_BYTES = 60;
+
+export interface IArMember {
+  /**
+   * The header name with trailing spaces removed. Nothing else is resolved: a
+   * GNU trailing `/` stays, and GNU (`//`) or BSD (`#1/`) long-name forms are
+   * ordinary member names, as Gitea's `blakesmith/ar` reports them.
+   */
+  name: string;
+  size: number;
+  /** Exactly `size` bytes; reading past the input's end fails with `truncated`. */
+  body: LimitedReader;
+}
+
+/**
+ * A bounded reader for `ar` archives (the `.deb` container). It checks the
+ * `!<arch>\n` magic, the header terminator and the decimal size, which
+ * `blakesmith/ar` does not; every real `ar` writer produces them. A missing
+ * padding byte at the very end of the input ends the archive cleanly, as in
+ * `blakesmith/ar`.
+ */
+export class ArReader {
+  private started = false;
+  private done = false;
+  private current: LimitedReader | null = null;
+  private pad = 0;
+  private members = 0;
+
+  constructor(
+    private readonly input: IByteReader,
+    private readonly maxEntries: number,
+  ) {}
+
+  public async next(): Promise<IArMember | null> {
+    if (this.done) {
+      return null;
+    }
+    if (!this.started) {
+      await this.readMagic();
+      this.started = true;
+    }
+    if (this.current !== null) {
+      await this.current.skipRest();
+      this.current = null;
+    }
+    if (this.pad > 0) {
+      this.pad = 0;
+      const padding = await readUpTo(this.input, 1);
+      if (padding.length === 0) {
+        this.done = true;
+        return null;
+      }
+    }
+    const header = await readUpTo(this.input, AR_HEADER_BYTES);
+    if (header.length === 0) {
+      this.done = true;
+      return null;
+    }
+    if (header.length < AR_HEADER_BYTES) {
+      throw truncated('ar member header');
+    }
+    if (header[58] !== 0x60 || header[59] !== 0x0a) {
+      throw malformed('ar member header has no "`\\n" terminator');
+    }
+    const name = decodeGoUtf8(header.subarray(0, 16)).replace(/ +$/, '');
+    const sizeText = decodeLatin1(header.subarray(48, 58)).replace(/ +$/, '');
+    if (!/^[0-9]+$/.test(sizeText)) {
+      throw malformed('ar member size is not a decimal number');
+    }
+    const size = Number(sizeText);
+    if (!Number.isSafeInteger(size)) {
+      throw malformed('ar member size is out of range');
+    }
+    this.members += 1;
+    if (this.members > this.maxEntries) {
+      throw limitExceeded('maxEntries', this.maxEntries, 'ar member count');
+    }
+    this.current = new LimitedReader(this.input, size, 'ar member data');
+    this.pad = size % 2;
+    return { name, size, body: this.current };
+  }
+
+  private async readMagic(): Promise<void> {
+    const magic = await readUpTo(this.input, AR_MAGIC.length);
+    const text = decodeLatin1(magic);
+    if (!AR_MAGIC.startsWith(text)) {
+      throw new LinuxRepoError('malformed-archive', 'input is not an ar archive', { field: 'ar magic' });
+    }
+    if (magic.length < AR_MAGIC.length) {
+      throw truncated('ar magic');
+    }
+  }
+}
+
+function malformed(messageArg: string): LinuxRepoError {
+  return new LinuxRepoError('malformed-archive', messageArg, { field: 'ar member header' });
+}

package/ts/readers/bytes.ts

@@ -0,0 +1,182 @@
+import { truncated } from '../shared/errors.js';
+
+/**
+ * A pull-based byte reader. `read` returns between 1 and `max` bytes, or
+ * `null` at the end of input. `unread` pushes bytes back so the next `read`
+ * returns them first (used where a decoder consumed less than it was given).
+ */
+export interface IByteReader {
+  read(maxArg: number): Promise<Uint8Array | null>;
+  unread(bytesArg: Uint8Array): void;
+}
+
+/** The read size the readers use when they pull without a structural bound. */
+export const READ_CHUNK_BYTES = 64 * 1024;
+
+export function concatBytes(chunksArg: Uint8Array[], totalArg?: number): Uint8Array {
+  if (chunksArg.length === 1) {
+    return chunksArg[0];
+  }
+  const total = totalArg ?? chunksArg.reduce((sum, chunk) => sum + chunk.length, 0);
+  const result = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunksArg) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return result;
+}
+
+/**
+ * Reads up to `lengthArg` bytes; returns fewer only when the input ends.
+ */
+export async function readUpTo(readerArg: IByteReader, lengthArg: number): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  while (total < lengthArg) {
+    const chunk = await readerArg.read(lengthArg - total);
+    if (chunk === null) {
+      break;
+    }
+    chunks.push(chunk);
+    total += chunk.length;
+  }
+  return chunks.length === 0 ? new Uint8Array(0) : concatBytes(chunks, total);
+}
+
+/**
+ * Reads exactly `lengthArg` bytes, or fails with `truncated` naming `whatArg`.
+ */
+export async function readExact(
+  readerArg: IByteReader,
+  lengthArg: number,
+  whatArg: string,
+): Promise<Uint8Array> {
+  const bytes = await readUpTo(readerArg, lengthArg);
+  if (bytes.length !== lengthArg) {
+    throw truncated(whatArg);
+  }
+  return bytes;
+}
+
+/**
+ * Skips exactly `lengthArg` bytes, or fails with `truncated` naming `whatArg`.
+ */
+export async function skipExact(
+  readerArg: IByteReader,
+  lengthArg: number,
+  whatArg: string,
+): Promise<void> {
+  let remaining = lengthArg;
+  while (remaining > 0) {
+    const chunk = await readerArg.read(Math.min(remaining, READ_CHUNK_BYTES));
+    if (chunk === null) {
+      throw truncated(whatArg);
+    }
+    remaining -= chunk.length;
+  }
+}
+
+/**
+ * Reads and discards `readerArg` until it reports its end. The reader's own
+ * bounds apply: a decompressing reader counts the output against its
+ * `maxOutputBytes` and reports a cut or corrupt stream when its decoder does.
+ */
+export async function skipToEnd(readerArg: IByteReader): Promise<void> {
+  while ((await readerArg.read(READ_CHUNK_BYTES)) !== null) {
+    // the bytes are discarded; the reader counts and checks them
+  }
+}
+
+/** A reader over bytes already in memory. */
+export class BufferReader implements IByteReader {
+  private bytes: Uint8Array;
+  private offset = 0;
+  private pushedBack: Uint8Array[] = [];
+
+  constructor(bytesArg: Uint8Array) {
+    this.bytes = bytesArg;
+  }
+
+  public async read(maxArg: number): Promise<Uint8Array | null> {
+    const pushed = this.pushedBack.pop();
+    if (pushed !== undefined) {
+      return takeFront(pushed, maxArg, this.pushedBack);
+    }
+    if (this.offset >= this.bytes.length) {
+      return null;
+    }
+    const end = Math.min(this.bytes.length, this.offset + Math.max(1, maxArg));
+    const chunk = this.bytes.subarray(this.offset, end);
+    this.offset = end;
+    return chunk;
+  }
+
+  public unread(bytesArg: Uint8Array): void {
+    if (bytesArg.length > 0) {
+      this.pushedBack.push(bytesArg);
+    }
+  }
+}
+
+/**
+ * Exposes exactly `lengthArg` bytes of an inner reader. Reading past the inner
+ * end fails with `truncated` naming `whatArg`; `skipRest` consumes what the
+ * consumer left.
+ */
+export class LimitedReader implements IByteReader {
+  private remaining: number;
+  private pushedBack: Uint8Array[] = [];
+
+  constructor(
+    private readonly inner: IByteReader,
+    lengthArg: number,
+    private readonly what: string,
+  ) {
+    this.remaining = lengthArg;
+  }
+
+  /** Bytes of the window the consumer has not read yet. */
+  public get unreadBytes(): number {
+    return this.remaining + this.pushedBack.reduce((sum, chunk) => sum + chunk.length, 0);
+  }
+
+  public async read(maxArg: number): Promise<Uint8Array | null> {
+    const pushed = this.pushedBack.pop();
+    if (pushed !== undefined) {
+      return takeFront(pushed, maxArg, this.pushedBack);
+    }
+    if (this.remaining === 0) {
+      return null;
+    }
+    const chunk = await this.inner.read(Math.min(Math.max(1, maxArg), this.remaining));
+    if (chunk === null) {
+      throw truncated(this.what);
+    }
+    this.remaining -= chunk.length;
+    return chunk;
+  }
+
+  public unread(bytesArg: Uint8Array): void {
+    if (bytesArg.length > 0) {
+      this.pushedBack.push(bytesArg);
+    }
+  }
+
+  public async skipRest(): Promise<void> {
+    this.pushedBack = [];
+    await skipExact(this.inner, this.remaining, this.what);
+    this.remaining = 0;
+  }
+}
+
+/** Splits `chunkArg` at `maxArg`, pushing the tail back onto `stackArg`. */
+export function takeFront(chunkArg: Uint8Array, maxArg: number, stackArg: Uint8Array[]): Uint8Array {
+  const max = Math.max(1, maxArg);
+  if (chunkArg.length <= max) {
+    return chunkArg;
+  }
+  stackArg.push(chunkArg.subarray(max));
+  return chunkArg.subarray(0, max);
+}
+

package/ts/readers/bytesource.ts

@@ -0,0 +1,164 @@
+import * as plugins from '../plugins.js';
+import { LinuxRepoError, limitExceeded } from '../shared/errors.js';
+import type { ILinuxPackageDigests } from '../shared/types.js';
+import { takeFront, type IByteReader } from './bytes.js';
+
+/**
+ * The digest tee: pulls the caller's package bytes once, counts them against
+ * `maxPackageBytes`, and feeds every byte exactly once into MD5, SHA-1,
+ * SHA-256 and SHA-512 at the moment it is pulled.
+ *
+ * - It pulls with `next()` only. It never calls `return()` or `throw()` on the
+ *   caller's iterator, so a web `ReadableStream` is never cancelled.
+ * - An error from the caller's iterator propagates unchanged. After any error
+ *   the source stops pulling for good.
+ * - `drain()` reads the rest of the input so the digests cover the whole file;
+ *   `digests()` is available only after the input has ended.
+ */
+export class PackageByteSource implements IByteReader {
+  private readonly source: AsyncIterable<Uint8Array>;
+  private readonly maxPackageBytes: number;
+  private iterator: AsyncIterator<Uint8Array> | null = null;
+  private readonly pushedBack: Uint8Array[] = [];
+  private readonly hashes = {
+    md5: plugins.crypto.createHash('md5'),
+    sha1: plugins.crypto.createHash('sha1'),
+    sha256: plugins.crypto.createHash('sha256'),
+    sha512: plugins.crypto.createHash('sha512'),
+  };
+  private pulledBytes = 0;
+  private deliveredBytes = 0;
+  private ended = false;
+  private failure: unknown = undefined;
+  private failed = false;
+  private finalDigests: ILinuxPackageDigests | null = null;
+
+  constructor(sourceArg: AsyncIterable<Uint8Array>, maxPackageBytesArg: number) {
+    if (
+      sourceArg === null ||
+      typeof sourceArg !== 'object' ||
+      typeof (sourceArg as Partial<AsyncIterable<Uint8Array>>)[Symbol.asyncIterator] !== 'function'
+    ) {
+      throw new LinuxRepoError('invalid-input', 'source must be an AsyncIterable<Uint8Array>', {
+        field: 'source',
+      });
+    }
+    this.source = sourceArg;
+    this.maxPackageBytes = maxPackageBytesArg;
+  }
+
+  /** Offset of the next byte `read` returns, counted from the start of the file. */
+  public get position(): number {
+    return this.deliveredBytes;
+  }
+
+  /** Bytes pulled from the caller so far. */
+  public get pulled(): number {
+    return this.pulledBytes;
+  }
+
+  public async read(maxArg: number): Promise<Uint8Array | null> {
+    const pushed = this.pushedBack.pop();
+    if (pushed !== undefined) {
+      const chunk = takeFront(pushed, maxArg, this.pushedBack);
+      this.deliveredBytes += chunk.length;
+      return chunk;
+    }
+    const pulled = await this.pull();
+    if (pulled === null) {
+      return null;
+    }
+    const chunk = takeFront(pulled, maxArg, this.pushedBack);
+    this.deliveredBytes += chunk.length;
+    return chunk;
+  }
+
+  public unread(bytesArg: Uint8Array): void {
+    if (bytesArg.length === 0) {
+      return;
+    }
+    this.pushedBack.push(bytesArg);
+    this.deliveredBytes -= bytesArg.length;
+  }
+
+  /** Reads and discards the rest of the input. */
+  public async drain(): Promise<void> {
+    this.deliveredBytes += this.pushedBack.reduce((sum, chunk) => sum + chunk.length, 0);
+    this.pushedBack.length = 0;
+    while ((await this.pull()) !== null) {
+      // every pulled chunk is hashed and counted inside pull()
+    }
+    this.deliveredBytes = this.pulledBytes;
+  }
+
+  /** Size and digests of the whole input; only after the input has ended. */
+  public digests(): ILinuxPackageDigests {
+    if (!this.ended || this.failed) {
+      throw new Error('PackageByteSource.digests() called before the input ended');
+    }
+    if (this.finalDigests === null) {
+      this.finalDigests = {
+        size: this.pulledBytes,
+        md5: this.hashes.md5.digest('hex'),
+        sha1: this.hashes.sha1.digest('hex'),
+        sha256: this.hashes.sha256.digest('hex'),
+        sha512: this.hashes.sha512.digest('hex'),
+      };
+    }
+    return { ...this.finalDigests };
+  }
+
+  private async pull(): Promise<Uint8Array | null> {
+    if (this.failed) {
+      throw this.failure;
+    }
+    if (this.ended) {
+      return null;
+    }
+    try {
+      this.iterator ??= this.source[Symbol.asyncIterator]();
+      for (;;) {
+        let result: IteratorResult<Uint8Array>;
+        try {
+          result = await this.iterator.next();
+        } catch (error) {
+          // the caller's error, unchanged
+          this.fail(error);
+          throw error;
+        }
+        if (result.done) {
+          this.ended = true;
+          return null;
+        }
+        const value: unknown = result.value;
+        if (!(value instanceof Uint8Array)) {
+          throw new LinuxRepoError('invalid-input', 'source yielded a chunk that is not a Uint8Array', {
+            field: 'source',
+          });
+        }
+        if (value.length === 0) {
+          continue;
+        }
+        if (this.pulledBytes + value.length > this.maxPackageBytes) {
+          throw limitExceeded('maxPackageBytes', this.maxPackageBytes, 'package input');
+        }
+        this.pulledBytes += value.length;
+        this.hashes.md5.update(value);
+        this.hashes.sha1.update(value);
+        this.hashes.sha256.update(value);
+        this.hashes.sha512.update(value);
+        return value;
+      }
+    } catch (error) {
+      this.fail(error);
+      throw error;
+    }
+  }
+
+  private fail(errorArg: unknown): void {
+    if (!this.failed) {
+      this.failed = true;
+      this.failure = errorArg;
+    }
+  }
+}