@push.rocks/smartlinuxrepo 1.1.2 → 1.2.0

package/ts/alpine/apkindexreader.ts

@@ -0,0 +1,265 @@
+import { LinuxRepoError, truncated } from '../shared/errors.js';
+import { resolveLinuxPackageLimits, type ILinuxPackageLimits } from '../shared/limits.js';
+import { BufferReader } from '../readers/bytes.js';
+import {
+  assertReaderBytes,
+  assertReaderInput,
+  checkUnitBytes,
+  countUnit,
+  lines,
+  readIndexDocument,
+} from '../readers/document.js';
+import { readGzipMember } from '../readers/gzipmember.js';
+import { readTarEntryData, TarReader, type ITarEntry } from '../readers/tar.js';
+import { decodeGoUtf8 } from '../readers/text.js';
+
+/** One `.SIGN.<type>.<keyName>` entry of an index's signature member. */
+export interface IAlpineIndexSignatureEntry {
+  /** The text between `.SIGN.` and the next `.`: `'RSA'` for the `.SIGN.RSA.<keyName>` that Gitea and the codec write. */
+  readonly type: string;
+  /** The rest of the entry name: the key name apk looks the public key up by. */
+  readonly keyName: string;
+  /** The entry's data, padding excluded. */
+  readonly signature: Uint8Array;
+}
+
+/** One regular file of the signed stream's tar. */
+export interface IAlpineIndexMember {
+  readonly name: string;
+  readonly bytes: Uint8Array;
+}
+
+/** One block of the `APKINDEX` member. */
+export interface IAlpineIndexBlock {
+  /** The block's lines and the empty line that ends it (a block cut by the end of the text has none). */
+  readonly bytes: Uint8Array;
+  /** The block's offset in the `APKINDEX` member's bytes. */
+  readonly offset: number;
+  /** The `P:`, `V:` and `A:` values; `''` when absent. */
+  readonly name: string;
+  readonly version: string;
+  readonly architecture: string;
+}
+
+/** An `APKINDEX.tar.gz` read into its signature entries, signed bytes, members and blocks. */
+export interface IAlpineIndex {
+  /** The `.SIGN.*` entries of the first gzip member; `[]` when the index is unsigned. */
+  readonly signatures: readonly IAlpineIndexSignatureEntry[];
+  /** The input after the signature member (all of it when unsigned): the bytes the signature covers. */
+  readonly signedBytes: Uint8Array;
+  /** The regular files of the signed stream's tar in order (`APKINDEX`, `DESCRIPTION`, ...). */
+  readonly members: readonly IAlpineIndexMember[];
+  /** The blocks of the one `APKINDEX` member, in order, duplicates included. */
+  readonly blocks: readonly IAlpineIndexBlock[];
+}
+
+/** The signature part of an index: its `.SIGN.*` entries and where the signed bytes start. */
+export interface IAlpineIndexSignatureSplit {
+  signatures: IAlpineIndexSignatureEntry[];
+  /** Offset of the signed bytes in the input: the signature member's length, or 0 when unsigned. */
+  signedOffset: number;
+}
+
+const SIGN_PREFIX = '.SIGN.';
+const APKINDEX = 'APKINDEX';
+const COLON = 0x3a;
+
+/**
+ * Splits an `APKINDEX.tar.gz` into its signature entries and the signed bytes,
+ * as apk decides it: the first gzip member is read with its exact length
+ * (`readGzipMember`, bounded as `readAlpinePackage` bounds the members before
+ * the control segment: `maxMetadataBytes` for the member, and
+ * `maxDecompressedMetadataBytes` for its tar), and when its tar's first entry
+ * is named `.SIGN.*` that member is the signature member: every entry in it
+ * must be a regular `.SIGN.<type>.<keyName>` file with a non-empty type and
+ * key name (`malformed-signature`). Otherwise the index is unsigned and the
+ * signed bytes are the whole input. Nothing after the first member is read
+ * here, so the signed bytes are never decoded. Empty input, or a signature
+ * member with nothing after it, is `truncated`; apk v3 (ADB) input is
+ * `unsupported-package-format`. Internal: shared by `readAlpineIndex` and the
+ * Alpine verifier.
+ */
+export async function splitAlpineIndexSignature(
+  bytesArg: Uint8Array,
+  limitsArg: Readonly<ILinuxPackageLimits>,
+): Promise<IAlpineIndexSignatureSplit> {
+  if (bytesArg.length === 0) {
+    throw truncated('apk index');
+  }
+  if (bytesArg.length >= 3 && bytesArg[0] === 0x41 && bytesArg[1] === 0x44 && bytesArg[2] === 0x42) {
+    throw new LinuxRepoError('unsupported-package-format', 'apk v3 (ADB) indexes are not supported', {
+      field: 'apk index',
+    });
+  }
+  const member = await readGzipMember(new BufferReader(bytesArg), {
+    maxMemberBytes: limitsArg.maxMetadataBytes,
+    memberLimit: 'maxMetadataBytes',
+    maxOutputBytes: limitsArg.maxDecompressedMetadataBytes,
+    outputLimit: 'maxDecompressedMetadataBytes',
+    maxHeaderBytes: limitsArg.maxHeaderBytes,
+  });
+  if (member === null) {
+    throw truncated('apk index');
+  }
+  const tar = new TarReader(new BufferReader(member.output), {
+    maxEntries: limitsArg.maxEntries,
+    maxHeaderBytes: limitsArg.maxHeaderBytes,
+  });
+  // the first entry's name decides, as in apk
+  const first = await tar.next();
+  if (first === null || !first.name.startsWith(SIGN_PREFIX)) {
+    return { signatures: [], signedOffset: 0 };
+  }
+  const signatures: IAlpineIndexSignatureEntry[] = [];
+  for (let entry: ITarEntry | null = first; entry !== null; entry = await tar.next()) {
+    const parsed = parseSignatureEntryName(entry);
+    signatures.push({
+      ...parsed,
+      signature: await readTarEntryData(entry, limitsArg.maxDecompressedMetadataBytes, 'maxDecompressedMetadataBytes'),
+    });
+  }
+  if (member.memberBytes === bytesArg.length) {
+    throw truncated('apk index signed stream');
+  }
+  return { signatures, signedOffset: member.memberBytes };
+}
+
+function parseSignatureEntryName(entryArg: ITarEntry): { type: string; keyName: string } {
+  const refuse = (whatArg: string): never => {
+    throw new LinuxRepoError('malformed-signature', `the index signature member ${whatArg}`, {
+      field: 'apk index signature',
+      value: entryArg.name,
+    });
+  };
+  if (entryArg.typeflag !== '0' || entryArg.sparse) {
+    refuse('holds an entry that is not a regular file');
+  }
+  if (!entryArg.name.startsWith(SIGN_PREFIX)) {
+    refuse('holds an entry not named .SIGN.*');
+  }
+  const rest = entryArg.name.slice(SIGN_PREFIX.length);
+  const dot = rest.indexOf('.');
+  const type = dot < 0 ? rest : rest.slice(0, dot);
+  const keyName = dot < 0 ? '' : rest.slice(dot + 1);
+  if (type === '' || keyName === '') {
+    refuse('holds a .SIGN.* entry without a type or key name');
+  }
+  return { type, keyName };
+}
+
+/**
+ * Reads an `APKINDEX.tar.gz` into its signature entries, the signed bytes,
+ * the members of the signed stream and the blocks of its `APKINDEX` member.
+ *
+ * - The signature split is apk's first-entry rule: the first gzip member is
+ *   the signature member when its tar's first entry is named `.SIGN.*`; it is
+ *   bounded by `maxMetadataBytes` and `maxDecompressedMetadataBytes`, which
+ *   an unsigned index's first member must meet too. An unsigned index has
+ *   `signatures: []`. Verify a signature with `createAlpineVerifier`.
+ * - The signed bytes are decoded as a gzip stream (bounded by
+ *   `maxIndexBytes`) and read as a tar: regular files become `members`, PAX
+ *   global headers are skipped, a sparse entry is
+ *   `unsupported-package-format` and any other type `malformed-index`.
+ *   Exactly one `APKINDEX` member is required: none is `missing-field`
+ *   (`APKINDEX`), two are `malformed-index`. Tar headers are not returned.
+ * - `APKINDEX` lines end in LF, the last one too, and every non-empty line is
+ *   one byte other than `:`, then `:`, then the value (`malformed-index`). A
+ *   block ends at an empty line, extra empty lines are skipped, and a block
+ *   with two `P:`, `V:` or `A:` lines is `malformed-index`.
+ *
+ * Bounds: `maxEntries` for members and blocks, `maxMetadataBytes` for a
+ * member's data and one block, `maxHeaderBytes` for a line. Every returned
+ * array is a view into buffers the reader allocated, never the caller's input.
+ */
+export async function readAlpineIndex(inputArg: {
+  bytes: Uint8Array;
+  limits?: Partial<ILinuxPackageLimits>;
+}): Promise<IAlpineIndex> {
+  assertReaderInput(inputArg);
+  assertReaderBytes(inputArg.bytes);
+  const limits = resolveLinuxPackageLimits(inputArg.limits);
+  const split = await splitAlpineIndexSignature(inputArg.bytes, limits);
+  const signedBytes = inputArg.bytes.slice(split.signedOffset);
+  const decoded = await readIndexDocument(signedBytes, 'gzip', limits);
+
+  const members: IAlpineIndexMember[] = [];
+  const tar = new TarReader(new BufferReader(decoded), { maxEntries: limits.maxEntries, maxHeaderBytes: limits.maxHeaderBytes });
+  for (let entry = await tar.next(); entry !== null; entry = await tar.next()) {
+    if (entry.sparse) {
+      await readTarEntryData(entry, limits.maxMetadataBytes, 'maxMetadataBytes'); // refused: sparse data
+    }
+    if (entry.typeflag === 'g') {
+      continue;
+    }
+    if (entry.typeflag !== '0') {
+      throw new LinuxRepoError('malformed-index', `the APKINDEX tar holds an entry of type ${JSON.stringify(entry.typeflag)}`, {
+        field: 'APKINDEX tar',
+        value: entry.name,
+      });
+    }
+    members.push({ name: entry.name, bytes: await readTarEntryData(entry, limits.maxMetadataBytes, 'maxMetadataBytes') });
+  }
+  const indexMembers = members.filter((member) => member.name === APKINDEX);
+  if (indexMembers.length === 0) {
+    throw new LinuxRepoError('missing-field', 'the index has no APKINDEX member', { field: APKINDEX });
+  }
+  if (indexMembers.length > 1) {
+    throw new LinuxRepoError('malformed-index', 'the index holds more than one APKINDEX member', { field: APKINDEX });
+  }
+  return { signatures: split.signatures, signedBytes, members, blocks: readBlocks(indexMembers[0].bytes, limits) };
+}
+
+interface IBlockState {
+  start: number;
+  name: string | null;
+  version: string | null;
+  architecture: string | null;
+}
+
+function readBlocks(textArg: Uint8Array, limitsArg: Readonly<ILinuxPackageLimits>): IAlpineIndexBlock[] {
+  const blocks: IAlpineIndexBlock[] = [];
+  let block: IBlockState | null = null;
+  const finish = (endArg: number) => {
+    if (block !== null) {
+      const current: IBlockState = block;
+      block = null;
+      blocks.push({
+        bytes: textArg.subarray(current.start, endArg),
+        offset: current.start,
+        name: current.name ?? '',
+        version: current.version ?? '',
+        architecture: current.architecture ?? '',
+      });
+    }
+  };
+  for (const line of lines(textArg, limitsArg.maxHeaderBytes, 'APKINDEX')) {
+    if (line.end === line.start) {
+      finish(line.end + 1);
+      continue;
+    }
+    if (line.end - line.start < 2 || textArg[line.start] === COLON || textArg[line.start + 1] !== COLON) {
+      throw new LinuxRepoError('malformed-index', 'an APKINDEX line is not "<letter>:<value>"', {
+        field: APKINDEX,
+        value: line.start,
+      });
+    }
+    if (block === null) {
+      countUnit(blocks.length + 1, limitsArg, 'APKINDEX block');
+      block = { start: line.start, name: null, version: null, architecture: null };
+    }
+    checkUnitBytes(line.end + 1 - block.start, limitsArg, 'APKINDEX block');
+    const key = textArg[line.start];
+    const slot = key === 0x50 ? 'name' : key === 0x56 ? 'version' : key === 0x41 ? 'architecture' : null;
+    if (slot !== null) {
+      if (block[slot] !== null) {
+        throw new LinuxRepoError('malformed-index', `an APKINDEX block holds two ${String.fromCharCode(key)}: lines`, {
+          field: APKINDEX,
+          value: line.start,
+        });
+      }
+      block[slot] = decodeGoUtf8(textArg.subarray(line.start + 2, line.end));
+    }
+  }
+  finish(textArg.length);
+  return blocks;
+}

package/ts/arch/indexreader.ts

@@ -0,0 +1,66 @@
+import { LinuxRepoError } from '../shared/errors.js';
+import { resolveLinuxPackageLimits, type ILinuxPackageLimits } from '../shared/limits.js';
+import { BufferReader } from '../readers/bytes.js';
+import type { TLinuxCompression } from '../shared/types.js';
+import { assertLinuxCompression, assertReaderBytes, assertReaderInput, readIndexDocument } from '../readers/document.js';
+import { readTarEntryData, TarReader } from '../readers/tar.js';
+
+/** One regular file of a `packages.db` (`<name>-<version>/desc` or `/files`). */
+export interface IArchIndexMember {
+  /** The tar entry name. */
+  readonly name: string;
+  /** The entry's data, without header or padding (a view into `decoded`). */
+  readonly bytes: Uint8Array;
+}
+
+/** A `packages.db` read into its members. */
+export interface IArchIndex {
+  /** The decompressed tar (a copy for `'none'`). */
+  readonly decoded: Uint8Array;
+  /** Every regular file in archive order, duplicates included. */
+  readonly members: readonly IArchIndexMember[];
+}
+
+/**
+ * Reads an Arch repository database (`packages.db`, `<repo>.db.tar.gz`,
+ * `.files`; the caller names `compression`) into its members. The tar is
+ * read with Go `archive/tar` semantics: regular files become `members`,
+ * directory entries (pacman's `repo-add` writes them) and PAX global headers
+ * are skipped, a sparse entry is `unsupported-package-format`, and any other
+ * entry type (links, devices, FIFOs) is `malformed-index`. The member data is
+ * returned raw; `desc` and `files` are not parsed. Verify `packages.db.sig`
+ * with `createOpenPgpVerifier().verifyDetachedSignature` over the compressed
+ * file's bytes.
+ *
+ * Bounds: the decoded tar by `maxIndexBytes`, entries by `maxEntries`, one
+ * member's data by `maxMetadataBytes`, one header chain by `maxHeaderBytes`.
+ */
+export async function readArchIndex(inputArg: {
+  bytes: Uint8Array;
+  compression: TLinuxCompression;
+  limits?: Partial<ILinuxPackageLimits>;
+}): Promise<IArchIndex> {
+  assertReaderInput(inputArg);
+  assertReaderBytes(inputArg.bytes);
+  assertLinuxCompression(inputArg.compression);
+  const limits = resolveLinuxPackageLimits(inputArg.limits);
+  const decoded = await readIndexDocument(inputArg.bytes, inputArg.compression, limits);
+  const members: IArchIndexMember[] = [];
+  const tar = new TarReader(new BufferReader(decoded), { maxEntries: limits.maxEntries, maxHeaderBytes: limits.maxHeaderBytes });
+  for (let entry = await tar.next(); entry !== null; entry = await tar.next()) {
+    if (entry.sparse) {
+      await readTarEntryData(entry, limits.maxMetadataBytes, 'maxMetadataBytes'); // refused: sparse data
+    }
+    if (entry.typeflag === '5' || entry.typeflag === 'g') {
+      continue;
+    }
+    if (entry.typeflag !== '0') {
+      throw new LinuxRepoError('malformed-index', `the Arch database holds an entry of type ${JSON.stringify(entry.typeflag)}`, {
+        field: 'Arch database',
+        value: entry.name,
+      });
+    }
+    members.push({ name: entry.name, bytes: await readTarEntryData(entry, limits.maxMetadataBytes, 'maxMetadataBytes') });
+  }
+  return { decoded, members };
+}

package/ts/arch/reader.ts

@@ -5,7 +5,8 @@ import type { IArchPackageMetadata, IArchPackageRecord } from '../shared/types.j
 import { isValidUrl } from '../shared/url.js';
 import { readUpTo, skipToEnd } from '../readers/bytes.js';
 import { PackageByteSource } from '../readers/bytesource.js';
-import { withDecompressingReader, type TLinuxCompression } from '../readers/decompress.js';
+import { withDecompressingReader } from '../readers/decompress.js';
+import type { TLinuxCompression } from '../shared/types.js';
 import { readTarEntryData, TarReader, type ITarEntry } from '../readers/tar.js';
 import { decodeGoUtf8 } from '../readers/text.js';
 import { goParseInt64, goTrimSpace } from '../shared/gostrings.js';

package/ts/debian/indexbuilder.ts

@@ -6,10 +6,13 @@ import { assertString, assertTimestamp } from '../shared/validation.js';
 import type { IOpenPgpSigner } from '../signing/signers.js';
 import { createCleartextSignedMessage, createDetachedArmoredSignature } from '../signing/openpgpframing.js';
 import { compareUtf8, goTrimSpace, utf8ByteString } from './gostring.js';
-import { DEBIAN_SYMBOL_PATTERN, hasPoolFileNameRefusedCharacter, isDebianReleaseValue, isDebianSymbol } from './rules.js';
-
-/** The fields the `Packages` stanza writes itself; a control file's own copies are dropped. */
-const INDEX_OWNED_FIELDS = new Set(['filename', 'size', 'md5sum', 'sha1', 'sha256', 'sha512']);
+import {
+  DEBIAN_SYMBOL_PATTERN,
+  hasPoolFileNameRefusedCharacter,
+  isDebianReleaseValue,
+  isDebianSymbol,
+  stripDebianIndexOwnedFields,
+} from './rules.js';
 
 const PACKAGES_FILES = ['Packages', 'Packages.gz', 'Packages.xz'] as const;
 
@@ -197,24 +200,11 @@ export function renderDebianPackagesStanza(input: {
   assertSymbol(input.distribution, 'distribution');
   assertSymbol(input.component, 'component');
   assertRecord(input.record);
-  const kept: string[] = [];
-  let dropping = false;
-  // Gitea trims the stored stanza again when it writes it (repository.go:202)
-  for (const line of goTrimSpaceText(input.record.metadata.control).split('\n')) {
-    if (line[0] === ' ' || line[0] === '\t') {
-      if (!dropping) {
-        kept.push(line);
-      }
-      continue;
-    }
-    const colon = line.indexOf(':');
-    const field = colon < 0 ? null : goTrimSpaceText(line.slice(0, colon)).toLowerCase();
-    dropping = field !== null && INDEX_OWNED_FIELDS.has(field);
-    if (!dropping) {
-      kept.push(line);
-    }
-  }
-  const control = kept.join('\n');
+  // Gitea trims the stored stanza again when it writes it (repository.go:202);
+  // the trimmed text plus its final LF is one stanza, which the index-owned
+  // field rule walks (`stripDebianIndexOwnedFields`)
+  const stanza = new TextEncoder().encode(`${goTrimSpaceText(input.record.metadata.control)}\n`);
+  const control = new TextDecoder().decode(stripDebianIndexOwnedFields(stanza));
   if (control === '') {
     throw new LinuxRepoError('invalid-input', 'the record control stanza has no fields besides the index-owned ones', {
       field: 'record.metadata.control',
@@ -222,7 +212,7 @@ export function renderDebianPackagesStanza(input: {
   }
   const { digests } = input.record;
   return (
-    `${control}\n` +
+    control +
     `Filename: pool/${input.distribution}/${input.component}/${input.record.fileName}\n` +
     `Size: ${digests.size}\n` +
     `MD5sum: ${digests.md5}\n` +