@push.rocks/smartlinuxrepo 0.0.0-stage → 1.0.0

package/ts/debian/indexbuilder.ts

@@ -0,0 +1,355 @@
+import * as plugins from '../plugins.js';
+import { isIndexableArchitecture } from '../shared/architecture.js';
+import { LinuxRepoError } from '../shared/errors.js';
+import type { IDebianPackageRecord, ILinuxIndexFile } from '../shared/types.js';
+import { assertSegmentName, assertString, assertTimestamp, hasControlCharacter } from '../shared/validation.js';
+import type { IOpenPgpSigner } from '../signing/signers.js';
+import { createCleartextSignedMessage, createDetachedArmoredSignature } from '../signing/openpgpframing.js';
+import { DEBIAN_SYMBOL_PATTERN } from './control.js';
+import { compareUtf8, goTrimSpace, utf8ByteString } from './gostring.js';
+
+/** The fields the `Packages` stanza writes itself; a control file's own copies are dropped. */
+const INDEX_OWNED_FIELDS = new Set(['filename', 'size', 'md5sum', 'sha1', 'sha256', 'sha512']);
+
+const PACKAGES_FILES = ['Packages', 'Packages.gz', 'Packages.xz'] as const;
+
+const MEDIA_TYPES: Record<string, string> = {
+  Packages: 'text/plain; charset=utf-8',
+  'Packages.gz': 'application/gzip',
+  'Packages.xz': 'application/x-xz',
+  Release: 'text/plain; charset=utf-8',
+  'Release.gpg': 'application/pgp-signature',
+  InRelease: 'text/plain; charset=utf-8',
+};
+
+const HASH_PATTERNS = {
+  md5: /^[0-9a-f]{32}$/,
+  sha1: /^[0-9a-f]{40}$/,
+  sha256: /^[0-9a-f]{64}$/,
+  sha512: /^[0-9a-f]{128}$/,
+};
+
+export interface IDebianIndexEntry {
+  component: string;
+  record: IDebianPackageRecord;
+}
+
+/**
+ * Builds the index files of one Debian distribution as Gitea 28.2.0 does
+ * (`services/packages/debian/repository.go:147-397`):
+ *
+ * - per component and architecture with packages:
+ *   `dists/<dist>/<component>/binary-<arch>/Packages`, `.gz` and `.xz`,
+ *   each with `by-hash/{MD5Sum,SHA1,SHA256,SHA512}/<hex>` aliases;
+ * - `dists/<dist>/Release`, `Release.gpg` (armored detached signature) and
+ *   `InRelease` (cleartext-signed `Release`), all signed at `timestamp`.
+ *
+ * An `all` package is listed only under `binary-all`, as in Gitea. Stanzas are
+ * sorted by name, version, architecture and file name (bytewise; Gitea uses
+ * database order), and the Release file lists follow component, architecture
+ * and file order. The output is the complete set of files for the
+ * distribution; an empty entry set returns `[]`.
+ */
+export async function buildDebianIndex(input: {
+  distribution: string;
+  entries: Array<{ component: string; record: IDebianPackageRecord }>;
+  origin: string;
+  label: string;
+  signer: IOpenPgpSigner;
+  timestamp: Date;
+}): Promise<ILinuxIndexFile[]> {
+  if (input === null || typeof input !== 'object') {
+    throw new LinuxRepoError('invalid-input', 'input must be an object', { field: 'input' });
+  }
+  assertSymbol(input.distribution, 'distribution');
+  assertReleaseValue(input.origin, 'origin');
+  assertReleaseValue(input.label, 'label');
+  assertTimestamp(input.timestamp);
+  if (input.signer === null || typeof input.signer !== 'object') {
+    throw new LinuxRepoError('invalid-input', 'signer must be an IOpenPgpSigner', { field: 'signer' });
+  }
+  if (!Array.isArray(input.entries)) {
+    throw new LinuxRepoError('invalid-input', 'entries must be an array', { field: 'entries' });
+  }
+  if (input.entries.length === 0) {
+    return [];
+  }
+
+  const groups = new Map<string, { component: string; architecture: string; entries: IDebianIndexEntry[] }>();
+  const poolPaths = new Set<string>();
+  for (const entry of input.entries) {
+    if (entry === null || typeof entry !== 'object') {
+      throw new LinuxRepoError('invalid-input', 'each entry must be an object', { field: 'entries' });
+    }
+    assertSymbol(entry.component, 'component');
+    assertRecord(entry.record);
+    const poolPath = `${entry.component}/${entry.record.fileName}`;
+    if (poolPaths.has(poolPath)) {
+      throw new LinuxRepoError('invalid-input', `duplicate package file pool/${input.distribution}/${poolPath}`, {
+        field: 'entries',
+        value: entry.record.fileName,
+      });
+    }
+    poolPaths.add(poolPath);
+    const groupKey = `${entry.component}\n${entry.record.architecture}`;
+    let group = groups.get(groupKey);
+    if (group === undefined) {
+      group = { component: entry.component, architecture: entry.record.architecture, entries: [] };
+      groups.set(groupKey, group);
+    }
+    group.entries.push(entry);
+  }
+
+  const sortedGroups = [...groups.values()].sort(
+    (a, b) => compareUtf8(a.component, b.component) || compareUtf8(a.architecture, b.architecture),
+  );
+  const contentFiles: ILinuxIndexFile[] = [];
+  const releaseLines: Record<'md5' | 'sha1' | 'sha256' | 'sha512', string[]> = { md5: [], sha1: [], sha256: [], sha512: [] };
+  for (const group of sortedGroups) {
+    const stanzas = group.entries
+      .slice()
+      .sort((a, b) => compareRecords(a.record, b.record))
+      .map((entry) => renderDebianPackagesStanza({ distribution: input.distribution, component: entry.component, record: entry.record }));
+    // one blank line between stanzas (repository.go:197-199)
+    const packages = new TextEncoder().encode(stanzas.join('\n'));
+    const directory = `dists/${input.distribution}/${group.component}/binary-${group.architecture}`;
+    const encoded: Record<(typeof PACKAGES_FILES)[number], Uint8Array> = {
+      Packages: packages,
+      'Packages.gz': await gzip(packages),
+      'Packages.xz': new Uint8Array(await plugins.lzma.xz.compress(packages)),
+    };
+    for (const fileName of PACKAGES_FILES) {
+      const bytes = encoded[fileName];
+      const hashes = digestsOf(bytes);
+      const releasePath = `${group.component}/binary-${group.architecture}/${fileName}`;
+      for (const algorithm of ['md5', 'sha1', 'sha256', 'sha512'] as const) {
+        releaseLines[algorithm].push(` ${hashes[algorithm]} ${bytes.length} ${releasePath}`);
+      }
+      contentFiles.push({
+        path: `${directory}/${fileName}`,
+        bytes,
+        sha256: hashes.sha256,
+        mediaType: MEDIA_TYPES[fileName],
+        stage: 'content',
+        aliases: [
+          `${directory}/by-hash/MD5Sum/${hashes.md5}`,
+          `${directory}/by-hash/SHA1/${hashes.sha1}`,
+          `${directory}/by-hash/SHA256/${hashes.sha256}`,
+          `${directory}/by-hash/SHA512/${hashes.sha512}`,
+        ],
+      });
+    }
+  }
+
+  const components = distinctSorted(sortedGroups.map((group) => group.component));
+  const architectures = distinctSorted(sortedGroups.map((group) => group.architecture));
+  // repository.go:322-352
+  const release =
+    `Origin: ${input.origin}\n` +
+    `Label: ${input.label}\n` +
+    `Suite: ${input.distribution}\n` +
+    `Codename: ${input.distribution}\n` +
+    `Components: ${components.join(' ')}\n` +
+    `Architectures: ${architectures.join(' ')}\n` +
+    `Date: ${formatRfc1123Utc(input.timestamp)}\n` +
+    'Acquire-By-Hash: yes\n' +
+    `MD5Sum:\n${releaseLines.md5.map((line) => `${line}\n`).join('')}` +
+    `SHA1:\n${releaseLines.sha1.map((line) => `${line}\n`).join('')}` +
+    `SHA256:\n${releaseLines.sha256.map((line) => `${line}\n`).join('')}` +
+    `SHA512:\n${releaseLines.sha512.map((line) => `${line}\n`).join('')}`;
+  const releaseBytes = new TextEncoder().encode(release);
+  const releaseGpg = new TextEncoder().encode(
+    await createDetachedArmoredSignature({ data: releaseBytes, signer: input.signer, timestamp: input.timestamp }),
+  );
+  const inRelease = new TextEncoder().encode(
+    await createCleartextSignedMessage({ text: release, signer: input.signer, timestamp: input.timestamp }),
+  );
+  const pointerFiles: ILinuxIndexFile[] = [
+    ['Release', releaseBytes],
+    ['Release.gpg', releaseGpg],
+    ['InRelease', inRelease],
+  ].map(([fileName, bytes]) => ({
+    path: `dists/${input.distribution}/${fileName as string}`,
+    bytes: bytes as Uint8Array,
+    sha256: plugins.crypto.createHash('sha256').update(bytes as Uint8Array).digest('hex'),
+    mediaType: MEDIA_TYPES[fileName as string],
+    stage: 'pointer',
+    aliases: [],
+  }));
+  return [...contentFiles, ...pointerFiles];
+}
+
+/**
+ * One `Packages` stanza, without the separating blank line: Gitea's trimmed
+ * control stanza without the six index-owned fields (`Filename`, `Size`,
+ * `MD5sum`, `SHA1`, `SHA256`, `SHA512`, matched case-insensitively, with their
+ * continuation lines), then those fields from the record
+ * (`repository.go:202-209`). A control file that sets them would point apt at
+ * other bytes; Gitea would write them twice. Internal: 1.0.0 exports no
+ * per-entry renderer.
+ */
+export function renderDebianPackagesStanza(input: {
+  distribution: string;
+  component: string;
+  record: IDebianPackageRecord;
+}): string {
+  assertSymbol(input.distribution, 'distribution');
+  assertSymbol(input.component, 'component');
+  assertRecord(input.record);
+  const kept: string[] = [];
+  let dropping = false;
+  // Gitea trims the stored stanza again when it writes it (repository.go:202)
+  for (const line of goTrimSpaceText(input.record.metadata.control).split('\n')) {
+    if (line[0] === ' ' || line[0] === '\t') {
+      if (!dropping) {
+        kept.push(line);
+      }
+      continue;
+    }
+    const colon = line.indexOf(':');
+    const field = colon < 0 ? null : goTrimSpaceText(line.slice(0, colon)).toLowerCase();
+    dropping = field !== null && INDEX_OWNED_FIELDS.has(field);
+    if (!dropping) {
+      kept.push(line);
+    }
+  }
+  const control = kept.join('\n');
+  if (control === '') {
+    throw new LinuxRepoError('invalid-input', 'the record control stanza has no fields besides the index-owned ones', {
+      field: 'record.metadata.control',
+    });
+  }
+  const { digests } = input.record;
+  return (
+    `${control}\n` +
+    `Filename: pool/${input.distribution}/${input.component}/${input.record.fileName}\n` +
+    `Size: ${digests.size}\n` +
+    `MD5sum: ${digests.md5}\n` +
+    `SHA1: ${digests.sha1}\n` +
+    `SHA256: ${digests.sha256}\n` +
+    `SHA512: ${digests.sha512}\n`
+  );
+}
+
+/**
+ * Refuses a record the index cannot list: wrong format, an architecture that
+ * is not a path segment, a file name that is not one, digests that are not
+ * hex of the right length, or a control text that is not one stanza (empty,
+ * or holding a blank line that would start another stanza).
+ */
+function assertRecord(recordArg: IDebianPackageRecord): void {
+  if (recordArg === null || typeof recordArg !== 'object' || recordArg.format !== 'debian') {
+    throw new LinuxRepoError('invalid-input', 'record must be a Debian package record', { field: 'record' });
+  }
+  if (!isIndexableArchitecture(recordArg.architecture)) {
+    throw new LinuxRepoError(
+      'invalid-input',
+      `${String(recordArg.fileName)}: architecture "${String(recordArg.architecture)}" cannot name an index path`,
+      { field: 'record.architecture', value: String(recordArg.fileName) },
+    );
+  }
+  assertSegmentName(recordArg.fileName, 'record.fileName');
+  if (/\s/.test(recordArg.fileName)) {
+    throw new LinuxRepoError('invalid-input', 'record.fileName must not contain whitespace', {
+      field: 'record.fileName',
+      value: recordArg.fileName,
+    });
+  }
+  const digests = recordArg.digests;
+  if (digests === null || typeof digests !== 'object' || !Number.isSafeInteger(digests.size) || digests.size < 0) {
+    throw new LinuxRepoError('invalid-input', 'record.digests.size must be a non-negative safe integer', {
+      field: 'record.digests',
+    });
+  }
+  for (const algorithm of ['md5', 'sha1', 'sha256', 'sha512'] as const) {
+    if (typeof digests[algorithm] !== 'string' || !HASH_PATTERNS[algorithm].test(digests[algorithm])) {
+      throw new LinuxRepoError('invalid-input', `record.digests.${algorithm} must be lowercase hex`, {
+        field: `record.digests.${algorithm}`,
+      });
+    }
+  }
+  const metadata = recordArg.metadata;
+  if (metadata === null || typeof metadata !== 'object') {
+    throw new LinuxRepoError('invalid-input', 'record.metadata must be an object', { field: 'record.metadata' });
+  }
+  assertString(metadata.control, 'record.metadata.control');
+  if (goTrimSpaceText(metadata.control).split('\n').some((line) => goTrimSpaceText(line) === '')) {
+    throw new LinuxRepoError('invalid-input', 'record.metadata.control must be one stanza without blank lines', {
+      field: 'record.metadata.control',
+    });
+  }
+}
+
+function assertSymbol(valueArg: unknown, fieldArg: string): asserts valueArg is string {
+  assertString(valueArg, fieldArg);
+  if (!DEBIAN_SYMBOL_PATTERN.test(valueArg)) {
+    throw new LinuxRepoError('invalid-input', `${fieldArg} must match ${DEBIAN_SYMBOL_PATTERN.source}`, {
+      field: fieldArg,
+      value: valueArg,
+    });
+  }
+}
+
+/** A Release header value: one line, non-empty, no leading or trailing whitespace (canonical cleartext). */
+function assertReleaseValue(valueArg: unknown, fieldArg: string): asserts valueArg is string {
+  assertString(valueArg, fieldArg);
+  if (valueArg === '' || hasControlCharacter(valueArg) || valueArg !== valueArg.trim()) {
+    throw new LinuxRepoError(
+      'invalid-input',
+      `${fieldArg} must be non-empty, without control characters or surrounding whitespace`,
+      { field: fieldArg },
+    );
+  }
+}
+
+/** Go `strings.TrimSpace` on a Unicode string. */
+function goTrimSpaceText(valueArg: string): string {
+  return Buffer.from(goTrimSpace(utf8ByteString(valueArg)), 'latin1').toString('utf8');
+}
+
+function compareRecords(aArg: IDebianPackageRecord, bArg: IDebianPackageRecord): number {
+  return (
+    compareUtf8(aArg.name, bArg.name) ||
+    compareUtf8(aArg.version, bArg.version) ||
+    compareUtf8(aArg.architecture, bArg.architecture) ||
+    compareUtf8(aArg.fileName, bArg.fileName)
+  );
+}
+
+function distinctSorted(valuesArg: string[]): string[] {
+  return [...new Set(valuesArg)].sort(compareUtf8);
+}
+
+function digestsOf(bytesArg: Uint8Array): Record<'md5' | 'sha1' | 'sha256' | 'sha512', string> {
+  return {
+    md5: plugins.crypto.createHash('md5').update(bytesArg).digest('hex'),
+    sha1: plugins.crypto.createHash('sha1').update(bytesArg).digest('hex'),
+    sha256: plugins.crypto.createHash('sha256').update(bytesArg).digest('hex'),
+    sha512: plugins.crypto.createHash('sha512').update(bytesArg).digest('hex'),
+  };
+}
+
+function gzip(bytesArg: Uint8Array): Promise<Uint8Array> {
+  return new Promise((resolve, reject) => {
+    plugins.zlib.gzip(bytesArg, (error, result) => {
+      if (error !== null) {
+        reject(error);
+      } else {
+        resolve(new Uint8Array(result.buffer, result.byteOffset, result.byteLength));
+      }
+    });
+  });
+}
+
+const WEEKDAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
+const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+
+/** Go `time.Time.UTC().Format(time.RFC1123)`: `Mon, 02 Jan 2006 15:04:05 UTC`. */
+function formatRfc1123Utc(timestampArg: Date): string {
+  const pad = (valueArg: number) => String(valueArg).padStart(2, '0');
+  return (
+    `${WEEKDAYS[timestampArg.getUTCDay()]}, ${pad(timestampArg.getUTCDate())} ${MONTHS[timestampArg.getUTCMonth()]} ` +
+    `${timestampArg.getUTCFullYear()} ${pad(timestampArg.getUTCHours())}:${pad(timestampArg.getUTCMinutes())}:` +
+    `${pad(timestampArg.getUTCSeconds())} UTC`
+  );
+}