@push.rocks/smartlinuxrepo 0.0.0-stage → 0.1.0

package/ts/readers/text.ts

@@ -0,0 +1,91 @@
+const strictUtf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
+
+/**
+ * Decodes bytes as Go maps them to text: valid UTF-8 unchanged (a leading BOM
+ * is kept), and every byte of an invalid sequence replaced by one U+FFFD, as
+ * Go's `utf8.DecodeRune` reports invalid input with width 1. The WHATWG
+ * decoder differs for truncated multi-byte sequences (one U+FFFD for the
+ * maximal subpart).
+ *
+ * Gitea 28.2.0 stores package metadata with `encoding/json/v2`, which refuses
+ * invalid UTF-8 (`modules/json/jsonv2.go:22-29`,
+ * `services/packages/packages.go:166`), so 28.2.0 rejects such a package.
+ * Packages stored under older Gitea are still served, so the codec
+ * deliberately accepts more than 28.2.0 to keep them importable.
+ */
+export function decodeGoUtf8(bytesArg: Uint8Array): string {
+  try {
+    return strictUtf8.decode(bytesArg);
+  } catch {
+    return decodeGoUtf8Slow(bytesArg);
+  }
+}
+
+function decodeGoUtf8Slow(bytesArg: Uint8Array): string {
+  const parts: string[] = [];
+  let i = 0;
+  while (i < bytesArg.length) {
+    const b0 = bytesArg[i];
+    if (b0 < 0x80) {
+      parts.push(String.fromCharCode(b0));
+      i += 1;
+      continue;
+    }
+    let need: number;
+    let codePoint: number;
+    let low = 0x80;
+    let high = 0xbf;
+    if (b0 >= 0xc2 && b0 <= 0xdf) {
+      need = 1;
+      codePoint = b0 & 0x1f;
+    } else if (b0 >= 0xe0 && b0 <= 0xef) {
+      need = 2;
+      codePoint = b0 & 0x0f;
+      if (b0 === 0xe0) {
+        low = 0xa0;
+      } else if (b0 === 0xed) {
+        high = 0x9f;
+      }
+    } else if (b0 >= 0xf0 && b0 <= 0xf4) {
+      need = 3;
+      codePoint = b0 & 0x07;
+      if (b0 === 0xf0) {
+        low = 0x90;
+      } else if (b0 === 0xf4) {
+        high = 0x8f;
+      }
+    } else {
+      parts.push('\uFFFD');
+      i += 1;
+      continue;
+    }
+    let valid = i + need < bytesArg.length;
+    for (let k = 1; valid && k <= need; k++) {
+      const b = bytesArg[i + k];
+      const min = k === 1 ? low : 0x80;
+      const max = k === 1 ? high : 0xbf;
+      if (b < min || b > max) {
+        valid = false;
+      } else {
+        codePoint = (codePoint << 6) | (b & 0x3f);
+      }
+    }
+    if (!valid) {
+      parts.push('\uFFFD');
+      i += 1;
+      continue;
+    }
+    parts.push(String.fromCodePoint(codePoint));
+    i += need + 1;
+  }
+  return parts.join('');
+}
+
+/** ASCII/Latin-1 view of header bytes (magic values, numeric fields). */
+export function decodeLatin1(bytesArg: Uint8Array): string {
+  let text = '';
+  for (let i = 0; i < bytesArg.length; i++) {
+    text += String.fromCharCode(bytesArg[i]);
+  }
+  return text;
+}

package/ts/rpm/builder.ts

@@ -0,0 +1,253 @@
+import * as plugins from '../plugins.js';
+import { isIndexableArchitecture } from '../shared/architecture.js';
+import { LinuxRepoError } from '../shared/errors.js';
+import type { ILinuxIndexFile, IRpmPackageRecord, IRpmUpdate } from '../shared/types.js';
+import { assertTimestamp, isPathSegment } from '../shared/validation.js';
+import { createDetachedArmoredSignature } from '../signing/openpgpframing.js';
+import type { IOpenPgpSigner } from '../signing/signers.js';
+import { renderFilelists, renderOther, renderPrimary, renderRepomd, rpmPackageLocation, type IRepomdData } from './repodata.js';
+import { assertRpmUpdates, combineRpmUpdates, renderUpdateInfo } from './updates.js';
+import { compareCodePoints } from './xml.js';
+
+/**
+ * Builds the repodata of one RPM repository group, as Gitea 28.2.0's
+ * `BuildSpecificRepositoryFiles` does (`services/packages/rpm/repository.go:136-309`):
+ * `primary`, `filelists` and `other` (gzip), `updateinfo` only when there are
+ * updates, then `repomd.xml` and its armored detached signature
+ * `repomd.xml.asc`. Paths are relative to the RPM root: `<group>/repodata/…`,
+ * or `repodata/…` for the empty group. `repodataNaming` has no default:
+ * `'plain'` is Gitea's `repodata/<type>.xml.gz`, `'content-addressed'` is
+ * createrepo_c's default `repodata/<sha256 of the .gz>-<type>.xml.gz`
+ * (`--unique-md-filenames`, `src/cmd_parser.c:37`, `src/repomd.c:576-636`).
+ *
+ * Packages are sorted by name, version, architecture and file name; updates
+ * are combined as `combineRpmUpdates` describes. Every `repomd.xml` timestamp
+ * is `timestamp` in whole seconds. An empty entry set returns `[]` (Gitea
+ * deletes the repodata of a group without packages). An entry whose
+ * architecture fails `isIndexableArchitecture`, or whose name or version is not
+ * a path segment, is `invalid-input`, as are two entries with the same file.
+ */
+export async function buildRpmIndex(input: {
+  group: string;
+  entries: IRpmPackageRecord[];
+  updates: IRpmUpdate[];
+  repodataNaming: 'plain' | 'content-addressed';
+  signer: IOpenPgpSigner;
+  timestamp: Date;
+}): Promise<ILinuxIndexFile[]> {
+  if (input === null || typeof input !== 'object') {
+    throw new LinuxRepoError('invalid-input', 'input must be an object', { field: 'input' });
+  }
+  const prefix = groupPrefix(input.group);
+  if (input.repodataNaming !== 'plain' && input.repodataNaming !== 'content-addressed') {
+    throw new LinuxRepoError('invalid-input', "repodataNaming must be 'plain' or 'content-addressed'", {
+      field: 'repodataNaming',
+    });
+  }
+  if (
+    input.signer === null ||
+    typeof input.signer !== 'object' ||
+    typeof input.signer.signBinary !== 'function' ||
+    typeof input.signer.signText !== 'function'
+  ) {
+    throw new LinuxRepoError('invalid-input', 'signer must implement IOpenPgpSigner', { field: 'signer' });
+  }
+  assertTimestamp(input.timestamp);
+  assertRpmRecords(input.entries);
+  assertRpmUpdates(input.updates, 'updates');
+  if (input.entries.length === 0) {
+    return [];
+  }
+
+  const entries = [...input.entries].sort(
+    (a, b) =>
+      compareCodePoints(a.name, b.name) ||
+      compareCodePoints(a.version, b.version) ||
+      compareCodePoints(a.architecture, b.architecture) ||
+      compareCodePoints(a.fileName, b.fileName),
+  );
+  const seconds = Math.floor(input.timestamp.getTime() / 1000);
+  const documents: Array<[string, string]> = [
+    ['primary', renderPrimary(entries)],
+    ['filelists', renderFilelists(entries)],
+    ['other', renderOther(entries)],
+  ];
+  const updates = combineRpmUpdates(input.updates);
+  if (updates.length > 0) {
+    documents.push(['updateinfo', renderUpdateInfo(updates)]);
+  }
+
+  const files: ILinuxIndexFile[] = [];
+  const data: IRepomdData[] = [];
+  for (const [type, xml] of documents) {
+    const open = new TextEncoder().encode(xml);
+    const gzipped = gzip(open);
+    const checksum = sha256(gzipped);
+    const name = input.repodataNaming === 'plain' ? `${type}.xml.gz` : `${checksum}-${type}.xml.gz`;
+    files.push(indexFile(`${prefix}repodata/${name}`, gzipped, checksum, 'application/gzip', 'content'));
+    data.push({
+      type,
+      checksum,
+      openChecksum: sha256(open),
+      href: `repodata/${name}`,
+      timestamp: seconds,
+      size: gzipped.length,
+      openSize: open.length,
+    });
+  }
+  const repomd = new TextEncoder().encode(renderRepomd(data));
+  const signature = new TextEncoder().encode(
+    await createDetachedArmoredSignature({ data: repomd, signer: input.signer, timestamp: input.timestamp }),
+  );
+  files.push(indexFile(`${prefix}repodata/repomd.xml`, repomd, sha256(repomd), 'application/xml', 'pointer'));
+  files.push(indexFile(`${prefix}repodata/repomd.xml.asc`, signature, sha256(signature), 'application/pgp-signature', 'pointer'));
+  return files;
+}
+
+/** `''` for the root group, else `<group>/`; a group is `/`-separated path segments. */
+function groupPrefix(groupArg: unknown): string {
+  if (typeof groupArg !== 'string') {
+    throw new LinuxRepoError('invalid-input', 'group must be a string', { field: 'group' });
+  }
+  if (groupArg === '') {
+    return '';
+  }
+  for (const segment of groupArg.split('/')) {
+    if (!isPathSegment(segment)) {
+      throw new LinuxRepoError('invalid-input', 'group must be "/"-separated non-empty segments other than "." and ".."', {
+        field: 'group',
+      });
+    }
+  }
+  return `${groupArg}/`;
+}
+
+function indexFile(
+  pathArg: string,
+  bytesArg: Uint8Array,
+  sha256Arg: string,
+  mediaTypeArg: string,
+  stageArg: ILinuxIndexFile['stage'],
+): ILinuxIndexFile {
+  return { path: pathArg, bytes: bytesArg, sha256: sha256Arg, mediaType: mediaTypeArg, stage: stageArg, aliases: [] };
+}
+
+function gzip(bytesArg: Uint8Array): Uint8Array {
+  const compressed = plugins.zlib.gzipSync(bytesArg);
+  return new Uint8Array(compressed.buffer, compressed.byteOffset, compressed.byteLength);
+}
+
+function sha256(bytesArg: Uint8Array): string {
+  return plugins.crypto.createHash('sha256').update(bytesArg).digest('hex');
+}
+
+/**
+ * Checks the record fields the renderers read and the layout rules: an
+ * indexable architecture, a name and version that are path segments, and no
+ * two entries with the same package location.
+ */
+function assertRpmRecords(entriesArg: unknown): asserts entriesArg is IRpmPackageRecord[] {
+  if (!Array.isArray(entriesArg)) {
+    throw new LinuxRepoError('invalid-input', 'entries must be an array of RPM package records', { field: 'entries' });
+  }
+  const locations = new Set<string>();
+  entriesArg.forEach((entry: unknown, i) => {
+    const at = `entries[${i}]`;
+    const record = entry as IRpmPackageRecord;
+    const fail = (whatArg: string): never => {
+      throw new LinuxRepoError('invalid-input', `${at} ${whatArg}`, { field: at });
+    };
+    if (record === null || typeof record !== 'object' || record.format !== 'rpm') {
+      fail('must be an RPM package record');
+    }
+    for (const key of ['name', 'version', 'architecture', 'fileName'] as const) {
+      if (typeof record[key] !== 'string') {
+        fail(`.${key} must be a string`);
+      }
+    }
+    if (!isIndexableArchitecture(record.architecture)) {
+      fail(`(${record.fileName}) has an architecture that cannot name an index path: ${JSON.stringify(record.architecture)}`);
+    }
+    if (!isPathSegment(record.name) || !isPathSegment(record.version)) {
+      fail(`(${record.fileName}) has a name or version that cannot name a package path`);
+    }
+    const digests = record.digests;
+    if (digests === null || typeof digests !== 'object' || typeof digests.sha256 !== 'string' || !isCount(digests.size)) {
+      fail('.digests must hold sha256 and size');
+    }
+    const metadata = record.metadata;
+    if (metadata === null || typeof metadata !== 'object') {
+      fail('.metadata must be an object');
+    }
+    for (const key of [
+      'epoch',
+      'upstreamVersion',
+      'release',
+      'summary',
+      'description',
+      'license',
+      'projectUrl',
+      'vendor',
+      'group',
+      'packager',
+      'sourceRpm',
+      'buildHost',
+    ] as const) {
+      if (typeof metadata[key] !== 'string') {
+        fail(`.metadata.${key} must be a string`);
+      }
+    }
+    for (const key of ['buildTime', 'fileTime', 'installedSize', 'archiveSize'] as const) {
+      if (!isCount(metadata[key])) {
+        fail(`.metadata.${key} must be a non-negative safe integer`);
+      }
+    }
+    for (const key of ['provides', 'requires', 'conflicts', 'obsoletes'] as const) {
+      const list: unknown = metadata[key];
+      if (
+        !Array.isArray(list) ||
+        !list.every(
+          (dependency) =>
+            dependency !== null &&
+            typeof dependency === 'object' &&
+            ['name', 'flags', 'epoch', 'version', 'release'].every((field) => typeof dependency[field] === 'string'),
+        )
+      ) {
+        fail(`.metadata.${key} must be a list of dependencies`);
+      }
+    }
+    const files: unknown = metadata.files;
+    if (
+      !Array.isArray(files) ||
+      !files.every(
+        (file) =>
+          file !== null && typeof file === 'object' && typeof file.path === 'string' && typeof file.type === 'string' && typeof file.isExecutable === 'boolean',
+      )
+    ) {
+      fail('.metadata.files must be a list of files');
+    }
+    const changelogs: unknown = metadata.changelogs;
+    if (
+      !Array.isArray(changelogs) ||
+      !changelogs.every(
+        (changelog) =>
+          changelog !== null &&
+          typeof changelog === 'object' &&
+          typeof changelog.author === 'string' &&
+          typeof changelog.text === 'string' &&
+          isCount(changelog.date),
+      )
+    ) {
+      fail('.metadata.changelogs must be a list of changelog entries');
+    }
+    const location = rpmPackageLocation(record);
+    if (locations.has(location)) {
+      fail(`duplicates the file ${record.fileName}`);
+    }
+    locations.add(location);
+  });
+}
+
+function isCount(valueArg: unknown): boolean {
+  return typeof valueArg === 'number' && Number.isSafeInteger(valueArg) && valueArg >= 0;
+}