@tutti-os/app-release-tools 0.0.78 → 0.0.80

package/README.md

@@ -6,25 +6,43 @@ Command-line tools for publishing Tutti workspace apps into App Center release m
 
 ```sh
 build-tutti-app-release --app-id vibe-design --package-dir dist/tutti-app/vibe-design --base-url https://cdn.example.test/tutti-app-releases
-build-tutti-app-catalog --release-file ./apps/vibe-design/latest.json --output ./catalog.json
-build-tutti-app-catalog --existing-catalog ./catalog.json --release-file ./apps/vibe-design/latest.json --output ./catalog.json
+build-tutti-app-versions --release-file ./apps/vibe-design/0.2.0/release.json --min-tutti-version 0.12.0 --output ./apps/vibe-design/versions.json
+build-tutti-app-catalog --versions-file ./apps/vibe-design/versions.json --output ./catalog.json
+build-tutti-app-catalog --existing-catalog ./catalog.json --versions-file ./apps/vibe-design/versions.json --output ./catalog.json
 verify-tutti-app-release-artifacts --release-file ./apps/vibe-design/latest.json
-verify-tutti-app-release-artifacts --catalog-file ./catalog.json --release-file ./apps/vibe-design/latest.json
+verify-tutti-app-release-artifacts --catalog-file ./catalog.json --versions-file ./apps/vibe-design/versions.json
 ```
 
-The release command validates a complete Tutti app package, creates a zip,
-writes immutable `release.json`, and writes mutable `latest.json`. When the
-manifest declares `localizationInfo`, the release metadata includes the
-referenced manifest locale files so App Center can localize uninstalled remote
-apps without downloading the package.
+The release command validates a complete Tutti app package, creates a
+reproducible zip with stable entry ordering and timestamps, writes immutable
+`release.json`, and writes mutable `latest.json`. Rebuilding the same package
+version produces the same artifact SHA-256, so an interrupted release can
+safely verify and reuse its immutable upload. When the manifest declares
+`localizationInfo`, the release metadata includes the referenced manifest
+locale files so App Center can localize uninstalled remote apps without
+downloading the package.
 
-The catalog command merges one or more release files into `tutti.app.catalog.v1`.
-Pass `--existing-catalog` to preserve existing catalog apps and update only the
-apps represented by the release files. With `--existing-catalog`, release files
-are optional, which allows rewriting an existing catalog without changing its app
-set.
+The versions command maintains one mutable `tutti.app.versions.v1` index per
+app. Every catalog-eligible release has an explicit minimum Tutti version and
+an `active` or `withdrawn` status. Reusing an immutable app version with changed
+release or compatibility metadata is rejected.
+
+The catalog command merges version indexes into `tutti.app.catalog.v1`.
+`apps[]` contains the highest active app version whose minimum Tutti version is
+`0.0.0`, so old Tutti clients remain safe. `compatibility.apps` contains only
+the compact compatibility frontier needed by new Tutti clients, not every
+historical release. Catalog output is rejected when it exceeds the legacy 1 MiB
+reader limit. The older `--release-file` form remains available for legacy
+tooling but does not create compatibility metadata.
 
 The verify command checks release and catalog metadata against the referenced
-artifact downloads. When both `--catalog-file` and `--release-file` are passed,
-catalog entries for those apps must exactly match the latest release metadata
-before artifact SHA-256 and size checks run.
+artifact downloads. When `--catalog-file` is combined with `--release-file` or
+`--versions-file`, catalog entries for those apps must exactly match the
+corresponding release metadata before artifact SHA-256 and size checks run.
+
+`publish-tutti-app-metadata` and `publish-tutti-app-catalog` update mutable S3
+JSON objects with ETag conditional writes and retry on concurrent changes. They
+bootstrap a missing versions index from the exact immutable release currently
+present in the legacy catalog; they never infer legacy compatibility from a
+newer `latest.json` object. Catalog publication validates all metadata and
+downloads referenced artifacts before attempting the conditional S3 write.

package/bin/build-tutti-app-catalog.mjs

@@ -7,85 +7,108 @@ import {
   releaseToCatalogApp,
   validateRelease
 } from "./build-tutti-app-release.mjs";
+import { validateVersionsDocument } from "./build-tutti-app-versions.mjs";
+import {
+  compareReleaseVersions,
+  compareSemver
+} from "./tutti-app-versioning.mjs";
 
-const catalogSchemaVersion = "tutti.app.catalog.v1";
+export const catalogSchemaVersion = "tutti.app.catalog.v1";
+export const maxCatalogBytes = 1024 * 1024;
 
 export async function buildTuttiAppCatalog(options) {
   const existingCatalogPath = options.existingCatalogPath
     ? path.resolve(String(options.existingCatalogPath))
     : null;
-  const releaseFiles = normalizeReleaseFiles(options.releaseFiles);
+  const releaseFiles = normalizeFiles(options.releaseFiles);
+  const versionsFiles = normalizeFiles(options.versionsFiles);
   const outputPath = path.resolve(
     options.outputPath
       ? String(options.outputPath)
       : "dist/tutti-app-catalog/catalog.json"
   );
 
-  const seenAppIDs = new Set();
-  const seenReleaseAppIDs = new Set();
   const appsByID = new Map();
-
+  const compatibilityByAppID = new Map();
   if (existingCatalogPath) {
     const existingCatalog = JSON.parse(
       await readFile(existingCatalogPath, "utf8")
     );
     validateCatalog(existingCatalog);
     for (const app of existingCatalog.apps) {
-      const appID = app.manifest.appId;
-      if (seenAppIDs.has(appID)) {
-        throw new Error(`duplicate catalog appId ${appID}`);
-      }
-      seenAppIDs.add(appID);
-      appsByID.set(appID, app);
+      appsByID.set(app.manifest.appId, app);
+    }
+    for (const [appId, entries] of Object.entries(
+      existingCatalog.compatibility?.apps ?? {}
+    )) {
+      compatibilityByAppID.set(appId, entries);
     }
   }
 
+  const seenVersionsAppIDs = new Set();
+  for (const versionsFile of versionsFiles) {
+    const versions = JSON.parse(await readFile(versionsFile, "utf8"));
+    validateVersionsDocument(versions);
+    if (seenVersionsAppIDs.has(versions.appId)) {
+      throw new Error(`duplicate versions appId ${versions.appId}`);
+    }
+    seenVersionsAppIDs.add(versions.appId);
+    applyVersionsDocument(appsByID, compatibilityByAppID, versions);
+  }
+
+  const seenReleaseAppIDs = new Set();
   for (const releaseFile of releaseFiles) {
     const release = JSON.parse(await readFile(releaseFile, "utf8"));
     validateRelease(release);
     if (seenReleaseAppIDs.has(release.appId)) {
       throw new Error(`duplicate release appId ${release.appId}`);
     }
-    seenReleaseAppIDs.add(release.appId);
-    if (!seenAppIDs.has(release.appId)) {
-      seenAppIDs.add(release.appId);
+    if (seenVersionsAppIDs.has(release.appId)) {
+      throw new Error(
+        `appId ${release.appId} cannot be supplied as both release and versions metadata`
+      );
     }
+    seenReleaseAppIDs.add(release.appId);
     appsByID.set(release.appId, releaseToCatalogApp(release));
   }
 
-  if (appsByID.size === 0) {
+  if (appsByID.size === 0 && compatibilityByAppID.size === 0) {
     throw new Error(
-      "at least one release file or existing catalog app is required"
+      "at least one versions file, release file, or existing catalog app is required"
     );
   }
 
-  const apps = [...appsByID.values()];
-  apps.sort((left, right) =>
+  const apps = [...appsByID.values()].sort((left, right) =>
     left.manifest.appId.localeCompare(right.manifest.appId)
   );
-
+  const compatibilityApps = Object.fromEntries(
+    [...compatibilityByAppID.entries()]
+      .filter(([, entries]) => entries.length > 0)
+      .sort(([left], [right]) => left.localeCompare(right))
+  );
   const catalog = {
     schemaVersion: catalogSchemaVersion,
-    apps
+    apps,
+    ...(Object.keys(compatibilityApps).length > 0
+      ? { compatibility: { apps: compatibilityApps } }
+      : {})
   };
+  validateCatalog(catalog);
+
+  const encoded = `${JSON.stringify(catalog, null, 2)}\n`;
+  if (Buffer.byteLength(encoded) > maxCatalogBytes) {
+    throw new Error(
+      `catalog exceeds legacy ${maxCatalogBytes}-byte response limit`
+    );
+  }
 
   await mkdir(path.dirname(outputPath), { recursive: true });
-  await writeFile(outputPath, `${JSON.stringify(catalog, null, 2)}\n`);
+  await writeFile(outputPath, encoded);
   return { outputPath, catalog };
 }
 
-function normalizeReleaseFiles(value) {
-  const files = Array.isArray(value)
-    ? value
-    : String(value ?? "")
-        .split(/[\n,]/)
-        .map((file) => file.trim())
-        .filter(Boolean);
-  return files.map((file) => path.resolve(file));
-}
-
-function validateCatalog(catalog) {
-  if (!catalog || typeof catalog !== "object") {
+export function validateCatalog(catalog) {
+  if (!catalog || typeof catalog !== "object" || Array.isArray(catalog)) {
     throw new Error("catalog must be an object");
   }
   if (catalog.schemaVersion !== catalogSchemaVersion) {
@@ -94,51 +117,188 @@ function validateCatalog(catalog) {
   if (!Array.isArray(catalog.apps)) {
     throw new Error("catalog apps must be an array");
   }
+
+  const seenLegacyAppIDs = new Set();
   for (const [index, app] of catalog.apps.entries()) {
-    if (!app || typeof app !== "object") {
-      throw new Error(`catalog apps[${index}] must be an object`);
+    const appId = validateCatalogApp(app, `catalog apps[${index}]`);
+    if (seenLegacyAppIDs.has(appId)) {
+      throw new Error(`duplicate catalog appId ${appId}`);
     }
+    seenLegacyAppIDs.add(appId);
+  }
+
+  const compatibility = catalog.compatibility;
+  if (compatibility === undefined) {
+    return catalog;
+  }
+  if (
+    !compatibility ||
+    typeof compatibility !== "object" ||
+    Array.isArray(compatibility) ||
+    !compatibility.apps ||
+    typeof compatibility.apps !== "object" ||
+    Array.isArray(compatibility.apps)
+  ) {
+    throw new Error("catalog compatibility.apps must be an object");
+  }
+  for (const [appId, entries] of Object.entries(compatibility.apps)) {
+    if (!Array.isArray(entries) || entries.length === 0) {
+      throw new Error(`catalog compatibility app ${appId} must be non-empty`);
+    }
+    const seenVersions = new Set();
+    const seenMinimums = new Set();
+    for (const [index, entry] of entries.entries()) {
+      const label = `catalog compatibility app ${appId}[${index}]`;
+      if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+        throw new Error(`${label} must be an object`);
+      }
+      const minTuttiVersion = String(entry.minTuttiVersion ?? "").trim();
+      compareSemver(minTuttiVersion, minTuttiVersion);
+      if (seenMinimums.has(minTuttiVersion)) {
+        throw new Error(`${label} has duplicate minTuttiVersion`);
+      }
+      seenMinimums.add(minTuttiVersion);
+      const entryAppId = validateCatalogApp(entry.app, `${label}.app`);
+      if (entryAppId !== appId) {
+        throw new Error(`${label}.app manifest.appId must match map key`);
+      }
+      const version = entry.app.manifest.version;
+      if (seenVersions.has(version)) {
+        throw new Error(`${label} has duplicate app version ${version}`);
+      }
+      seenVersions.add(version);
+    }
+  }
+  return catalog;
+}
+
+export function validateCatalogApp(app, label) {
+  if (!app || typeof app !== "object" || Array.isArray(app)) {
+    throw new Error(`${label} must be an object`);
+  }
+  const appId = String(app.manifest?.appId ?? "").trim();
+  const version = String(app.manifest?.version ?? "").trim();
+  if (!appId) {
+    throw new Error(`${label}.manifest.appId is required`);
+  }
+  compareSemver(version, version);
+  const distribution = app.distribution;
+  if (!distribution || distribution.kind !== "remote") {
+    throw new Error(`${label}.distribution.kind must be remote`);
+  }
+  for (const key of ["artifactUrl", "artifactSha256", "iconUrl"]) {
     if (
-      !app.manifest ||
-      typeof app.manifest !== "object" ||
-      typeof app.manifest.appId !== "string" ||
-      app.manifest.appId.trim() === ""
+      typeof distribution[key] !== "string" ||
+      distribution[key].trim() === ""
     ) {
-      throw new Error(`catalog apps[${index}].manifest.appId is required`);
+      throw new Error(`${label}.distribution.${key} is required`);
     }
   }
+  if (!/^[a-f0-9]{64}$/iu.test(distribution.artifactSha256)) {
+    throw new Error(`${label}.distribution.artifactSha256 must be sha256`);
+  }
+  return appId;
+}
+
+function applyVersionsDocument(appsByID, compatibilityByAppID, versions) {
+  const activeRecords = versions.versions.filter(
+    (record) => record.status === "active"
+  );
+  const legacyRecords = activeRecords.filter(
+    (record) => record.minTuttiVersion === "0.0.0"
+  );
+  const legacy = highestReleaseRecord(legacyRecords);
+  if (legacy) {
+    appsByID.set(versions.appId, releaseToCatalogApp(legacy.release));
+  } else {
+    appsByID.delete(versions.appId);
+  }
+
+  const entries = compatibilityFrontier(activeRecords).map((record) => ({
+    minTuttiVersion: record.minTuttiVersion,
+    app: releaseToCatalogApp(record.release)
+  }));
+  if (entries.length > 0) {
+    compatibilityByAppID.set(versions.appId, entries);
+  } else {
+    compatibilityByAppID.delete(versions.appId);
+  }
+}
+
+export function compatibilityFrontier(records) {
+  const highestByMinimum = new Map();
+  for (const record of records) {
+    const existing = highestByMinimum.get(record.minTuttiVersion);
+    if (
+      !existing ||
+      compareReleaseVersions(record.release, existing.release) > 0
+    ) {
+      highestByMinimum.set(record.minTuttiVersion, record);
+    }
+  }
+
+  const candidates = [...highestByMinimum.values()].sort((left, right) => {
+    const minimum = compareSemver(left.minTuttiVersion, right.minTuttiVersion);
+    if (minimum !== 0) {
+      return minimum;
+    }
+    return compareReleaseVersions(left.release, right.release);
+  });
+  const frontier = [];
+  let highest = null;
+  for (const candidate of candidates) {
+    if (
+      !highest ||
+      compareReleaseVersions(candidate.release, highest.release) > 0
+    ) {
+      frontier.push(candidate);
+      highest = candidate;
+    }
+  }
+  return frontier;
+}
+
+function highestReleaseRecord(records) {
+  return [...records]
+    .sort((left, right) => compareReleaseVersions(left.release, right.release))
+    .at(-1);
+}
+
+function normalizeFiles(value) {
+  const files = Array.isArray(value)
+    ? value
+    : String(value ?? "")
+        .split(/[\n,]/u)
+        .map((file) => file.trim())
+        .filter(Boolean);
+  return files.map((file) => path.resolve(file));
 }
 
 function parseArgs(argv) {
-  const result = {
-    releaseFiles: []
-  };
+  const result = { releaseFiles: [], versionsFiles: [] };
   for (let index = 0; index < argv.length; index += 1) {
     const arg = argv[index];
-    if (arg === "--release-file") {
-      const value = argv[index + 1];
-      if (!value || value.startsWith("--")) {
-        throw new Error("missing value for --release-file");
-      }
-      result.releaseFiles.push(value);
-      index += 1;
-      continue;
-    }
-    if (arg === "--existing-catalog") {
-      const value = argv[index + 1];
+    const value = argv[index + 1];
+    if (
+      [
+        "--release-file",
+        "--versions-file",
+        "--existing-catalog",
+        "--output"
+      ].includes(arg)
+    ) {
       if (!value || value.startsWith("--")) {
-        throw new Error("missing value for --existing-catalog");
+        throw new Error(`missing value for ${arg}`);
       }
-      result.existingCatalogPath = value;
-      index += 1;
-      continue;
-    }
-    if (arg === "--output") {
-      const value = argv[index + 1];
-      if (!value || value.startsWith("--")) {
-        throw new Error("missing value for --output");
+      if (arg === "--release-file") {
+        result.releaseFiles.push(value);
+      } else if (arg === "--versions-file") {
+        result.versionsFiles.push(value);
+      } else if (arg === "--existing-catalog") {
+        result.existingCatalogPath = value;
+      } else {
+        result.outputPath = value;
       }
-      result.outputPath = value;
       index += 1;
       continue;
     }

package/bin/build-tutti-app-release.mjs

@@ -2,9 +2,22 @@
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { pathToFileURL } from "node:url";
-import { cp, mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import {
+  cp,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readdir,
+  rm,
+  stat,
+  utimes,
+  writeFile
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
 import path from "node:path";
 
+import { requireSemver } from "./tutti-app-versioning.mjs";
+
 const manifestSchemaVersion = "tutti.app.manifest.v1";
 const cliManifestSchemaVersion = "tutti.app.cli.v1";
 const releaseSchemaVersion = "tutti.app.release.v1";
@@ -37,6 +50,7 @@ export async function buildTuttiAppRelease(options) {
     "version"
   );
   requireSafePathSegment(version, "version");
+  requireSemver(version, "version");
   manifest.version = version;
   validateManifest(manifest, manifestPath);
   await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
@@ -72,7 +86,7 @@ export async function buildTuttiAppRelease(options) {
 
   const artifactName = `${appId}-${version}.zip`;
   const artifactPath = path.join(releaseDir, artifactName);
-  createZip(packageDir, artifactPath);
+  await createZip(packageDir, artifactPath);
   const artifact = await fileDigestAndSize(artifactPath);
 
   const iconName = path.basename(sourceIconPath);
@@ -496,6 +510,7 @@ export function validateRelease(release) {
   ]) {
     requireManifestString(release, key, "release");
   }
+  requireSemver(release.version, "release version");
   validateManifest(release.manifest, "release.manifest");
   if (release.manifest.appId !== release.appId) {
     throw new Error("release manifest.appId must match release appId");
@@ -619,17 +634,71 @@ function validateReleaseLocalizations(localizations, sourceLabel) {
   }
 }
 
-function createZip(packageDir, artifactPath) {
-  const result = spawnSync("zip", ["-qr", artifactPath, "."], {
-    cwd: packageDir,
-    encoding: "utf8"
-  });
-  if (result.error) {
-    throw result.error;
+async function createZip(packageDir, artifactPath) {
+  const stagingDir = await mkdtemp(
+    path.join(tmpdir(), "tutti-app-release-archive-")
+  );
+  const archiveRoot = path.join(stagingDir, "package");
+  try {
+    await cp(packageDir, archiveRoot, {
+      recursive: true,
+      dereference: true
+    });
+    const entries = await normalizeArchiveEntries(archiveRoot);
+    await rm(artifactPath, { force: true });
+    const result = spawnSync("zip", ["-X", "-q", artifactPath, "-@"], {
+      cwd: archiveRoot,
+      encoding: "utf8",
+      env: {
+        ...process.env,
+        LC_ALL: "C",
+        TZ: "UTC"
+      },
+      input: `${entries.join("\n")}\n`
+    });
+    if (result.error) {
+      throw result.error;
+    }
+    if (result.status !== 0) {
+      throw new Error(
+        result.stderr || `zip exited with status ${result.status}`
+      );
+    }
+  } finally {
+    await rm(stagingDir, { recursive: true, force: true });
   }
-  if (result.status !== 0) {
-    throw new Error(result.stderr || `zip exited with status ${result.status}`);
+}
+
+async function normalizeArchiveEntries(rootDir, relativeDir = "") {
+  const fixedTimestamp = new Date("1980-01-01T00:00:00.000Z");
+  const directoryEntries = await readdir(path.join(rootDir, relativeDir), {
+    withFileTypes: true
+  });
+  directoryEntries.sort((left, right) =>
+    left.name < right.name ? -1 : left.name > right.name ? 1 : 0
+  );
+
+  const result = [];
+  for (const entry of directoryEntries) {
+    if (entry.name.includes("\n") || entry.name.includes("\r")) {
+      throw new Error(
+        `package path contains an unsupported newline: ${path.join(relativeDir, entry.name)}`
+      );
+    }
+    const relativePath = path.join(relativeDir, entry.name);
+    const absolutePath = path.join(rootDir, relativePath);
+    await utimes(absolutePath, fixedTimestamp, fixedTimestamp);
+    if (entry.isDirectory()) {
+      result.push(`${relativePath}/`);
+      result.push(...(await normalizeArchiveEntries(rootDir, relativePath)));
+      continue;
+    }
+    if (!entry.isFile()) {
+      throw new Error(`unsupported package entry type: ${relativePath}`);
+    }
+    result.push(relativePath);
   }
+  return result;
 }
 
 async function fileDigestAndSize(filePath) {