@auraindustry/aurajs 0.0.6 → 0.1.0

package/src/external-asset-cache.mjs

@@ -0,0 +1,587 @@
+import { createHash } from 'node:crypto';
+import {
+  copyFileSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { request as httpRequest } from 'node:http';
+import { request as httpsRequest } from 'node:https';
+import { homedir } from 'node:os';
+import { basename, dirname, join, resolve } from 'node:path';
+
+import { SELF_HOSTED_EXTERNAL_ASSETS_CONFIG } from './external-asset-policy.mjs';
+import {
+  SELF_HOSTED_ASSETS_SCHEMA,
+  SELF_HOSTED_CONFIG_SCHEMA,
+} from './self-hosted-assets.mjs';
+
+const EXTERNAL_ASSET_CACHE_ROOT_ENV = 'AURAJS_EXTERNAL_ASSET_CACHE_DIR';
+const LOCAL_HTTP_HOSTS = new Set(['localhost', '127.0.0.1', '::1']);
+
+export class ExternalAssetCacheError extends Error {
+  constructor(reasonCode, message, details = {}) {
+    super(message);
+    this.name = 'ExternalAssetCacheError';
+    this.reasonCode = typeof reasonCode === 'string' && reasonCode
+      ? reasonCode
+      : 'external_asset_cache_failed';
+    this.details = details && typeof details === 'object' ? details : {};
+  }
+}
+
+function sha256Buffer(buffer) {
+  const hash = createHash('sha256');
+  hash.update(buffer);
+  return hash.digest('hex');
+}
+
+function sha1Buffer(buffer) {
+  const hash = createHash('sha1');
+  hash.update(buffer);
+  return hash.digest('hex');
+}
+
+function sha256Text(value) {
+  return sha256Buffer(Buffer.from(String(value || ''), 'utf8'));
+}
+
+function readJsonFile(path, label) {
+  try {
+    return JSON.parse(readFileSync(path, 'utf8'));
+  } catch (error) {
+    throw new ExternalAssetCacheError(
+      'external_asset_json_parse_failed',
+      `Unable to parse ${label} at ${path}: ${error instanceof Error ? error.message : String(error)}`,
+      { path, label },
+    );
+  }
+}
+
+function readPackageMetadata(projectRoot) {
+  const packagePath = resolve(projectRoot, 'package.json');
+  if (!existsSync(packagePath)) {
+    return {
+      name: basename(projectRoot),
+      version: '0.0.0',
+    };
+  }
+  const projectPackage = readJsonFile(packagePath, 'package.json');
+  return {
+    name: typeof projectPackage?.name === 'string' && projectPackage.name.trim()
+      ? projectPackage.name.trim()
+      : basename(projectRoot),
+    version: typeof projectPackage?.version === 'string' && projectPackage.version.trim()
+      ? projectPackage.version.trim()
+      : '0.0.0',
+  };
+}
+
+function normalizeSlug(value, fallback = 'cache') {
+  const normalized = String(value || '')
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-+|-+$/g, '');
+  return normalized || fallback;
+}
+
+function defaultCacheRoot() {
+  return process.env[EXTERNAL_ASSET_CACHE_ROOT_ENV]
+    || join(homedir(), '.cache', 'aurajs', 'external-assets');
+}
+
+function ensurePublicUrl(rawValue, label) {
+  let parsed;
+  try {
+    parsed = new URL(String(rawValue || '').trim());
+  } catch (error) {
+    throw new ExternalAssetCacheError(
+      'external_asset_url_invalid',
+      `${label} is invalid: ${error instanceof Error ? error.message : String(error)}`,
+      { value: rawValue, label },
+    );
+  }
+
+  if (parsed.protocol === 'https:') {
+    return parsed;
+  }
+  if (parsed.protocol === 'http:' && LOCAL_HTTP_HOSTS.has(parsed.hostname)) {
+    return parsed;
+  }
+
+  throw new ExternalAssetCacheError(
+    'external_asset_url_invalid',
+    `${label} must use https://, or http:// only for localhost testing.`,
+    { value: rawValue, label },
+  );
+}
+
+function validateRelativeAssetPath(pathLike) {
+  const normalized = String(pathLike || '').replace(/\\/g, '/').trim();
+  if (!normalized) {
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_invalid',
+      'External asset manifest entry path must be a non-empty relative path.',
+    );
+  }
+  if (normalized.startsWith('/') || /^[A-Za-z]:\//.test(normalized)) {
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_invalid',
+      `External asset path "${normalized}" must be relative.`,
+      { path: normalized },
+    );
+  }
+  for (const segment of normalized.split('/')) {
+    if (segment === '..') {
+      throw new ExternalAssetCacheError(
+        'external_asset_manifest_invalid',
+        `External asset path "${normalized}" may not contain "..".`,
+        { path: normalized },
+      );
+    }
+  }
+  return normalized;
+}
+
+function loadExternalAssetsConfig(projectRoot) {
+  const configPath = resolve(projectRoot, SELF_HOSTED_EXTERNAL_ASSETS_CONFIG);
+  if (!existsSync(configPath)) {
+    return null;
+  }
+
+  const config = readJsonFile(configPath, 'aura.external-assets.json');
+  if (config?.schema !== SELF_HOSTED_CONFIG_SCHEMA) {
+    throw new ExternalAssetCacheError(
+      'external_asset_config_invalid',
+      `Expected ${SELF_HOSTED_CONFIG_SCHEMA} in ${configPath}.`,
+      { configPath, schema: config?.schema || null },
+    );
+  }
+  if (config?.mode !== 'self-hosted') {
+    throw new ExternalAssetCacheError(
+      'external_asset_config_invalid',
+      `Unsupported external asset mode "${config?.mode || '<missing>'}" in ${configPath}.`,
+      { configPath, mode: config?.mode || null },
+    );
+  }
+  if (typeof config?.assetsManifestUrl !== 'string' || !config.assetsManifestUrl.trim()) {
+    throw new ExternalAssetCacheError(
+      'external_asset_config_invalid',
+      `${configPath} is missing assetsManifestUrl.`,
+      { configPath },
+    );
+  }
+
+  ensurePublicUrl(config.assetsManifestUrl, 'assetsManifestUrl');
+  return {
+    path: configPath,
+    value: config,
+  };
+}
+
+function requestBuffer(urlLike, headers = {}) {
+  const url = urlLike instanceof URL ? urlLike : new URL(String(urlLike));
+  const requestImpl = url.protocol === 'https:' ? httpsRequest : httpRequest;
+
+  return new Promise((resolveRequest, rejectRequest) => {
+    const request = requestImpl(url, {
+      method: 'GET',
+      agent: false,
+      headers: {
+        connection: 'close',
+        ...headers,
+      },
+    }, (response) => {
+      const chunks = [];
+      response.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+      response.on('end', () => {
+        response.socket?.destroy();
+        resolveRequest({
+          statusCode: response.statusCode ?? 0,
+          body: Buffer.concat(chunks),
+        });
+      });
+    });
+
+    request.on('socket', (socket) => {
+      socket.unref();
+    });
+    request.on('error', rejectRequest);
+    request.end();
+  });
+}
+
+async function fetchJsonWithCache(url, cachedPath) {
+  try {
+    const response = await requestBuffer(url, {
+      accept: 'application/json',
+    });
+    if (response.statusCode < 200 || response.statusCode >= 300) {
+      throw new ExternalAssetCacheError(
+        'external_asset_manifest_fetch_failed',
+        `Failed to fetch external asset manifest from ${url}: HTTP ${response.statusCode}.`,
+        { url, status: response.statusCode },
+      );
+    }
+    const payload = JSON.parse(response.body.toString('utf8'));
+    mkdirSync(dirname(cachedPath), { recursive: true });
+    writeFileSync(cachedPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    return {
+      payload,
+      source: 'remote',
+      fetchWarning: null,
+    };
+  } catch (error) {
+    if (existsSync(cachedPath)) {
+      return {
+        payload: readJsonFile(cachedPath, 'cached external asset manifest'),
+        source: 'cached',
+        fetchWarning: error instanceof Error ? error.message : String(error),
+      };
+    }
+    if (error instanceof ExternalAssetCacheError) {
+      throw error;
+    }
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_fetch_failed',
+      `Failed to fetch external asset manifest from ${url}: ${error instanceof Error ? error.message : String(error)}`,
+      { url },
+    );
+  }
+}
+
+function validateExternalAssetsManifest(manifest, manifestUrl) {
+  if (manifest?.schema !== SELF_HOSTED_ASSETS_SCHEMA) {
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_invalid',
+      `Expected ${SELF_HOSTED_ASSETS_SCHEMA} from ${manifestUrl}.`,
+      { manifestUrl, schema: manifest?.schema || null },
+    );
+  }
+  if (manifest?.assetMode !== 'sibling') {
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_unsupported',
+      `Packaged play/join currently requires self-hosted sibling assets. Received assetMode "${manifest?.assetMode || '<missing>'}" from ${manifestUrl}.`,
+      { manifestUrl, assetMode: manifest?.assetMode || null },
+    );
+  }
+  if (manifest?.runtimeResolver !== 'disk') {
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_unsupported',
+      `Packaged play/join currently requires runtimeResolver "disk". Received "${manifest?.runtimeResolver || '<missing>'}" from ${manifestUrl}.`,
+      { manifestUrl, runtimeResolver: manifest?.runtimeResolver || null },
+    );
+  }
+  if (!Array.isArray(manifest.entries)) {
+    throw new ExternalAssetCacheError(
+      'external_asset_manifest_invalid',
+      `External asset manifest from ${manifestUrl} is missing entries[].`,
+      { manifestUrl },
+    );
+  }
+
+  const entries = manifest.entries.map((entry, index) => {
+    const path = validateRelativeAssetPath(entry?.path);
+    const blobUrl = typeof entry?.blobUrl === 'string' ? entry.blobUrl.trim() : '';
+    if (!blobUrl) {
+      throw new ExternalAssetCacheError(
+        'external_asset_manifest_invalid',
+        `External asset entry "${path}" is missing blobUrl.`,
+        { manifestUrl, index, path },
+      );
+    }
+    ensurePublicUrl(blobUrl, `blobUrl for ${path}`);
+
+    const sha256 = typeof entry?.sha256 === 'string' && entry.sha256.trim()
+      ? entry.sha256.trim().toLowerCase()
+      : null;
+    const sha1 = typeof entry?.sha1 === 'string' && entry.sha1.trim()
+      ? entry.sha1.trim().toLowerCase()
+      : null;
+    if (!sha256 && !sha1) {
+      throw new ExternalAssetCacheError(
+        'external_asset_manifest_invalid',
+        `External asset entry "${path}" must include sha256 or sha1.`,
+        { manifestUrl, index, path },
+      );
+    }
+
+    return {
+      path,
+      blobUrl,
+      sizeBytes: Number.isFinite(entry?.sizeBytes) ? entry.sizeBytes : null,
+      sha256,
+      sha1,
+    };
+  });
+
+  return {
+    ...manifest,
+    entries,
+  };
+}
+
+function readFileHash(path, algorithm) {
+  const hash = createHash(algorithm);
+  hash.update(readFileSync(path));
+  return hash.digest('hex');
+}
+
+function fileMatchesEntry(path, entry) {
+  if (!existsSync(path)) return false;
+  const stats = statSync(path);
+  if (!stats.isFile()) return false;
+  if (entry.sizeBytes !== null && stats.size !== entry.sizeBytes) {
+    return false;
+  }
+  if (entry.sha256) {
+    return readFileHash(path, 'sha256') === entry.sha256;
+  }
+  if (entry.sha1) {
+    return readFileHash(path, 'sha1') === entry.sha1;
+  }
+  return false;
+}
+
+function walkFiles(root) {
+  if (!existsSync(root)) return [];
+  const files = [];
+  const stack = [root];
+  while (stack.length > 0) {
+    const current = stack.pop();
+    for (const entry of readdirSync(current, { withFileTypes: true })) {
+      const fullPath = join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(fullPath);
+        continue;
+      }
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
+function removeUnexpectedFiles(root, expectedPaths) {
+  for (const filePath of walkFiles(root)) {
+    const relativePath = filePath.slice(root.length + 1).replace(/\\/g, '/');
+    if (!expectedPaths.has(relativePath)) {
+      rmSync(filePath, { force: true });
+    }
+  }
+}
+
+async function downloadEntry(entry, destPath) {
+  const response = await requestBuffer(entry.blobUrl);
+  if (response.statusCode < 200 || response.statusCode >= 300) {
+    throw new ExternalAssetCacheError(
+      'external_asset_download_failed',
+      `Failed to download external asset "${entry.path}" from ${entry.blobUrl}: HTTP ${response.statusCode}.`,
+      { path: entry.path, url: entry.blobUrl, status: response.statusCode },
+    );
+  }
+
+  const buffer = response.body;
+  if (entry.sizeBytes !== null && buffer.length !== entry.sizeBytes) {
+    throw new ExternalAssetCacheError(
+      'external_asset_hash_mismatch',
+      `External asset "${entry.path}" size mismatch: expected ${entry.sizeBytes} bytes, got ${buffer.length}.`,
+      { path: entry.path, expectedSizeBytes: entry.sizeBytes, actualSizeBytes: buffer.length },
+    );
+  }
+
+  const expectedSha256 = entry.sha256;
+  if (expectedSha256 && sha256Buffer(buffer) !== expectedSha256) {
+    throw new ExternalAssetCacheError(
+      'external_asset_hash_mismatch',
+      `External asset "${entry.path}" failed sha256 validation after download.`,
+      { path: entry.path, expectedSha256 },
+    );
+  }
+  if (!expectedSha256 && entry.sha1 && sha1Buffer(buffer) !== entry.sha1) {
+    throw new ExternalAssetCacheError(
+      'external_asset_hash_mismatch',
+      `External asset "${entry.path}" failed sha1 validation after download.`,
+      { path: entry.path, expectedSha1: entry.sha1 },
+    );
+  }
+
+  mkdirSync(dirname(destPath), { recursive: true });
+  const tempPath = `${destPath}.tmp-${process.pid}-${Date.now()}`;
+  writeFileSync(tempPath, buffer);
+  try {
+    rmSync(destPath, { force: true });
+  } catch {}
+  copyFileSync(tempPath, destPath);
+  rmSync(tempPath, { force: true });
+}
+
+async function hydrateAssetFiles(assetRoot, manifest) {
+  mkdirSync(assetRoot, { recursive: true });
+  removeUnexpectedFiles(assetRoot, new Set(manifest.entries.map((entry) => entry.path)));
+
+  let downloadedCount = 0;
+  let reusedCount = 0;
+  for (const entry of manifest.entries) {
+    const destPath = resolve(assetRoot, ...entry.path.split('/'));
+    if (fileMatchesEntry(destPath, entry)) {
+      reusedCount += 1;
+      continue;
+    }
+    await downloadEntry(entry, destPath);
+    if (!fileMatchesEntry(destPath, entry)) {
+      throw new ExternalAssetCacheError(
+        'external_asset_hash_mismatch',
+        `External asset "${entry.path}" failed validation after download was written to cache.`,
+        { path: entry.path, destPath },
+      );
+    }
+    downloadedCount += 1;
+  }
+
+  return {
+    downloadedCount,
+    reusedCount,
+  };
+}
+
+function resolveNodeModulesRoot(projectRoot) {
+  const directNodeModules = resolve(projectRoot, 'node_modules');
+  if (existsSync(directNodeModules)) {
+    try {
+      if (statSync(directNodeModules).isDirectory()) {
+        return directNodeModules;
+      }
+    } catch {}
+  }
+
+  let current = resolve(projectRoot);
+  while (true) {
+    const parent = dirname(current);
+    if (parent === current) {
+      return null;
+    }
+    current = parent;
+    if (basename(current) === 'node_modules') {
+      return current;
+    }
+  }
+}
+
+function linkOrCopy(sourcePath, destPath, isDirectory) {
+  mkdirSync(dirname(destPath), { recursive: true });
+  try {
+    const linkType = isDirectory
+      ? (process.platform === 'win32' ? 'junction' : 'dir')
+      : 'file';
+    symlinkSync(resolve(sourcePath), destPath, linkType);
+    return;
+  } catch {}
+
+  if (isDirectory) {
+    cpSync(sourcePath, destPath, {
+      recursive: true,
+      force: true,
+    });
+    return;
+  }
+
+  copyFileSync(sourcePath, destPath);
+}
+
+function buildRuntimeWorkspace(projectRoot, runtimeRoot, assetRoot) {
+  rmSync(runtimeRoot, { recursive: true, force: true });
+  mkdirSync(runtimeRoot, { recursive: true });
+
+  const skippedNames = new Set(['assets', 'build', '.aura', '.logs', 'node_modules']);
+  for (const entry of readdirSync(projectRoot, { withFileTypes: true })) {
+    if (skippedNames.has(entry.name)) continue;
+    const sourcePath = join(projectRoot, entry.name);
+    const destPath = join(runtimeRoot, entry.name);
+    linkOrCopy(sourcePath, destPath, entry.isDirectory());
+  }
+
+  const nodeModulesRoot = resolveNodeModulesRoot(projectRoot);
+  if (nodeModulesRoot) {
+    linkOrCopy(nodeModulesRoot, join(runtimeRoot, 'node_modules'), true);
+  }
+  linkOrCopy(assetRoot, join(runtimeRoot, 'assets'), true);
+}
+
+function ensureCacheLayout(projectRoot, config, manifest) {
+  const packageMeta = readPackageMetadata(projectRoot);
+  const packageSlug = normalizeSlug(packageMeta.name, 'game');
+  const versionSlug = normalizeSlug(packageMeta.version, '0.0.0');
+  const manifestUrlHash = sha256Text(config.assetsManifestUrl).slice(0, 16);
+  const manifestHash = normalizeSlug(
+    manifest.sourceManifestHash || sha256Text(JSON.stringify(manifest)),
+    'manifest',
+  );
+  const cacheBase = resolve(defaultCacheRoot(), packageSlug, versionSlug, manifestUrlHash, manifestHash);
+  return {
+    packageMeta,
+    cacheBase,
+    assetRoot: join(cacheBase, 'assets'),
+    runtimeRoot: join(cacheBase, 'runtime'),
+    cachedManifestPath: join(defaultCacheRoot(), packageSlug, versionSlug, 'manifests', `${manifestUrlHash}.json`),
+  };
+}
+
+export async function prepareExternalAssetRuntime({
+  projectRoot = process.cwd(),
+  commandName = 'play',
+  stdout = process.stdout,
+} = {}) {
+  const configRecord = loadExternalAssetsConfig(projectRoot);
+  if (!configRecord) {
+    return {
+      enabled: false,
+      runtimeProjectRoot: projectRoot,
+      downloadSummary: null,
+      manifestSource: null,
+      cacheBase: null,
+    };
+  }
+
+  const config = configRecord.value;
+  const packageMeta = readPackageMetadata(projectRoot);
+  const manifestUrlHash = sha256Text(config.assetsManifestUrl).slice(0, 16);
+  const cachedManifestPath = join(
+    defaultCacheRoot(),
+    normalizeSlug(packageMeta.name, 'game'),
+    normalizeSlug(packageMeta.version, '0.0.0'),
+    'manifests',
+    `${manifestUrlHash}.json`,
+  );
+
+  const fetchedManifest = await fetchJsonWithCache(config.assetsManifestUrl, cachedManifestPath);
+  const manifest = validateExternalAssetsManifest(fetchedManifest.payload, config.assetsManifestUrl);
+  const cacheLayout = ensureCacheLayout(projectRoot, config, manifest);
+  const downloadSummary = await hydrateAssetFiles(cacheLayout.assetRoot, manifest);
+  buildRuntimeWorkspace(projectRoot, cacheLayout.runtimeRoot, cacheLayout.assetRoot);
+
+  stdout?.write(`  Self-hosted assets: using cached runtime workspace for ${commandName}.\n`);
+  if (fetchedManifest.source === 'cached') {
+    stdout?.write('  External asset manifest: using cached copy because the remote manifest was unavailable.\n');
+  }
+  stdout?.write(
+    `  Asset cache: downloaded=${downloadSummary.downloadedCount} reused=${downloadSummary.reusedCount} root=${cacheLayout.assetRoot}\n`,
+  );
+
+  return {
+    enabled: true,
+    runtimeProjectRoot: cacheLayout.runtimeRoot,
+    downloadSummary,
+    manifestSource: fetchedManifest.source,
+    manifestFetchWarning: fetchedManifest.fetchWarning,
+    cacheBase: cacheLayout.cacheBase,
+    assetRoot: cacheLayout.assetRoot,
+  };
+}