@triflare/fx 1.0.3 → 2.0.0

package/src/config-edit.js

@@ -54,7 +54,8 @@ function childKey(lines, start, end, key) {
 }
 
 function childIndent(lines, start, end) {
-  for (let index = start + 1; index < end; index++) {
+  const limit = Math.min(end, lines.length);
+  for (let index = start + 1; index < limit; index++) {
     const match = /^([ \t]+)\S/.exec(content(lines[index]));
     if (match) return match[1];
   }
@@ -336,3 +337,281 @@ export function removeProjectExtension(configPath, target) {
     unsandboxedRemoved: unsandboxedRemoved.length > 0,
   };
 }
+
+/**
+ * Remove `project.extensions` entirely (used by migration).
+ *
+ * Unlike {@link removeProjectExtension}, this drops the key itself instead of
+ * leaving an empty list, so a fully migrated file has no legacy section left.
+ */
+export function removeProjectExtensions(configPath) {
+  const text = fs.readFileSync(configPath, "utf8");
+  const lines = splitLines(text);
+  const project = topLevel(lines, "project");
+  if (!project) return [];
+  const { value: rest, comment } = splitComment(project.rest);
+
+  if (rest.startsWith("{")) {
+    if (!rest.endsWith("}"))
+      throw new Error("project must be a mapping to edit it");
+    const mapping = parseValue(rest);
+    if (!mapping || typeof mapping !== "object" || Array.isArray(mapping))
+      throw new Error("project must be a mapping to edit it");
+    if (!Object.hasOwn(mapping, "extensions")) return [];
+
+    const removed = stringList(mapping.extensions);
+    delete mapping.extensions;
+    const line = lines[project.index];
+    lines[project.index] =
+      `project: ${JSON.stringify(mapping)}` +
+      `${comment ? ` ${comment}` : ""}${ending(line) || eolOf(text)}`;
+    writeDocument(configPath, lines.join(""));
+    return removed;
+  }
+
+  const bounds = projectBounds(lines);
+  if (!bounds) return [];
+  const list = readList(lines, "extensions", bounds.index, bounds.end);
+  if (!list) return [];
+
+  const removed = [...list.values];
+  if (list.style === "block") {
+    const end = Math.max(list.keyLine.index, list.lastLine) + 1;
+    lines.splice(list.keyLine.index, end - list.keyLine.index);
+  } else {
+    lines.splice(list.keyLine.index, 1);
+  }
+  writeDocument(configPath, lines.join(""));
+  return removed;
+}
+
+// --------------------------------------------------------------- vendor edits
+
+const SAFE_KEY = /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/;
+
+/** Quote a mapping key when YAML would otherwise parse it as non-text. */
+function quoteKey(key) {
+  // YAML 1.1 treats bare yes/no/on/off/null as non-strings.
+  return SAFE_KEY.test(key) &&
+    !/^(?:y|yes|n|no|on|off|null|true|false)$/i.test(key)
+    ? key
+    : JSON.stringify(String(key));
+}
+
+/** Serialize one vendor entry, expanding options only when needed. */
+function vendorEntryLines(name, source, options, indent, eol) {
+  const extra = [];
+  if (options.unsandboxed) extra.push(`${indent}  unsandboxed: true${eol}`);
+  if (options.volatile !== undefined)
+    extra.push(`${indent}  volatile: ${Boolean(options.volatile)}${eol}`);
+  if (extra.length === 0)
+    return [`${indent}${quoteKey(name)}: ${quote(source)}${eol}`];
+  return [
+    `${indent}${quoteKey(name)}:${eol}`,
+    `${indent}  url: ${quote(source)}${eol}`,
+    ...extra,
+  ];
+}
+
+/** Locate a top-level mapping, accepting an inline flow mapping. */
+function topLevelBounds(lines, key) {
+  const found = topLevel(lines, key);
+  if (!found) return null;
+  const { value: rest, comment } = splitComment(found.rest);
+  if (rest !== "") {
+    // A flow mapping can be expanded losslessly (comments on the line aside).
+    if (rest.startsWith("{") && rest.endsWith("}")) {
+      const mapping = parseValue(rest);
+      if (mapping && typeof mapping === "object" && !Array.isArray(mapping))
+        return {
+          index: found.index,
+          end: found.index + 1,
+          inline: rest,
+          comment,
+        };
+    }
+    throw new Error(`${key} must be a block mapping to edit it`);
+  }
+  return {
+    index: found.index,
+    end: sectionEnd(lines, found.index),
+    inline: null,
+    comment,
+  };
+}
+
+/** Find direct child entries and their line ranges in a mapping. */
+function mappingEntrySpans(lines, start, end) {
+  const indent = childIndent(lines, start, end);
+  const spans = [];
+  const limit = Math.min(end, lines.length);
+  for (let index = start + 1; index < limit; index++) {
+    const text = content(lines[index]);
+    if (text.trim() === "" || /^[ \t]*#/.test(text)) continue;
+    const outer = /^([ \t]+)(.*)$/.exec(text);
+    if (!outer || outer[1] !== indent) continue;
+    const match = /^(.+?)[ \t]*:[ \t]*(.*)$/.exec(outer[2]);
+    if (!match) continue;
+    let key;
+    try {
+      const parsed = yaml.load(`${match[1]}: null`);
+      key =
+        parsed && typeof parsed === "object" ? Object.keys(parsed)[0] : null;
+    } catch {
+      continue;
+    }
+    if (key == null) continue;
+    spans.push({ index, indent, key: String(key), rest: match[2] });
+  }
+  for (let index = 0; index < spans.length; index++)
+    spans[index].end = index + 1 < spans.length ? spans[index + 1].index : end;
+  return spans;
+}
+
+/** Replace a half-open line range in place. */
+function replaceRange(lines, start, end, added) {
+  lines.splice(start, end - start, ...added);
+}
+
+/**
+ * Add or replace a `vendor:` entry without reformatting the rest of fx.yml.
+ *
+ * @param {string} configPath
+ * @param {string} name
+ * @param {string} source
+ * @param {{unsandboxed?: boolean, volatile?: boolean}} [options]
+ * @returns {{added: boolean, updated: boolean}}
+ */
+export function addVendorEntry(configPath, name, source, options = {}) {
+  const text = fs.readFileSync(configPath, "utf8");
+  const lines = splitLines(text);
+  const eol = eolOf(text);
+  const key = String(name);
+  let added = false;
+  let updated = false;
+
+  const bounds = topLevelBounds(lines, "vendor");
+  if (!bounds) {
+    if (lines.length > 0 && !lines[lines.length - 1].endsWith("\n"))
+      lines[lines.length - 1] += eol;
+    lines.push(
+      `vendor:${eol}`,
+      ...vendorEntryLines(key, source, options, "  ", eol),
+    );
+    added = true;
+  } else if (bounds.inline) {
+    const mapping = parseValue(bounds.inline) || {};
+    updated = Object.hasOwn(mapping, key);
+    added = !updated;
+    mapping[key] =
+      options.unsandboxed || options.volatile !== undefined
+        ? {
+            url: source,
+            ...(options.unsandboxed ? { unsandboxed: true } : {}),
+            ...(options.volatile !== undefined
+              ? { volatile: Boolean(options.volatile) }
+              : {}),
+          }
+        : source;
+    const dumped = yaml
+      .dump({ vendor: mapping }, { lineWidth: -1, noRefs: true })
+      .split("\n")
+      .filter((line, index, all) => !(index === all.length - 1 && line === ""))
+      .map((line) => `${line}${eol}`);
+    if (bounds.comment) dumped[0] = `vendor: ${bounds.comment}${eol}`;
+    replaceRange(lines, bounds.index, bounds.index + 1, dumped);
+  } else {
+    const spans = mappingEntrySpans(lines, bounds.index, bounds.end);
+    const existing = spans.find((span) => span.key === key);
+    if (existing) {
+      replaceRange(
+        lines,
+        existing.index,
+        existing.end,
+        vendorEntryLines(key, source, options, existing.indent, eol),
+      );
+      updated = true;
+    } else {
+      const indent = childIndent(lines, bounds.index, bounds.end);
+      insertAt(
+        lines,
+        bounds.end,
+        vendorEntryLines(key, source, options, indent, eol),
+        eol,
+      );
+      added = true;
+    }
+  }
+
+  if (added || updated) writeDocument(configPath, lines.join(""));
+  return { added, updated };
+}
+
+/**
+ * Remove a `vendor:` entry.
+ * @param {string} configPath
+ * @param {string} name
+ * @returns {{removed: string|null}}
+ */
+export function removeVendorEntry(configPath, name) {
+  const text = fs.readFileSync(configPath, "utf8");
+  const lines = splitLines(text);
+  const eol = eolOf(text);
+  const key = String(name);
+  const bounds = topLevelBounds(lines, "vendor");
+  if (!bounds) return { removed: null };
+
+  if (bounds.inline) {
+    const mapping = parseValue(bounds.inline);
+    if (!mapping || typeof mapping !== "object" || Array.isArray(mapping))
+      return { removed: null };
+    if (!Object.hasOwn(mapping, key)) return { removed: null };
+    const source =
+      typeof mapping[key] === "string"
+        ? mapping[key]
+        : (mapping[key]?.url ?? mapping[key]?.source ?? null);
+    delete mapping[key];
+    const rest = Object.keys(mapping).length
+      ? ` ${JSON.stringify(mapping)}`
+      : " {}";
+    const comment = bounds.comment ? ` ${bounds.comment}` : "";
+    const line = lines[bounds.index];
+    lines[bounds.index] = `vendor:${rest}${comment}${
+      /\r?\n$/.test(line) ? ending(line) : eol
+    }`;
+    writeDocument(configPath, lines.join(""));
+    return { removed: source };
+  }
+
+  const spans = mappingEntrySpans(lines, bounds.index, bounds.end);
+  const existing = spans.find((span) => span.key === key);
+  if (!existing) return { removed: null };
+
+  let source = null;
+  try {
+    const parsed = yaml.load(
+      [
+        content(lines[bounds.index]),
+        ...lines.slice(existing.index, existing.end),
+      ]
+        .join("\n")
+        .replace(/^vendor:/, "vendor:"),
+    );
+    const entry = parsed?.vendor?.[key];
+    source =
+      typeof entry === "string"
+        ? entry
+        : (entry?.url ?? entry?.source ?? entry?.path ?? null);
+  } catch {
+    // The entry is removed either way.
+  }
+
+  replaceRange(lines, existing.index, existing.end, []);
+  const end = bounds.end - (existing.end - existing.index);
+  const remaining = mappingEntrySpans(lines, bounds.index, end);
+  if (remaining.length === 0) {
+    replaceRange(lines, bounds.index, end, [`vendor: {}${eol}`]);
+  }
+  writeDocument(configPath, lines.join(""));
+  return { removed: source ?? key };
+}

package/src/config.js

@@ -31,8 +31,112 @@ export const DEFAULT_PROJECT = {
  */
 export const DEFAULT_FX = {
   partialSprites: false,
+  vendorDir: "extensions",
 };
 
+/** Default directory that receives vendored extension sources. */
+export const DEFAULT_VENDOR_DIR = "extensions";
+
+/** Source schemes that identify a remote extension. */
+const REMOTE_SOURCE = /^https?:\/\//i;
+const DATA_SOURCE = /^data:/i;
+
+/** True for http(s) extension sources. */
+function isRemoteSource(source) {
+  return REMOTE_SOURCE.test(String(source));
+}
+
+/** True for embedded data-URI extension sources. */
+function isDataSource(source) {
+  return DATA_SOURCE.test(String(source));
+}
+
+/** Localhost/loopback URLs can point at a development server that changes. */
+export function isLocalhostSource(source) {
+  if (!isRemoteSource(source)) return false;
+  try {
+    const url = new URL(String(source));
+    const host = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
+    return (
+      host === "localhost" ||
+      host.endsWith(".localhost") ||
+      host === "0.0.0.0" ||
+      host === "::1" ||
+      /^127\./.test(host)
+    );
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Normalize one `vendor:` entry. A string is shorthand for `{ url }`; a
+ * mapping may add `unsandboxed`, `volatile` and any future fields.
+ *
+ * @param {string} name
+ * @param {unknown} raw
+ * @returns {object}
+ */
+export function normalizeVendorEntry(name, raw) {
+  const key = String(name);
+  let source;
+  let options = {};
+  if (typeof raw === "string" || typeof raw === "number") {
+    source = String(raw);
+  } else if (isPlainObject(raw)) {
+    source = raw.url ?? raw.source ?? raw.path ?? raw.file;
+    options = { ...raw };
+  } else {
+    throw new Error(
+      `${configFilename()}: vendor.${key} must be a URL, path, or mapping`,
+    );
+  }
+  if (typeof source !== "string" || source.trim() === "")
+    throw new Error(
+      `${configFilename()}: vendor.${key} needs a "url" (or "source"/"path")`,
+    );
+  source = source.trim();
+
+  const remote = isRemoteSource(source);
+  const local = !remote && !isDataSource(source);
+  const unsandboxed =
+    options.unsandboxed === true ||
+    (typeof options.unsandboxed === "string" &&
+      options.unsandboxed.trim() !== "");
+  const volatile =
+    options.volatile == null
+      ? remote && isLocalhostSource(source)
+      : Boolean(options.volatile);
+
+  return {
+    ...options,
+    name: key,
+    source,
+    url: source,
+    remote,
+    local,
+    data: isDataSource(source),
+    volatile,
+    unsandboxed,
+  };
+}
+
+/**
+ * Normalize the top-level `vendor:` mapping. Order is load order, so this is an
+ * array rather than a plain object.
+ *
+ * @param {unknown} raw
+ * @returns {object[]}
+ */
+export function normalizeVendor(raw = {}) {
+  if (raw == null) return [];
+  if (!isPlainObject(raw))
+    throw new Error(`${configFilename()}: "vendor" must be a mapping`);
+  return Object.entries(raw).map(([name, entry]) =>
+    normalizeVendorEntry(name, entry),
+  );
+}
+
 /**
  * Packaging defaults. These never reach PenguinMod itself; they are handed to
  * the PenguinMod Packager when `fx package` or `fx dev` runs.
@@ -281,6 +385,11 @@ export function normalizeFx(raw = {}) {
   if (raw && typeof raw === "object" && !Array.isArray(raw)) {
     if (raw.partialSprites != null)
       fx.partialSprites = Boolean(raw.partialSprites);
+    if (raw.vendorDir != null) {
+      if (typeof raw.vendorDir !== "string" || raw.vendorDir.trim() === "")
+        throw new Error(`${configFilename()}: fx.vendorDir must be a path`);
+      fx.vendorDir = raw.vendorDir.trim();
+    }
   }
   return fx;
 }
@@ -288,7 +397,7 @@ export function normalizeFx(raw = {}) {
 /**
  * Parse a YAML config string.
  * @param {string} text
- * @returns {{project: typeof DEFAULT_PROJECT, fx: typeof DEFAULT_FX, raw: object}}
+ * @returns {{project: typeof DEFAULT_PROJECT, fx: typeof DEFAULT_FX, package: typeof DEFAULT_PACKAGE, vendor: object[], vendorDir: string, raw: object}}
  */
 export function parseConfig(text) {
   const raw = yaml.load(text) || {};
@@ -298,10 +407,20 @@ export function parseConfig(text) {
     );
   if (raw.fx != null && (typeof raw.fx !== "object" || Array.isArray(raw.fx)))
     throw new Error(`${configFilename()} must contain an "fx" mapping`);
+  const fx = normalizeFx(raw.fx || {});
+  // `vendorDir` is documented at the top level, but accepting it under `fx:`
+  // keeps older drafts and long configs working.
+  if (raw.vendorDir != null) {
+    if (typeof raw.vendorDir !== "string" || raw.vendorDir.trim() === "")
+      throw new Error(`${configFilename()}: vendorDir must be a path`);
+    fx.vendorDir = raw.vendorDir.trim();
+  }
   return {
     project: normalizeProject(raw.project || {}),
-    fx: normalizeFx(raw.fx || {}),
+    fx,
     package: normalizePackage(raw.package || {}),
+    vendor: normalizeVendor(raw.vendor || {}),
+    vendorDir: fx.vendorDir,
     raw,
   };
 }
@@ -339,14 +458,26 @@ export function loadConfigFrom(dir) {
       project,
       fx,
       package: pkg,
+      vendor,
+      vendorDir,
       raw,
     } = parseConfig(fs.readFileSync(found, "utf8"));
-    return { project, fx, package: pkg, raw, path: found };
+    return {
+      project,
+      fx,
+      package: pkg,
+      vendor,
+      vendorDir,
+      raw,
+      path: found,
+    };
   }
   return {
     project: { ...DEFAULT_PROJECT },
     fx: { ...DEFAULT_FX },
     package: normalizePackage({}),
+    vendor: [],
+    vendorDir: DEFAULT_VENDOR_DIR,
     raw: {},
     path: null,
   };
@@ -394,7 +525,10 @@ export function configComment(project) {
 
 export {
   addProjectExtension,
+  addVendorEntry,
   extensionKey,
   readProjectExtensions,
   removeProjectExtension,
+  removeProjectExtensions,
+  removeVendorEntry,
 } from "./config-edit.js";

package/src/extensions.js

@@ -757,6 +757,7 @@ const BASE64_PAYLOAD = /^[A-Za-z0-9+/=\s]*$/;
  * @param {string} source
  * @returns {string}
  */
+/** Encode extension source as a self-contained JavaScript data URI. */
 export function extensionDataUri(source) {
   const base64 = Buffer.from(String(source), "utf8").toString("base64");
   return `data:text/javascript;base64,${base64}`;
@@ -782,8 +783,13 @@ export function readExtensionDataUri(uri) {
   }
 }
 
-/** Resolve a config entry to source text. */
-async function resolveSource(entry, baseDir, fetchImpl) {
+/**
+ * Resolve a config entry to source text and the original bytes.
+ *
+ * The raw buffer is retained so vendoring can lock exactly what the server
+ * returned, even when the text decoder strips a byte-order mark.
+ */
+export async function resolveSource(entry, baseDir, fetchImpl) {
   const value = String(entry);
   if (/^data:/i.test(value)) {
     const source = readExtensionDataUri(value);
@@ -791,7 +797,13 @@ async function resolveSource(entry, baseDir, fetchImpl) {
       throw new Error(
         "unsupported data URI; expected base64 or percent-encoded JavaScript",
       );
-    return { source, remote: false, dataUri: true, url: value };
+    return {
+      source,
+      buffer: Buffer.from(source, "utf8"),
+      remote: false,
+      dataUri: true,
+      url: value,
+    };
   }
   if (/^https?:\/\//i.test(value)) {
     const fetchFn = fetchImpl || globalThis.fetch;
@@ -800,13 +812,24 @@ async function resolveSource(entry, baseDir, fetchImpl) {
     const response = await fetchFn(value);
     if (!response.ok)
       throw new Error(`HTTP ${response.status} ${response.statusText}`.trim());
-    return { source: await response.text(), remote: true, url: value };
+    const buffer =
+      typeof response.arrayBuffer === "function"
+        ? Buffer.from(await response.arrayBuffer())
+        : Buffer.from(await response.text(), "utf8");
+    return {
+      source: buffer.toString("utf8").replace(/^\uFEFF/, ""),
+      buffer,
+      remote: true,
+      url: value,
+    };
   }
   const resolved = path.isAbsolute(value)
     ? value
     : path.resolve(baseDir, value);
+  const buffer = fs.readFileSync(resolved);
   return {
-    source: fs.readFileSync(resolved, "utf8"),
+    source: buffer.toString("utf8"),
+    buffer,
     remote: false,
     url: value,
     file: resolved,
@@ -850,19 +873,39 @@ async function evaluateExtensionAny(source, filename) {
   }
 }
 
+/**
+ * Inspect extension source without running it in the Node.js process.
+ * Useful to propose an extension id while vendoring or migrating.
+ * @param {string} source
+ * @param {string} [filename]
+ * @returns {Promise<{infos: object[], unsandboxed: boolean}>}
+ */
+export async function inspectExtension(source, filename = "extension") {
+  return evaluateExtensionAny(source, filename);
+}
+
 /**
  * Load a list of extension references.
  *
  * @param {Array<string>} list Paths, `http(s)://` URLs or `data:` URIs.
- * @param {{baseDir?: string, fetchImpl?: typeof fetch, expectUnsandboxed?: string|string[]}} [options]
+ * @param {{baseDir?: string, fetchImpl?: typeof fetch, expectUnsandboxed?: string|string[], reportUnmatchedUnsandboxed?: boolean}} [options]
  *   `expectUnsandboxed` acknowledges extensions that are known to require
- *   unsandboxed mode, silencing the warning for them.
- * @returns {Promise<{catalog: Record<string, object>, extensionUrls: Record<string, string>, warnings: string[], ids: string[], unsandboxed: string[]}>}
+ *   unsandboxed mode, silencing the warning for them. Set
+ *   `reportUnmatchedUnsandboxed` to false when a caller combines declarations
+ *   from several loaders and reports unmatched names itself.
+ * @returns {Promise<{catalog: Record<string, object>, extensionUrls: Record<string, string>, warnings: string[], ids: string[], unsandboxed: string[], matchedUnsandboxed: string[], unnecessaryUnsandboxed: string[]}>}
  *   `unsandboxed` holds the ids of extensions that only load unsandboxed.
+ *   `matchedUnsandboxed` and `unnecessaryUnsandboxed` let callers combine
+ *   declarations across multiple loaders without double-reporting them.
  */
 export async function loadExtensions(
   list,
-  { baseDir = process.cwd(), fetchImpl, expectUnsandboxed = [] } = {},
+  {
+    baseDir = process.cwd(),
+    fetchImpl,
+    expectUnsandboxed = [],
+    reportUnmatchedUnsandboxed = true,
+  } = {},
 ) {
   const catalog = {};
   const extensionUrls = {};
@@ -941,17 +984,27 @@ export async function loadExtensions(
         "project.unsandboxed in fx.yml once you trust it)",
     );
   }
-  for (const name of declared) {
-    if (declaredNotNeeded.has(name)) {
-      warnings.push(
-        `project.unsandboxed entry "${name}" loads in the sandbox; remove it from fx.yml`,
-      );
-    } else if (!declaredSeen.has(name)) {
-      warnings.push(
-        `project.unsandboxed entry "${name}" does not match any loaded extension`,
-      );
+  if (reportUnmatchedUnsandboxed) {
+    for (const name of declared) {
+      if (declaredNotNeeded.has(name)) {
+        warnings.push(
+          `project.unsandboxed entry "${name}" loads in the sandbox; remove it from fx.yml`,
+        );
+      } else if (!declaredSeen.has(name)) {
+        warnings.push(
+          `project.unsandboxed entry "${name}" does not match any loaded extension`,
+        );
+      }
     }
   }
 
-  return { catalog, extensionUrls, warnings, ids, unsandboxed };
+  return {
+    catalog,
+    extensionUrls,
+    warnings,
+    ids,
+    unsandboxed,
+    matchedUnsandboxed: [...declaredSeen],
+    unnecessaryUnsandboxed: [...declaredNotNeeded],
+  };
 }

package/src/index.js

@@ -29,6 +29,27 @@ export {
   resolvePackageConfig,
   slugify,
 } from "./package.js";
+export {
+  LOCK_FILENAME,
+  VendorError,
+  defaultVendorName,
+  fetchVendorSource,
+  formatDiff,
+  installVendor,
+  installVendorEntry,
+  loadVendorExtensions,
+  migrateProjectExtensions,
+  outdatedVendor,
+  readLock,
+  removeVendoredExtension,
+  resolveVendorFiles,
+  sha256,
+  uniqueVendorName,
+  updateVendor,
+  vendorDirPath,
+  verifyVendor,
+  writeLock,
+} from "./vendor.js";
 export { formatDevUrls, injectDevClient, startDevServer } from "./dev.js";
 export * from "./catalog.js";
 export * from "./config.js";