goldsync 0.1.16 → 0.1.34

package/src/store.mjs

@@ -29,6 +29,17 @@ function decodePathSegment(segment) {
   );
 }
 
+function decodeTreeSegment(segment) {
+  const numbered = /^(.*) \(([1-9][0-9]{0,8})\)$/.exec(segment);
+  if (numbered) return { name: decodePathSegment(numbered[1]), id: numbered[2] };
+  const match = /^(.*)~([0-9a-f]{32})$/.exec(segment);
+  return { name: decodePathSegment(match ? match[1] : segment), id: match?.[2] ?? "" };
+}
+
+function treePathKey(studioPath, instanceIds = []) {
+  return JSON.stringify(studioPath.map((name, index) => [name, instanceIds[index] ?? ""]));
+}
+
 async function findSplitFiles(directory, depth, relative = []) {
   let entries;
   try {
@@ -92,9 +103,11 @@ export class SnapshotStore {
   constructor(config) {
     this.config = config;
     this.roots = new Map(config.roots.map((root) => [root.id, root]));
+    this.rootsByFile = new Map(config.roots.map((root) => [root.absoluteFile.toLowerCase(), root]));
     this.stateFile = path.join(config.stateDirectory, "state.json");
     this.backupRoot = path.join(config.stateDirectory, "backups");
     this.state = { version: 1, roots: {} };
+    this.treeScanNeeded = true;
   }
 
   makeSplitRoot(splitRoot, relativePath) {
@@ -128,6 +141,8 @@ export class SnapshotStore {
       return false;
     }
     this.roots.set(root.id, root);
+    this.manifestRoots = null;
+    this.rootsByFile.set(root.absoluteFile.toLowerCase(), root);
     await mkdir(path.dirname(root.absoluteFile), { recursive: true });
     await this.refresh(root.id, false);
     return true;
@@ -154,6 +169,7 @@ export class SnapshotStore {
   }
 
   async refreshSplitRoots() {
+    await this.refreshTreeRoots();
     for (const splitRoot of this.config.splitRoots ?? []) {
       await mkdir(splitRoot.absoluteDirectory, { recursive: true });
       const paths = await findSplitFiles(splitRoot.absoluteDirectory, splitRoot.splitDepth);
@@ -161,6 +177,224 @@ export class SnapshotStore {
     }
   }
 
+  makeTreeRoot(tree, relativePath, container, relativeIds) {
+    if (!Array.isArray(relativePath) || typeof container !== "boolean"
+      || tree.studioPath.length + relativePath.length > this.config.maxPathDepth
+      || relativePath.some((segment) => typeof segment !== "string" || !segment || segment === "." || segment === ".." || segment.toLowerCase() === "_root")) {
+      throw new Error(`tree root ${tree.id} received an invalid path`);
+    }
+    if (relativePath.length === 0 && !container) throw new Error("tree roots must be containers");
+    relativeIds ??= relativePath.map(() => "");
+    if (!Array.isArray(relativeIds) || relativeIds.length !== relativePath.length
+      || relativeIds.some((id) => typeof id !== "string" || (id !== "" && !/^(?:[1-9][0-9]{0,8}|[0-9a-f]{32})$/.test(id)))) {
+      throw new Error("instanceIds must match the path and contain valid sibling identifiers");
+    }
+    const studioPath = [...tree.studioPath, ...relativePath];
+    const instanceIds = [...tree.studioPath.map(() => ""), ...relativeIds];
+    const identity = relativeIds.some(Boolean) ? [studioPath, instanceIds] : studioPath;
+    const suffix = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 16);
+    const directory = path.join(tree.directory, ...relativePath.map((name, index) =>
+      encodePathSegment(name).replaceAll("~", "%7e").replaceAll("(", "%28").replaceAll(")", "%29")
+        + (relativeIds[index] ? (relativeIds[index].length <= 9 ? ` (${relativeIds[index]})` : `~${relativeIds[index]}`) : "")));
+    const file = (container ? path.join(directory, "_root.rbxm") : directory + ".rbxm").replaceAll(path.sep, "/");
+    return {
+      id: `${tree.id}-${suffix}`, studioPath, file,
+      absoluteFile: path.resolve(this.config.projectRoot, file),
+      treeRootId: tree.id, container, instanceIds,
+    };
+  }
+
+  async discoverTreePaths(treeId, entries) {
+    const tree = (this.config.treeRoots ?? []).find((candidate) => candidate.id === treeId);
+    if (!tree) throw new Error(`unknown tree root: ${treeId}`);
+    const roots = entries.map((entry) => this.makeTreeRoot(tree, entry.path, entry.container, entry.instanceIds));
+    const byPath = new Map();
+    const byFile = new Map();
+    for (const root of [...this.roots.values(), ...roots]) {
+      const key = treePathKey(root.studioPath, root.instanceIds);
+      const existing = byPath.get(key);
+      if (existing && existing.container !== root.container) throw new Error(`conflicting file and directory at ${root.file}`);
+      const fileKey = root.absoluteFile.toLowerCase();
+      if (byFile.has(fileKey) && byFile.get(fileKey) !== key) throw new Error(`duplicate tree output file: ${root.file}`);
+      byPath.set(key, root);
+      byFile.set(fileKey, key);
+    }
+    for (const root of roots) {
+      for (let depth = 2; depth < root.studioPath.length; depth += 1) {
+        const ancestor = byPath.get(treePathKey(root.studioPath.slice(0, depth), root.instanceIds));
+        if (ancestor && ancestor.container === false) throw new Error(`tree path is inside an existing asset: ${root.file}`);
+      }
+    }
+    let changed = false;
+    for (const root of roots) changed = (await this.addRoot(root)) || changed;
+    if (changed) await this.persistState();
+    return changed;
+  }
+
+  async replaceBoundary(id, entries, expected) {
+    if (this.treeScan) await this.treeScan;
+    if (this.boundaryChange) throw new Error("Another boundary conversion is running");
+    const original = this.getRoot(id);
+    const tree = this.config.treeRoots.find((candidate) => candidate.id === original.treeRootId);
+    if (!tree || !Array.isArray(entries)) throw new Error("Invalid boundary conversion");
+    const removing = entries.length === 0;
+    if (removing && original.studioPath.length === tree.studioPath.length) throw new Error("Cannot remove a configured tree root");
+    const prefix = treePathKey(original.studioPath, original.instanceIds);
+    const affected = this.getRoots().filter((root) => root.treeRootId === tree.id
+      && treePathKey(root.studioPath.slice(0, original.studioPath.length), root.instanceIds) === prefix);
+    const replacements = entries.map((entry) => ({
+      root: this.makeTreeRoot(tree, entry.path, entry.container, entry.instanceIds), bytes: entry.bytes,
+    }));
+    const replacement = replacements.find(({ root }) => root.id === id)?.root;
+    if (!removing && (!replacement || replacement.container === original.container
+      || (!replacement.container && replacements.length !== 1))) throw new Error("Expected a changed boundary");
+    const destination = removing ? null : replacement.container ? path.dirname(replacement.absoluteFile) : replacement.absoluteFile;
+    const source = original.container ? path.dirname(original.absoluteFile) : original.absoluteFile;
+    const outputFiles = new Set();
+    const byPath = new Map(replacements.map(({ root }) => [treePathKey(root.studioPath, root.instanceIds), root]));
+    for (const { root, bytes } of replacements) {
+      const relative = path.relative(replacement.container ? destination : path.dirname(destination), root.absoluteFile);
+      if (treePathKey(root.studioPath.slice(0, original.studioPath.length), root.instanceIds) !== prefix
+        || relative.startsWith("..") || path.isAbsolute(relative) || outputFiles.has(root.absoluteFile.toLowerCase())
+        || !Buffer.isBuffer(bytes) || (!root.container && bytes.length === 0)) throw new Error("Invalid replacement snapshot");
+      outputFiles.add(root.absoluteFile.toLowerCase());
+      if (root.id !== id) {
+        const parent = byPath.get(treePathKey(root.studioPath.slice(0, -1), root.instanceIds));
+        if (!parent?.container) throw new Error("Replacement child must have a container parent");
+      }
+    }
+    if (affected.length !== Object.keys(expected ?? {}).length) throw new ConflictError(this.getManifest(id));
+    const transaction = path.join(this.backupRoot, `boundary-${Date.now()}-${randomUUID()}`);
+    const staged = path.join(transaction, "new");
+    const backup = path.join(transaction, "original");
+    const previousRoots = new Map(this.roots);
+    const previousFiles = new Map(this.rootsByFile);
+    const previousState = structuredClone(this.state);
+    this.boundaryChange = true;
+    let moved = false;
+    let installed = false;
+    try {
+      if (original.container) {
+        const tracked = new Set(affected.map((root) => root.absoluteFile.toLowerCase()));
+        const inspect = async (directory) => {
+          for (const entry of await readdir(directory, { withFileTypes: true })) {
+            const file = path.join(directory, entry.name);
+            if (entry.isSymbolicLink() || (!entry.isDirectory() && !tracked.has(file.toLowerCase()))) throw new Error(`Untracked content prevents conversion: ${entry.name}`);
+            if (entry.isDirectory()) await inspect(file);
+          }
+        };
+        await inspect(source);
+      }
+      await mkdir(transaction, { recursive: true });
+      if (replacement?.container) await mkdir(staged);
+      for (const { root, bytes } of replacements) {
+        const file = replacement.container ? path.join(staged, path.relative(destination, root.absoluteFile)) : staged;
+        await mkdir(path.dirname(file), { recursive: true });
+        if (bytes.length) await atomicWrite(file, bytes);
+      }
+      if (!removing) {
+        try {
+          await stat(destination);
+          throw new Error("Replacement destination already exists");
+        } catch (error) { if (error.code !== "ENOENT") throw error; }
+      }
+      for (const root of affected) {
+        const snapshot = await readSnapshot(root.absoluteFile);
+        if (!Object.hasOwn(expected, root.id) || (snapshot?.hash ?? false) !== expected[root.id]) throw new ConflictError(this.getManifest(root.id));
+      }
+      await rename(source, backup);
+      moved = true;
+      if (!removing) {
+        await rename(staged, destination);
+        installed = true;
+      }
+      for (const root of affected) {
+        this.roots.delete(root.id);
+        this.rootsByFile.delete(root.absoluteFile.toLowerCase());
+        delete this.state.roots[root.id];
+      }
+      for (const { root } of replacements) {
+        this.roots.set(root.id, root);
+        this.rootsByFile.set(root.absoluteFile.toLowerCase(), root);
+        const snapshot = await readSnapshot(root.absoluteFile);
+        this.state.roots[root.id] = {
+          exists: snapshot !== null, deleted: false, hash: snapshot?.hash ?? null,
+          revision: (previousState.roots[root.id]?.revision ?? 0) + 1,
+          size: snapshot?.size ?? 0, mtimeMs: snapshot?.mtimeMs ?? null,
+        };
+      }
+      this.manifestRoots = null;
+      await this.persistState();
+      this.treeScanNeeded = true;
+      return replacements.map(({ root }) => this.getManifest(root.id));
+    } catch (error) {
+      if (installed) await rename(destination, staged);
+      if (moved) await rename(backup, source);
+      this.roots = previousRoots;
+      this.rootsByFile = previousFiles;
+      this.state = previousState;
+      this.manifestRoots = null;
+      throw error;
+    } finally {
+      this.boundaryChange = false;
+    }
+  }
+
+  async refreshTreeRoots() {
+    if (this.boundaryChange) return;
+    if (this.treeScan) return this.treeScan;
+    if (!this.treeScanNeeded && Date.now() < this.nextTreeScan) return;
+    this.treeScanNeeded = false;
+    this.treeScan = this.scanTreeRoots();
+    try {
+      await this.treeScan;
+      this.nextTreeScan = Date.now() + 30000;
+    } catch (error) {
+      this.treeScanNeeded = true;
+      throw error;
+    } finally {
+      this.treeScan = null;
+    }
+  }
+
+  async scanTreeRoots() {
+    for (const tree of this.config.treeRoots ?? []) {
+      await mkdir(tree.absoluteDirectory, { recursive: true });
+      const entries = [];
+      const visit = async (directory, relativePath, instanceIds) => {
+        if (tree.studioPath.length + relativePath.length > this.config.maxPathDepth) throw new Error(`tree root ${tree.id} exceeds maxPathDepth`);
+        const start = entries.length;
+        let metadata = false;
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+          if (entry.isSymbolicLink()) throw new Error(`tree roots cannot contain symbolic links: ${entry.name}`);
+          if (entry.isDirectory()) {
+            const segment = decodeTreeSegment(entry.name);
+            await visit(path.join(directory, entry.name), [...relativePath, segment.name], [...instanceIds, segment.id]);
+          } else if (entry.isFile() && entry.name === "_root.rbxm") {
+            metadata = true;
+          } else if (entry.isFile() && entry.name.toLowerCase().endsWith(".rbxm")) {
+            const segment = decodeTreeSegment(entry.name.slice(0, -5));
+            entries.push({ path: [...relativePath, segment.name], instanceIds: [...instanceIds, segment.id], container: false });
+          }
+        }
+        if (relativePath.length === 0 || metadata || entries.length > start) {
+          entries.push({ path: relativePath, instanceIds, container: true });
+        }
+      };
+      await visit(tree.absoluteDirectory, [], []);
+      const containers = new Set(entries.filter(entry => entry.container)
+        .map(entry => this.makeTreeRoot(tree, entry.path, true, entry.instanceIds).id));
+      for (const [id, root] of this.roots) {
+        if (root.treeRootId !== tree.id || !root.diskContainer || containers.has(id)) continue;
+        this.roots.delete(id);
+        this.rootsByFile.delete(root.absoluteFile.toLowerCase());
+        this.manifestRoots = null;
+      }
+      await this.discoverTreePaths(tree.id, entries);
+      for (const id of containers) this.roots.get(id).diskContainer = true;
+    }
+  }
+
   async initialize() {
     await mkdir(this.config.stateDirectory, { recursive: true });
     try {
@@ -204,13 +438,16 @@ export class SnapshotStore {
       size: current?.size ?? 0,
       mtimeMs: current?.mtimeMs ?? null,
       splitRootId: root.splitRootId ?? null,
+      treeRootId: root.treeRootId ?? null,
+      container: root.container === true,
+      instanceIds: root.instanceIds ?? null,
     };
   }
 
   getManifests() {
-    return [...this.roots.values()]
-      .sort((left, right) => left.studioPath.join(".").localeCompare(right.studioPath.join(".")))
-      .map((root) => this.getManifest(root.id));
+    this.manifestRoots ??= [...this.roots.values()]
+      .sort((left, right) => left.studioPath.join(".").localeCompare(right.studioPath.join(".")));
+    return this.manifestRoots.map((root) => this.getManifest(root.id));
   }
 
   getRoots() {
@@ -258,8 +495,9 @@ export class SnapshotStore {
     return this.getManifest(id);
   }
 
-  async delete(id, expectedHash) {
+  async delete(id, expectedHash, plainFolder = false) {
     const root = this.getRoot(id);
+    if (root.container && !plainFolder) throw new Error("container snapshots must be restored or removed through the project configuration");
     const current = await readSnapshot(root.absoluteFile);
     if (!current || expectedHash !== current.hash) {
       throw new ConflictError(this.getManifest(id));
@@ -270,7 +508,7 @@ export class SnapshotStore {
     const previousRevision = this.state.roots[id]?.revision ?? 0;
     this.state.roots[id] = {
       exists: false,
-      deleted: true,
+      deleted: !root.container,
       hash: null,
       revision: previousRevision + 1,
       size: 0,
@@ -279,6 +517,8 @@ export class SnapshotStore {
     const deletedManifest = this.getManifest(id);
     if (root.splitRootId) {
       this.roots.delete(id);
+      this.manifestRoots = null;
+      this.rootsByFile.delete(root.absoluteFile.toLowerCase());
       delete this.state.roots[id];
     }
     await this.persistState();
@@ -286,6 +526,7 @@ export class SnapshotStore {
   }
 
   async refresh(id, persist = true) {
+    if (this.boundaryChange || !this.roots.has(id)) return false;
     const root = this.getRoot(id);
     const snapshot = await readSnapshot(root.absoluteFile);
     const previous = this.state.roots[id];