goldsync 0.1.11

package/src/server.mjs

@@ -0,0 +1,308 @@
+import { watch } from "node:fs";
+import { createServer } from "node:http";
+import path from "node:path";
+import { ConflictError } from "./store.mjs";
+
+const MAX_SNAPSHOT_BYTES = 128 * 1024 * 1024;
+
+function sendJson(response, status, value) {
+  const body = Buffer.from(JSON.stringify(value));
+  response.writeHead(status, {
+    "Content-Type": "application/json",
+    "Content-Length": body.length,
+    "Cache-Control": "no-store",
+  });
+  response.end(body);
+}
+
+async function readBody(request) {
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of request) {
+    size += chunk.length;
+    if (size > MAX_SNAPSHOT_BYTES) {
+      const error = new Error("snapshot exceeds the 128 MiB limit");
+      error.statusCode = 413;
+      throw error;
+    }
+    chunks.push(chunk);
+  }
+  return Buffer.concat(chunks);
+}
+
+function requireStudioClient(request) {
+  if (request.headers.origin !== undefined) {
+    return false;
+  }
+  return request.headers["x-goldsync-client"] === "studio-plugin";
+}
+
+export class GoldSyncServer {
+  constructor(config, store, gitConflicts = null) {
+    this.config = config;
+    this.store = store;
+    this.gitConflicts = gitConflicts;
+    this.server = createServer((request, response) => {
+      this.handle(request, response).catch((error) => {
+        sendJson(response, error.statusCode ?? 500, { error: error.message });
+      });
+    });
+    this.watchers = [];
+    this.refreshTimers = new Map();
+    this.syncEnabled = true;
+    this.sessionToken = null;
+  }
+
+  setSyncEnabled(enabled) {
+    this.syncEnabled = enabled;
+  }
+
+  setSessionToken(token) {
+    this.sessionToken = token;
+  }
+
+  async start(options = {}) {
+    if (options.listen !== false) {
+      await new Promise((resolve, reject) => {
+        this.server.once("error", reject);
+        this.server.listen(this.config.port, this.config.host, () => {
+          this.server.off("error", reject);
+          resolve();
+        });
+      });
+    }
+
+    const watchedDirectories = new Map();
+    for (const root of this.config.roots) {
+      watchedDirectories.set(path.dirname(root.absoluteFile), false);
+    }
+    for (const splitRoot of this.config.splitRoots ?? []) {
+      watchedDirectories.set(splitRoot.absoluteDirectory, true);
+    }
+    for (const [directory, recursive] of watchedDirectories) {
+      const watcher = watch(directory, { recursive }, (_event, filename) => {
+        if (!filename) {
+          return;
+        }
+        const changedFile = path.join(directory, filename.toString()).toLowerCase();
+        let matched = false;
+        for (const root of this.store.getRoots()) {
+          if (root.absoluteFile.toLowerCase() === changedFile) {
+            matched = true;
+            this.scheduleRefresh(root.id);
+          }
+        }
+        if (!matched && recursive && path.extname(changedFile) === ".rbxm") {
+          this.store
+            .refreshSplitRoots()
+            .then(() => this.gitConflicts?.syncRoots?.(this.store.getRoots()))
+            .catch((error) => console.error(`[GoldSync] failed to discover split root for ${changedFile}:`, error));
+        }
+      });
+      watcher.on("error", (error) => console.error(`[GoldSync] watcher failed for ${directory}:`, error));
+      this.watchers.push(watcher);
+    }
+  }
+
+  async stop() {
+    for (const timer of this.refreshTimers.values()) {
+      clearTimeout(timer);
+    }
+    this.refreshTimers.clear();
+    for (const watcher of this.watchers) {
+      watcher.close();
+    }
+    this.watchers = [];
+    if (this.server.listening) {
+      await new Promise((resolve) => this.server.close(resolve));
+    }
+  }
+
+  get port() {
+    const address = this.server.address();
+    return typeof address === "object" && address !== null ? address.port : null;
+  }
+
+  scheduleRefresh(id) {
+    clearTimeout(this.refreshTimers.get(id));
+    this.refreshTimers.set(
+      id,
+      setTimeout(() => {
+        this.refreshTimers.delete(id);
+        this.store.refresh(id).catch((error) => console.error(`[GoldSync] failed to refresh ${id}:`, error));
+      }, 100),
+    );
+  }
+
+  async handle(request, response) {
+    if (!requireStudioClient(request)) {
+      sendJson(response, 403, { error: "request was not sent by the GoldSync Studio plugin" });
+      return;
+    }
+    if (!this.sessionToken || request.headers["x-goldsync-session"] !== this.sessionToken) {
+      sendJson(response, 409, { error: "this Studio place is not bound to the current Rojo session" });
+      return;
+    }
+    const url = new URL(request.url, `http://${this.config.host}:${this.config.port}`);
+    if (request.method === "GET" && url.pathname === "/v1/config") {
+      await this.store.refreshSplitRoots();
+      this.gitConflicts?.syncRoots?.(this.store.getRoots());
+      const conflicts = this.gitConflicts ? await this.gitConflicts.list() : new Map();
+      sendJson(response, 200, {
+        version: 1,
+        name: this.config.name,
+        projectId: this.config.projectId,
+        workspaceId: this.config.workspaceId,
+        maxPathDepth: this.config.maxPathDepth,
+        pollIntervalMs: this.config.pollIntervalMs,
+        debounceMs: this.config.debounceMs,
+        rojoConnected: this.syncEnabled,
+        splitRoots: (this.config.splitRoots ?? []).map((root) => ({
+          id: root.id,
+          studioPath: root.studioPath,
+          splitDepth: root.splitDepth,
+        })),
+        roots: this.store.getManifests().map((manifest) => {
+          const conflict = conflicts.get(manifest.id);
+          return {
+            ...manifest,
+            gitConflict: conflict !== undefined,
+            conflictToken: conflict?.token ?? null,
+            conflictStages: conflict?.stages ?? null,
+          };
+        }),
+      });
+      return;
+    }
+
+    const splitMatch = /^\/v1\/splits\/([a-z0-9-]+)\/discover$/.exec(url.pathname);
+    if (request.method === "POST" && splitMatch) {
+      const payload = JSON.parse((await readBody(request)).toString("utf8"));
+      if (!Array.isArray(payload.paths) || payload.paths.length > 10000) {
+        sendJson(response, 400, { error: "paths must be an array with at most 10000 entries" });
+        return;
+      }
+      await this.store.discoverSplitPaths(splitMatch[1], payload.paths);
+      this.gitConflicts?.syncRoots?.(this.store.getRoots());
+      sendJson(response, 200, { discovered: payload.paths.length });
+      return;
+    }
+
+    const conflictMatch = /^\/v1\/roots\/([a-z0-9-]+)\/conflict\/(base|ours|theirs)$/.exec(url.pathname);
+    if (request.method === "GET" && conflictMatch) {
+      if (!this.gitConflicts) {
+        sendJson(response, 404, { error: "Git conflict viewer is unavailable" });
+        return;
+      }
+      const bytes = await this.gitConflicts.readStage(conflictMatch[1], conflictMatch[2]);
+      if (!bytes) {
+        sendJson(response, 404, { error: "conflict stage does not exist" });
+        return;
+      }
+      response.writeHead(200, {
+        "Content-Type": "application/octet-stream",
+        "Content-Length": bytes.length,
+        "Cache-Control": "no-store",
+      });
+      response.end(bytes);
+      return;
+    }
+
+    const resolveMatch = /^\/v1\/roots\/([a-z0-9-]+)\/conflict\/resolve$/.exec(url.pathname);
+    if (request.method === "PUT" && resolveMatch) {
+      if (!this.gitConflicts) {
+        sendJson(response, 404, { error: "Git conflict viewer is unavailable" });
+        return;
+      }
+      const token = request.headers["x-goldsync-conflict-token"];
+      if (typeof token !== "string") {
+        sendJson(response, 428, { error: "X-GoldSync-Conflict-Token is required" });
+        return;
+      }
+      const manifest = await this.gitConflicts.resolve(resolveMatch[1], await readBody(request), token, this.store);
+      sendJson(response, 200, manifest);
+      return;
+    }
+
+    const match = /^\/v1\/roots\/([a-z0-9-]+)\/snapshot$/.exec(url.pathname);
+    if (!match) {
+      sendJson(response, 404, { error: "not found" });
+      return;
+    }
+
+    const id = match[1];
+    if (!this.store.roots.has(id)) {
+      sendJson(response, 404, { error: "unknown sync root" });
+      return;
+    }
+    if (this.gitConflicts && (await this.gitConflicts.list()).has(id)) {
+      sendJson(response, 409, { error: "resolve the Git conflict before synchronizing this root" });
+      return;
+    }
+    if (!this.syncEnabled) {
+      sendJson(response, 503, { error: "normal synchronization is paused until the configured Rojo project is served" });
+      return;
+    }
+    if (request.method === "DELETE") {
+      const expectedHashHeader = request.headers["if-match"];
+      if (typeof expectedHashHeader !== "string" || expectedHashHeader === "*") {
+        sendJson(response, 428, { error: "DELETE requires the last synchronized hash in If-Match" });
+        return;
+      }
+      const expectedHash = expectedHashHeader.replace(/^"|"$/g, "");
+      try {
+        sendJson(response, 200, await this.store.delete(id, expectedHash));
+      } catch (error) {
+        if (error instanceof ConflictError) {
+          sendJson(response, 409, { error: error.message, current: error.current });
+          return;
+        }
+        throw error;
+      }
+      return;
+    }
+    if (request.method === "GET") {
+      const snapshot = await this.store.get(id);
+      if (!snapshot) {
+        sendJson(response, 404, { error: "snapshot does not exist" });
+        return;
+      }
+      response.writeHead(200, {
+        "Content-Type": "application/octet-stream",
+        "Content-Length": snapshot.bytes.length,
+        ETag: `"${snapshot.hash}"`,
+        "Cache-Control": "no-store",
+      });
+      response.end(snapshot.bytes);
+      return;
+    }
+
+    if (request.method === "PUT") {
+      const expectedHashHeader = request.headers["if-match"];
+      if (typeof expectedHashHeader !== "string") {
+        sendJson(response, 428, { error: "If-Match is required" });
+        return;
+      }
+      const expectedHash = expectedHashHeader === "*" ? "*" : expectedHashHeader.replace(/^"|"$/g, "");
+      const bytes = await readBody(request);
+      if (bytes.length === 0) {
+        sendJson(response, 400, { error: "snapshot body is empty" });
+        return;
+      }
+
+      try {
+        const manifest = await this.store.put(id, bytes, expectedHash);
+        sendJson(response, 200, manifest);
+      } catch (error) {
+        if (error instanceof ConflictError) {
+          sendJson(response, 409, { error: error.message, current: error.current });
+          return;
+        }
+        throw error;
+      }
+      return;
+    }
+
+    sendJson(response, 405, { error: "method not allowed" });
+  }
+}

package/src/session-marker.mjs

@@ -0,0 +1,27 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+export async function readSessionMarker(file) {
+  try {
+    const marker = JSON.parse(await readFile(file, "utf8"));
+    return typeof marker?.properties?.Value === "string" ? marker.properties.Value : null;
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  }
+}
+
+export async function writeSessionMarker(file, token) {
+  if ((await readSessionMarker(file)) === token) {
+    return;
+  }
+  await mkdir(path.dirname(file), { recursive: true });
+  const temporaryFile = `${file}.${process.pid}.tmp`;
+  await writeFile(
+    temporaryFile,
+    `${JSON.stringify({ className: "StringValue", properties: { Value: token } }, null, 2)}\n`,
+  );
+  await rename(temporaryFile, file);
+}

package/src/store.mjs

@@ -0,0 +1,337 @@
+import { createHash, randomUUID } from "node:crypto";
+import { copyFile, mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";
+import path from "node:path";
+
+export class ConflictError extends Error {
+  constructor(current) {
+    super("snapshot changed since the client last synchronized");
+    this.name = "ConflictError";
+    this.current = current;
+  }
+}
+
+function hash(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+function encodePathSegment(segment) {
+  const encoded = segment.replace(/[%<>:"/\\|?*\u0000-\u001f]/g, (character) =>
+    [...Buffer.from(character)].map((byte) => `%${byte.toString(16).padStart(2, "0")}`).join(""),
+  );
+  return encoded.replace(/[. ]+$/g, (suffix) =>
+    [...Buffer.from(suffix)].map((byte) => `%${byte.toString(16).padStart(2, "0")}`).join(""),
+  );
+}
+
+function decodePathSegment(segment) {
+  return segment.replace(/(?:%[0-9a-f]{2})+/gi, (encoded) =>
+    Buffer.from(encoded.match(/[0-9a-f]{2}/gi).map((byte) => Number.parseInt(byte, 16))).toString("utf8"),
+  );
+}
+
+async function findSplitFiles(directory, depth, relative = []) {
+  let entries;
+  try {
+    entries = await readdir(directory, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+
+  const files = [];
+  for (const entry of entries) {
+    if (depth === 1 && entry.isFile() && path.extname(entry.name).toLowerCase() === ".rbxm") {
+      files.push([...relative, decodePathSegment(path.basename(entry.name, ".rbxm"))]);
+    } else if (depth > 1 && entry.isDirectory()) {
+      files.push(...(await findSplitFiles(path.join(directory, entry.name), depth - 1, [...relative, decodePathSegment(entry.name)])));
+    }
+  }
+  return files;
+}
+
+async function readSnapshot(file) {
+  try {
+    const bytes = await readFile(file);
+    const fileStat = await stat(file);
+    return {
+      bytes,
+      hash: hash(bytes),
+      size: bytes.length,
+      mtimeMs: fileStat.mtimeMs,
+    };
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  }
+}
+
+async function atomicWrite(file, bytes) {
+  await mkdir(path.dirname(file), { recursive: true });
+  const temporaryFile = `${file}.${process.pid}.${randomUUID()}.tmp`;
+  const handle = await open(temporaryFile, "wx");
+  try {
+    await handle.writeFile(bytes);
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+
+  try {
+    await rename(temporaryFile, file);
+  } catch (error) {
+    await rm(temporaryFile, { force: true });
+    throw error;
+  }
+}
+
+export class SnapshotStore {
+  constructor(config) {
+    this.config = config;
+    this.roots = new Map(config.roots.map((root) => [root.id, root]));
+    this.stateFile = path.join(config.stateDirectory, "state.json");
+    this.backupRoot = path.join(config.stateDirectory, "backups");
+    this.state = { version: 1, roots: {} };
+  }
+
+  makeSplitRoot(splitRoot, relativePath) {
+    if (
+      !Array.isArray(relativePath) ||
+      relativePath.length !== splitRoot.splitDepth ||
+      relativePath.some((segment) => typeof segment !== "string" || segment.length === 0)
+    ) {
+      throw new Error(`split root ${splitRoot.id} received an invalid relative path`);
+    }
+    const encodedPath = relativePath.map(encodePathSegment);
+    const relativeFile = path.join(splitRoot.directory, ...encodedPath) + ".rbxm";
+    const slug = relativePath
+      .join("-")
+      .toLowerCase()
+      .replace(/[^a-z0-9-]+/g, "-")
+      .replace(/^-+|-+$/g, "")
+      .slice(0, 40) || "root";
+    const suffix = createHash("sha256").update(JSON.stringify(relativePath)).digest("hex").slice(0, 8);
+    return {
+      id: `${splitRoot.id}-${slug}-${suffix}`,
+      studioPath: [...splitRoot.studioPath, ...relativePath],
+      file: relativeFile.replaceAll(path.sep, "/"),
+      absoluteFile: path.resolve(this.config.projectRoot, relativeFile),
+      splitRootId: splitRoot.id,
+    };
+  }
+
+  async addRoot(root) {
+    if (this.roots.has(root.id)) {
+      return false;
+    }
+    this.roots.set(root.id, root);
+    await mkdir(path.dirname(root.absoluteFile), { recursive: true });
+    await this.refresh(root.id, false);
+    return true;
+  }
+
+  async discoverSplitPaths(splitRootId, relativePaths) {
+    const splitRoot = (this.config.splitRoots ?? []).find((candidate) => candidate.id === splitRootId);
+    if (!splitRoot) {
+      throw new Error(`unknown split root: ${splitRootId}`);
+    }
+    let changed = false;
+    for (const relativePath of relativePaths) {
+      changed = (await this.addRoot(this.makeSplitRoot(splitRoot, relativePath))) || changed;
+    }
+    if (changed) {
+      await this.persistState();
+    }
+    return changed;
+  }
+
+  async refreshSplitRoots() {
+    for (const splitRoot of this.config.splitRoots ?? []) {
+      await mkdir(splitRoot.absoluteDirectory, { recursive: true });
+      const paths = await findSplitFiles(splitRoot.absoluteDirectory, splitRoot.splitDepth);
+      await this.discoverSplitPaths(splitRoot.id, paths);
+    }
+  }
+
+  async initialize() {
+    await mkdir(this.config.stateDirectory, { recursive: true });
+    try {
+      const state = JSON.parse(await readFile(this.stateFile, "utf8"));
+      if (state.version === 1 && typeof state.roots === "object") {
+        this.state = state;
+      }
+    } catch (error) {
+      if (error.code !== "ENOENT") {
+        throw new Error(`could not read ${this.stateFile}: ${error.message}`);
+      }
+    }
+
+    for (const root of this.config.roots) {
+      await mkdir(path.dirname(root.absoluteFile), { recursive: true });
+      await this.refresh(root.id, false);
+    }
+    await this.refreshSplitRoots();
+    await this.persistState();
+  }
+
+  getRoot(id) {
+    const root = this.roots.get(id);
+    if (!root) {
+      throw new Error(`unknown sync root: ${id}`);
+    }
+    return root;
+  }
+
+  getManifest(id) {
+    const root = this.getRoot(id);
+    const current = this.state.roots[id];
+    return {
+      id,
+      studioPath: root.studioPath,
+      file: root.file,
+      exists: current?.exists === true,
+      deleted: current?.deleted === true,
+      hash: current?.hash ?? null,
+      revision: current?.revision ?? 0,
+      size: current?.size ?? 0,
+      mtimeMs: current?.mtimeMs ?? null,
+      splitRootId: root.splitRootId ?? null,
+    };
+  }
+
+  getManifests() {
+    return [...this.roots.values()]
+      .sort((left, right) => left.studioPath.join(".").localeCompare(right.studioPath.join(".")))
+      .map((root) => this.getManifest(root.id));
+  }
+
+  getRoots() {
+    return [...this.roots.values()];
+  }
+
+  async get(id) {
+    const root = this.getRoot(id);
+    const snapshot = await readSnapshot(root.absoluteFile);
+    if (!snapshot) {
+      return null;
+    }
+    return snapshot;
+  }
+
+  async put(id, bytes, expectedHash) {
+    const root = this.getRoot(id);
+    const current = await readSnapshot(root.absoluteFile);
+    const currentHash = current?.hash ?? null;
+    const expectsMissing = expectedHash === "*";
+    if ((expectsMissing && current !== null) || (!expectsMissing && expectedHash !== currentHash)) {
+      throw new ConflictError(this.getManifest(id));
+    }
+
+    const nextHash = hash(bytes);
+    if (currentHash === nextHash) {
+      return this.getManifest(id);
+    }
+
+    if (current) {
+      await this.backup(id, current.hash);
+    }
+    await atomicWrite(root.absoluteFile, bytes);
+    const next = await readSnapshot(root.absoluteFile);
+    const previousRevision = this.state.roots[id]?.revision ?? 0;
+    this.state.roots[id] = {
+      exists: true,
+      deleted: false,
+      hash: next.hash,
+      revision: previousRevision + 1,
+      size: next.size,
+      mtimeMs: next.mtimeMs,
+    };
+    await this.persistState();
+    return this.getManifest(id);
+  }
+
+  async delete(id, expectedHash) {
+    const root = this.getRoot(id);
+    const current = await readSnapshot(root.absoluteFile);
+    if (!current || expectedHash !== current.hash) {
+      throw new ConflictError(this.getManifest(id));
+    }
+
+    await this.backup(id, current.hash);
+    await rm(root.absoluteFile);
+    const previousRevision = this.state.roots[id]?.revision ?? 0;
+    this.state.roots[id] = {
+      exists: false,
+      deleted: true,
+      hash: null,
+      revision: previousRevision + 1,
+      size: 0,
+      mtimeMs: null,
+    };
+    const deletedManifest = this.getManifest(id);
+    if (root.splitRootId) {
+      this.roots.delete(id);
+      delete this.state.roots[id];
+    }
+    await this.persistState();
+    return deletedManifest;
+  }
+
+  async refresh(id, persist = true) {
+    const root = this.getRoot(id);
+    const snapshot = await readSnapshot(root.absoluteFile);
+    const previous = this.state.roots[id];
+    const nextHash = snapshot?.hash ?? null;
+    if (previous === undefined && snapshot === null) {
+      this.state.roots[id] = {
+        exists: false,
+        deleted: false,
+        hash: null,
+        revision: 0,
+        size: 0,
+        mtimeMs: null,
+      };
+      if (persist) {
+        await this.persistState();
+      }
+      return false;
+    }
+    if ((previous?.hash ?? null) === nextHash && previous?.exists === (snapshot !== null)) {
+      return false;
+    }
+
+    this.state.roots[id] = {
+      exists: snapshot !== null,
+      deleted: false,
+      hash: nextHash,
+      revision: (previous?.revision ?? 0) + 1,
+      size: snapshot?.size ?? 0,
+      mtimeMs: snapshot?.mtimeMs ?? null,
+    };
+    if (persist) {
+      await this.persistState();
+    }
+    return true;
+  }
+
+  async backup(id, snapshotHash) {
+    const directory = path.join(this.backupRoot, id);
+    await mkdir(directory, { recursive: true });
+    const backupFile = path.join(directory, `${Date.now()}-${snapshotHash.slice(0, 12)}.rbxm`);
+    const root = this.getRoot(id);
+    await copyFile(root.absoluteFile, backupFile);
+
+    const backups = (await readdir(directory))
+      .filter((file) => file.endsWith(".rbxm"))
+      .sort()
+      .reverse();
+    await Promise.all(backups.slice(10).map((file) => rm(path.join(directory, file), { force: true })));
+  }
+
+  async persistState() {
+    await atomicWrite(this.stateFile, Buffer.from(`${JSON.stringify(this.state, null, 2)}\n`));
+  }
+}