@tinyrack/devsync 1.0.0

package/src/services/local-materialization.ts

@@ -0,0 +1,421 @@
+import { lstat, mkdir, mkdtemp, rm, symlink } from "node:fs/promises";
+import { basename, dirname, join, posix } from "node:path";
+
+import {
+  type ResolvedSyncConfig,
+  type ResolvedSyncConfigEntry,
+  resolveManagedSyncMode,
+} from "#app/config/sync.ts";
+import { DevsyncError } from "./error.ts";
+import {
+  copyFilesystemNode,
+  getPathStats,
+  listDirectoryEntries,
+  removePathAtomically,
+  replacePathAtomically,
+  writeFileNode,
+  writeSymlinkNode,
+} from "./filesystem.ts";
+import type { FileLikeSnapshotNode, SnapshotNode } from "./local-snapshot.ts";
+import { buildDirectoryKey } from "./paths.ts";
+
+type EntryMaterialization =
+  | Readonly<{
+      desiredKeys: ReadonlySet<string>;
+      type: "absent";
+    }>
+  | Readonly<{
+      desiredKeys: ReadonlySet<string>;
+      node: FileLikeSnapshotNode;
+      type: "file";
+    }>
+  | Readonly<{
+      desiredKeys: ReadonlySet<string>;
+      nodes: ReadonlyMap<string, FileLikeSnapshotNode>;
+      type: "directory";
+    }>;
+
+const copyIgnoredLocalNodesToDirectory = async (
+  sourceDirectory: string,
+  targetDirectory: string,
+  config: ResolvedSyncConfig,
+  repoPathPrefix: string,
+): Promise<number> => {
+  const stats = await getPathStats(sourceDirectory);
+
+  if (stats === undefined || !stats.isDirectory()) {
+    return 0;
+  }
+
+  let copiedNodeCount = 0;
+  const entries = await listDirectoryEntries(sourceDirectory);
+  const directoryMode = resolveManagedSyncMode(config, repoPathPrefix);
+
+  if (directoryMode === "ignore") {
+    await mkdir(targetDirectory, { recursive: true });
+    copiedNodeCount += 1;
+  }
+
+  for (const entry of entries) {
+    const sourcePath = join(sourceDirectory, entry.name);
+    const targetPath = join(targetDirectory, entry.name);
+    const repoPath = posix.join(repoPathPrefix, entry.name);
+    const entryStats = await lstat(sourcePath);
+
+    if (entryStats.isDirectory()) {
+      copiedNodeCount += await copyIgnoredLocalNodesToDirectory(
+        sourcePath,
+        targetPath,
+        config,
+        repoPath,
+      );
+      continue;
+    }
+
+    if (resolveManagedSyncMode(config, repoPath) !== "ignore") {
+      continue;
+    }
+
+    await mkdir(dirname(targetPath), { recursive: true });
+    await copyFilesystemNode(sourcePath, targetPath, entryStats);
+    copiedNodeCount += 1;
+  }
+
+  return copiedNodeCount;
+};
+
+const stageAndReplaceFilePath = async (
+  targetPath: string,
+  node: FileLikeSnapshotNode,
+) => {
+  await mkdir(dirname(targetPath), { recursive: true });
+  const stagingDirectory = await mkdtemp(
+    join(dirname(targetPath), `.${basename(targetPath)}.devsync-sync-`),
+  );
+  const stagedPath = join(stagingDirectory, basename(targetPath));
+
+  try {
+    if (node.type === "symlink") {
+      await symlink(node.linkTarget, stagedPath);
+    } else {
+      await writeFileNode(stagedPath, node);
+    }
+
+    await replacePathAtomically(targetPath, stagedPath);
+  } finally {
+    await rm(stagingDirectory, { force: true, recursive: true });
+  }
+};
+
+const stageAndReplaceMergedDirectoryPath = async (
+  entry: ResolvedSyncConfigEntry,
+  config: ResolvedSyncConfig,
+  desiredNodes: ReadonlyMap<string, FileLikeSnapshotNode>,
+) => {
+  await mkdir(dirname(entry.localPath), { recursive: true });
+  const stagingDirectory = await mkdtemp(
+    join(
+      dirname(entry.localPath),
+      `.${basename(entry.localPath)}.devsync-sync-`,
+    ),
+  );
+
+  try {
+    const preservedIgnoredNodeCount = await copyIgnoredLocalNodesToDirectory(
+      entry.localPath,
+      stagingDirectory,
+      config,
+      entry.repoPath,
+    );
+
+    for (const relativePath of [...desiredNodes.keys()].sort((left, right) => {
+      return left.localeCompare(right);
+    })) {
+      const node = desiredNodes.get(relativePath);
+
+      if (node === undefined) {
+        continue;
+      }
+
+      const targetNodePath = join(stagingDirectory, ...relativePath.split("/"));
+
+      if (node.type === "symlink") {
+        await writeSymlinkNode(targetNodePath, node.linkTarget);
+      } else {
+        await writeFileNode(targetNodePath, node);
+      }
+    }
+
+    if (preservedIgnoredNodeCount === 0 && desiredNodes.size === 0) {
+      await removePathAtomically(entry.localPath);
+
+      return;
+    }
+
+    await replacePathAtomically(entry.localPath, stagingDirectory);
+  } finally {
+    await rm(stagingDirectory, { force: true, recursive: true });
+  }
+};
+
+export const buildEntryMaterialization = (
+  entry: ResolvedSyncConfigEntry,
+  snapshot: ReadonlyMap<string, SnapshotNode>,
+): EntryMaterialization => {
+  if (entry.kind === "file") {
+    const node = snapshot.get(entry.repoPath);
+
+    if (node === undefined) {
+      return {
+        desiredKeys: new Set<string>(),
+        type: "absent",
+      };
+    }
+
+    if (node.type === "directory") {
+      throw new DevsyncError(
+        `File sync entry resolves to a directory in the repository: ${entry.repoPath}`,
+      );
+    }
+
+    return {
+      desiredKeys: new Set<string>([entry.repoPath]),
+      node,
+      type: "file",
+    };
+  }
+
+  const rootNode = snapshot.get(entry.repoPath);
+
+  if (rootNode !== undefined && rootNode.type !== "directory") {
+    throw new DevsyncError(
+      `Directory sync entry resolves to a file in the repository: ${entry.repoPath}`,
+    );
+  }
+
+  const nodes = new Map<string, FileLikeSnapshotNode>();
+  const desiredKeys = new Set<string>();
+
+  for (const [repoPath, node] of snapshot.entries()) {
+    if (!repoPath.startsWith(`${entry.repoPath}/`)) {
+      continue;
+    }
+
+    if (node.type === "directory") {
+      continue;
+    }
+
+    const relativePath = repoPath.slice(entry.repoPath.length + 1);
+
+    nodes.set(relativePath, node);
+    desiredKeys.add(repoPath);
+  }
+
+  if (rootNode === undefined && nodes.size === 0) {
+    return {
+      desiredKeys,
+      type: "absent",
+    };
+  }
+
+  desiredKeys.add(buildDirectoryKey(entry.repoPath));
+
+  return {
+    desiredKeys,
+    nodes,
+    type: "directory",
+  };
+};
+
+const collectLocalLeafKeys = async (
+  targetPath: string,
+  repoPathPrefix: string,
+  keys: Set<string>,
+  prefix?: string,
+) => {
+  const stats = await getPathStats(targetPath);
+
+  if (stats === undefined) {
+    return;
+  }
+
+  if (!stats.isDirectory()) {
+    keys.add(repoPathPrefix);
+
+    return;
+  }
+
+  keys.add(buildDirectoryKey(repoPathPrefix));
+
+  const entries = await listDirectoryEntries(targetPath);
+
+  for (const entry of entries) {
+    const absolutePath = join(targetPath, entry.name);
+    const relativePath =
+      prefix === undefined ? entry.name : `${prefix}/${entry.name}`;
+    const childStats = await lstat(absolutePath);
+
+    if (childStats?.isDirectory()) {
+      await collectLocalLeafKeys(
+        absolutePath,
+        repoPathPrefix,
+        keys,
+        relativePath,
+      );
+      continue;
+    }
+
+    keys.add(posix.join(repoPathPrefix, relativePath));
+  }
+};
+
+const collectIgnoredLocalKeys = async (
+  targetPath: string,
+  repoPath: string,
+  config: ResolvedSyncConfig,
+  keys: Set<string>,
+): Promise<boolean> => {
+  const stats = await getPathStats(targetPath);
+
+  if (stats === undefined) {
+    return false;
+  }
+
+  const mode = resolveManagedSyncMode(config, repoPath);
+
+  if (!stats.isDirectory()) {
+    if (mode !== "ignore") {
+      return false;
+    }
+
+    keys.add(repoPath);
+
+    return true;
+  }
+
+  let preservedIgnoredChildren = mode === "ignore";
+  const entries = await listDirectoryEntries(targetPath);
+
+  for (const entry of entries) {
+    const childPath = join(targetPath, entry.name);
+    const childRepoPath = posix.join(repoPath, entry.name);
+
+    preservedIgnoredChildren =
+      (await collectIgnoredLocalKeys(childPath, childRepoPath, config, keys)) ||
+      preservedIgnoredChildren;
+  }
+
+  if (mode === "ignore" || preservedIgnoredChildren) {
+    keys.add(buildDirectoryKey(repoPath));
+  }
+
+  return mode === "ignore" || preservedIgnoredChildren;
+};
+
+export const countDeletedLocalNodes = async (
+  entry: ResolvedSyncConfigEntry,
+  desiredKeys: ReadonlySet<string>,
+  config: ResolvedSyncConfig,
+) => {
+  const existingKeys = new Set<string>();
+  const preservedIgnoredKeys = new Set<string>();
+
+  await collectLocalLeafKeys(entry.localPath, entry.repoPath, existingKeys);
+  await collectIgnoredLocalKeys(
+    entry.localPath,
+    entry.repoPath,
+    config,
+    preservedIgnoredKeys,
+  );
+
+  return [...existingKeys].filter((key) => {
+    return !desiredKeys.has(key) && !preservedIgnoredKeys.has(key);
+  }).length;
+};
+
+export const applyEntryMaterialization = async (
+  entry: ResolvedSyncConfigEntry,
+  materialization: EntryMaterialization,
+  config: ResolvedSyncConfig,
+) => {
+  if (
+    entry.kind === "file" &&
+    resolveManagedSyncMode(config, entry.repoPath) === "ignore"
+  ) {
+    return;
+  }
+
+  if (materialization.type === "absent") {
+    if (entry.kind === "directory") {
+      await stageAndReplaceMergedDirectoryPath(entry, config, new Map());
+
+      return;
+    }
+
+    await removePathAtomically(entry.localPath);
+
+    return;
+  }
+
+  if (materialization.type === "file") {
+    await stageAndReplaceFilePath(entry.localPath, materialization.node);
+
+    return;
+  }
+
+  await stageAndReplaceMergedDirectoryPath(
+    entry,
+    config,
+    materialization.nodes,
+  );
+};
+
+export const buildPullCounts = (
+  materializations: readonly EntryMaterialization[],
+) => {
+  let decryptedFileCount = 0;
+  let directoryCount = 0;
+  let plainFileCount = 0;
+  let symlinkCount = 0;
+
+  for (const materialization of materializations) {
+    if (materialization === undefined) {
+      continue;
+    }
+
+    if (materialization.type === "file") {
+      if (materialization.node.type === "symlink") {
+        symlinkCount += 1;
+      } else if (materialization.node.secret) {
+        decryptedFileCount += 1;
+      } else {
+        plainFileCount += 1;
+      }
+
+      continue;
+    }
+
+    if (materialization.type !== "directory") {
+      continue;
+    }
+
+    directoryCount += 1;
+
+    for (const node of materialization.nodes.values()) {
+      if (node.type === "symlink") {
+        symlinkCount += 1;
+      } else if (node.secret) {
+        decryptedFileCount += 1;
+      } else {
+        plainFileCount += 1;
+      }
+    }
+  }
+
+  return {
+    decryptedFileCount,
+    directoryCount,
+    plainFileCount,
+    symlinkCount,
+  };
+};

package/src/services/local-snapshot.ts

@@ -0,0 +1,173 @@
+import { lstat, readFile, readlink } from "node:fs/promises";
+import { join, posix } from "node:path";
+
+import {
+  type ResolvedSyncConfig,
+  resolveManagedSyncMode,
+} from "#app/config/sync.ts";
+import { DevsyncError } from "./error.ts";
+import {
+  getPathStats,
+  isExecutableMode,
+  listDirectoryEntries,
+} from "./filesystem.ts";
+import { assertStorageSafeRepoPath } from "./repo-artifacts.ts";
+
+export type SnapshotNode =
+  | Readonly<{
+      type: "directory";
+    }>
+  | Readonly<{
+      executable: boolean;
+      secret: boolean;
+      type: "file";
+      contents: Uint8Array;
+    }>
+  | Readonly<{
+      linkTarget: string;
+      type: "symlink";
+    }>;
+
+export type FileSnapshotNode = Extract<
+  SnapshotNode,
+  Readonly<{ type: "file" }>
+>;
+
+export type FileLikeSnapshotNode = Extract<
+  SnapshotNode,
+  Readonly<{ type: "file" | "symlink" }>
+>;
+
+export const addSnapshotNode = (
+  snapshot: Map<string, SnapshotNode>,
+  repoPath: string,
+  node: SnapshotNode,
+) => {
+  if (snapshot.has(repoPath)) {
+    throw new DevsyncError(`Duplicate sync path generated for ${repoPath}`);
+  }
+
+  snapshot.set(repoPath, node);
+};
+
+const addLocalNode = async (
+  snapshot: Map<string, SnapshotNode>,
+  config: ResolvedSyncConfig,
+  repoPath: string,
+  path: string,
+  stats: Awaited<ReturnType<typeof lstat>>,
+) => {
+  assertStorageSafeRepoPath(repoPath);
+  const mode = resolveManagedSyncMode(config, repoPath);
+
+  if (mode === "ignore") {
+    return;
+  }
+
+  if (stats.isDirectory()) {
+    throw new DevsyncError(
+      `Expected a file-like path but found a directory: ${path}`,
+    );
+  }
+
+  if (stats.isSymbolicLink()) {
+    if (mode === "secret") {
+      throw new DevsyncError(
+        `Secret sync paths must be regular files, not symlinks: ${repoPath}`,
+      );
+    }
+
+    addSnapshotNode(snapshot, repoPath, {
+      linkTarget: await readlink(path),
+      type: "symlink",
+    });
+
+    return;
+  }
+
+  if (!stats.isFile()) {
+    throw new DevsyncError(`Unsupported filesystem entry: ${path}`);
+  }
+
+  addSnapshotNode(snapshot, repoPath, {
+    contents: await readFile(path),
+    executable: isExecutableMode(stats.mode),
+    secret: mode === "secret",
+    type: "file",
+  });
+};
+
+const walkLocalDirectory = async (
+  snapshot: Map<string, SnapshotNode>,
+  config: ResolvedSyncConfig,
+  localDirectory: string,
+  repoPathPrefix: string,
+) => {
+  const entries = await listDirectoryEntries(localDirectory);
+
+  for (const entry of entries) {
+    const localPath = join(localDirectory, entry.name);
+    const repoPath = posix.join(repoPathPrefix, entry.name);
+    const stats = await lstat(localPath);
+
+    if (stats.isDirectory()) {
+      assertStorageSafeRepoPath(repoPath);
+      await walkLocalDirectory(snapshot, config, localPath, repoPath);
+      continue;
+    }
+
+    await addLocalNode(snapshot, config, repoPath, localPath, stats);
+  }
+};
+
+export const buildLocalSnapshot = async (config: ResolvedSyncConfig) => {
+  const snapshot = new Map<string, SnapshotNode>();
+
+  for (const entry of config.entries) {
+    const stats = await getPathStats(entry.localPath);
+
+    if (stats === undefined) {
+      continue;
+    }
+
+    const entryMode = resolveManagedSyncMode(config, entry.repoPath);
+
+    if (entry.kind === "file") {
+      if (entryMode === "ignore") {
+        continue;
+      }
+
+      if (stats.isDirectory()) {
+        throw new DevsyncError(
+          `Sync entry ${entry.name} expects a file, but found a directory: ${entry.localPath}`,
+        );
+      }
+
+      await addLocalNode(
+        snapshot,
+        config,
+        entry.repoPath,
+        entry.localPath,
+        stats,
+      );
+      continue;
+    }
+
+    if (!stats.isDirectory()) {
+      throw new DevsyncError(
+        `Sync entry ${entry.name} expects a directory: ${entry.localPath}`,
+      );
+    }
+
+    const snapshotSizeBeforeWalk = snapshot.size;
+    await walkLocalDirectory(snapshot, config, entry.localPath, entry.repoPath);
+
+    if (entryMode !== "ignore" || snapshot.size > snapshotSizeBeforeWalk) {
+      addSnapshotNode(snapshot, entry.repoPath, {
+        type: "directory",
+      });
+    }
+  }
+
+  return snapshot;
+};

package/src/services/paths.ts

@@ -0,0 +1,98 @@
+import { isAbsolute, relative, resolve } from "node:path";
+
+import { normalizeSyncRepoPath } from "#app/config/sync.ts";
+import { expandHomePath } from "#app/config/xdg.ts";
+
+import { DevsyncError } from "./error.ts";
+
+export const buildDirectoryKey = (repoPath: string) => {
+  return `${repoPath}/`;
+};
+
+export const isPathEqualOrNested = (path: string, rootPath: string) => {
+  const rootToPath = relative(rootPath, path);
+
+  return (
+    rootToPath === "" ||
+    (!isAbsolute(rootToPath) &&
+      !rootToPath.startsWith("..") &&
+      rootToPath !== "..")
+  );
+};
+
+export const doPathsOverlap = (leftPath: string, rightPath: string) => {
+  return (
+    isPathEqualOrNested(leftPath, rightPath) ||
+    isPathEqualOrNested(rightPath, leftPath)
+  );
+};
+
+export const isExplicitLocalPath = (target: string) => {
+  return (
+    target === "." ||
+    target === ".." ||
+    target === "~" ||
+    target.startsWith("./") ||
+    target.startsWith("../") ||
+    target.startsWith("~/") ||
+    isAbsolute(target)
+  );
+};
+
+export const resolveCommandTargetPath = (
+  target: string,
+  environment: NodeJS.ProcessEnv,
+  cwd: string,
+) => {
+  return resolve(cwd, expandHomePath(target, environment));
+};
+
+export const buildRepoPathWithinRoot = (
+  absolutePath: string,
+  rootPath: string,
+  description: string,
+) => {
+  const relativePath = relative(rootPath, absolutePath);
+
+  if (relativePath === "") {
+    throw new DevsyncError(
+      `${description} must be inside ${rootPath}, not the root itself: ${absolutePath}`,
+    );
+  }
+
+  if (
+    isAbsolute(relativePath) ||
+    relativePath.startsWith("..") ||
+    relativePath === ".."
+  ) {
+    throw new DevsyncError(
+      `${description} must be inside ${rootPath}: ${absolutePath}`,
+    );
+  }
+
+  return normalizeSyncRepoPath(relativePath);
+};
+
+export const buildConfiguredHomeLocalPath = (repoPath: string) => {
+  return `~/${repoPath}`;
+};
+
+export const tryBuildRepoPathWithinRoot = (
+  absolutePath: string,
+  rootPath: string,
+  description: string,
+) => {
+  try {
+    return buildRepoPathWithinRoot(absolutePath, rootPath, description);
+  } catch {
+    return undefined;
+  }
+};
+
+export const tryNormalizeRepoPathInput = (value: string) => {
+  try {
+    return normalizeSyncRepoPath(value);
+  } catch {
+    return undefined;
+  }
+};