@tinyrack/devsync 1.0.0

package/src/services/pull.ts

@@ -0,0 +1,75 @@
+import { readSyncConfig } from "#app/config/sync.ts";
+
+import {
+  applyEntryMaterialization,
+  buildEntryMaterialization,
+  buildPullCounts,
+  countDeletedLocalNodes,
+} from "./local-materialization.ts";
+import { buildRepositorySnapshot } from "./repo-snapshot.ts";
+import { ensureSyncRepository, type SyncContext } from "./runtime.ts";
+
+export type SyncPullRequest = Readonly<{
+  dryRun: boolean;
+}>;
+
+export type SyncPullResult = Readonly<{
+  configPath: string;
+  decryptedFileCount: number;
+  deletedLocalCount: number;
+  directoryCount: number;
+  dryRun: boolean;
+  plainFileCount: number;
+  symlinkCount: number;
+  syncDirectory: string;
+}>;
+
+export const pullSync = async (
+  request: SyncPullRequest,
+  context: SyncContext,
+): Promise<SyncPullResult> => {
+  await ensureSyncRepository(context);
+
+  const config = await readSyncConfig(
+    context.paths.syncDirectory,
+    context.environment,
+  );
+  const snapshot = await buildRepositorySnapshot(
+    context.paths.syncDirectory,
+    config,
+  );
+  const materializations = config.entries.map((entry) => {
+    return buildEntryMaterialization(entry, snapshot);
+  });
+
+  let deletedLocalCount = 0;
+
+  for (let index = 0; index < config.entries.length; index += 1) {
+    const entry = config.entries[index];
+    const materialization = materializations[index];
+
+    if (entry === undefined || materialization === undefined) {
+      continue;
+    }
+
+    deletedLocalCount += await countDeletedLocalNodes(
+      entry,
+      materialization.desiredKeys,
+      config,
+    );
+
+    if (!request.dryRun) {
+      await applyEntryMaterialization(entry, materialization, config);
+    }
+  }
+
+  const counts = buildPullCounts(materializations);
+
+  return {
+    configPath: context.paths.configPath,
+    deletedLocalCount,
+    dryRun: request.dryRun,
+    syncDirectory: context.paths.syncDirectory,
+    ...counts,
+  };
+};

package/src/services/push.ts

@@ -0,0 +1,121 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { join } from "node:path";
+
+import {
+  readSyncConfig,
+  resolveSyncArtifactsDirectoryPath,
+} from "#app/config/sync.ts";
+
+import { replacePathAtomically } from "./filesystem.ts";
+import { buildLocalSnapshot, type SnapshotNode } from "./local-snapshot.ts";
+import {
+  buildArtifactKey,
+  buildRepoArtifacts,
+  collectExistingArtifactKeys,
+  writeArtifactsToDirectory,
+} from "./repo-artifacts.ts";
+import { ensureSyncRepository, type SyncContext } from "./runtime.ts";
+
+export type SyncPushRequest = Readonly<{
+  dryRun: boolean;
+}>;
+
+export type SyncPushResult = Readonly<{
+  configPath: string;
+  deletedArtifactCount: number;
+  directoryCount: number;
+  dryRun: boolean;
+  encryptedFileCount: number;
+  plainFileCount: number;
+  symlinkCount: number;
+  syncDirectory: string;
+}>;
+
+const buildPushCounts = (snapshot: ReadonlyMap<string, SnapshotNode>) => {
+  let directoryCount = 0;
+  let encryptedFileCount = 0;
+  let plainFileCount = 0;
+  let symlinkCount = 0;
+
+  for (const node of snapshot.values()) {
+    if (node.type === "directory") {
+      directoryCount += 1;
+      continue;
+    }
+
+    if (node.type === "symlink") {
+      symlinkCount += 1;
+      continue;
+    }
+
+    if (node.secret) {
+      encryptedFileCount += 1;
+    } else {
+      plainFileCount += 1;
+    }
+  }
+
+  return {
+    directoryCount,
+    encryptedFileCount,
+    plainFileCount,
+    symlinkCount,
+  };
+};
+
+export const pushSync = async (
+  request: SyncPushRequest,
+  context: SyncContext,
+): Promise<SyncPushResult> => {
+  await ensureSyncRepository(context);
+
+  const config = await readSyncConfig(
+    context.paths.syncDirectory,
+    context.environment,
+  );
+  const snapshot = await buildLocalSnapshot(config);
+  const artifacts = await buildRepoArtifacts(snapshot, config);
+  const desiredArtifactKeys = new Set(
+    artifacts.map((artifact) => {
+      return buildArtifactKey(artifact);
+    }),
+  );
+  const existingArtifactKeys = await collectExistingArtifactKeys(
+    context.paths.syncDirectory,
+    config,
+  );
+  const deletedArtifactCount = [...existingArtifactKeys].filter((key) => {
+    return !desiredArtifactKeys.has(key);
+  }).length;
+
+  if (!request.dryRun) {
+    const stagingRoot = await mkdtemp(
+      join(context.paths.syncDirectory, ".devsync-sync-push-"),
+    );
+    const nextArtifactsDirectory = join(stagingRoot, "files");
+
+    try {
+      await writeArtifactsToDirectory(nextArtifactsDirectory, artifacts);
+
+      await replacePathAtomically(
+        resolveSyncArtifactsDirectoryPath(context.paths.syncDirectory),
+        nextArtifactsDirectory,
+      );
+    } finally {
+      await rm(stagingRoot, {
+        force: true,
+        recursive: true,
+      });
+    }
+  }
+
+  const counts = buildPushCounts(snapshot);
+
+  return {
+    configPath: context.paths.configPath,
+    deletedArtifactCount,
+    dryRun: request.dryRun,
+    syncDirectory: context.paths.syncDirectory,
+    ...counts,
+  };
+};

package/src/services/repo-artifacts.ts

@@ -0,0 +1,262 @@
+import { lstat, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+
+import {
+  hasReservedSyncArtifactSuffixSegment,
+  type ResolvedSyncConfig,
+  resolveSyncArtifactsDirectoryPath,
+  syncSecretArtifactSuffix,
+} from "#app/config/sync.ts";
+
+import { encryptSecretFile } from "./crypto.ts";
+import { DevsyncError } from "./error.ts";
+import {
+  getPathStats,
+  listDirectoryEntries,
+  pathExists,
+  writeFileNode,
+  writeSymlinkNode,
+} from "./filesystem.ts";
+import type { SnapshotNode } from "./local-snapshot.ts";
+import { buildDirectoryKey } from "./paths.ts";
+
+export type RepoArtifact =
+  | Readonly<{
+      category: "plain";
+      kind: "directory";
+      repoPath: string;
+    }>
+  | Readonly<{
+      category: "plain";
+      kind: "file";
+      repoPath: string;
+      contents: Uint8Array;
+      executable: boolean;
+    }>
+  | Readonly<{
+      category: "plain";
+      kind: "symlink";
+      repoPath: string;
+      linkTarget: string;
+    }>
+  | Readonly<{
+      category: "secret";
+      kind: "file";
+      repoPath: string;
+      contents: string;
+      executable: boolean;
+    }>;
+
+export const buildArtifactKey = (artifact: RepoArtifact) => {
+  const relativePath = resolveArtifactRelativePath(artifact);
+
+  return artifact.kind === "directory"
+    ? buildDirectoryKey(relativePath)
+    : relativePath;
+};
+
+export const isSecretArtifactPath = (relativePath: string) => {
+  return relativePath.endsWith(syncSecretArtifactSuffix);
+};
+
+export const stripSecretArtifactSuffix = (relativePath: string) => {
+  if (!isSecretArtifactPath(relativePath)) {
+    return undefined;
+  }
+
+  return relativePath.slice(0, -syncSecretArtifactSuffix.length);
+};
+
+export const assertStorageSafeRepoPath = (repoPath: string) => {
+  if (!hasReservedSyncArtifactSuffixSegment(repoPath)) {
+    return;
+  }
+
+  throw new DevsyncError(
+    `Tracked sync paths must not use the reserved suffix ${syncSecretArtifactSuffix}: ${repoPath}`,
+  );
+};
+
+export const resolveArtifactRelativePath = (
+  artifact: Pick<RepoArtifact, "category" | "repoPath">,
+) => {
+  return artifact.category === "secret"
+    ? `${artifact.repoPath}${syncSecretArtifactSuffix}`
+    : artifact.repoPath;
+};
+
+export const buildRepoArtifacts = async (
+  snapshot: ReadonlyMap<string, SnapshotNode>,
+  config: ResolvedSyncConfig,
+) => {
+  const artifacts: RepoArtifact[] = [];
+  const seenArtifactKeys = new Set<string>();
+
+  for (const repoPath of [...snapshot.keys()].sort((left, right) => {
+    return left.localeCompare(right);
+  })) {
+    assertStorageSafeRepoPath(repoPath);
+    const node = snapshot.get(repoPath);
+
+    if (node === undefined) {
+      continue;
+    }
+
+    if (node.type === "directory") {
+      const artifact = {
+        category: "plain",
+        kind: "directory",
+        repoPath,
+      } satisfies RepoArtifact;
+      const key = buildArtifactKey(artifact);
+
+      if (seenArtifactKeys.has(key)) {
+        throw new DevsyncError(
+          `Duplicate repository artifact generated for ${key}`,
+        );
+      }
+
+      seenArtifactKeys.add(key);
+      artifacts.push(artifact);
+      continue;
+    }
+
+    if (node.type === "symlink") {
+      const artifact = {
+        category: "plain",
+        kind: "symlink",
+        linkTarget: node.linkTarget,
+        repoPath,
+      } satisfies RepoArtifact;
+      const key = buildArtifactKey(artifact);
+
+      if (seenArtifactKeys.has(key)) {
+        throw new DevsyncError(
+          `Duplicate repository artifact generated for ${key}`,
+        );
+      }
+
+      seenArtifactKeys.add(key);
+      artifacts.push(artifact);
+      continue;
+    }
+
+    if (!node.secret) {
+      const artifact = {
+        category: "plain",
+        contents: node.contents,
+        executable: node.executable,
+        kind: "file",
+        repoPath,
+      } satisfies RepoArtifact;
+      const key = buildArtifactKey(artifact);
+
+      if (seenArtifactKeys.has(key)) {
+        throw new DevsyncError(
+          `Duplicate repository artifact generated for ${key}`,
+        );
+      }
+
+      seenArtifactKeys.add(key);
+      artifacts.push(artifact);
+      continue;
+    }
+
+    const artifact = {
+      category: "secret",
+      contents: await encryptSecretFile(node.contents, config.age.recipients),
+      executable: node.executable,
+      kind: "file",
+      repoPath,
+    } satisfies RepoArtifact;
+    const key = buildArtifactKey(artifact);
+
+    if (seenArtifactKeys.has(key)) {
+      throw new DevsyncError(
+        `Duplicate repository artifact generated for ${key}`,
+      );
+    }
+
+    seenArtifactKeys.add(key);
+    artifacts.push(artifact);
+  }
+
+  return artifacts;
+};
+
+const collectArtifactLeafKeys = async (
+  rootDirectory: string,
+  keys: Set<string>,
+  prefix?: string,
+) => {
+  if (!(await pathExists(rootDirectory))) {
+    return;
+  }
+
+  const entries = await listDirectoryEntries(rootDirectory);
+
+  for (const entry of entries) {
+    const absolutePath = join(rootDirectory, entry.name);
+    const relativePath =
+      prefix === undefined ? entry.name : `${prefix}/${entry.name}`;
+    const stats = await lstat(absolutePath);
+
+    if (stats?.isDirectory()) {
+      await collectArtifactLeafKeys(absolutePath, keys, relativePath);
+      continue;
+    }
+
+    keys.add(relativePath);
+  }
+};
+
+export const collectExistingArtifactKeys = async (
+  syncDirectory: string,
+  config: ResolvedSyncConfig,
+) => {
+  const keys = new Set<string>();
+  const artifactsDirectory = resolveSyncArtifactsDirectoryPath(syncDirectory);
+
+  await collectArtifactLeafKeys(artifactsDirectory, keys);
+
+  for (const entry of config.entries) {
+    if (entry.kind !== "directory") {
+      continue;
+    }
+
+    const path = join(artifactsDirectory, ...entry.repoPath.split("/"));
+    const stats = await getPathStats(path);
+
+    if (stats?.isDirectory()) {
+      keys.add(buildDirectoryKey(entry.repoPath));
+    }
+  }
+
+  return keys;
+};
+
+export const writeArtifactsToDirectory = async (
+  rootDirectory: string,
+  artifacts: readonly RepoArtifact[],
+) => {
+  await mkdir(rootDirectory, { recursive: true });
+
+  for (const artifact of artifacts) {
+    const artifactPath = join(
+      rootDirectory,
+      ...resolveArtifactRelativePath(artifact).split("/"),
+    );
+
+    if (artifact.kind === "directory") {
+      await mkdir(artifactPath, { recursive: true });
+      continue;
+    }
+
+    if (artifact.kind === "symlink") {
+      await writeSymlinkNode(artifactPath, artifact.linkTarget);
+      continue;
+    }
+
+    await writeFileNode(artifactPath, artifact);
+  }
+};

package/src/services/repo-snapshot.ts

@@ -0,0 +1,197 @@
+import { lstat, readFile, readlink } from "node:fs/promises";
+import { join } from "node:path";
+
+import {
+  findOwningSyncEntry,
+  type ResolvedSyncConfig,
+  resolveManagedSyncMode,
+  resolveSyncArtifactsDirectoryPath,
+} from "#app/config/sync.ts";
+
+import { decryptSecretFile } from "./crypto.ts";
+import { DevsyncError } from "./error.ts";
+import {
+  getPathStats,
+  isExecutableMode,
+  listDirectoryEntries,
+  pathExists,
+} from "./filesystem.ts";
+import { addSnapshotNode, type SnapshotNode } from "./local-snapshot.ts";
+import {
+  assertStorageSafeRepoPath,
+  isSecretArtifactPath,
+  stripSecretArtifactSuffix,
+} from "./repo-artifacts.ts";
+
+const readPlainSnapshotNode = async (
+  absolutePath: string,
+  repoPath: string,
+  config: ResolvedSyncConfig,
+  snapshot: Map<string, SnapshotNode>,
+) => {
+  assertStorageSafeRepoPath(repoPath);
+  const mode = resolveManagedSyncMode(config, repoPath);
+
+  if (mode === "ignore") {
+    return;
+  }
+
+  if (findOwningSyncEntry(config, repoPath) === undefined) {
+    throw new DevsyncError(
+      `Unmanaged plain sync path found in repository: ${repoPath}`,
+    );
+  }
+
+  if (mode === "secret") {
+    throw new DevsyncError(
+      `Secret sync path is stored in plain text in the repository: ${repoPath}`,
+    );
+  }
+
+  const stats = await lstat(absolutePath);
+
+  if (stats.isSymbolicLink()) {
+    addSnapshotNode(snapshot, repoPath, {
+      linkTarget: await readlink(absolutePath),
+      type: "symlink",
+    });
+
+    return;
+  }
+
+  if (!stats.isFile()) {
+    throw new DevsyncError(
+      `Unsupported plain repository entry: ${absolutePath}`,
+    );
+  }
+
+  addSnapshotNode(snapshot, repoPath, {
+    contents: await readFile(absolutePath),
+    executable: isExecutableMode(stats.mode),
+    secret: false,
+    type: "file",
+  });
+};
+
+const readRepositoryTree = async (
+  rootDirectory: string,
+  config: ResolvedSyncConfig,
+  snapshot: Map<string, SnapshotNode>,
+  prefix?: string,
+) => {
+  if (!(await pathExists(rootDirectory))) {
+    return;
+  }
+
+  const entries = await listDirectoryEntries(rootDirectory);
+
+  for (const entry of entries) {
+    const absolutePath = join(rootDirectory, entry.name);
+    const relativePath =
+      prefix === undefined ? entry.name : `${prefix}/${entry.name}`;
+    const stats = await lstat(absolutePath);
+
+    if (stats.isDirectory()) {
+      assertStorageSafeRepoPath(relativePath);
+      await readRepositoryTree(absolutePath, config, snapshot, relativePath);
+      continue;
+    }
+
+    if (stats.isSymbolicLink()) {
+      if (isSecretArtifactPath(relativePath)) {
+        throw new DevsyncError(
+          `Secret repository entries must be regular files, not symlinks: ${relativePath}`,
+        );
+      }
+
+      await readPlainSnapshotNode(absolutePath, relativePath, config, snapshot);
+      continue;
+    }
+
+    if (isSecretArtifactPath(relativePath)) {
+      const repoPath = stripSecretArtifactSuffix(relativePath);
+
+      if (repoPath === undefined || repoPath.length === 0) {
+        throw new DevsyncError(
+          `Secret repository files must include a path before ${relativePath}`,
+        );
+      }
+
+      assertStorageSafeRepoPath(repoPath);
+      const mode = resolveManagedSyncMode(config, repoPath);
+
+      if (findOwningSyncEntry(config, repoPath) === undefined) {
+        throw new DevsyncError(
+          `Unmanaged secret sync path found in repository: ${repoPath}`,
+        );
+      }
+
+      if (mode === "ignore") {
+        continue;
+      }
+
+      if (mode !== "secret") {
+        throw new DevsyncError(
+          `Plain sync path is stored in secret form in the repository: ${repoPath}`,
+        );
+      }
+
+      addSnapshotNode(snapshot, repoPath, {
+        contents: await decryptSecretFile(
+          await readFile(absolutePath, "utf8"),
+          config.age.identityFile,
+        ),
+        executable: isExecutableMode(stats.mode),
+        secret: true,
+        type: "file",
+      });
+      continue;
+    }
+
+    if (!stats.isFile()) {
+      throw new DevsyncError(
+        `Unsupported plain repository entry: ${absolutePath}`,
+      );
+    }
+
+    await readPlainSnapshotNode(absolutePath, relativePath, config, snapshot);
+  }
+};
+
+export const buildRepositorySnapshot = async (
+  syncDirectory: string,
+  config: ResolvedSyncConfig,
+) => {
+  const snapshot = new Map<string, SnapshotNode>();
+  const artifactsDirectory = resolveSyncArtifactsDirectoryPath(syncDirectory);
+
+  await readRepositoryTree(artifactsDirectory, config, snapshot);
+
+  for (const entry of config.entries) {
+    if (entry.kind !== "directory") {
+      continue;
+    }
+
+    const artifactPath = join(artifactsDirectory, ...entry.repoPath.split("/"));
+    const stats = await getPathStats(artifactPath);
+
+    if (stats !== undefined && !stats.isDirectory()) {
+      throw new DevsyncError(
+        `Directory sync entry is not stored as a directory in the repository: ${entry.repoPath}`,
+      );
+    }
+
+    const mode = resolveManagedSyncMode(config, entry.repoPath);
+    const hasTrackedChildren = [...snapshot.keys()].some((repoPath) => {
+      return repoPath.startsWith(`${entry.repoPath}/`);
+    });
+
+    if (stats?.isDirectory() && (mode !== "ignore" || hasTrackedChildren)) {
+      addSnapshotNode(snapshot, entry.repoPath, {
+        type: "directory",
+      });
+    }
+  }
+
+  return snapshot;
+};

package/src/services/runtime.ts

@@ -0,0 +1,57 @@
+import {
+  resolveSyncArtifactsDirectoryPath,
+  resolveSyncConfigFilePath,
+} from "#app/config/sync.ts";
+import {
+  resolveDevsyncSyncDirectory,
+  resolveHomeDirectory,
+} from "#app/config/xdg.ts";
+
+import { ensureGitRepository } from "./git.ts";
+
+export type SyncPaths = Readonly<{
+  artifactsDirectory: string;
+  configPath: string;
+  homeDirectory: string;
+  syncDirectory: string;
+}>;
+
+export type SyncContext = Readonly<{
+  cwd: string;
+  environment: NodeJS.ProcessEnv;
+  paths: SyncPaths;
+}>;
+
+export const createSyncPaths = (
+  environment: NodeJS.ProcessEnv = process.env,
+): SyncPaths => {
+  const syncDirectory = resolveDevsyncSyncDirectory(environment);
+
+  return {
+    artifactsDirectory: resolveSyncArtifactsDirectoryPath(syncDirectory),
+    configPath: resolveSyncConfigFilePath(syncDirectory),
+    homeDirectory: resolveHomeDirectory(environment),
+    syncDirectory,
+  };
+};
+
+export const createSyncContext = (
+  options: Readonly<{
+    cwd?: string;
+    environment?: NodeJS.ProcessEnv;
+  }> = {},
+): SyncContext => {
+  const environment = options.environment ?? process.env;
+
+  return {
+    cwd: options.cwd ?? process.cwd(),
+    environment,
+    paths: createSyncPaths(environment),
+  };
+};
+
+export const ensureSyncRepository = async (
+  context: Pick<SyncContext, "paths">,
+) => {
+  await ensureGitRepository(context.paths.syncDirectory);
+};