@hanseltime/template-repo-sync 1.0.1

package/src/template-sync.spec.ts

@@ -0,0 +1,216 @@
+import { copy } from "fs-extra";
+import { mkdtemp, readFile, rm } from "fs/promises";
+import { templateSync } from "./template-sync";
+import { tempDir, TEST_FIXTURES_DIR } from "./test-utils";
+import { join, resolve } from "path";
+import { existsSync, readFileSync, writeFileSync } from "fs";
+
+// Just return the test-fixture directory
+const dummyCloneDriver = async () => {
+  return resolve(TEST_FIXTURES_DIR, "template");
+};
+
+const downstreamDir = resolve(TEST_FIXTURES_DIR, "downstream");
+
+describe("templateSync", () => {
+  let tmpDir: string;
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(tempDir());
+    await copy(downstreamDir, tmpDir);
+  });
+  afterEach(async () => {
+    await rm(tmpDir, {
+      force: true,
+      recursive: true,
+    });
+  });
+  it("appropriately merges according to just the templatesync config file into an empty dir", async () => {
+    const emptyTmpDir = await mkdtemp(tempDir());
+    expect(
+      await templateSync({
+        tmpCloneDir: "stubbed-by-driver",
+        cloneDriver: dummyCloneDriver,
+        repoUrl: "not-important",
+        repoDir: emptyTmpDir,
+      }),
+    ).toEqual({
+      // Expect no changes since there was no local sync file
+      localSkipFiles: [],
+      localFileChanges: {},
+    });
+
+    // Verify the files
+    await fileMatchTemplate(emptyTmpDir, "templatesync.json");
+    await fileMatchTemplate(emptyTmpDir, "package.json");
+    await fileMatchTemplate(emptyTmpDir, "src/templated.ts");
+
+    // Expect the ignores to not be a problem
+    expect(existsSync(resolve(emptyTmpDir, "src/index.ts"))).toBeFalsy();
+    expect(existsSync(resolve(emptyTmpDir, "src/custom-bin"))).toBeFalsy();
+  });
+  it("appropriately merges according to just the templatesync config file in an existing repo", async () => {
+    // Remove the local sync overrides
+    await rm(join(tmpDir, "templatesync.local.json"));
+
+    const result = await templateSync({
+      tmpCloneDir: "stubbed-by-driver",
+      cloneDriver: dummyCloneDriver,
+      repoUrl: "not-important",
+      repoDir: tmpDir,
+    });
+
+    expect(result.localSkipFiles).toEqual([]);
+    expect(result.localFileChanges).toEqual({});
+
+    // Verify the files
+    await fileMatchTemplate(tmpDir, "templatesync.json");
+    await fileMatchTemplate(tmpDir, "src/templated.ts");
+    const packageJson = JSON.parse(
+      readFileSync(resolve(tmpDir, "package.json")).toString(),
+    );
+
+    expect(packageJson).toEqual({
+      name: "mypkg",
+      description: "my description",
+      dependencies: {
+        mypackage: "^1.2.0",
+        newpacakge: "^22.2.2",
+        package2: "3.22.1",
+        huh: "~1.0.0",
+      },
+      engines: {
+        node: ">=15",
+      },
+      scripts: {
+        build: "build",
+        test: "jest",
+        myscript: "somescript",
+      },
+      // By default we add new top-level fields
+      version: "new-version",
+    });
+
+    // Expect the ignores to not be a problem
+    await fileMatchDownstream(tmpDir, "src/index.ts");
+    await fileMatchDownstream(tmpDir, "plugins/custom-plugin.js");
+  });
+  it("appropriately merges according to the templatesync config file and the local config in an existing repo", async () => {
+    // Remove the local sync overrides
+    await rm(join(tmpDir, "templatesync.local.json"));
+
+    writeFileSync(
+      join(tmpDir, "templatesync.local.json"),
+      JSON.stringify({
+        ignore: [
+          // Ignores the templated.ts
+          "**/*.ts",
+          // We don't have a need for this in here, but it's an example of keeping things cleaner for our custom plugins
+          "plugins/**",
+        ],
+        merge: {
+          ".json": {
+            // Let's nuke package.json with this plugin
+            plugin: "plugins/custom-plugin.js",
+            rules: [
+              {
+                glob: "package.json",
+                options: {},
+              },
+            ],
+          },
+        },
+      }),
+    );
+
+    const result = await templateSync({
+      tmpCloneDir: "stubbed-by-driver",
+      cloneDriver: dummyCloneDriver,
+      repoUrl: "not-important",
+      repoDir: tmpDir,
+    });
+
+    expect(result.localSkipFiles).toEqual(["src/templated.ts"]);
+    // TODO: more rigorous testing around diff changes
+    expect(result.localFileChanges).toEqual(
+      expect.objectContaining({
+        "package.json": expect.arrayContaining([]),
+      }),
+    );
+
+    // Verify the files
+    await fileMatchTemplate(tmpDir, "templatesync.json");
+    await fileMatchDownstream(tmpDir, "src/templated.ts");
+    const packageJson = JSON.parse(
+      readFileSync(resolve(tmpDir, "package.json")).toString(),
+    );
+
+    // The plugin nuked this
+    expect(packageJson).toEqual({
+      downstream: true,
+    });
+
+    // Expect the ignores to not be a problem
+    await fileMatchDownstream(tmpDir, "src/index.ts");
+    await fileMatchDownstream(tmpDir, "plugins/custom-plugin.js");
+  });
+  it("appropriately merges according to the templatesync config file and the local config in an existing repo with afterRef", async () => {
+    // Remove the local sync overrides
+    await rm(join(tmpDir, "templatesync.local.json"));
+
+    writeFileSync(
+      join(tmpDir, "templatesync.local.json"),
+      JSON.stringify({
+        afterRef: "dummySha",
+        ignore: [
+          // We don't have a need for this in here, but it's an example of keeping things cleaner for our custom plugins
+          "plugins/**",
+        ],
+      }),
+    );
+
+    // We will only update the templated.ts
+    const mockDiffDriver = jest
+      .fn()
+      .mockImplementation(async () => ["src/templated.ts"]);
+    const result = await templateSync({
+      tmpCloneDir: "stubbed-by-driver",
+      cloneDriver: dummyCloneDriver,
+      repoUrl: "not-important",
+      repoDir: tmpDir,
+      diffDriver: mockDiffDriver,
+    });
+
+    // since there was no override for this file, not changes from the local file
+    expect(result.localFileChanges).toEqual(expect.objectContaining({}));
+
+    // Verify the files
+    await fileMatchTemplate(tmpDir, "templatesync.json");
+    await fileMatchTemplate(tmpDir, "src/templated.ts");
+
+    // Expect the none of the diff files to work
+    await fileMatchDownstream(tmpDir, "src/index.ts");
+    await fileMatchDownstream(tmpDir, "plugins/custom-plugin.js");
+    await fileMatchDownstream(tmpDir, "package.json");
+  });
+});
+
+// helper
+async function fileMatchTemplate(tmpDir: string, relPath: string) {
+  return fileMatch(tmpDir, relPath, "template");
+}
+
+async function fileMatchDownstream(tmpDir: string, relPath: string) {
+  return fileMatch(tmpDir, relPath, "downstream");
+}
+
+async function fileMatch(
+  tmpDir: string,
+  relPath: string,
+  source: "downstream" | "template",
+) {
+  const dir =
+    source === "downstream" ? downstreamDir : await dummyCloneDriver();
+  expect((await readFile(resolve(tmpDir, relPath))).toString()).toEqual(
+    (await readFile(resolve(dir, relPath))).toString(),
+  );
+}

package/src/template-sync.ts

@@ -0,0 +1,113 @@
+import { join } from "path";
+import { existsSync, readFileSync } from "fs";
+import { getAllFilesInDir } from "./match";
+import { Config, LocalConfig } from "./types";
+import { mergeFile } from "./merge-file";
+import { gitClone } from "./clone-drivers/git-clone";
+import { Change } from "diff";
+import { TemplateCloneDriverFn } from "./clone-drivers";
+import { TemplateDiffDriverFn, gitDiff } from "./diff-drivers";
+
+export interface TemplateSyncOptions {
+  repoUrl: string;
+
+  /**
+   * The directory for cloning our template repo into via the cloneDriver
+   */
+  tmpCloneDir: string;
+
+  /**
+   * The repo directory path that we are going to merge toward
+   */
+  repoDir: string;
+
+  /**
+   * Defaults to using git clone
+   */
+  cloneDriver?: TemplateCloneDriverFn;
+
+  /**
+   * Defaults to using git diff
+   */
+  diffDriver?: TemplateDiffDriverFn;
+}
+
+export interface TemplateSyncReturn {
+  /**
+   * An array of files that were skipped outright due to a templatesync.local ignore glob
+   */
+  localSkipFiles: string[];
+  /**
+   * An object mapping all file paths to any merge rules that would've overridden the merge rules
+   * of the template sync file.
+   *
+   * Note: right now, this shows non-changed diffs as well so you have to look for added or removed
+   */
+  localFileChanges: {
+    [filePath: string]: Change[];
+  };
+}
+
+export const TEMPLATE_SYNC_CONFIG = "templatesync";
+export const TEMPLATE_SYNC_LOCAL_CONFIG = "templatesync.local";
+
+export async function templateSync(
+  options: TemplateSyncOptions,
+): Promise<TemplateSyncReturn> {
+  const cloneDriver = options.cloneDriver ?? gitClone;
+  const diffDriver = options.diffDriver ?? gitDiff;
+  const tempCloneDir = await cloneDriver(options.tmpCloneDir, options.repoUrl);
+
+  // Get the clone Config
+  const cloneConfigPath = join(tempCloneDir, `${TEMPLATE_SYNC_CONFIG}.json`);
+  const templateSyncConfig: Config = existsSync(cloneConfigPath)
+    ? JSON.parse(readFileSync(cloneConfigPath).toString())
+    : {};
+
+  const localConfigPath = join(
+    options.repoDir,
+    `${TEMPLATE_SYNC_LOCAL_CONFIG}.json`,
+  );
+  const localTemplateSyncConfig = existsSync(localConfigPath)
+    ? JSON.parse(readFileSync(localConfigPath).toString())
+    : ({} as LocalConfig);
+
+  let filesToSync: string[];
+  if (localTemplateSyncConfig.afterRef) {
+    filesToSync = await diffDriver(
+      tempCloneDir,
+      localTemplateSyncConfig.afterRef,
+    );
+  } else {
+    filesToSync = getAllFilesInDir(tempCloneDir, [
+      ...templateSyncConfig.ignore,
+      ".git/**",
+    ]);
+  }
+
+  const localSkipFiles: string[] = [];
+  const localFileChanges: {
+    [filePath: string]: Change[];
+  } = {};
+
+  await Promise.all(
+    filesToSync.map(async (f) => {
+      const result = await mergeFile(f, {
+        localTemplateSyncConfig,
+        templateSyncConfig,
+        tempCloneDir,
+        cwd: options.repoDir,
+      });
+      if (result.ignoredDueToLocal) {
+        localSkipFiles.push(f);
+      } else if (result?.localChanges && result.localChanges.length > 0) {
+        localFileChanges[f] = result.localChanges;
+      }
+    }),
+  );
+
+  return {
+    localSkipFiles,
+    localFileChanges,
+  };
+}

package/src/test-utils/index.ts

@@ -0,0 +1,13 @@
+import { resolve } from "path";
+import { tmpdir } from "os";
+
+export const TEST_FIXTURES_DIR = resolve(
+  __dirname,
+  "..",
+  "..",
+  "test-fixtures",
+);
+
+export function tempDir(): string {
+  return process.env.RUNNER_TEMP ?? tmpdir();
+}

package/src/types.ts

@@ -0,0 +1,124 @@
+/**
+ * How we want to merge json
+ *
+ * overwrite - the template sync overwrites completely
+ * merge-template - we merge template into the current json with overrides on matching values happening from the template
+ * merge-current - we merge the current json into the template json with overrides on matching values happening from the current json
+ */
+export type BaseJsonMergeOptions =
+  | "overwrite"
+  | "merge-template"
+  | "merge-current";
+export interface JsonPathOverrides {
+  /**
+   * If set to true, this means we won't add new properties from the template
+   */
+  ignoreNewProperties?: boolean;
+  /**
+   * If set to true, overwrite will apply undefined values as deleted for the jsonpaths
+   * or for values that are supposed to be merged on top of other values
+   */
+  missingIsDelete?: boolean;
+  /**
+   * Note, if multiple json paths match a rule, we pick the first one in the list that matches
+   */
+  paths: /**
+   * We only override jsonpaths.  Anything not specified is kept the same.
+   */
+  [jsonPath: `$.${string}`, options: BaseJsonMergeOptions][];
+}
+export type JsonFileMergeOptions = BaseJsonMergeOptions | JsonPathOverrides;
+
+// Sum of all basic file merge options
+type MergePluginOptions = JsonFileMergeOptions;
+
+/**
+ * Configuration object for a given file type merge configuration
+ */
+export interface MergeConfig<T> {
+  /**
+   * The node module, available on the calling node context, that you want to run.
+   * If not provided, we try to use any built-in merge plugins.
+   */
+  plugin?: string;
+  /**
+   * An array of first match file globs that will
+   */
+  rules: { glob: string; options: MergePluginOptions | T }[];
+}
+
+/**
+ * The shape of a template sync config file
+ */
+export interface Config<T = unknown> {
+  ignore: string[];
+  /**
+   * If there is no merge config, then we will always just overwrite the file for the diff
+   */
+  merge?: {
+    [fileExt: string]: MergeConfig<T>;
+  };
+}
+
+/**
+ * The shape of a local template sync config file that overrides the root template repo
+ */
+export interface LocalConfig<T = unknown> {
+  /**
+   * This is the ref that we compare against.  If empty, we use all files that have changed since the
+   * beginning of the template repo.
+   */
+  afterRef?: string;
+
+  ignore: string[];
+  /**
+   * If there is no merge config, then we will always just overwrite the file for the diff
+   */
+  merge?: {
+    [fileExt: string]: MergeConfig<T>;
+  };
+}
+
+/**
+ * Information around the file we are trying to merge and that arguments for that merge
+ * from matching a merge configuration
+ */
+export interface MergeContext<PluginOptions = unknown> {
+  relFilePath: string;
+  /**
+   * If you have provided a custom merge plugin, this will be the "options" section of
+   * the matching glob
+   */
+  mergeArguments: PluginOptions;
+  /**
+   * If this run is for the local config.  Keep in mind that any local config overrides will
+   * be run after the template sync options and we would like to report any changes that
+   * are done
+   */
+  isLocalOptions?: boolean;
+}
+
+export interface MergePlugin<PluginOptions> {
+  /**
+   * This method will be called when a file from the template and it's analog in the downstream repo
+   * have some differences.  The plugin must perform the merge and return the appropriate file contents
+   * as a string
+   *
+   * TODO: we may create a V2 plugin that could deal with large files and not pass around strings in memory,
+   * but for now, this is the current implementation
+   *
+   * @param current - The downstream repos current file contents
+   * @param fromTemplateRepo - the current
+   * @param context - an object defining the context around the file and the specific options
+   */
+  merge(
+    current: string,
+    fromTemplateRepo: string,
+    context: MergeContext<PluginOptions>,
+  ): Promise<string>;
+  /**
+   * Given an options object for the merge, this validates the options object and returns error messages if there is anything wrong.
+   * @param options any json value that the user provided - must be validated against the expected options
+   */
+  validate(options: unknown): string[] | undefined;
+}

package/templatesync.local.json

@@ -0,0 +1,15 @@
+{
+  "ignore": ["**/*.md"],
+  "merge": {
+    ".json": {
+      "rules": [
+        {
+          "glob": "package.json",
+          "options": {
+            "paths": [["$.repository", "merge-template"]]
+          }
+        }
+      ]
+    }
+  }
+}

package/test-fixtures/downstream/README.md

@@ -0,0 +1,3 @@
+# My custom readme
+
+This is a custom readme for my file

package/test-fixtures/downstream/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "mypkg",
+  "description": "my description",
+  "dependencies": {
+    "mypackage": "^1.2.0",
+    "newpacakge": "^22.2.2",
+    "package2": "3.22.1",
+    "huh": "^2.30.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "build",
+    "test": "jest",
+    "myscript": "somescript"
+  }
+}

package/test-fixtures/downstream/plugins/custom-plugin.js

@@ -0,0 +1,11 @@
+module.exports = {
+  merge: () => {
+    return JSON.stringify(
+      {
+        downstream: true,
+      },
+      null,
+      4,
+    );
+  },
+};

package/test-fixtures/downstream/src/index.js

@@ -0,0 +1,2 @@
+"use strict";
+console.log("This is my custom stuff");

package/test-fixtures/downstream/src/index.ts

@@ -0,0 +1 @@
+console.log("This is my custom stuff");

package/test-fixtures/downstream/src/templated.js

@@ -0,0 +1,2 @@
+"use strict";
+console.log("this is templated stuff that should not change");

package/test-fixtures/downstream/src/templated.ts

@@ -0,0 +1 @@
+console.log("this is templated stuff that should not change");

package/test-fixtures/downstream/templatesync.json

@@ -0,0 +1,19 @@
+{
+  "ignore": ["src/!(templated).ts", "custom-bin/**"],
+  "merge": {
+    ".json": {
+      "rules": [
+        {
+          "glob": "package.json",
+          "options": {
+            "paths": [
+              ["$.dependencies", "merge-template"],
+              ["$.devDependencies", "merge-template"],
+              ["$.engines", "merge-template"]
+            ]
+          }
+        }
+      ]
+    }
+  }
+}

package/test-fixtures/downstream/templatesync.local.json

@@ -0,0 +1,14 @@
+{
+  "ignore": ["custom-bin/**"],
+  "merge": {
+    ".json": {
+      "plugin": "plugins/custom-plugin.js",
+      "rules": [
+        {
+          "glob": "package.json",
+          "options": {}
+        }
+      ]
+    }
+  }
+}

package/test-fixtures/dummy-plugin.js

@@ -0,0 +1,8 @@
+module.exports = {
+  merge: () => {
+    return JSON.stringify({ tested: true }, null, 4);
+  },
+  validate: () => {
+    return [];
+  },
+};

package/test-fixtures/glob-test/folder1/something.js

@@ -0,0 +1 @@
+"use strict";

package/test-fixtures/template/custom-bin/something.txt

@@ -0,0 +1 @@
+This is a file that is only meant for the base template maintainers

package/test-fixtures/template/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "some-stub-name",
+  "description": "some-stub-description",
+  "dependencies": {
+    "mypackage": "^1.2.0",
+    "package2": "3.22.1",
+    "huh": "~1.0.0"
+  },
+  "engines": {
+    "node": ">=15"
+  },
+  "scripts": {
+    "build": "build",
+    "test": "fill this in yourself"
+  },
+  "version": "new-version"
+}

package/test-fixtures/template/src/index.js

@@ -0,0 +1,2 @@
+"use strict";
+console.log("replace this in the downstream");

package/test-fixtures/template/src/index.ts

@@ -0,0 +1 @@
+console.log("replace this in the downstream");

package/test-fixtures/template/src/templated.js

@@ -0,0 +1,2 @@
+"use strict";
+console.log("we changed this up and want to push the changes");

package/test-fixtures/template/src/templated.ts

@@ -0,0 +1 @@
+console.log("we changed this up and want to push the changes");

package/test-fixtures/template/templatesync.json

@@ -0,0 +1,19 @@
+{
+  "ignore": ["src/!(templated).ts", "custom-bin/**"],
+  "merge": {
+    ".json": {
+      "rules": [
+        {
+          "glob": "package.json",
+          "options": {
+            "paths": [
+              ["$.dependencies", "merge-template"],
+              ["$.devDependencies", "merge-template"],
+              ["$.engines", "merge-template"]
+            ]
+          }
+        }
+      ]
+    }
+  }
+}

package/tsconfig.cjs.json

@@ -0,0 +1,12 @@
+// Overrides for building to cjs
+{
+  "extends": "./tsconfig.esm.json",
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    "module": "CommonJS" /* Specify what module code is generated. */,
+    "outDir": "./lib/cjs",
+    "moduleResolution": "Node",
+    "declaration": true
+  }
+}

package/tsconfig.esm.json

@@ -0,0 +1,10 @@
+// Base ESM tsconfig file for builds
+{
+  "extends": "./tsconfig.json",
+  "exclude": [
+    "node_modules/**/*",
+    "test-fixtures/**/*",
+    "lib/**/*",
+    "src/**/*.spec.ts"
+  ]
+}