@tahminator/pipeline 1.0.75-beta.6f4bf3b7 → 1.0.76-beta.f1e3e65b

package/README.md

@@ -38,6 +38,7 @@ import {
   EnvClient,
   GitHubClient,
   NPMClient,
+  ProtobufCompilerClient,
   PulumiClient,
   SonarScannerClient,
   Utils,
@@ -53,6 +54,7 @@ Jump to client documentation
 
 - [`GitHubClient`](#githubclient)
 - [`DockerClient`](#dockerclient)
+- [`ProtobufCompilerClient`](#protobufcompilerclient)
 - [`NPMClient`](#npmclient)
 - [`SonarScannerClient`](#sonarscannerclient)
 - [`Utils`](#utils)
@@ -178,6 +180,32 @@ await client.promoteDockerImage({
 });
 ```
 
+### `ProtobufCompilerClient`
+
+Generate protobuf clients with the [Buf CLI](https://buf.build/docs/installation/) and its remote plugins, then optionally upload them to a GitHub repository path. The client installs Buf globally with npm when necessary.
+
+```ts
+const gh = await GitHubClient.createWithGithubAppToken({
+  appId: process.env.GH_APP_ID!,
+  privateKey: await Utils.decodeBase64EncodedString(
+    process.env.GH_PRIVATE_KEY_B64!,
+  ),
+  installationId: process.env.GH_INSTALLATION_ID!,
+});
+const protobuf = new ProtobufCompilerClient(gh);
+
+await protobuf.compile({
+  sourceLanguage: ProtobufSourceLanguage.RUST,
+  bufToken: process.env.BUF_TOKEN,
+  targetLanguages: {
+    [ProtobufTargetLanguage.JAVA]: {
+      uploadLocation: "Patina-Network/proto-clients/java",
+    },
+  },
+  protoFilesLocation: "./proto",
+});
+```
+
 ### `NPMClient`
 
 Interface with NPM registry in order to publish packages to `npmjs.com`

package/dist/gh/client.d.ts

@@ -118,6 +118,13 @@ export declare class GitHubClient {
         pull_requests: import("@octokit/openapi-types").components["schemas"]["pull-request-minimal"][];
         deployment?: import("@octokit/openapi-types").components["schemas"]["deployment-simple"];
     } | null>;
+    /**
+     * Uploads a local directory into a repository path using the authenticated GitHub API.
+     * Existing files under `directory` that are not present locally are deleted.
+     *
+     * **PLEASE DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU ARE DOING.**
+     */
+    DANGEROUS__uploadDirectoryContents(...args: Parameters<GitHubPRManager["uploadDirectoryContents"]>): Promise<void>;
     /**
      * Confirms whether the given user is an active member of a team within an org.
      *

package/dist/gh/client.js

@@ -126,6 +126,15 @@ export class GitHubClient {
     statusCheck(...args) {
         return this.prManager.statusCheck(...args);
     }
+    /**
+     * Uploads a local directory into a repository path using the authenticated GitHub API.
+     * Existing files under `directory` that are not present locally are deleted.
+     *
+     * **PLEASE DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU ARE DOING.**
+     */
+    DANGEROUS__uploadDirectoryContents(...args) {
+        return this.prManager.uploadDirectoryContents(...args);
+    }
     /**
      * Confirms whether the given user is an active member of a team within an org.
      *

package/dist/gh/pr/index.d.ts

@@ -7,6 +7,14 @@ type CheckRunOutput = {
     summary: string;
     text?: string;
 };
+type UploadDirectoryContentsArgs = {
+    owner: string;
+    repository: string;
+    sourceDirectory: string;
+    directory?: string;
+    message?: string;
+    branch?: string;
+};
 type StatusCheckArgs = {
     action: "create";
     owner: string;
@@ -42,6 +50,7 @@ type StatusCheckArgs = {
 export declare class GitHubPRManager {
     private readonly client;
     constructor(client: Octokit);
+    uploadDirectoryContents({ owner, repository, sourceDirectory, directory, message, branch, }: UploadDirectoryContentsArgs): Promise<void>;
     updateK8sTagWithPR({ newTag, imageName, kustomizationFilePath, environment, originRepo, manifestRepo, }: {
         newTag: string;
         imageName: string;
@@ -89,5 +98,9 @@ export declare class GitHubPRManager {
         repository: string;
         message: string;
     }): Promise<void>;
+    private createDirectoryTree;
+    private createDirectoryBlobEntries;
+    private getDirectoryPathPrefix;
+    private collectDirectoryFiles;
 }
 export {};

package/dist/gh/pr/index.js

@@ -1,3 +1,5 @@
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
 import yaml from "yaml";
 import { Utils } from "../../utils";
 import { kustomizeSchema } from "./schema";
@@ -6,6 +8,47 @@ export class GitHubPRManager {
     constructor(client) {
         this.client = client;
     }
+    async uploadDirectoryContents({ owner, repository, sourceDirectory, directory = "", message = "chore: update generated files", branch, }) {
+        const { data: repo } = await this.client.rest.repos.get({
+            owner,
+            repo: repository,
+        });
+        const targetBranch = branch ?? repo.default_branch;
+        const { data: ref } = await this.client.rest.git.getRef({
+            owner,
+            repo: repository,
+            ref: `heads/${targetBranch}`,
+        });
+        const { data: baseCommit } = await this.client.rest.git.getCommit({
+            owner,
+            repo: repository,
+            commit_sha: ref.object.sha,
+        });
+        const treeSha = await this.createDirectoryTree({
+            directory,
+            owner,
+            repository,
+            sourceDirectory,
+            baseTreeSha: baseCommit.tree.sha,
+        });
+        if (treeSha === baseCommit.tree.sha) {
+            console.log("No directory changes to upload.");
+            return;
+        }
+        const { data: commit } = await this.client.rest.git.createCommit({
+            owner,
+            repo: repository,
+            message,
+            tree: treeSha,
+            parents: [ref.object.sha],
+        });
+        await this.client.rest.git.updateRef({
+            owner,
+            repo: repository,
+            ref: `heads/${targetBranch}`,
+            sha: commit.sha,
+        });
+    }
     async updateK8sTagWithPR({ newTag, imageName, kustomizationFilePath, environment, originRepo, manifestRepo, }) {
         const [manifestOwner, manifestRepository] = manifestRepo;
         const [originOwner, originRepository] = originRepo;
@@ -131,4 +174,84 @@ export class GitHubPRManager {
             body: message,
         });
     }
+    async createDirectoryTree({ owner, repository, sourceDirectory, directory, baseTreeSha, }) {
+        const pathPrefix = this.getDirectoryPathPrefix(directory);
+        const files = await this.collectDirectoryFiles(sourceDirectory);
+        const localPaths = new Set(files.map(({ relativePath }) => `${pathPrefix}${relativePath}`));
+        const { data: existingTree } = await this.client.rest.git.getTree({
+            owner,
+            repo: repository,
+            tree_sha: baseTreeSha,
+            recursive: "true",
+        });
+        const deletionEntries = existingTree.tree.flatMap((entry) => {
+            if (entry.type !== "blob" ||
+                !entry.path?.startsWith(pathPrefix) ||
+                localPaths.has(entry.path)) {
+                return [];
+            }
+            return [
+                {
+                    path: entry.path,
+                    mode: "100644",
+                    type: "blob",
+                    sha: null,
+                },
+            ];
+        });
+        const blobEntries = await this.createDirectoryBlobEntries({
+            files,
+            owner,
+            repository,
+            pathPrefix,
+        });
+        const { data: tree } = await this.client.rest.git.createTree({
+            owner,
+            repo: repository,
+            base_tree: baseTreeSha,
+            tree: [...deletionEntries, ...blobEntries],
+        });
+        return tree.sha;
+    }
+    async createDirectoryBlobEntries({ owner, repository, files, pathPrefix, }) {
+        return Promise.all(files.map(async ({ absolutePath, relativePath }) => {
+            const content = await readFile(absolutePath);
+            const { data: blob } = await this.client.rest.git.createBlob({
+                owner,
+                repo: repository,
+                content: content.toString("base64"),
+                encoding: "base64",
+            });
+            return {
+                path: `${pathPrefix}${relativePath}`,
+                mode: "100644",
+                type: "blob",
+                sha: blob.sha,
+            };
+        }));
+    }
+    getDirectoryPathPrefix(directory) {
+        const normalizedDirectory = directory.replace(/^\/+|\/+$/g, "");
+        return normalizedDirectory ? `${normalizedDirectory}/` : "";
+    }
+    async collectDirectoryFiles(directory, baseDirectory = directory) {
+        const files = [];
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            const absolutePath = path.join(directory, entry.name);
+            if (entry.isDirectory()) {
+                files.push(...(await this.collectDirectoryFiles(absolutePath, baseDirectory)));
+                continue;
+            }
+            if (entry.isFile()) {
+                files.push({
+                    absolutePath,
+                    relativePath: path
+                        .relative(baseDirectory, absolutePath)
+                        .split(path.sep)
+                        .join("/"),
+                });
+            }
+        }
+        return files;
+    }
 }

package/dist/index.d.ts

@@ -3,6 +3,7 @@ export * from "./docker";
 export * from "./npm";
 export * from "./sonar";
 export * from "./pulumi";
+export * from "./protobuf";
 export * from "./types";
 export * from "./env";
 export * from "./postgres";

package/dist/index.js

@@ -3,6 +3,7 @@ export * from "./docker";
 export * from "./npm";
 export * from "./sonar";
 export * from "./pulumi";
+export * from "./protobuf";
 export * from "./types";
 export * from "./env";
 export * from "./postgres";

package/dist/protobuf/client.d.ts

@@ -0,0 +1,16 @@
+import type { GitHubClient } from "../gh";
+import { type ProtobufCompilerCompileArgs, type ProtobufCompilerCompileResult } from "./types";
+export declare class ProtobufCompilerClient {
+    private readonly githubClient;
+    constructor(githubClient: GitHubClient);
+    compile({ bufToken, targetLanguages, protoFilesLocation, outputDirectory, }: ProtobufCompilerCompileArgs): Promise<ProtobufCompilerCompileResult>;
+    installToolchains(): Promise<void>;
+    private generateWithBuf;
+    private getCompilationTargets;
+    private getConfiguredTargetLanguages;
+    private getTargetLanguageStrategy;
+    private getBufGenerateInput;
+    private findBufModuleDirectory;
+    private uploadGeneratedDirectory;
+    private parseGitHubRepository;
+}

package/dist/protobuf/client.js

@@ -0,0 +1,154 @@
+import { $ } from "bun";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import yaml from "yaml";
+import { isCmdAvailable } from "../utils/cmd";
+import { GoProtobufTargetLanguageStrategy, JavaProtobufTargetLanguageStrategy, RustProtobufTargetLanguageStrategy, } from "./strategy";
+import { ProtobufTargetLanguage, } from "./types";
+export class ProtobufCompilerClient {
+    githubClient;
+    constructor(githubClient) {
+        this.githubClient = githubClient;
+    }
+    async compile({ bufToken, targetLanguages, protoFilesLocation, outputDirectory, }) {
+        await this.installToolchains();
+        const outputDirectoryPath = path.resolve(outputDirectory ??
+            (await mkdtemp(path.join(tmpdir(), "protobuf-compiler-"))));
+        const targets = this.getCompilationTargets(targetLanguages);
+        await mkdir(outputDirectoryPath, { recursive: true });
+        await this.generateWithBuf({
+            bufToken,
+            outputDirectory: outputDirectoryPath,
+            protoFilesLocation,
+            targetStrategies: targets.map(({ strategy }) => strategy),
+        });
+        const generatedDirectories = {};
+        for (const { language, strategy } of targets) {
+            generatedDirectories[language] = path.join(outputDirectoryPath, strategy.outputDirectoryName);
+        }
+        for (const { language, options } of targets) {
+            if (!options.uploadLocation)
+                continue;
+            const generatedDirectory = generatedDirectories[language];
+            if (!generatedDirectory) {
+                throw new Error("Generated code directory was not created.");
+            }
+            await this.uploadGeneratedDirectory({
+                generatedDirectory,
+                options,
+            });
+        }
+        return { generatedDirectories, outputDirectory: outputDirectoryPath };
+    }
+    async installToolchains() {
+        if (!(await isCmdAvailable("buf"))) {
+            console.log("Buf is missing, installing globally via NPM...");
+            await $ `npm i -g @bufbuild/buf`;
+        }
+    }
+    async generateWithBuf({ bufToken, protoFilesLocation, outputDirectory, targetStrategies, }) {
+        const templateDirectory = await mkdtemp(path.join(tmpdir(), "protobuf-buf-template-"));
+        const templatePath = path.join(templateDirectory, "buf.gen.yaml");
+        const { input, pathFilter } = await this.getBufGenerateInput(protoFilesLocation);
+        const pathFlag = pathFilter ? ["--path", pathFilter] : [];
+        await writeFile(templatePath, yaml.stringify({
+            version: "v2",
+            plugins: targetStrategies.map((strategy) => ({
+                out: strategy.outputDirectoryName,
+                remote: strategy.remotePlugin,
+            })),
+        }));
+        try {
+            const command = bufToken ?
+                $.env({
+                    ...process.env,
+                    BUF_TOKEN: bufToken,
+                }) `buf generate ${input} --template ${templatePath} ${pathFlag}`.cwd(outputDirectory)
+                : $ `buf generate ${input} --template ${templatePath} ${pathFlag}`.cwd(outputDirectory);
+            await command;
+        }
+        finally {
+            await rm(templateDirectory, { force: true, recursive: true });
+        }
+    }
+    getCompilationTargets(targetLanguages) {
+        return this.getConfiguredTargetLanguages(targetLanguages).map(([language, options]) => ({
+            language,
+            options,
+            strategy: this.getTargetLanguageStrategy(language),
+        }));
+    }
+    getConfiguredTargetLanguages(targetLanguages) {
+        const targets = [];
+        for (const [language, options] of Object.entries(targetLanguages)) {
+            if (!options)
+                continue;
+            targets.push([Number(language), options]);
+        }
+        return targets;
+    }
+    getTargetLanguageStrategy(targetLanguage) {
+        switch (targetLanguage) {
+            case ProtobufTargetLanguage.JAVA:
+                return new JavaProtobufTargetLanguageStrategy();
+            case ProtobufTargetLanguage.GO:
+                return new GoProtobufTargetLanguageStrategy();
+            case ProtobufTargetLanguage.RUST:
+                return new RustProtobufTargetLanguageStrategy();
+        }
+    }
+    async getBufGenerateInput(protoFilesLocation) {
+        const resolvedLocation = path.resolve(protoFilesLocation);
+        const stats = await Bun.file(resolvedLocation).stat();
+        if (stats.isFile() && !resolvedLocation.endsWith(".proto")) {
+            throw new Error(`${protoFilesLocation} is not a .proto file.`);
+        }
+        const sourceDirectory = stats.isFile() ? path.dirname(resolvedLocation) : resolvedLocation;
+        const moduleDirectory = (await this.findBufModuleDirectory(sourceDirectory)) ?? sourceDirectory;
+        return {
+            input: moduleDirectory,
+            pathFilter: moduleDirectory === resolvedLocation ? undefined : resolvedLocation,
+        };
+    }
+    async findBufModuleDirectory(directory) {
+        let currentDirectory = path.resolve(directory);
+        while (true) {
+            if (await Bun.file(path.join(currentDirectory, "buf.yaml")).exists()) {
+                return currentDirectory;
+            }
+            const parentDirectory = path.dirname(currentDirectory);
+            if (parentDirectory === currentDirectory) {
+                return null;
+            }
+            currentDirectory = parentDirectory;
+        }
+    }
+    async uploadGeneratedDirectory({ generatedDirectory, options, }) {
+        if (!options.uploadLocation) {
+            throw new Error("An upload location is required to upload generated code.");
+        }
+        const repository = this.parseGitHubRepository(options.uploadLocation, options.directory);
+        await this.githubClient.DANGEROUS__uploadDirectoryContents({
+            owner: repository.owner,
+            repository: repository.repository,
+            directory: repository.directory,
+            sourceDirectory: generatedDirectory,
+            message: options.commitMessage ?? "chore: update generated protobuf clients",
+            branch: options.branch,
+        });
+    }
+    parseGitHubRepository(uploadLocation, directory) {
+        const [owner, repository, ...pathParts] = uploadLocation
+            .split("/")
+            .filter(Boolean);
+        if (!owner || !repository) {
+            throw new Error("GitHub upload location must be in the form `owner/repo` or `owner/repo/path`.");
+        }
+        return {
+            directory: directory ?? pathParts.join("/"),
+            owner,
+            repository,
+        };
+    }
+}

package/dist/protobuf/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./client";
+export * from "./types";

package/dist/protobuf/index.js

@@ -0,0 +1,2 @@
+export * from "./client";
+export * from "./types";

package/dist/protobuf/strategy/go.d.ts

@@ -0,0 +1,5 @@
+import type { IProtobufTargetLanguageStrategy } from "./types";
+export declare class GoProtobufTargetLanguageStrategy implements IProtobufTargetLanguageStrategy {
+    readonly outputDirectoryName = "go";
+    readonly remotePlugin = "buf.build/protocolbuffers/go";
+}

package/dist/protobuf/strategy/go.js

@@ -0,0 +1,4 @@
+export class GoProtobufTargetLanguageStrategy {
+    outputDirectoryName = "go";
+    remotePlugin = "buf.build/protocolbuffers/go";
+}

package/dist/protobuf/strategy/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./go";
+export * from "./java";
+export * from "./rust";
+export * from "./types";

package/dist/protobuf/strategy/index.js

@@ -0,0 +1,4 @@
+export * from "./go";
+export * from "./java";
+export * from "./rust";
+export * from "./types";

package/dist/protobuf/strategy/java.d.ts

@@ -0,0 +1,5 @@
+import type { IProtobufTargetLanguageStrategy } from "./types";
+export declare class JavaProtobufTargetLanguageStrategy implements IProtobufTargetLanguageStrategy {
+    readonly outputDirectoryName = "java";
+    readonly remotePlugin = "buf.build/protocolbuffers/java";
+}

package/dist/protobuf/strategy/java.js

@@ -0,0 +1,4 @@
+export class JavaProtobufTargetLanguageStrategy {
+    outputDirectoryName = "java";
+    remotePlugin = "buf.build/protocolbuffers/java";
+}

package/dist/protobuf/strategy/rust.d.ts

@@ -0,0 +1,5 @@
+import type { IProtobufTargetLanguageStrategy } from "./types";
+export declare class RustProtobufTargetLanguageStrategy implements IProtobufTargetLanguageStrategy {
+    readonly outputDirectoryName = "rust";
+    readonly remotePlugin = "buf.build/community/neoeinstein-prost";
+}

package/dist/protobuf/strategy/rust.js

@@ -0,0 +1,4 @@
+export class RustProtobufTargetLanguageStrategy {
+    outputDirectoryName = "rust";
+    remotePlugin = "buf.build/community/neoeinstein-prost";
+}

package/dist/protobuf/strategy/types.d.ts

@@ -0,0 +1,4 @@
+export interface IProtobufTargetLanguageStrategy {
+    readonly outputDirectoryName: string;
+    readonly remotePlugin: string;
+}

package/dist/protobuf/types.d.ts

@@ -0,0 +1,47 @@
+export declare enum ProtobufSourceLanguage {
+    RUST = 0
+}
+export declare enum ProtobufTargetLanguage {
+    JAVA = 0,
+    RUST = 1,
+    GO = 2
+}
+export interface ProtobufTargetLanguageOptions {
+    /**
+     * Either `owner/repo` or `owner/repo/path/inside/repo`.
+     *
+     * @example "Patina-Network/proto-clients/java"
+     */
+    uploadLocation?: string;
+    /** Overrides the path parsed from `uploadLocation`. */
+    directory?: string;
+    branch?: string;
+    commitMessage?: string;
+}
+export type ProtobufTargetLanguages = Partial<Record<ProtobufTargetLanguage, ProtobufTargetLanguageOptions>>;
+export interface ProtobufCompilerCompileArgs {
+    sourceLanguage: ProtobufSourceLanguage;
+    /** Optional BSR API token used for Buf remote plugins. */
+    bufToken?: string;
+    /**
+     * A map of target language to target-specific options.
+     *
+     * @example
+     * ```ts
+     * {
+     *   [ProtobufTargetLanguage.JAVA]: {
+     *     uploadLocation: "Patina-Network/proto-clients/java",
+     *   },
+     * }
+     * ```
+     */
+    targetLanguages: ProtobufTargetLanguages;
+    /** Directory containing proto files, or one specific `.proto` file. */
+    protoFilesLocation: string;
+    /** Defaults to a temporary directory. */
+    outputDirectory?: string;
+}
+export interface ProtobufCompilerCompileResult {
+    outputDirectory: string;
+    generatedDirectories: Partial<Record<ProtobufTargetLanguage, string>>;
+}

package/dist/protobuf/types.js

@@ -0,0 +1,10 @@
+export var ProtobufSourceLanguage;
+(function (ProtobufSourceLanguage) {
+    ProtobufSourceLanguage[ProtobufSourceLanguage["RUST"] = 0] = "RUST";
+})(ProtobufSourceLanguage || (ProtobufSourceLanguage = {}));
+export var ProtobufTargetLanguage;
+(function (ProtobufTargetLanguage) {
+    ProtobufTargetLanguage[ProtobufTargetLanguage["JAVA"] = 0] = "JAVA";
+    ProtobufTargetLanguage[ProtobufTargetLanguage["RUST"] = 1] = "RUST";
+    ProtobufTargetLanguage[ProtobufTargetLanguage["GO"] = 2] = "GO";
+})(ProtobufTargetLanguage || (ProtobufTargetLanguage = {}));

package/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "author": "Tahmid Ahmed",
   "description": "A collection of Bun shell scripts that can be re-used in various CICD pipelines.",
-  "version": "1.0.75-beta.6f4bf3b7",
+  "version": "1.0.76-beta.f1e3e65b",
   "repository": {
     "url": "git+https://github.com/tahminator/pipeline.git"
   },
@@ -21,14 +21,13 @@
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "bun run typecheck && bun run fmt && bun run lint",
+    "test": "bun run typecheck && bun run fmt && bun run lint && bun test",
     "fmt": "bun run prettier",
     "lint": "bun run eslint",
     "eslint": "eslint .",
     "eslint:fix": "eslint . --fix",
     "prettier": "prettier --check .",
     "prettier:fix": "prettier --write .",
-    "vt": "vitest run",
     "build": "tsc -p tsconfig.build.json"
   },
   "devDependencies": {