@avi2dg/checks 0.31.0 → 0.32.0

package/src/delivery/release.ts

@@ -0,0 +1,79 @@
+import { Effect, Schema } from "effect";
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { groupOf } from "./changelog.ts";
+import { git } from "../core/git.ts";
+
+class ReleaseUnreadable extends Schema.TaggedError<ReleaseUnreadable>()("ReleaseUnreadable", {
+  message: Schema.String,
+}) {}
+
+class BuildFailed extends Schema.TaggedError<BuildFailed>()("BuildFailed", {
+  message: Schema.String,
+}) {}
+
+export type Pending = {
+  readonly since: string | undefined;
+  readonly unreleased: readonly string[];
+};
+
+const RELEASE_TAG = "v[0-9]*.[0-9]*.[0-9]*";
+const PLAIN_VERSION = /^(\d+)\.(\d+)\.(\d+)$/;
+const RELEASE_SUBJECT = /^chore: release (\d+\.\d+\.\d+)(?: \(#\d+\))?$/;
+
+export function unreleasedOf(subjects: readonly string[]): readonly string[] {
+  return subjects.filter((subject) => groupOf(subject) !== undefined);
+}
+
+// Under 0.x a breaking change moves the minor, as semver leaves a 0.x version free to break.
+export function nextVersion(version: string, unreleased: readonly string[]): string | undefined {
+  const [, major, minor, patch] = PLAIN_VERSION.exec(version)?.map(Number) ?? [];
+  if (major === undefined || minor === undefined || patch === undefined) return undefined;
+  const groups = new Set(unreleased.map(groupOf));
+  if (groups.has("Breaking changes") && major > 0) return `${major + 1}.0.0`;
+  if (groups.has("Breaking changes") || groups.has("Features")) return `${major}.${minor + 1}.0`;
+  return `${major}.${minor}.${patch + 1}`;
+}
+
+// Rewrites only the version's own text, so the rest of the manifest keeps the formatting its repository gave it.
+export function withVersion(manifest: string, from: string, to: string): string | undefined {
+  const field = new RegExp(`("version"\\s*:\\s*")${from.replaceAll(".", "\\.")}(")`);
+  return field.test(manifest) ? manifest.replace(field, `$1${to}$2`) : undefined;
+}
+
+export function releaseTitle(version: string): string {
+  return `chore: release ${version}`;
+}
+
+export function releasedVersionOf(subject: string): string | undefined {
+  return RELEASE_SUBJECT.exec(subject)?.[1];
+}
+
+export const runBuild = Effect.fn("runBuild")(function* (root: string) {
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const exitCode = yield* spawner.exitCode(
+    ChildProcess.make(process.execPath, ["run", "build"], { cwd: root, stdin: "ignore", stdout: "inherit", stderr: "inherit" }),
+  );
+  if (exitCode !== ChildProcessSpawner.ExitCode(0)) return yield* new BuildFailed({ message: `bun run build exited ${exitCode}` });
+});
+
+export const lastTag = Effect.fn("lastTag")(function* (root: string) {
+  const described = yield* git(["describe", "--tags", "--abbrev=0", `--match=${RELEASE_TAG}`, "HEAD"], root).pipe(
+    Effect.map((tag) => tag.trim()),
+    Effect.catchTag("GitFailure", () => Effect.succeed("")),
+  );
+  return described === "" ? undefined : described;
+});
+
+const subjectsSince = Effect.fn("subjectsSince")(function* (root: string, since: string | undefined) {
+  const range = since === undefined ? "HEAD" : `${since}..HEAD`;
+  const log = yield* git(["log", "--topo-order", "--no-merges", "--format=%s", range], root);
+  return log.split("\n").filter((subject) => subject !== "");
+});
+
+export const readPending = Effect.fn("readPending")(function* (root: string) {
+  if ((yield* git(["rev-parse", "--is-shallow-repository"], root)).trim() === "true") {
+    return yield* new ReleaseUnreadable({ message: "the checkout is shallow, so the tag it sees may not be the last one; fetch all of it" });
+  }
+  const since = yield* lastTag(root);
+  return { since, unreleased: unreleasedOf(yield* subjectsSince(root, since)) } satisfies Pending;
+});

package/src/delivery/secrets.ts

@@ -0,0 +1,86 @@
+#!/usr/bin/env bun
+import { Console, Effect, FileSystem, Path, Schema } from "effect";
+import { commitOf, git, refArgs } from "../core/git.ts";
+import { runMain } from "../core/main.ts";
+import { cacheRoot } from "../dependencies/cache-root.ts";
+import { pinnedGitleaks, scanCommits, type Leak } from "./gitleaks.ts";
+
+const NAME = "secrets";
+const USAGE = "usage: secrets.ts <ref> | <base-ref> <head-ref>";
+const SHORT_SHA = 8;
+const NEW_SIDE_PREFIX = "b/";
+const OCTOPUS_PARENTS = 3;
+
+class OctopusMerge extends Schema.TaggedError<OctopusMerge>()("OctopusMerge", {
+  message: Schema.String,
+}) {}
+
+// git reads a lone commit as its whole ancestry, and -1 limits only what a walk prints, so a single ref is bounded with --no-walk.
+const revisionsOf = Effect.fn("revisionsOf")(function* (args: readonly string[], root: string) {
+  const { first, second } = yield* refArgs(args, USAGE);
+  if (second === undefined) return ["--no-walk", yield* commitOf(first, root)];
+  const head = yield* commitOf(second, root);
+  const base = (yield* git(["merge-base", yield* commitOf(first, root), head], root)).trim();
+  return [`${base}..${head}`];
+});
+
+// Without --remerge-diff git log prints no diff for a merge, and a first-parent diff would rescan what the merge brings in.
+// git skips --remerge-diff for an octopus merge with only a warning, so the scan would pass a commit it never read.
+const logOptionsOf = Effect.fn("logOptionsOf")(function* (args: readonly string[], root: string) {
+  const revisions = yield* revisionsOf(args, root);
+  const [octopus] = (yield* git(["rev-list", `--min-parents=${OCTOPUS_PARENTS}`, ...revisions], root)).split("\n");
+  if (octopus !== undefined && octopus !== "") {
+    return yield* new OctopusMerge({
+      message: `git gives no resolution diff for the octopus merge ${octopus}, so the gate cannot scan it`,
+    });
+  }
+  return ["--remerge-diff", ...revisions].join(" ");
+});
+
+// gitleaks names a file under a remerge conflict header by its `+++ b/` line, so a path the commit does not hold carries that prefix.
+const atRepositoryPath = Effect.fn("atRepositoryPath")(function* (leak: Leak, root: string) {
+  if (!leak.File.startsWith(NEW_SIDE_PREFIX)) return leak;
+  const held = yield* git(["cat-file", "-e", `${leak.Commit}:${leak.File}`], root).pipe(
+    Effect.as(true),
+    Effect.catchTag("GitFailure", () => Effect.succeed(false)),
+  );
+  return held ? leak : { ...leak, File: leak.File.slice(NEW_SIDE_PREFIX.length) };
+});
+
+function leakLine({ File, StartLine, RuleID, Commit, Description }: Leak): string {
+  return `  ${File}:${StartLine} ${RuleID} in ${Commit.slice(0, SHORT_SHA)}: ${Description}`;
+}
+
+function byPlace(left: Leak, right: Leak): number {
+  return (
+    left.File.localeCompare(right.File) ||
+    left.StartLine - right.StartLine ||
+    left.RuleID.localeCompare(right.RuleID) ||
+    left.Commit.localeCompare(right.Commit)
+  );
+}
+
+export function report(leaks: readonly Leak[]): string {
+  if (leaks.length === 0) return `${NAME}: the range adds no secret`;
+  return [
+    `${NAME}: the range adds ${leaks.length} secret(s); put a placeholder such as <private-key> in its place in the commit that added it, and rotate any secret that left this machine:`,
+    ...leaks.toSorted(byPlace).map(leakLine),
+  ].join("\n");
+}
+
+const secrets = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const root = (yield* git(["rev-parse", "--show-toplevel"])).trim();
+  const logOptions = yield* logOptionsOf(process.argv.slice(2), root);
+  const binary = yield* pinnedGitleaks(yield* cacheRoot());
+  const config = path.join(import.meta.dir, "gitleaks.toml");
+  const ignoreDir = yield* fs.makeTempDirectoryScoped({ prefix: "checks-secrets-" });
+  const gitDir = (yield* git(["rev-parse", "--absolute-git-dir"], root)).trim();
+  const reported = yield* scanCommits({ binary, gitDir, logOptions, config, ignoreDir });
+  const leaks = yield* Effect.forEach(reported, (leak) => atRepositoryPath(leak, root));
+  yield* Console.log(report(leaks));
+  return leaks.length === 0;
+}).pipe(Effect.scoped);
+
+if (import.meta.main) runMain(NAME, secrets);

package/src/dependencies/osv-scanner.ts

@@ -1,11 +1,9 @@
-import { Clock, Context, Crypto, Effect, Encoding, FileSystem, Layer, Option, Path, Schema } from "effect";
-import { HttpClient, HttpClientResponse } from "effect/unstable/http";
-import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient";
+import { Clock, Context, Effect, FileSystem, Layer, Option, Path, Schema } from "effect";
 import { collect } from "../core/git.ts";
+import { installPinned } from "./pinned-binary.ts";
 
 export const OSV_SCANNER_VERSION = "2.6.0";
 const RELEASES = `https://github.com/google/osv-scanner/releases/download/v${OSV_SCANNER_VERSION}`;
-const EXECUTABLE_MODE = 0o755;
 const HOUR_MS = 3_600_000;
 const DAY_MS = 24 * HOUR_MS;
 export const REFRESH_HOURS = 24;
@@ -33,46 +31,6 @@ export function buildFor(platform: string, arch: string): Option.Option<Build> {
   return Option.fromNullishOr(BUILDS[`${platform}-${arch}`]);
 }
 
-const sha256Of = Effect.fn("sha256Of")(function* (bytes: Uint8Array) {
-  const crypto = yield* Crypto.Crypto;
-  return Encoding.encodeHex(yield* crypto.digest("SHA-256", bytes));
-});
-
-const verified = Effect.fn("verified")(function* (binary: string, sha256: string) {
-  const fs = yield* FileSystem.FileSystem;
-  const found = yield* sha256Of(yield* fs.readFile(binary));
-  if (found !== sha256) {
-    return yield* new OsvScannerError({ message: `${binary} has SHA-256 ${found}, not the pinned ${sha256}; delete it and rerun` });
-  }
-  return binary;
-});
-
-const download = Effect.fn("download")(function* (url: string) {
-  const response = yield* HttpClient.get(url).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk));
-  return new Uint8Array(yield* response.arrayBuffer);
-}, Effect.provide(FetchHttpClient.layer));
-
-// The binary lands through a rename in its own directory, so a concurrent run sees it whole or not at all.
-export const installPinned = Effect.fn("installPinned")(function* (url: string, sha256: string, binary: string) {
-  const fs = yield* FileSystem.FileSystem;
-  const path = yield* Path.Path;
-  if (yield* fs.exists(binary)) return yield* verified(binary, sha256);
-  const bytes = yield* download(url).pipe(
-    Effect.mapError((cause) => new OsvScannerError({ message: `cannot download ${url}: ${cause.message}` })),
-  );
-  const found = yield* sha256Of(bytes);
-  if (found !== sha256) {
-    return yield* new OsvScannerError({ message: `${url} has SHA-256 ${found}, not the pinned ${sha256}, so nothing was installed` });
-  }
-  yield* fs.makeDirectory(path.dirname(binary), { recursive: true });
-  const staging = yield* fs.makeTempDirectoryScoped({ directory: path.dirname(binary), prefix: `.${path.basename(binary)}-` });
-  const staged = path.join(staging, path.basename(binary));
-  yield* fs.writeFile(staged, bytes);
-  yield* fs.chmod(staged, EXECUTABLE_MODE);
-  yield* fs.rename(staged, binary);
-  return binary;
-}, Effect.scoped);
-
 const pinnedBinary = Effect.fn("pinnedBinary")(function* (cache: string) {
   const build = buildFor(process.platform, process.arch);
   if (Option.isNone(build)) {
@@ -80,7 +38,7 @@ const pinnedBinary = Effect.fn("pinnedBinary")(function* (cache: string) {
   }
   const path = yield* Path.Path;
   const { asset, sha256 } = build.value;
-  return yield* installPinned(`${RELEASES}/${asset}`, sha256, path.join(cache, "osv-scanner", OSV_SCANNER_VERSION, asset));
+  return yield* installPinned({ kind: "binary", url: `${RELEASES}/${asset}`, sha256 }, path.join(cache, "osv-scanner", OSV_SCANNER_VERSION, asset));
 });
 
 type PinnedBinary = ReturnType<typeof pinnedBinary>;

package/src/dependencies/pinned-binary.ts

@@ -0,0 +1,79 @@
+import { Crypto, Effect, Encoding, FileSystem, Path, Schema } from "effect";
+import { HttpClient, HttpClientResponse } from "effect/unstable/http";
+import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient";
+import { collect } from "../core/git.ts";
+
+const EXECUTABLE_MODE = 0o755;
+
+export type Asset =
+  | { readonly kind: "binary"; readonly url: string; readonly sha256: string }
+  | { readonly kind: "tar.gz"; readonly url: string; readonly sha256: string; readonly member: string; readonly memberSha256: string };
+
+class PinnedBinaryError extends Schema.TaggedError<PinnedBinaryError>()("PinnedBinaryError", {
+  message: Schema.String,
+}) {}
+
+function installedSha256(asset: Asset): string {
+  return asset.kind === "binary" ? asset.sha256 : asset.memberSha256;
+}
+
+const sha256Of = Effect.fn("sha256Of")(function* (bytes: Uint8Array) {
+  const crypto = yield* Crypto.Crypto;
+  return Encoding.encodeHex(yield* crypto.digest("SHA-256", bytes));
+});
+
+const verified = Effect.fn("verified")(function* (binary: string, sha256: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const found = yield* sha256Of(yield* fs.readFile(binary));
+  if (found !== sha256) {
+    return yield* new PinnedBinaryError({ message: `${binary} has SHA-256 ${found}, not the pinned ${sha256}; delete it and rerun` });
+  }
+  return binary;
+});
+
+const download = Effect.fn("download")(function* (url: string) {
+  const response = yield* HttpClient.get(url).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk));
+  return new Uint8Array(yield* response.arrayBuffer);
+}, Effect.provide(FetchHttpClient.layer));
+
+const unpacked = Effect.fn("unpacked")(function* (asset: Asset, bytes: Uint8Array, staging: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const staged = path.join(staging, path.basename(asset.url));
+  yield* fs.writeFile(staged, bytes);
+  if (asset.kind === "binary") return staged;
+  const { stderr, exitCode } = yield* collect("tar", ["-xzf", staged, "-C", staging, asset.member]).pipe(
+    Effect.mapError((cause) => new PinnedBinaryError({ message: `cannot run tar: ${cause.message}` })),
+  );
+  if (exitCode !== 0) {
+    return yield* new PinnedBinaryError({ message: `cannot unpack ${asset.member} from ${asset.url}: ${stderr.trim()}` });
+  }
+  const member = path.join(staging, asset.member);
+  const found = yield* sha256Of(yield* fs.readFile(member));
+  if (found !== asset.memberSha256) {
+    return yield* new PinnedBinaryError({
+      message: `${asset.member} in ${asset.url} has SHA-256 ${found}, not the pinned ${asset.memberSha256}, so nothing was installed`,
+    });
+  }
+  return member;
+});
+
+// The binary lands through a rename in its own directory, so a concurrent run sees it whole or not at all.
+export const installPinned = Effect.fn("installPinned")(function* (asset: Asset, binary: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  if (yield* fs.exists(binary)) return yield* verified(binary, installedSha256(asset));
+  const bytes = yield* download(asset.url).pipe(
+    Effect.mapError((cause) => new PinnedBinaryError({ message: `cannot download ${asset.url}: ${cause.message}` })),
+  );
+  const found = yield* sha256Of(bytes);
+  if (found !== asset.sha256) {
+    return yield* new PinnedBinaryError({ message: `${asset.url} has SHA-256 ${found}, not the pinned ${asset.sha256}, so nothing was installed` });
+  }
+  yield* fs.makeDirectory(path.dirname(binary), { recursive: true });
+  const staging = yield* fs.makeTempDirectoryScoped({ directory: path.dirname(binary), prefix: `.${path.basename(binary)}-` });
+  const staged = yield* unpacked(asset, bytes, staging);
+  yield* fs.chmod(staged, EXECUTABLE_MODE);
+  yield* fs.rename(staged, binary);
+  return binary;
+}, Effect.scoped);