@pepys/source-github 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,33 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/source-github</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/source-github"><img src="https://img.shields.io/npm/v/@pepys/source-github" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a> •
+    <a href="../../docs/design/v1-plug-ins.md">The plug-ins of v1</a>
+  </p>
+</div>
+
+The pepys GitHub plug-in, for Bun.
+
+The plug-in receives GitHub's webhook deliveries as facts, and builds pull request rows from them
+with its consumer.
+
+## Install
+
+```sh
+bun add @pepys/source-github
+```
+
+## Features
+
+- `githubPlugin`: the plug-in, to add to a deployment.
+- `githubPullRequestConsumer`: the consumer of pull request facts, with its SQL schema.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of the plug-ins.

package/package.json

@@ -1,6 +1,35 @@
 {
   "name": "@pepys/source-github",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "The pepys GitHub plug-in: pull requests, releases, commits and deploys",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/source-github"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pepys/api": "0.1.0",
+    "@pepys/ingest": "0.1.0",
+    "@pepys/kit": "0.1.0",
+    "@zgeoff/std": "0.4.3",
+    "kysely": "0.29.8",
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@msw/data": "1.1.8",
+    "@pepys/testing": "0.1.0",
+    "msw": "3.0.3",
+    "tiny-invariant": "1.3.3"
+  }
+}

package/src/api/serve-api.ts

@@ -0,0 +1,8 @@
+import { serveAPI } from '@pepys/api';
+import { defineDeployment } from '@pepys/kit';
+import { githubPlugin } from '../github-plugin.ts';
+
+serveAPI({
+  deployment: defineDeployment({ name: 'github', plugins: [githubPlugin] }),
+  environment: process.env,
+});

package/src/consumers/github-deploy.ts

@@ -0,0 +1,199 @@
+import {
+  initGithubDeployModel,
+  readGithubDeployRow,
+  settings,
+  toGithubDeployID,
+  updateGithubDeployModel,
+} from '../generated/github-deploy-boundary.ts';
+import type {
+  EntityID,
+  GithubDeployMessage,
+  GithubDeployModel,
+  GithubDeployOperation,
+  GithubDeployRow,
+} from '../generated/github-deploy-boundary.ts';
+import type { DeploySignal } from '../utils/deploy-signals.ts';
+import { findField } from '../utils/find-field.ts';
+
+export type {
+  GithubDeployMessage,
+  GithubDeployModel,
+} from '../generated/github-deploy-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): GithubDeployModel {
+  return initGithubDeployModel();
+}
+
+export function update(model: GithubDeployModel, message: GithubDeployMessage): GithubDeployModel {
+  return updateGithubDeployModel(model, message);
+}
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+interface DeployReport {
+  repositoryID: number;
+  environment: string;
+  target: string | null;
+  sha: string;
+  signal: DeploySignal;
+  at: string;
+}
+
+// The row says "this signal reported this commit", so it keeps the earliest time: a second report
+// of one commit, in any order, changes nothing.
+function mapFact(factJSON: string): GithubDeployOperation[] {
+  const fact: unknown = JSON.parse(factJSON);
+  const report = pickDeployReport(fact);
+
+  if (report === undefined) {
+    return [];
+  }
+
+  const target = report.target === null ? '' : `:${report.target}`;
+
+  const id = toGithubDeployID(
+    `${report.repositoryID}:${report.environment}:${report.signal}:${report.sha}${target}`,
+  );
+
+  const stored = readGithubDeployRow(id);
+
+  if (stored !== undefined && stored.deployed_at <= report.at) {
+    return [];
+  }
+
+  const row: GithubDeployRow = {
+    id,
+    repository_id: toEntityID(`github:repo:${report.repositoryID}`),
+    environment: report.environment,
+    target: report.target,
+    sha: report.sha,
+    signal: report.signal,
+    deployed_at: report.at,
+  };
+
+  return [{ upsert: { github_deploy: row } }];
+}
+
+function pickDeployReport(fact: unknown): DeployReport | undefined {
+  const type = findField(fact, 'type');
+  const at = findField(fact, 'at');
+  const payload = findField(fact, 'payload');
+
+  if (typeof at !== 'string') {
+    return undefined;
+  }
+
+  if (type === 'github.ref.pushed') {
+    return pickTagReport(payload, at);
+  }
+
+  if (type === 'github.deployment.status_created') {
+    return pickStatusReport(payload, at);
+  }
+
+  return undefined;
+}
+
+// A push that moves a tag of the deploy tag pattern to a commit. A push that deletes the tag names
+// no commit.
+function pickTagReport(payload: unknown, at: string): DeployReport | undefined {
+  const ref = findField(payload, 'ref');
+  const sha = findField(payload, 'head_commit', 'id');
+  const repositoryID = findField(payload, 'repository', 'id');
+
+  if (typeof ref !== 'string' || !ref.startsWith('refs/tags/') || typeof sha !== 'string') {
+    return undefined;
+  }
+
+  if (findField(payload, 'deleted') === true || typeof repositoryID !== 'number') {
+    return undefined;
+  }
+
+  const tag = findTagDeploy(ref.slice('refs/tags/'.length), settings.deployTagPattern);
+
+  if (tag === undefined) {
+    return undefined;
+  }
+
+  return { repositoryID, ...tag, sha, signal: 'tag', at };
+}
+
+// The tag is the pattern with an environment in place of `<environment>`, and may end with
+// `/<target>`. Neither part is empty or holds a `/`.
+function findTagDeploy(
+  tag: string,
+  pattern: string,
+): { environment: string; target: string | null } | undefined {
+  const marker = '<environment>';
+  const markerAt = pattern.indexOf(marker);
+  const before = pattern.slice(0, markerAt);
+  const after = pattern.slice(markerAt + marker.length);
+
+  if (markerAt === -1 || !tag.startsWith(before)) {
+    return undefined;
+  }
+
+  const rest = tag.slice(before.length);
+  const environmentEnd = after === '' ? getPartEnd(rest) : rest.indexOf(after);
+
+  if (environmentEnd <= 0) {
+    return undefined;
+  }
+
+  const environment = rest.slice(0, environmentEnd);
+  const tail = rest.slice(environmentEnd + after.length);
+
+  if (environment.includes('/')) {
+    return undefined;
+  }
+
+  if (tail === '') {
+    return { environment, target: null };
+  }
+
+  const target = tail.slice(1);
+
+  if (!tail.startsWith('/') || target === '' || target.includes('/')) {
+    return undefined;
+  }
+
+  return { environment, target };
+}
+
+function getPartEnd(text: string): number {
+  const slash = text.indexOf('/');
+
+  return slash === -1 ? text.length : slash;
+}
+
+function pickStatusReport(payload: unknown, at: string): DeployReport | undefined {
+  const state = findField(payload, 'deployment_status', 'state');
+  const environment = findField(payload, 'deployment', 'environment');
+  const sha = findField(payload, 'deployment', 'sha');
+  const repositoryID = findField(payload, 'repository', 'id');
+
+  if (state !== 'success' || typeof environment !== 'string' || environment === '') {
+    return undefined;
+  }
+
+  if (typeof sha !== 'string' || typeof repositoryID !== 'number') {
+    return undefined;
+  }
+
+  return { repositoryID, environment, target: null, sha, signal: 'deployment_status', at };
+}
+
+function isEntityID(text: string): text is EntityID {
+  return text.length > 0;
+}
+
+function toEntityID(text: string): EntityID {
+  if (!isEntityID(text)) {
+    throw new Error('an entity ID is never empty');
+  }
+
+  return text;
+}

package/src/consumers/github-pull-request.ts

@@ -0,0 +1,275 @@
+import {
+  initGithubPullRequestModel,
+  readGithubPullRequestRow,
+  toGithubPullRequestID,
+  updateGithubPullRequestModel,
+} from '../generated/github-pull-request-boundary.ts';
+import type {
+  EntityID,
+  GithubPullRequestMessage,
+  GithubPullRequestModel,
+  GithubPullRequestOperation,
+  GithubPullRequestRow,
+  GithubReleaseBumpRow,
+} from '../generated/github-pull-request-boundary.ts';
+import { findField } from '../utils/find-field.ts';
+
+export type {
+  GithubPullRequestMessage,
+  GithubPullRequestModel,
+} from '../generated/github-pull-request-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): GithubPullRequestModel {
+  return initGithubPullRequestModel();
+}
+
+export function update(
+  model: GithubPullRequestModel,
+  message: GithubPullRequestMessage,
+): GithubPullRequestModel {
+  return updateGithubPullRequestModel(model, message);
+}
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+function mapFact(factJSON: string): GithubPullRequestOperation[] {
+  const row = readRow(factJSON);
+
+  if (row === undefined) {
+    return [];
+  }
+
+  const stored = readGithubPullRequestRow(row.id);
+
+  if (stored !== undefined && isStale(stored, row)) {
+    return [];
+  }
+
+  // The bump rows of the replaced body come from the stored payload, so a late fact of an older
+  // body cannot bring back a bump that a newer body dropped.
+  const given = collectReleaseBumps(row.id, row.payload);
+  const replaced = stored === undefined ? [] : collectReleaseBumps(row.id, stored.payload);
+
+  const givenKeys = new Set(given.map((bump) => JSON.stringify(bump)));
+  const replacedKeys = new Set(replaced.map((bump) => JSON.stringify(bump)));
+
+  const removed: GithubPullRequestOperation[] = replaced
+    .filter((bump) => !givenKeys.has(JSON.stringify(bump)))
+    .map((bump) => ({ delete: { github_release_bump: bump } }));
+
+  const added: GithubPullRequestOperation[] = given
+    .filter((bump) => !replacedKeys.has(JSON.stringify(bump)))
+    .map((bump) => ({ upsert: { github_release_bump: bump } }));
+
+  return [{ upsert: { github_pull_request: row } }, ...removed, ...added];
+}
+
+const releaseURLStart = 'https://github.com/';
+const releaseURLEnds = [' ', '\n', '\r', '\t', ')', ']', '>', '"', "'", '`', '#', '?'];
+
+// One row per `https://github.com/<owner>/<name>/releases/tag/<tag>` URL in the body of a pull
+// request whose author is a bot, each once, with the tag percent-decoded.
+function collectReleaseBumps(
+  pullRequestID: GithubReleaseBumpRow['pull_request_id'],
+  pullRequest: unknown,
+): GithubReleaseBumpRow[] {
+  const body = findField(pullRequest, 'body');
+
+  if (findField(pullRequest, 'user', 'type') !== 'Bot' || typeof body !== 'string') {
+    return [];
+  }
+
+  const bumps: GithubReleaseBumpRow[] = [];
+  const seen: string[] = [];
+
+  for (const part of body.split(releaseURLStart).slice(1)) {
+    const bump = parseReleaseURLPath(part);
+
+    if (bump === undefined || seen.includes(`${bump.repository}\n${bump.tag}`)) {
+      continue;
+    }
+
+    seen.push(`${bump.repository}\n${bump.tag}`);
+
+    bumps.push({
+      pull_request_id: pullRequestID,
+      release_repository: bump.repository,
+      release_tag: bump.tag,
+    });
+  }
+
+  return bumps;
+}
+
+// `<owner>/<name>/releases/tag/<tag>`, up to the first character that ends a URL in text.
+function parseReleaseURLPath(text: string): { repository: string; tag: string } | undefined {
+  let end = text.length;
+
+  for (const mark of releaseURLEnds) {
+    const at = text.indexOf(mark);
+
+    if (at !== -1 && at < end) {
+      end = at;
+    }
+  }
+
+  const parts = text.slice(0, end).split('/');
+  const [owner, name, releases, tagWord, tag] = parts;
+
+  if (parts.length !== 5 || releases !== 'releases' || tagWord !== 'tag') {
+    return undefined;
+  }
+
+  if (owner === undefined || name === undefined || tag === undefined) {
+    return undefined;
+  }
+
+  if (owner === '' || name === '' || tag === '') {
+    return undefined;
+  }
+
+  const decoded = decodeTag(tag);
+
+  return decoded === undefined ? undefined : { repository: `${owner}/${name}`, tag: decoded };
+}
+
+function decodeTag(tag: string): string | undefined {
+  try {
+    return decodeURIComponent(tag);
+  } catch {
+    return undefined;
+  }
+}
+
+// Two updates can share one second. A repeat of the stored row changes nothing, and a different
+// row with the same update time is a distinct update, so the later fact in the stream wins. The
+// host skips a fact it has applied before, so a redelivery never reaches this check.
+function isStale(stored: GithubPullRequestRow, row: GithubPullRequestRow): boolean {
+  if (stored.updated_at !== row.updated_at) {
+    return stored.updated_at > row.updated_at;
+  }
+
+  return JSON.stringify(stored) === JSON.stringify(row);
+}
+
+function readRow(factJSON: string): GithubPullRequestRow | undefined {
+  const fact: unknown = JSON.parse(factJSON);
+
+  if (!isObject(fact) || !('payload' in fact) || !isObject(fact.payload)) {
+    return undefined;
+  }
+
+  if (!('pull_request' in fact.payload) || !isObject(fact.payload.pull_request)) {
+    return undefined;
+  }
+
+  const pullRequest = fact.payload.pull_request;
+
+  if (!('id' in pullRequest) || typeof pullRequest.id !== 'number') {
+    return undefined;
+  }
+
+  if (!('number' in pullRequest) || typeof pullRequest.number !== 'number') {
+    return undefined;
+  }
+
+  if (!('title' in pullRequest) || typeof pullRequest.title !== 'string') {
+    return undefined;
+  }
+
+  if (!('state' in pullRequest) || typeof pullRequest.state !== 'string') {
+    return undefined;
+  }
+
+  if (!('html_url' in pullRequest) || typeof pullRequest.html_url !== 'string') {
+    return undefined;
+  }
+
+  if (!('created_at' in pullRequest) || typeof pullRequest.created_at !== 'string') {
+    return undefined;
+  }
+
+  if (!('updated_at' in pullRequest) || typeof pullRequest.updated_at !== 'string') {
+    return undefined;
+  }
+
+  const repositoryID = readRepositoryID(pullRequest);
+
+  if (repositoryID === undefined) {
+    return undefined;
+  }
+
+  return {
+    id: toGithubPullRequestID(`${pullRequest.id}`),
+    number: pullRequest.number,
+    repository_id: toEntityID(`github:repo:${repositoryID}`),
+    title: pullRequest.title,
+    state: pullRequest.state,
+    is_draft: 'draft' in pullRequest && pullRequest.draft === true,
+    is_merged: 'merged' in pullRequest && pullRequest.merged === true,
+    author_id: readAuthorID(pullRequest),
+    url: pullRequest.html_url,
+    merge_commit_sha:
+      'merge_commit_sha' in pullRequest && typeof pullRequest.merge_commit_sha === 'string'
+        ? pullRequest.merge_commit_sha
+        : null,
+    created_at: pullRequest.created_at,
+    updated_at: pullRequest.updated_at,
+    closed_at:
+      'closed_at' in pullRequest && typeof pullRequest.closed_at === 'string'
+        ? pullRequest.closed_at
+        : null,
+    merged_at:
+      'merged_at' in pullRequest && typeof pullRequest.merged_at === 'string'
+        ? pullRequest.merged_at
+        : null,
+    payload: pullRequest,
+  };
+}
+
+function readRepositoryID(pullRequest: object): number | undefined {
+  if (!('base' in pullRequest) || !isObject(pullRequest.base)) {
+    return undefined;
+  }
+
+  if (!('repo' in pullRequest.base) || !isObject(pullRequest.base.repo)) {
+    return undefined;
+  }
+
+  if (!('id' in pullRequest.base.repo) || typeof pullRequest.base.repo.id !== 'number') {
+    return undefined;
+  }
+
+  return pullRequest.base.repo.id;
+}
+
+function readAuthorID(pullRequest: object): EntityID | null {
+  if (!('user' in pullRequest) || !isObject(pullRequest.user)) {
+    return null;
+  }
+
+  if (!('id' in pullRequest.user) || typeof pullRequest.user.id !== 'number') {
+    return null;
+  }
+
+  return toEntityID(`github:user:${pullRequest.user.id}`);
+}
+
+function isObject(value: unknown): value is object {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isEntityID(text: string): text is EntityID {
+  return text.length > 0;
+}
+
+function toEntityID(text: string): EntityID {
+  if (!isEntityID(text)) {
+    throw new Error('an entity ID is never empty');
+  }
+
+  return text;
+}

package/src/consumers/github-ref.ts

@@ -0,0 +1,88 @@
+import { initGithubRefModel, updateGithubRefModel } from '../generated/github-ref-boundary.ts';
+import type {
+  EntityID,
+  GithubCommitRow,
+  GithubRefMessage,
+  GithubRefModel,
+  GithubRefOperation,
+} from '../generated/github-ref-boundary.ts';
+import { findField } from '../utils/find-field.ts';
+
+export type { GithubRefMessage, GithubRefModel } from '../generated/github-ref-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): GithubRefModel {
+  return initGithubRefModel();
+}
+
+export function update(model: GithubRefModel, message: GithubRefMessage): GithubRefModel {
+  return updateGithubRefModel(model, message);
+}
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+// A commit's time is a property of the commit, so a row never changes, and a force push that drops
+// a commit leaves a row that is still true.
+function mapFact(factJSON: string): GithubRefOperation[] {
+  const payload = findField(JSON.parse(factJSON), 'payload');
+  const ref = findField(payload, 'ref');
+  const repositoryID = findField(payload, 'repository', 'id');
+  const defaultBranch = findField(payload, 'repository', 'default_branch');
+
+  if (typeof ref !== 'string' || typeof repositoryID !== 'number') {
+    return [];
+  }
+
+  const commits = collectPushedCommits(payload, ref, defaultBranch);
+  const repository = toEntityID(`github:repo:${repositoryID}`);
+
+  return commits.flatMap((commit) => {
+    const row = buildCommitRow(repository, commit);
+
+    return row === undefined ? [] : [{ upsert: { github_commit: row } }];
+  });
+}
+
+// A tag push gives its head commit; a push to the default branch gives every commit it adds.
+function collectPushedCommits(payload: unknown, ref: string, defaultBranch: unknown): unknown[] {
+  if (ref.startsWith('refs/tags/')) {
+    return [findField(payload, 'head_commit')];
+  }
+
+  if (typeof defaultBranch === 'string' && ref === `refs/heads/${defaultBranch}`) {
+    return toList(findField(payload, 'commits'));
+  }
+
+  return [];
+}
+
+function toList(value: unknown): unknown[] {
+  return Array.isArray(value) ? value.map((item: unknown) => item) : [];
+}
+
+// The time as GitHub writes it, with the committer's offset: scriptc fences `Date`, so a reader
+// compares these times as times, never as text.
+function buildCommitRow(repository: EntityID, commit: unknown): GithubCommitRow | undefined {
+  const sha = findField(commit, 'id');
+  const timestamp = findField(commit, 'timestamp');
+
+  if (typeof sha !== 'string' || typeof timestamp !== 'string') {
+    return undefined;
+  }
+
+  return { repository_id: repository, sha, committed_at: timestamp };
+}
+
+function isEntityID(text: string): text is EntityID {
+  return text.length > 0;
+}
+
+function toEntityID(text: string): EntityID {
+  if (!isEntityID(text)) {
+    throw new Error('an entity ID is never empty');
+  }
+
+  return text;
+}