@pepys/source-github 0.0.1 → 0.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/source-github",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "The pepys GitHub plug-in: pull requests, releases, commits and deploys",
   "license": "MIT",
   "repository": {
@@ -19,14 +19,17 @@
     "access": "public"
   },
   "dependencies": {
-    "@pepys/api": "0.0.1",
-    "@pepys/ingest": "0.0.1",
-    "@pepys/kit": "0.0.1",
-    "@pepys/schema": "0.0.1",
+    "@pepys/api": "0.1.0",
+    "@pepys/ingest": "0.1.0",
+    "@pepys/kit": "0.1.0",
     "@zgeoff/std": "0.4.3",
+    "kysely": "0.29.8",
     "zod": "4.6.7"
   },
   "devDependencies": {
+    "@msw/data": "1.1.8",
+    "@pepys/testing": "0.1.0",
+    "msw": "3.0.3",
     "tiny-invariant": "1.3.3"
   }
 }

package/src/api/serve-api.ts

@@ -1,37 +1,8 @@
-import { createApp, createStoreReader, openConsumerDatabase, readAPIConfig } from '@pepys/api';
-import { makeShutdownHandler } from '@pepys/schema';
-import { githubPullRequestConsumer } from '../github-pull-request-consumer.ts';
+import { serveAPI } from '@pepys/api';
+import { defineDeployment } from '@pepys/kit';
+import { githubPlugin } from '../github-plugin.ts';
 
-const config = readAPIConfig(process.env);
-
-if (!config.ok) {
-  console.error(config.failure.join('\n'));
-  process.exit(1);
-}
-
-const store = createStoreReader({
-  storeDirectory: config.value.storeDirectory,
-  openDatabase: openConsumerDatabase,
+serveAPI({
+  deployment: defineDeployment({ name: 'github', plugins: [githubPlugin] }),
+  environment: process.env,
 });
-
-const app = createApp({ store, consumers: [githubPullRequestConsumer] });
-
-app.listen({ hostname: '127.0.0.1', port: config.value.port });
-console.log(`api listening on ${app.server?.port}`);
-
-const shutdown = makeShutdownHandler({
-  stop: async () => {
-    await app.stop();
-    await store.close();
-  },
-  logError: (message, error) => {
-    console.error(message, error);
-  },
-  exit: (code) => process.exit(code),
-});
-
-for (const signal of ['SIGINT', 'SIGTERM'] as const) {
-  process.on(signal, () => {
-    void shutdown();
-  });
-}

package/src/consumers/github-deploy.ts

@@ -0,0 +1,199 @@
+import {
+  initGithubDeployModel,
+  readGithubDeployRow,
+  settings,
+  toGithubDeployID,
+  updateGithubDeployModel,
+} from '../generated/github-deploy-boundary.ts';
+import type {
+  EntityID,
+  GithubDeployMessage,
+  GithubDeployModel,
+  GithubDeployOperation,
+  GithubDeployRow,
+} from '../generated/github-deploy-boundary.ts';
+import type { DeploySignal } from '../utils/deploy-signals.ts';
+import { findField } from '../utils/find-field.ts';
+
+export type {
+  GithubDeployMessage,
+  GithubDeployModel,
+} from '../generated/github-deploy-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): GithubDeployModel {
+  return initGithubDeployModel();
+}
+
+export function update(model: GithubDeployModel, message: GithubDeployMessage): GithubDeployModel {
+  return updateGithubDeployModel(model, message);
+}
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+interface DeployReport {
+  repositoryID: number;
+  environment: string;
+  target: string | null;
+  sha: string;
+  signal: DeploySignal;
+  at: string;
+}
+
+// The row says "this signal reported this commit", so it keeps the earliest time: a second report
+// of one commit, in any order, changes nothing.
+function mapFact(factJSON: string): GithubDeployOperation[] {
+  const fact: unknown = JSON.parse(factJSON);
+  const report = pickDeployReport(fact);
+
+  if (report === undefined) {
+    return [];
+  }
+
+  const target = report.target === null ? '' : `:${report.target}`;
+
+  const id = toGithubDeployID(
+    `${report.repositoryID}:${report.environment}:${report.signal}:${report.sha}${target}`,
+  );
+
+  const stored = readGithubDeployRow(id);
+
+  if (stored !== undefined && stored.deployed_at <= report.at) {
+    return [];
+  }
+
+  const row: GithubDeployRow = {
+    id,
+    repository_id: toEntityID(`github:repo:${report.repositoryID}`),
+    environment: report.environment,
+    target: report.target,
+    sha: report.sha,
+    signal: report.signal,
+    deployed_at: report.at,
+  };
+
+  return [{ upsert: { github_deploy: row } }];
+}
+
+function pickDeployReport(fact: unknown): DeployReport | undefined {
+  const type = findField(fact, 'type');
+  const at = findField(fact, 'at');
+  const payload = findField(fact, 'payload');
+
+  if (typeof at !== 'string') {
+    return undefined;
+  }
+
+  if (type === 'github.ref.pushed') {
+    return pickTagReport(payload, at);
+  }
+
+  if (type === 'github.deployment.status_created') {
+    return pickStatusReport(payload, at);
+  }
+
+  return undefined;
+}
+
+// A push that moves a tag of the deploy tag pattern to a commit. A push that deletes the tag names
+// no commit.
+function pickTagReport(payload: unknown, at: string): DeployReport | undefined {
+  const ref = findField(payload, 'ref');
+  const sha = findField(payload, 'head_commit', 'id');
+  const repositoryID = findField(payload, 'repository', 'id');
+
+  if (typeof ref !== 'string' || !ref.startsWith('refs/tags/') || typeof sha !== 'string') {
+    return undefined;
+  }
+
+  if (findField(payload, 'deleted') === true || typeof repositoryID !== 'number') {
+    return undefined;
+  }
+
+  const tag = findTagDeploy(ref.slice('refs/tags/'.length), settings.deployTagPattern);
+
+  if (tag === undefined) {
+    return undefined;
+  }
+
+  return { repositoryID, ...tag, sha, signal: 'tag', at };
+}
+
+// The tag is the pattern with an environment in place of `<environment>`, and may end with
+// `/<target>`. Neither part is empty or holds a `/`.
+function findTagDeploy(
+  tag: string,
+  pattern: string,
+): { environment: string; target: string | null } | undefined {
+  const marker = '<environment>';
+  const markerAt = pattern.indexOf(marker);
+  const before = pattern.slice(0, markerAt);
+  const after = pattern.slice(markerAt + marker.length);
+
+  if (markerAt === -1 || !tag.startsWith(before)) {
+    return undefined;
+  }
+
+  const rest = tag.slice(before.length);
+  const environmentEnd = after === '' ? getPartEnd(rest) : rest.indexOf(after);
+
+  if (environmentEnd <= 0) {
+    return undefined;
+  }
+
+  const environment = rest.slice(0, environmentEnd);
+  const tail = rest.slice(environmentEnd + after.length);
+
+  if (environment.includes('/')) {
+    return undefined;
+  }
+
+  if (tail === '') {
+    return { environment, target: null };
+  }
+
+  const target = tail.slice(1);
+
+  if (!tail.startsWith('/') || target === '' || target.includes('/')) {
+    return undefined;
+  }
+
+  return { environment, target };
+}
+
+function getPartEnd(text: string): number {
+  const slash = text.indexOf('/');
+
+  return slash === -1 ? text.length : slash;
+}
+
+function pickStatusReport(payload: unknown, at: string): DeployReport | undefined {
+  const state = findField(payload, 'deployment_status', 'state');
+  const environment = findField(payload, 'deployment', 'environment');
+  const sha = findField(payload, 'deployment', 'sha');
+  const repositoryID = findField(payload, 'repository', 'id');
+
+  if (state !== 'success' || typeof environment !== 'string' || environment === '') {
+    return undefined;
+  }
+
+  if (typeof sha !== 'string' || typeof repositoryID !== 'number') {
+    return undefined;
+  }
+
+  return { repositoryID, environment, target: null, sha, signal: 'deployment_status', at };
+}
+
+function isEntityID(text: string): text is EntityID {
+  return text.length > 0;
+}
+
+function toEntityID(text: string): EntityID {
+  if (!isEntityID(text)) {
+    throw new Error('an entity ID is never empty');
+  }
+
+  return text;
+}

package/src/consumers/github-pull-request.ts

@@ -10,7 +10,9 @@ import type {
   GithubPullRequestModel,
   GithubPullRequestOperation,
   GithubPullRequestRow,
+  GithubReleaseBumpRow,
 } from '../generated/github-pull-request-boundary.ts';
+import { findField } from '../utils/find-field.ts';
 
 export type {
   GithubPullRequestMessage,
@@ -46,7 +48,100 @@ function mapFact(factJSON: string): GithubPullRequestOperation[] {
     return [];
   }
 
-  return [{ upsert: { github_pull_request: row } }];
+  // The bump rows of the replaced body come from the stored payload, so a late fact of an older
+  // body cannot bring back a bump that a newer body dropped.
+  const given = collectReleaseBumps(row.id, row.payload);
+  const replaced = stored === undefined ? [] : collectReleaseBumps(row.id, stored.payload);
+
+  const givenKeys = new Set(given.map((bump) => JSON.stringify(bump)));
+  const replacedKeys = new Set(replaced.map((bump) => JSON.stringify(bump)));
+
+  const removed: GithubPullRequestOperation[] = replaced
+    .filter((bump) => !givenKeys.has(JSON.stringify(bump)))
+    .map((bump) => ({ delete: { github_release_bump: bump } }));
+
+  const added: GithubPullRequestOperation[] = given
+    .filter((bump) => !replacedKeys.has(JSON.stringify(bump)))
+    .map((bump) => ({ upsert: { github_release_bump: bump } }));
+
+  return [{ upsert: { github_pull_request: row } }, ...removed, ...added];
+}
+
+const releaseURLStart = 'https://github.com/';
+const releaseURLEnds = [' ', '\n', '\r', '\t', ')', ']', '>', '"', "'", '`', '#', '?'];
+
+// One row per `https://github.com/<owner>/<name>/releases/tag/<tag>` URL in the body of a pull
+// request whose author is a bot, each once, with the tag percent-decoded.
+function collectReleaseBumps(
+  pullRequestID: GithubReleaseBumpRow['pull_request_id'],
+  pullRequest: unknown,
+): GithubReleaseBumpRow[] {
+  const body = findField(pullRequest, 'body');
+
+  if (findField(pullRequest, 'user', 'type') !== 'Bot' || typeof body !== 'string') {
+    return [];
+  }
+
+  const bumps: GithubReleaseBumpRow[] = [];
+  const seen: string[] = [];
+
+  for (const part of body.split(releaseURLStart).slice(1)) {
+    const bump = parseReleaseURLPath(part);
+
+    if (bump === undefined || seen.includes(`${bump.repository}\n${bump.tag}`)) {
+      continue;
+    }
+
+    seen.push(`${bump.repository}\n${bump.tag}`);
+
+    bumps.push({
+      pull_request_id: pullRequestID,
+      release_repository: bump.repository,
+      release_tag: bump.tag,
+    });
+  }
+
+  return bumps;
+}
+
+// `<owner>/<name>/releases/tag/<tag>`, up to the first character that ends a URL in text.
+function parseReleaseURLPath(text: string): { repository: string; tag: string } | undefined {
+  let end = text.length;
+
+  for (const mark of releaseURLEnds) {
+    const at = text.indexOf(mark);
+
+    if (at !== -1 && at < end) {
+      end = at;
+    }
+  }
+
+  const parts = text.slice(0, end).split('/');
+  const [owner, name, releases, tagWord, tag] = parts;
+
+  if (parts.length !== 5 || releases !== 'releases' || tagWord !== 'tag') {
+    return undefined;
+  }
+
+  if (owner === undefined || name === undefined || tag === undefined) {
+    return undefined;
+  }
+
+  if (owner === '' || name === '' || tag === '') {
+    return undefined;
+  }
+
+  const decoded = decodeTag(tag);
+
+  return decoded === undefined ? undefined : { repository: `${owner}/${name}`, tag: decoded };
+}
+
+function decodeTag(tag: string): string | undefined {
+  try {
+    return decodeURIComponent(tag);
+  } catch {
+    return undefined;
+  }
 }
 
 // Two updates can share one second. A repeat of the stored row changes nothing, and a different

package/src/consumers/github-ref.ts

@@ -0,0 +1,88 @@
+import { initGithubRefModel, updateGithubRefModel } from '../generated/github-ref-boundary.ts';
+import type {
+  EntityID,
+  GithubCommitRow,
+  GithubRefMessage,
+  GithubRefModel,
+  GithubRefOperation,
+} from '../generated/github-ref-boundary.ts';
+import { findField } from '../utils/find-field.ts';
+
+export type { GithubRefMessage, GithubRefModel } from '../generated/github-ref-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): GithubRefModel {
+  return initGithubRefModel();
+}
+
+export function update(model: GithubRefModel, message: GithubRefMessage): GithubRefModel {
+  return updateGithubRefModel(model, message);
+}
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+// A commit's time is a property of the commit, so a row never changes, and a force push that drops
+// a commit leaves a row that is still true.
+function mapFact(factJSON: string): GithubRefOperation[] {
+  const payload = findField(JSON.parse(factJSON), 'payload');
+  const ref = findField(payload, 'ref');
+  const repositoryID = findField(payload, 'repository', 'id');
+  const defaultBranch = findField(payload, 'repository', 'default_branch');
+
+  if (typeof ref !== 'string' || typeof repositoryID !== 'number') {
+    return [];
+  }
+
+  const commits = collectPushedCommits(payload, ref, defaultBranch);
+  const repository = toEntityID(`github:repo:${repositoryID}`);
+
+  return commits.flatMap((commit) => {
+    const row = buildCommitRow(repository, commit);
+
+    return row === undefined ? [] : [{ upsert: { github_commit: row } }];
+  });
+}
+
+// A tag push gives its head commit; a push to the default branch gives every commit it adds.
+function collectPushedCommits(payload: unknown, ref: string, defaultBranch: unknown): unknown[] {
+  if (ref.startsWith('refs/tags/')) {
+    return [findField(payload, 'head_commit')];
+  }
+
+  if (typeof defaultBranch === 'string' && ref === `refs/heads/${defaultBranch}`) {
+    return toList(findField(payload, 'commits'));
+  }
+
+  return [];
+}
+
+function toList(value: unknown): unknown[] {
+  return Array.isArray(value) ? value.map((item: unknown) => item) : [];
+}
+
+// The time as GitHub writes it, with the committer's offset: scriptc fences `Date`, so a reader
+// compares these times as times, never as text.
+function buildCommitRow(repository: EntityID, commit: unknown): GithubCommitRow | undefined {
+  const sha = findField(commit, 'id');
+  const timestamp = findField(commit, 'timestamp');
+
+  if (typeof sha !== 'string' || typeof timestamp !== 'string') {
+    return undefined;
+  }
+
+  return { repository_id: repository, sha, committed_at: timestamp };
+}
+
+function isEntityID(text: string): text is EntityID {
+  return text.length > 0;
+}
+
+function toEntityID(text: string): EntityID {
+  if (!isEntityID(text)) {
+    throw new Error('an entity ID is never empty');
+  }
+
+  return text;
+}

package/src/consumers/github-release.ts

@@ -0,0 +1,125 @@
+import {
+  initGithubReleaseModel,
+  readGithubReleaseRow,
+  toGithubReleaseID,
+  updateGithubReleaseModel,
+} from '../generated/github-release-boundary.ts';
+import type {
+  EntityID,
+  GithubReleaseMessage,
+  GithubReleaseModel,
+  GithubReleaseOperation,
+  GithubReleaseRow,
+} from '../generated/github-release-boundary.ts';
+import { findField } from '../utils/find-field.ts';
+
+export type {
+  GithubReleaseMessage,
+  GithubReleaseModel,
+} from '../generated/github-release-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): GithubReleaseModel {
+  return initGithubReleaseModel();
+}
+
+export function update(
+  model: GithubReleaseModel,
+  message: GithubReleaseMessage,
+): GithubReleaseModel {
+  return updateGithubReleaseModel(model, message);
+}
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+// GitHub never uses a release ID again, so a tombstone is never cleared. On a tie of times the
+// later fact in the stream wins: the host gives each fact ID once, so every replay keeps that order.
+function mapFact(factJSON: string): GithubReleaseOperation[] {
+  const fact: unknown = JSON.parse(factJSON);
+  const type = findField(fact, 'type');
+  const at = findField(fact, 'at');
+  const releaseID = findField(fact, 'payload', 'release', 'id');
+
+  if (typeof at !== 'string' || typeof releaseID !== 'number') {
+    return [];
+  }
+
+  const id = toGithubReleaseID(`${releaseID}`);
+  const stored = readGithubReleaseRow(id);
+  const isDelete = type === 'github.release.deleted';
+  const deletedAt = stored?.deleted_at ?? (isDelete ? at : null);
+  const given = buildReleaseRow(id, findField(fact, 'payload'), at);
+  const wins = given !== undefined && (stored === undefined || at >= stored.updated_at);
+  const row = wins ? given : stored;
+
+  if (row === undefined) {
+    return [];
+  }
+
+  const next: GithubReleaseRow = { ...row, deleted_at: deletedAt };
+
+  if (stored !== undefined && JSON.stringify(stored) === JSON.stringify(next)) {
+    return [];
+  }
+
+  return [{ upsert: { github_release: next } }];
+}
+
+// A fact that holds the release ID alone gives no row of its own.
+function buildReleaseRow(
+  id: GithubReleaseRow['id'],
+  payload: unknown,
+  at: string,
+): GithubReleaseRow | undefined {
+  const release = findField(payload, 'release');
+  const repositoryID = findField(payload, 'repository', 'id');
+  const fullName = findField(payload, 'repository', 'full_name');
+  const tagName = findField(release, 'tag_name');
+  const target = findField(release, 'target_commitish');
+  const name = findField(release, 'name');
+  const url = findField(release, 'html_url');
+  const createdAt = findField(release, 'created_at');
+  const publishedAt = findField(release, 'published_at');
+
+  if (typeof repositoryID !== 'number' || typeof fullName !== 'string') {
+    return undefined;
+  }
+
+  if (typeof tagName !== 'string' || typeof target !== 'string' || typeof url !== 'string') {
+    return undefined;
+  }
+
+  if (typeof createdAt !== 'string') {
+    return undefined;
+  }
+
+  return {
+    id,
+    repository_id: toEntityID(`github:repo:${repositoryID}`),
+    repository_full_name: fullName,
+    tag_name: tagName,
+    target_commitish: target,
+    name: typeof name === 'string' && name !== '' ? name : tagName,
+    url,
+    is_draft: findField(release, 'draft') === true,
+    is_prerelease: findField(release, 'prerelease') === true,
+    created_at: createdAt,
+    published_at: typeof publishedAt === 'string' ? publishedAt : null,
+    updated_at: at,
+    deleted_at: null,
+  };
+}
+
+function isEntityID(text: string): text is EntityID {
+  return text.length > 0;
+}
+
+function toEntityID(text: string): EntityID {
+  if (!isEntityID(text)) {
+    throw new Error('an entity ID is never empty');
+  }
+
+  return text;
+}

package/src/families/build-family-types.ts

@@ -0,0 +1,16 @@
+import type { z } from 'zod';
+
+// One fact type per action, `github.<family>.<action>`, each with the family's payload schema.
+export function buildFamilyTypes(
+  family: string,
+  actions: readonly string[],
+  schema: z.ZodType,
+): Readonly<Record<string, z.ZodType>> {
+  const types: Record<string, z.ZodType> = {};
+
+  for (const action of actions) {
+    types[`github.${family}.${action}`] = schema;
+  }
+
+  return types;
+}

package/src/families/deployment-family.ts

@@ -0,0 +1,15 @@
+import { defineFactFamily } from '@pepys/kit';
+import { z } from 'zod';
+
+const payloadSchema = z.looseObject({ deployment: z.looseObject({ id: z.number().int() }) });
+
+// A deployment and each of its statuses, as facts of the one deployment entity.
+export const deploymentFamily = defineFactFamily({
+  stream: 'facts.github.deployment',
+  api: 'github/2022-11-28',
+  entityKind: 'github:deployment',
+  types: {
+    'github.deployment.created': payloadSchema,
+    'github.deployment.status_created': payloadSchema,
+  },
+});

package/src/families/issue-family.ts

@@ -0,0 +1,38 @@
+import { defineFactFamily } from '@pepys/kit';
+import { z } from 'zod';
+import { buildFamilyTypes } from './build-family-types.ts';
+
+// The `action` values of GitHub's `issues` webhook event.
+const actions = [
+  'assigned',
+  'closed',
+  'deleted',
+  'demilestoned',
+  'edited',
+  'labeled',
+  'locked',
+  'milestoned',
+  'opened',
+  'pinned',
+  'reopened',
+  'transferred',
+  'typed',
+  'unassigned',
+  'unlabeled',
+  'unlocked',
+  'unpinned',
+  'untyped',
+] as const;
+
+const payloadSchema = z.looseObject({
+  action: z.string(),
+  issue: z.looseObject({ id: z.number().int() }),
+});
+
+// Stored with no consumer.
+export const issueFamily = defineFactFamily({
+  stream: 'facts.github.issue',
+  api: 'github/2022-11-28',
+  entityKind: 'github:issue',
+  types: buildFamilyTypes('issue', actions, payloadSchema),
+});