@pepys/source-github 0.0.0-stage → 0.1.0

package/src/generated/github-release-boundary.ts

@@ -0,0 +1,251 @@
+// Generated by `pepys build`. Do not edit.
+
+declare const idTag: unique symbol;
+declare const kindTag: unique symbol;
+
+export type EntityID = string & { readonly [idTag]: 'EntityID' };
+
+type KindedID<Kind extends string> = EntityID & { readonly [kindTag]: Kind };
+
+export type GithubReleaseID = KindedID<'github:release'>;
+
+export function toGithubReleaseID(id: string): GithubReleaseID {
+  const text = 'github:release:' + id;
+
+  if (!isGithubReleaseID(text)) {
+    throw new Error('an ID of the kind github:release is never empty');
+  }
+
+  return text;
+}
+
+export function parseGithubReleaseID(text: string): GithubReleaseID | undefined {
+  return isGithubReleaseID(text) ? text : undefined;
+}
+
+function isGithubReleaseID(text: string): text is GithubReleaseID {
+  return text.length > 'github:release:'.length && text.startsWith('github:release:');
+}
+
+export const settings = {
+  "deployTagPattern": "deployed/<environment>"
+} as const;
+
+export interface GithubReleaseModel {
+  input: string;
+}
+
+export type GithubReleaseMessage = { kind: 'apply'; input: string } | { kind: 'reset' };
+
+export function initGithubReleaseModel(): GithubReleaseModel {
+  return { input: '' };
+}
+
+export function updateGithubReleaseModel(model: GithubReleaseModel, message: GithubReleaseMessage): GithubReleaseModel {
+  return message.kind === 'reset' ? initGithubReleaseModel() : { ...model, input: message.input };
+}
+
+export interface GithubReleasePayload {
+  action: string;
+  release: GithubReleasePayloadRelease;
+}
+
+export interface GithubReleasePayloadRelease {
+  id: number;
+}
+
+export type GithubReleaseFact =
+  | { id: string; type: 'github.release.created' | 'github.release.deleted' | 'github.release.edited' | 'github.release.prereleased' | 'github.release.published' | 'github.release.released' | 'github.release.resynced' | 'github.release.unpublished'; entity: string; at: string; payload: GithubReleasePayload };
+
+export function parseGithubReleaseFact(factJSON: string): GithubReleaseFact | undefined {
+  const parsed: unknown = JSON.parse(factJSON);
+
+  return decodeGithubReleaseFact(parsed);
+}
+
+function decodeGithubReleaseFact(value: unknown): GithubReleaseFact | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const id = 'id' in value ? value.id : undefined;
+  const type = 'type' in value ? value.type : undefined;
+  const entity = 'entity' in value ? value.entity : undefined;
+  const at = 'at' in value ? value.at : undefined;
+  const payload = 'payload' in value ? value.payload : undefined;
+
+  if (typeof id !== 'string' || typeof entity !== 'string' || typeof at !== 'string') {
+    return undefined;
+  }
+
+  if (type === 'github.release.created' || type === 'github.release.deleted' || type === 'github.release.edited' || type === 'github.release.prereleased' || type === 'github.release.published' || type === 'github.release.released' || type === 'github.release.resynced' || type === 'github.release.unpublished') {
+    const decoded = decodeGithubReleasePayload(payload);
+
+    if (decoded === undefined) {
+      return undefined;
+    }
+
+    return { id, type, entity, at, payload: decoded };
+  }
+
+  return undefined;
+}
+
+function decodeGithubReleasePayload(value: unknown): GithubReleasePayload | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'action' in value ? value.action : undefined;
+  const field0 = typeof raw0 === 'string' ? raw0 : undefined;
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  const raw1 = 'release' in value ? value.release : undefined;
+  const field1 = decodeGithubReleasePayloadRelease(raw1);
+
+  if (field1 === undefined) {
+    return undefined;
+  }
+
+  return {
+    action: field0,
+    release: field1,
+  };
+}
+
+function decodeGithubReleasePayloadRelease(value: unknown): GithubReleasePayloadRelease | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'id' in value ? value.id : undefined;
+  const field0 = typeof raw0 === 'number' ? raw0 : undefined;
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  return {
+    id: field0,
+  };
+}
+
+export interface GithubReleaseRow {
+  id: GithubReleaseID;
+  repository_id: EntityID;
+  repository_full_name: string;
+  tag_name: string;
+  target_commitish: string;
+  name: string;
+  url: string;
+  is_draft: boolean;
+  is_prerelease: boolean;
+  created_at: string;
+  published_at: string | null;
+  updated_at: string;
+  deleted_at: string | null;
+}
+
+export interface GithubReleaseKey {
+  id: GithubReleaseID;
+}
+
+export type GithubReleaseOperation =
+  | { upsert: { github_release: GithubReleaseRow } }
+  | { delete: { github_release: GithubReleaseKey } };
+
+declare function readGithubReleaseLength(key: string): number;
+declare function readGithubReleaseInto(key: string, row: Uint8Array): void;
+
+export function readGithubReleaseRow(key: string): GithubReleaseRow | undefined {
+  const length = readGithubReleaseLength(key);
+
+  if (length < 0) {
+    return undefined;
+  }
+
+  const row = new Uint8Array(length);
+
+  readGithubReleaseInto(key, row);
+
+  const parsed: unknown = JSON.parse(new TextDecoder().decode(row));
+
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+    return undefined;
+  }
+
+  if (!('id' in parsed) || (typeof parsed.id !== 'string' || !isGithubReleaseID(parsed.id))) {
+    return undefined;
+  }
+
+  if (!('repository_id' in parsed) || (!isEntityID(parsed.repository_id))) {
+    return undefined;
+  }
+
+  if (!('repository_full_name' in parsed) || (typeof parsed.repository_full_name !== 'string')) {
+    return undefined;
+  }
+
+  if (!('tag_name' in parsed) || (typeof parsed.tag_name !== 'string')) {
+    return undefined;
+  }
+
+  if (!('target_commitish' in parsed) || (typeof parsed.target_commitish !== 'string')) {
+    return undefined;
+  }
+
+  if (!('name' in parsed) || (typeof parsed.name !== 'string')) {
+    return undefined;
+  }
+
+  if (!('url' in parsed) || (typeof parsed.url !== 'string')) {
+    return undefined;
+  }
+
+  if (!('is_draft' in parsed) || (typeof parsed.is_draft !== 'boolean')) {
+    return undefined;
+  }
+
+  if (!('is_prerelease' in parsed) || (typeof parsed.is_prerelease !== 'boolean')) {
+    return undefined;
+  }
+
+  if (!('created_at' in parsed) || (typeof parsed.created_at !== 'string')) {
+    return undefined;
+  }
+
+  if (!('published_at' in parsed) || (parsed.published_at !== null && (typeof parsed.published_at !== 'string'))) {
+    return undefined;
+  }
+
+  if (!('updated_at' in parsed) || (typeof parsed.updated_at !== 'string')) {
+    return undefined;
+  }
+
+  if (!('deleted_at' in parsed) || (parsed.deleted_at !== null && (typeof parsed.deleted_at !== 'string'))) {
+    return undefined;
+  }
+
+  return {
+    id: parsed.id,
+    repository_id: parsed.repository_id,
+    repository_full_name: parsed.repository_full_name,
+    tag_name: parsed.tag_name,
+    target_commitish: parsed.target_commitish,
+    name: parsed.name,
+    url: parsed.url,
+    is_draft: parsed.is_draft,
+    is_prerelease: parsed.is_prerelease,
+    created_at: parsed.created_at,
+    published_at: parsed.published_at,
+    updated_at: parsed.updated_at,
+    deleted_at: parsed.deleted_at,
+  };
+}
+
+function isEntityID(value: unknown): value is EntityID {
+  return typeof value === 'string';
+}

package/src/generated/github-release.sql

@@ -0,0 +1,32 @@
+-- Generated by `pepys build`. Do not edit.
+
+CREATE TABLE "github_release" (
+  "id" TEXT NOT NULL,
+  "repository_id" TEXT NOT NULL,
+  "repository_full_name" TEXT NOT NULL,
+  "tag_name" TEXT NOT NULL,
+  "target_commitish" TEXT NOT NULL,
+  "name" TEXT NOT NULL,
+  "url" TEXT NOT NULL,
+  "is_draft" INTEGER NOT NULL,
+  "is_prerelease" INTEGER NOT NULL,
+  "created_at" TEXT NOT NULL,
+  "published_at" TEXT,
+  "updated_at" TEXT NOT NULL,
+  "deleted_at" TEXT,
+  PRIMARY KEY ("id")
+);
+
+CREATE INDEX "github_release_repository_id_idx" ON "github_release" ("repository_id");
+
+CREATE INDEX "github_release_repository_full_name_tag_name_idx" ON "github_release" ("repository_full_name", "tag_name");
+
+CREATE VIEW "entity" AS
+SELECT
+  "id" AS "id",
+  'github:release' AS "kind",
+  "name" AS "title",
+  "url" AS "url",
+  NULL AS "actor",
+  "updated_at" AS "updated_at"
+FROM "github_release";

package/src/github-deploy-consumer.ts

@@ -0,0 +1,40 @@
+import { defineConsumer, defineTable } from '@pepys/kit';
+import { deploymentFamily } from './families/deployment-family.ts';
+import { refFamily } from './families/ref-family.ts';
+
+// One row per signal that reported a commit to an environment, with the earliest time it did.
+export const githubDeployConsumer = defineConsumer({
+  name: 'github_deploy',
+  entry: new URL('consumers/github-deploy.ts', import.meta.url),
+  input: { mode: 'fact', streams: [refFamily.stream, deploymentFamily.stream] },
+  tables: {
+    github_deploy: defineTable({
+      columns: {
+        // `github:deploy:<repository ID>:<environment>:<signal>:<sha>`, and `:<target>` after it
+        // for a row with a target, so two targets of one commit do not share a row
+        id: { kind: 'entityID' },
+        repository_id: { kind: 'entityID' },
+        environment: { kind: 'text' },
+        target: { kind: 'text', nullable: true },
+        sha: { kind: 'text' },
+        signal: { kind: 'text' },
+        deployed_at: { kind: 'timestamp' },
+      },
+      primaryKey: ['id'],
+      indexes: [{ columns: ['repository_id', 'environment'] }],
+    }),
+  },
+  entityViews: [
+    {
+      table: 'github_deploy',
+      kind: 'github:deploy',
+      id: 'id',
+      title: 'sha',
+      url: null,
+      actor: null,
+      updatedAt: 'deployed_at',
+    },
+  ],
+  reads: ['github_deploy'],
+  links: [],
+});

package/src/github-plugin.ts

@@ -0,0 +1,15 @@
+import { definePlugin } from '@pepys/kit';
+import { githubSettings } from './github-settings.ts';
+import { githubSource } from './github-source.ts';
+import { githubDeliveryConsumers } from './queries/github-delivery-consumers.ts';
+import { pullRequestDeliveryQuery } from './queries/pull-request-delivery-query.ts';
+
+// One plug-in for every deployment. A deployment that uses another tag pattern sets
+// `settings.github.deployTagPattern` in its deployment file.
+export const githubPlugin = definePlugin({
+  name: 'github',
+  settings: githubSettings,
+  sources: [githubSource],
+  consumers: [...githubDeliveryConsumers],
+  queries: [pullRequestDeliveryQuery],
+});

package/src/github-pull-request-consumer.ts

@@ -0,0 +1,56 @@
+import { defineConsumer, defineTable } from '@pepys/kit';
+import { pullRequestFamily } from './families/pull-request-family.ts';
+
+export const githubPullRequestConsumer = defineConsumer({
+  name: 'github_pull_request',
+  entry: new URL('consumers/github-pull-request.ts', import.meta.url),
+  input: { mode: 'fact', streams: [pullRequestFamily.stream] },
+  tables: {
+    github_pull_request: defineTable({
+      columns: {
+        id: { kind: 'entityID' },
+        number: { kind: 'integer' },
+        repository_id: { kind: 'entityID' },
+        title: { kind: 'text' },
+        state: { kind: 'text' },
+        is_draft: { kind: 'boolean' },
+        is_merged: { kind: 'boolean' },
+        author_id: { kind: 'entityID', nullable: true },
+        url: { kind: 'text' },
+        merge_commit_sha: { kind: 'text', nullable: true },
+        created_at: { kind: 'timestamp' },
+        updated_at: { kind: 'timestamp' },
+        closed_at: { kind: 'timestamp', nullable: true },
+        merged_at: { kind: 'timestamp', nullable: true },
+        payload: { kind: 'json' },
+      },
+      primaryKey: ['id'],
+      indexes: [{ columns: ['repository_id'] }],
+    }),
+
+    // One row per release that a bot's pull request names in its body, by the release URL. The rows
+    // follow the pull request row: a fact that loses writes none, and one that wins replaces them.
+    github_release_bump: defineTable({
+      columns: {
+        pull_request_id: { kind: 'entityID' },
+        release_repository: { kind: 'text' },
+        release_tag: { kind: 'text' },
+      },
+      primaryKey: ['pull_request_id', 'release_repository', 'release_tag'],
+      indexes: [{ columns: ['release_repository', 'release_tag'] }],
+    }),
+  },
+  entityViews: [
+    {
+      table: 'github_pull_request',
+      kind: pullRequestFamily.entityKind,
+      id: 'id',
+      title: 'title',
+      url: 'url',
+      actor: 'author_id',
+      updatedAt: 'updated_at',
+    },
+  ],
+  reads: ['github_pull_request'],
+  links: [],
+});

package/src/github-pull-request-schema-path.ts

@@ -0,0 +1,6 @@
+import { join } from 'node:path';
+
+export const githubPullRequestSchemaPath = join(
+  import.meta.dir,
+  'generated/github-pull-request.sql',
+);

package/src/github-ref-consumer.ts

@@ -0,0 +1,23 @@
+import { defineConsumer, defineTable } from '@pepys/kit';
+import { refFamily } from './families/ref-family.ts';
+
+// One row per commit of a push to the default branch, and one for the commit a pushed tag names.
+export const githubRefConsumer = defineConsumer({
+  name: 'github_ref',
+  entry: new URL('consumers/github-ref.ts', import.meta.url),
+  input: { mode: 'fact', streams: [refFamily.stream] },
+  tables: {
+    github_commit: defineTable({
+      columns: {
+        repository_id: { kind: 'entityID' },
+        sha: { kind: 'text' },
+        committed_at: { kind: 'timestamp' },
+      },
+      primaryKey: ['repository_id', 'sha'],
+      indexes: [],
+    }),
+  },
+  entityViews: [],
+  reads: [],
+  links: [],
+});

package/src/github-release-consumer.ts

@@ -0,0 +1,43 @@
+import { defineConsumer, defineTable } from '@pepys/kit';
+import { releaseFamily } from './families/release-family.ts';
+
+// One row per release, never deleted: a release GitHub deleted keeps its row with `deleted_at`.
+export const githubReleaseConsumer = defineConsumer({
+  name: 'github_release',
+  entry: new URL('consumers/github-release.ts', import.meta.url),
+  input: { mode: 'fact', streams: [releaseFamily.stream] },
+  tables: {
+    github_release: defineTable({
+      columns: {
+        id: { kind: 'entityID' },
+        repository_id: { kind: 'entityID' },
+        repository_full_name: { kind: 'text' },
+        tag_name: { kind: 'text' },
+        target_commitish: { kind: 'text' },
+        name: { kind: 'text' },
+        url: { kind: 'text' },
+        is_draft: { kind: 'boolean' },
+        is_prerelease: { kind: 'boolean' },
+        created_at: { kind: 'timestamp' },
+        published_at: { kind: 'timestamp', nullable: true },
+        updated_at: { kind: 'timestamp' },
+        deleted_at: { kind: 'timestamp', nullable: true },
+      },
+      primaryKey: ['id'],
+      indexes: [{ columns: ['repository_id'] }, { columns: ['repository_full_name', 'tag_name'] }],
+    }),
+  },
+  entityViews: [
+    {
+      table: 'github_release',
+      kind: releaseFamily.entityKind,
+      id: 'id',
+      title: 'name',
+      url: 'url',
+      actor: null,
+      updatedAt: 'updated_at',
+    },
+  ],
+  reads: ['github_release'],
+  links: [],
+});

package/src/github-settings.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+
+// `pepys build` writes the checked values into the glue of each consumer, so a changed pattern is a
+// new version of `github_deploy`. A tag with one more part, `<pattern>/<target>`, names a target.
+export const githubSettings = {
+  schema: z.object({
+    deployTagPattern: z
+      .string()
+      .includes('<environment>')
+      .refine((pattern) => pattern.split('<environment>').length === 2, {
+        message: 'the pattern holds <environment> once',
+      }),
+  }),
+  defaults: { deployTagPattern: 'deployed/<environment>' },
+};

package/src/github-source.ts

@@ -0,0 +1,56 @@
+import { defineSource } from '@pepys/kit';
+import { deploymentFamily } from './families/deployment-family.ts';
+import { issueFamily } from './families/issue-family.ts';
+import { pullRequestFamily } from './families/pull-request-family.ts';
+import { pullRequestReviewFamily } from './families/pull-request-review-family.ts';
+import { refFamily } from './families/ref-family.ts';
+import { releaseFamily } from './families/release-family.ts';
+import { workflowRunFamily } from './families/workflow-run-family.ts';
+import { readGithubDelivery } from './ingest/read-github-delivery.ts';
+import { pullFailedDeliveries } from './ingest/recovery/pull-failed-deliveries.ts';
+import { pullMarkedObjects } from './ingest/resync/pull-marked-objects.ts';
+
+// The GitHub App that both pulls call the API as.
+const appEnv = ['PEPYS_GITHUB_APP_ID', 'PEPYS_GITHUB_APP_PRIVATE_KEY'];
+
+// One webhook route, checked by GitHub's signature, for seven families. The recovery pull reads
+// each failed delivery every hour; the resync pull reads each marked object every minute.
+export const githubSource = defineSource({
+  name: 'github',
+  families: [
+    pullRequestFamily,
+    releaseFamily,
+    refFamily,
+    deploymentFamily,
+    workflowRunFamily,
+    issueFamily,
+    pullRequestReviewFamily,
+  ],
+  live: {
+    kind: 'receive',
+    route: '/webhook/github',
+    check: {
+      kind: 'hmac-sha256',
+      header: 'x-hub-signature-256',
+      secretEnv: 'PEPYS_GITHUB_WEBHOOK_SECRET',
+    },
+
+    // GitHub calls from the public internet
+    reach: 'public',
+    read: readGithubDelivery,
+  },
+  recovery: {
+    kind: 'pull',
+    name: 'redeliver',
+    intervalMs: 3_600_000,
+    env: appEnv,
+    pull: pullFailedDeliveries,
+  },
+  resync: {
+    kind: 'pull',
+    name: 'resync',
+    intervalMs: 60_000,
+    env: appEnv,
+    pull: pullMarkedObjects,
+  },
+});

package/src/index.ts

@@ -0,0 +1,13 @@
+export { githubPlugin } from './github-plugin.ts';
+export { githubPullRequestConsumer } from './github-pull-request-consumer.ts';
+export { githubPullRequestSchemaPath } from './github-pull-request-schema-path.ts';
+export { githubDeliveryConsumers } from './queries/github-delivery-consumers.ts';
+
+export {
+  parsePullRequestDeliveries,
+  selectPullRequestDelivery,
+} from './queries/select-pull-request-delivery.ts';
+
+export type { PullRequestDelivery } from './queries/select-pull-request-delivery.ts';
+export { deploySignals } from './utils/deploy-signals.ts';
+export type { DeploySignal } from './utils/deploy-signals.ts';

package/src/ingest/build-github-fact.ts

@@ -0,0 +1,55 @@
+import { parseFactEnvelope } from '@pepys/kit';
+import type { PendingFact, StreamName } from '@pepys/kit';
+import { toTagged } from '@zgeoff/std';
+import { buildBodyHash } from '../utils/build-body-hash.ts';
+import { githubEventRules } from './github-event-rules.ts';
+
+// `delivery`: a webhook body, from the receive path or the recovery pull. `observation`: an object
+// that the resync pull read from the API, in the shape of a webhook body.
+export type GithubFactOrigin = 'delivery' | 'observation';
+
+// One body of one webhook event as a fact, with an ID from the hash of the canonical body: GitHub
+// signs the body and not the delivery ID header, so only the body decides it. One delivery has one
+// ID on the receive path and on the recovery pull, and two reads of one state are one observation.
+export function buildGithubFact(
+  event: string,
+  body: unknown,
+  origin: GithubFactOrigin,
+): PendingFact | 'ignored' | 'invalid' {
+  const rule = Object.hasOwn(githubEventRules, event) ? githubEventRules[event] : undefined;
+
+  if (rule === undefined) {
+    return 'ignored';
+  }
+
+  const fields = rule.read(body);
+
+  if (fields === 'ignored' || fields === 'invalid') {
+    return fields;
+  }
+
+  const envelope = parseFactEnvelope({
+    id:
+      origin === 'delivery'
+        ? `webhook:github:h-${buildBodyHash(body)}`
+        : `observation:${buildBodyHash(body)}`,
+    type: fields.type,
+    v: 1,
+    api: rule.family.api,
+    entity: fields.entity,
+    ...(fields.actor === undefined ? {} : { actor: fields.actor }),
+    at: fields.at,
+    payload: body,
+  });
+
+  if (!envelope.ok) {
+    return 'invalid';
+  }
+
+  return {
+    stream: toTagged<StreamName>(rule.family.stream),
+    fact: envelope.value,
+    blobs: [],
+    ...(fields.locator === undefined ? {} : { locator: fields.locator }),
+  };
+}

package/src/ingest/github-app/build-app-jwt.ts

@@ -0,0 +1,20 @@
+import { createSign } from 'node:crypto';
+
+// The JWT a GitHub App signs to call the API as itself. GitHub refuses an `exp` more than ten
+// minutes after `iat`, and sets `iat` a minute back for clocks that drift. A private key kept in
+// one line of an env file holds `\n` in place of each line break.
+export function buildAppJWT(appID: string, privateKey: string, nowMs: number): string {
+  const nowSeconds = Math.floor(nowMs / 1000);
+  const header = toSegment({ alg: 'RS256', typ: 'JWT' });
+  const claims = toSegment({ iat: nowSeconds - 60, exp: nowSeconds + 540, iss: appID });
+
+  const signature = createSign('RSA-SHA256')
+    .update(`${header}.${claims}`)
+    .sign(privateKey.replaceAll(String.raw`\n`, '\n'), 'base64url');
+
+  return `${header}.${claims}.${signature}`;
+}
+
+function toSegment(value: object): string {
+  return Buffer.from(JSON.stringify(value)).toString('base64url');
+}

package/src/ingest/github-app/find-app-jwt.ts

@@ -0,0 +1,17 @@
+import { buildAppJWT } from './build-app-jwt.ts';
+
+// The App's JWT from the pull's env, or undefined when the private key cannot sign.
+export function findAppJWT(
+  env: Readonly<Record<string, string>>,
+  nowMs: number,
+): string | undefined {
+  try {
+    return buildAppJWT(
+      env['PEPYS_GITHUB_APP_ID'] ?? '',
+      env['PEPYS_GITHUB_APP_PRIVATE_KEY'] ?? '',
+      nowMs,
+    );
+  } catch {
+    return undefined;
+  }
+}

package/src/ingest/github-app/github-api-url.ts

@@ -0,0 +1 @@
+export const githubAPIURL = 'https://api.github.com';

package/src/ingest/github-app/pick-pull-failure.ts

@@ -0,0 +1,11 @@
+import type { Pulled } from '@pepys/kit';
+
+const retryAfterMs = 60_000;
+
+// What a pull answers for an API status that stopped it. A 401 means the App's credentials are
+// wrong, and no retry mends that; any other status, or 0 for a transport failure, may pass.
+export function pickPullFailure(status: number): Pulled {
+  return status === 401
+    ? { kind: 'rejected', reason: 'GitHub refused the App token' }
+    : { kind: 'unavailable', retryAfterMs };
+}