@pepys/source-github 0.1.0 → 0.2.0

package/README.md

@@ -27,6 +27,9 @@ bun add @pepys/source-github
 
 - `githubPlugin`: the plug-in, to add to a deployment.
 - `githubPullRequestConsumer`: the consumer of pull request facts, with its SQL schema.
+- `githubDelivery`: what a query of another plug-in needs to read when each pull request merged,
+  released and deployed. Pass `consumers` as the query's `reads`, run `select(input).execute(db)`,
+  and give the result's `rows` to `parse`.
 
 ## Documentation
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/source-github",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "The pepys GitHub plug-in: pull requests, releases, commits and deploys",
   "license": "MIT",
   "repository": {
@@ -19,16 +19,17 @@
     "access": "public"
   },
   "dependencies": {
-    "@pepys/api": "0.1.0",
-    "@pepys/ingest": "0.1.0",
-    "@pepys/kit": "0.1.0",
+    "@pepys/api": "0.2.0",
+    "@pepys/ingest": "0.2.0",
+    "@pepys/kit": "0.2.0",
     "@zgeoff/std": "0.4.3",
     "kysely": "0.29.8",
+    "ts-pattern": "5.9.2",
     "zod": "4.6.7"
   },
   "devDependencies": {
     "@msw/data": "1.1.8",
-    "@pepys/testing": "0.1.0",
+    "@pepys/testing": "0.2.0",
     "msw": "3.0.3",
     "tiny-invariant": "1.3.3"
   }

package/src/consumers/github-pull-request.ts

@@ -104,7 +104,8 @@ function collectReleaseBumps(
   return bumps;
 }
 
-// `<owner>/<name>/releases/tag/<tag>`, up to the first character that ends a URL in text.
+// `<owner>/<name>/releases/tag/<tag>`, up to the first character that ends a URL in text. GitHub
+// serves a tag that holds `/` both escaped and unescaped, so the tag is the rest of the path.
 function parseReleaseURLPath(text: string): { repository: string; tag: string } | undefined {
   let end = text.length;
 
@@ -117,17 +118,19 @@ function parseReleaseURLPath(text: string): { repository: string; tag: string }
   }
 
   const parts = text.slice(0, end).split('/');
-  const [owner, name, releases, tagWord, tag] = parts;
+  const [owner, name, releases, tagWord] = parts;
+  const tagParts = parts.slice(4);
+  const tag = tagParts.join('/');
 
-  if (parts.length !== 5 || releases !== 'releases' || tagWord !== 'tag') {
+  if (parts.length < 5 || releases !== 'releases' || tagWord !== 'tag') {
     return undefined;
   }
 
-  if (owner === undefined || name === undefined || tag === undefined) {
+  if (owner === undefined || name === undefined) {
     return undefined;
   }
 
-  if (owner === '' || name === '' || tag === '') {
+  if (owner === '' || name === '' || tagParts.includes('')) {
     return undefined;
   }
 

package/src/families/build-family-types.ts

@@ -1,6 +1,5 @@
 import type { z } from 'zod';
 
-// One fact type per action, `github.<family>.<action>`, each with the family's payload schema.
 export function buildFamilyTypes(
   family: string,
   actions: readonly string[],

package/src/families/issue-family.ts

@@ -29,7 +29,6 @@ const payloadSchema = z.looseObject({
   issue: z.looseObject({ id: z.number().int() }),
 });
 
-// Stored with no consumer.
 export const issueFamily = defineFactFamily({
   stream: 'facts.github.issue',
   api: 'github/2022-11-28',

package/src/families/pull-request-review-family.ts

@@ -9,7 +9,6 @@ const payloadSchema = z.looseObject({
   review: z.looseObject({ id: z.number().int() }),
 });
 
-// Stored with no consumer.
 export const pullRequestReviewFamily = defineFactFamily({
   stream: 'facts.github.pull_request_review',
   api: 'github/2022-11-28',

package/src/families/workflow-run-family.ts

@@ -9,7 +9,6 @@ const payloadSchema = z.looseObject({
   workflow_run: z.looseObject({ id: z.number().int() }),
 });
 
-// Stored with no consumer.
 export const workflowRunFamily = defineFactFamily({
   stream: 'facts.github.workflow_run',
   api: 'github/2022-11-28',

package/src/index.ts

@@ -1,6 +1,7 @@
 export { githubPlugin } from './github-plugin.ts';
 export { githubPullRequestConsumer } from './github-pull-request-consumer.ts';
 export { githubPullRequestSchemaPath } from './github-pull-request-schema-path.ts';
+export { githubDelivery } from './queries/github-delivery.ts';
 export { githubDeliveryConsumers } from './queries/github-delivery-consumers.ts';
 
 export {

package/src/ingest/build-github-fact.ts

@@ -1,16 +1,20 @@
 import { parseFactEnvelope } from '@pepys/kit';
 import type { PendingFact, StreamName } from '@pepys/kit';
 import { toTagged } from '@zgeoff/std';
+import { match } from 'ts-pattern';
 import { buildBodyHash } from '../utils/build-body-hash.ts';
 import { githubEventRules } from './github-event-rules.ts';
 
 // `delivery`: a webhook body, from the receive path or the recovery pull. `observation`: an object
-// that the resync pull read from the API, in the shape of a webhook body.
-export type GithubFactOrigin = 'delivery' | 'observation';
+// that the resync pull read from the API, in the shape of a webhook body, for a mark whose last
+// stored body has the digest `lastBodyDigest`.
+export type GithubFactOrigin =
+  | { kind: 'delivery' }
+  | { kind: 'observation'; lastBodyDigest: string | undefined };
 
 // One body of one webhook event as a fact, with an ID from the hash of the canonical body: GitHub
-// signs the body and not the delivery ID header, so only the body decides it. One delivery has one
-// ID on the receive path and on the recovery pull, and two reads of one state are one observation.
+// signs the body and not the delivery ID header, so only the body decides it. An observation's ID
+// also hashes the last stored body, so a read after a newer tied body is a fact the host applies.
 export function buildGithubFact(
   event: string,
   body: unknown,
@@ -29,10 +33,13 @@ export function buildGithubFact(
   }
 
   const envelope = parseFactEnvelope({
-    id:
-      origin === 'delivery'
-        ? `webhook:github:h-${buildBodyHash(body)}`
-        : `observation:${buildBodyHash(body)}`,
+    id: match(origin)
+      .with({ kind: 'delivery' }, () => `webhook:github:h-${buildBodyHash(body)}`)
+      .with(
+        { kind: 'observation' },
+        (observation) => `observation:${buildObservationHash(body, observation.lastBodyDigest)}`,
+      )
+      .exhaustive(),
     type: fields.type,
     v: 1,
     api: rule.family.api,
@@ -53,3 +60,9 @@ export function buildGithubFact(
     ...(fields.locator === undefined ? {} : { locator: fields.locator }),
   };
 }
+
+function buildObservationHash(body: unknown, lastBodyDigest: string | undefined): string {
+  return lastBodyDigest === undefined
+    ? buildBodyHash(body)
+    : buildBodyHash({ observed: body, lastBodyDigest });
+}

package/src/ingest/github-app/find-app-jwt.ts

@@ -1,6 +1,5 @@
 import { buildAppJWT } from './build-app-jwt.ts';
 
-// The App's JWT from the pull's env, or undefined when the private key cannot sign.
 export function findAppJWT(
   env: Readonly<Record<string, string>>,
   nowMs: number,

package/src/ingest/github-app/request-github-api.ts

@@ -1,8 +1,10 @@
 import { githubAPIURL } from './github-api-url.ts';
 
-// One answer of the GitHub REST API. `next` is the URL of the next page, from the `link` header.
+// One answer of the GitHub REST API. `text` is the body as sent, for a caller that must read a
+// number above 2^53 exactly. `next` is the URL of the next page, from the `link` header.
 export interface GithubAnswer {
   status: number;
+  text: string;
   body: unknown;
   next: string | undefined;
 }
@@ -19,13 +21,14 @@ export async function requestGithubAPI(
   const response = await sendRequest(url, token, method);
 
   if (response === undefined) {
-    return { status: 0, body: undefined, next: undefined };
+    return { status: 0, text: '', body: undefined, next: undefined };
   }
 
   const text = await response.text();
 
   return {
     status: response.status,
+    text,
     body: parseBody(text),
     next: findNextPage(response.headers.get('link')),
   };

package/src/ingest/read-github-delivery.ts

@@ -17,7 +17,7 @@ export function readGithubDelivery(delivery: Delivery): Received {
     return { kind: 'refused', status: 400, reason: 'the body is not JSON' };
   }
 
-  const fact = buildGithubFact(event, body.value, 'delivery');
+  const fact = buildGithubFact(event, body.value, { kind: 'delivery' });
 
   if (fact === 'ignored') {
     return { kind: 'ignored' };

package/src/ingest/recovery/pull-failed-deliveries.ts

@@ -1,3 +1,4 @@
+import { planAppends } from '@pepys/ingest';
 import type { PendingFact, PullContext, Pulled } from '@pepys/kit';
 import { z } from 'zod';
 import { buildGithubFact } from '../build-github-fact.ts';
@@ -24,7 +25,7 @@ export async function pullFailedDeliveries(
     return { kind: 'rejected', reason: 'the GitHub App private key cannot sign a token' };
   }
 
-  const seenID = cursor === null ? 0 : Number(cursor);
+  const seenID = cursor === null ? 0n : BigInt(cursor);
 
   const listed = await readDeliveriesAfter(seenID, appJWT);
 
@@ -43,20 +44,38 @@ export async function pullFailedDeliveries(
       return pickPullFailure(answer.status);
     }
 
-    const fact = buildGithubFact(detail.data.event, detail.data.request.payload, 'delivery');
+    const fact = buildGithubFact(detail.data.event, detail.data.request.payload, {
+      kind: 'delivery',
+    });
 
-    if (fact !== 'ignored' && fact !== 'invalid') {
-      facts.push(fact);
+    if (fact === 'ignored' || fact === 'invalid') {
+      continue;
     }
+
+    // The receive path answers 413 to a body this large and stores nothing; a batch that held it
+    // would be refused whole, with every other delivery and the cursor.
+    if (!planAppends([fact.fact]).ok) {
+      console.error(`the GitHub delivery ${id} is larger than one append of the log takes`);
+      continue;
+    }
+
+    facts.push(fact);
   }
 
-  const highestID = Math.max(seenID, ...listed.items.map((item) => item.id));
+  const highestID = listed.items.reduce(
+    (highest, item) => (item.id > highest ? item.id : highest),
+    seenID,
+  );
 
   return { kind: 'batch', batch: { facts }, cursor: `${highestID}`, isDone: true };
 }
 
+// A delivery ID is above 2^53, so it is read from its digits in the body, never from a number.
 const deliveryItemSchema = z.looseObject({
-  id: z.number().int(),
+  id: z
+    .string()
+    .regex(/^[1-9]\d*$/u)
+    .transform(BigInt),
   status_code: z.number().int(),
   event: z.string(),
 });
@@ -68,14 +87,14 @@ type DeliveryItem = z.infer<typeof deliveryItemSchema>;
 type DeliveryList = { kind: 'listed'; items: DeliveryItem[] } | { kind: 'failed'; status: number };
 
 // GitHub lists the newest delivery first, so the pages stop at the first one already seen.
-async function readDeliveriesAfter(seenID: number, appJWT: string): Promise<DeliveryList> {
+async function readDeliveriesAfter(seenID: bigint, appJWT: string): Promise<DeliveryList> {
   const items: DeliveryItem[] = [];
   let url: string | undefined = '/app/hook/deliveries?per_page=100';
 
   while (url !== undefined) {
     const answer: GithubAnswer = await requestGithubAPI(url, appJWT);
 
-    const page = deliveryListSchema.safeParse(answer.body);
+    const page = deliveryListSchema.safeParse(parseDeliveryList(answer.text));
 
     if (answer.status !== 200 || !page.success) {
       return { kind: 'failed', status: answer.status };
@@ -91,10 +110,29 @@ async function readDeliveriesAfter(seenID: number, appJWT: string): Promise<Deli
   return { kind: 'listed', items };
 }
 
+interface ReviverContext {
+  source?: string;
+}
+
+// `JSON.parse` rounds a number above 2^53, so each `id` keeps the source text the reviver is given.
+function parseDeliveryList(text: string): unknown {
+  try {
+    const list: unknown = JSON.parse(
+      text,
+      (key: string, value: unknown, context?: ReviverContext): unknown =>
+        key === 'id' && typeof value === 'number' ? context?.source : value,
+    );
+
+    return list;
+  } catch {
+    return undefined;
+  }
+}
+
 // oldest first, so the facts reach the log in the order GitHub sent them
-function collectFailedIDs(items: readonly DeliveryItem[]): number[] {
+function collectFailedIDs(items: readonly DeliveryItem[]): bigint[] {
   return items
     .filter((item) => item.status_code < 200 || item.status_code >= 300)
     .map((item) => item.id)
-    .toSorted((left, right) => left - right);
+    .toSorted((left, right) => Number(left - right));
 }

package/src/ingest/resync/collect-resync-targets.ts

@@ -1,11 +1,13 @@
 import type { ResyncMark } from '@pepys/kit';
 
-// One marked object that the resync pull reads again: its repository, `owner/name`, and its API path.
+// One marked object that the resync pull reads again: its repository, `owner/name`, its API path,
+// and the digest of the last body ingest stored for it.
 export interface ResyncTarget {
   kind: 'pull_request' | 'release';
   entity: string;
   repository: string;
   path: string;
+  lastBodyDigest: string | undefined;
 }
 
 const locatorPatterns = [
@@ -24,7 +26,15 @@ export function collectResyncTargets(marks: readonly ResyncMark[]): ResyncTarget
       const kind = locatorPattern.kind;
 
       if (repository !== undefined && mark.entity.startsWith(`github:${kind}:`)) {
-        return [{ kind, entity: mark.entity, repository, path: locator }];
+        return [
+          {
+            kind,
+            entity: mark.entity,
+            repository,
+            path: locator,
+            lastBodyDigest: mark.lastBodyDigest,
+          },
+        ];
       }
     }
 

package/src/ingest/resync/pull-marked-objects.ts

@@ -1,6 +1,7 @@
 import type { PendingFact, PullContext, Pulled } from '@pepys/kit';
 import { z } from 'zod';
 import { buildGithubFact } from '../build-github-fact.ts';
+import type { GithubFactOrigin } from '../build-github-fact.ts';
 import { findAppJWT } from '../github-app/find-app-jwt.ts';
 import { pickPullFailure } from '../github-app/pick-pull-failure.ts';
 import { readInstallationToken } from '../github-app/read-installation-token.ts';
@@ -97,13 +98,11 @@ async function readObject(target: ResyncTarget, token: string): Promise<ReadObje
     return { kind: 'failed', status: answer.status };
   }
 
+  const origin: GithubFactOrigin = { kind: 'observation', lastBodyDigest: target.lastBodyDigest };
+
   if (target.kind === 'pull_request') {
     return toReadObject(
-      buildGithubFact(
-        'pull_request',
-        { action: 'resynced', pull_request: answer.body },
-        'observation',
-      ),
+      buildGithubFact('pull_request', { action: 'resynced', pull_request: answer.body }, origin),
     );
   }
 
@@ -122,7 +121,7 @@ async function readObject(target: ResyncTarget, token: string): Promise<ReadObje
     repository: { id: parsed.data.id, full_name: parsed.data.full_name },
   };
 
-  return toReadObject(buildGithubFact('release', body, 'observation'));
+  return toReadObject(buildGithubFact('release', body, origin));
 }
 
 function toReadObject(fact: PendingFact | 'ignored' | 'invalid'): ReadObject {

package/src/queries/github-delivery.ts

@@ -0,0 +1,13 @@
+import { githubDeliveryConsumers } from './github-delivery-consumers.ts';
+import {
+  parsePullRequestDeliveries,
+  selectPullRequestDelivery,
+} from './select-pull-request-delivery.ts';
+
+// What a query of another plug-in needs to read deliveries: `consumers` for its `reads`, `select`
+// for the statement to run on its `db`, and `parse` for the rows of the result.
+export const githubDelivery = {
+  consumers: githubDeliveryConsumers,
+  select: selectPullRequestDelivery,
+  parse: parsePullRequestDeliveries,
+} as const;