@pepys/source-github 0.0.1 → 0.1.1

package/src/ingest/github-app/read-installation-token.ts

@@ -0,0 +1,43 @@
+import { z } from 'zod';
+import { requestGithubAPI } from './request-github-api.ts';
+
+// `not-installed`: the App has no installation on the repository, so it can read nothing there.
+export type InstallationToken =
+  | { kind: 'token'; token: string }
+  | { kind: 'not-installed' }
+  | { kind: 'failed'; status: number };
+
+const installationSchema = z.looseObject({ id: z.number().int() });
+const accessTokenSchema = z.looseObject({ token: z.string().min(1) });
+
+// The token of the App's installation on one repository, `owner/name`, from the App's JWT.
+export async function readInstallationToken(
+  repository: string,
+  appJWT: string,
+): Promise<InstallationToken> {
+  const installation = await requestGithubAPI(`/repos/${repository}/installation`, appJWT);
+
+  if (installation.status === 404) {
+    return { kind: 'not-installed' };
+  }
+
+  const parsedInstallation = installationSchema.safeParse(installation.body);
+
+  if (installation.status !== 200 || !parsedInstallation.success) {
+    return { kind: 'failed', status: installation.status };
+  }
+
+  const created = await requestGithubAPI(
+    `/app/installations/${parsedInstallation.data.id}/access_tokens`,
+    appJWT,
+    'POST',
+  );
+
+  const parsedToken = accessTokenSchema.safeParse(created.body);
+
+  if (created.status !== 201 || !parsedToken.success) {
+    return { kind: 'failed', status: created.status };
+  }
+
+  return { kind: 'token', token: parsedToken.data.token };
+}

package/src/ingest/github-app/request-github-api.ts

@@ -0,0 +1,75 @@
+import { githubAPIURL } from './github-api-url.ts';
+
+// One answer of the GitHub REST API. `next` is the URL of the next page, from the `link` header.
+export interface GithubAnswer {
+  status: number;
+  body: unknown;
+  next: string | undefined;
+}
+
+// One call to the GitHub REST API, with a path or a whole URL from a `link` header. A transport
+// failure answers status 0.
+export async function requestGithubAPI(
+  pathOrURL: string,
+  token: string,
+  method: 'GET' | 'POST' = 'GET',
+): Promise<GithubAnswer> {
+  const url = pathOrURL.startsWith('/') ? `${githubAPIURL}${pathOrURL}` : pathOrURL;
+
+  const response = await sendRequest(url, token, method);
+
+  if (response === undefined) {
+    return { status: 0, body: undefined, next: undefined };
+  }
+
+  const text = await response.text();
+
+  return {
+    status: response.status,
+    body: parseBody(text),
+    next: findNextPage(response.headers.get('link')),
+  };
+}
+
+async function sendRequest(
+  url: string,
+  token: string,
+  method: 'GET' | 'POST',
+): Promise<Response | undefined> {
+  try {
+    const response = await fetch(url, {
+      method,
+      headers: {
+        accept: 'application/vnd.github+json',
+        authorization: `Bearer ${token}`,
+        'user-agent': 'pepys-ingest',
+        'x-github-api-version': '2022-11-28',
+      },
+    });
+
+    return response;
+  } catch (error: unknown) {
+    console.error(`a GitHub API call to ${url} failed`, error);
+
+    return undefined;
+  }
+}
+
+function parseBody(text: string): unknown {
+  try {
+    const body: unknown = JSON.parse(text);
+
+    return body;
+  } catch {
+    return undefined;
+  }
+}
+
+function findNextPage(link: string | null): string | undefined {
+  const next = link
+    ?.split(',')
+    .map((part) => /<(?<url>[^>]+)>;\s*rel="next"/u.exec(part.trim()))
+    .find((found) => found !== null);
+
+  return next?.groups?.['url'];
+}

package/src/ingest/github-event-rules.ts

@@ -0,0 +1,324 @@
+import type { FactFamily } from '@pepys/kit';
+import { z } from 'zod';
+import { deploymentFamily } from '../families/deployment-family.ts';
+import { issueFamily } from '../families/issue-family.ts';
+import { pullRequestFamily } from '../families/pull-request-family.ts';
+import { pullRequestReviewFamily } from '../families/pull-request-review-family.ts';
+import { refFamily } from '../families/ref-family.ts';
+import { releaseFamily } from '../families/release-family.ts';
+import { workflowRunFamily } from '../families/workflow-run-family.ts';
+
+// What a rule reads from one delivery body. `locator` is the API path the resync pull reads the
+// entity at.
+interface GithubFactFields {
+  type: string;
+  entity: string;
+  actor: string | undefined;
+  at: string;
+  locator: string | undefined;
+}
+
+// `ignored`: a delivery pepys does not keep, such as a push to a feature branch, or an action that
+// the family does not list. `invalid`: the body is not what the event documents.
+export interface GithubEventRule {
+  readonly family: FactFamily;
+  readonly read: (body: unknown) => GithubFactFields | 'ignored' | 'invalid';
+}
+
+const senderSchema = z.looseObject({ id: z.number().int() }).optional();
+const timeSchema = z.string().min(1);
+
+const pullRequestBodySchema = z.looseObject({
+  action: z.string().min(1),
+  pull_request: z.looseObject({
+    id: z.number().int(),
+    updated_at: timeSchema,
+    url: z.string().optional(),
+  }),
+  sender: senderSchema,
+});
+
+const releaseBodySchema = z.looseObject({
+  action: z.string().min(1),
+  release: z.looseObject({
+    id: z.number().int(),
+    url: z.string().optional(),
+    created_at: timeSchema,
+    published_at: timeSchema.nullish(),
+    updated_at: timeSchema.nullish(),
+  }),
+  sender: senderSchema,
+});
+
+const pushedAtSchema = z.union([z.number(), z.string()]).nullish();
+
+const pushRepositorySchema = z.looseObject({
+  id: z.number().int(),
+  default_branch: z.string(),
+  pushed_at: pushedAtSchema,
+});
+
+const pushBodySchema = z.looseObject({
+  ref: z.string().min(1),
+  repository: pushRepositorySchema,
+  head_commit: z.looseObject({ timestamp: timeSchema }).nullish(),
+  sender: senderSchema,
+});
+
+const deploymentBodySchema = z.looseObject({
+  deployment: z.looseObject({ id: z.number().int(), created_at: timeSchema }),
+  sender: senderSchema,
+});
+
+const deploymentStatusBodySchema = z.looseObject({
+  deployment_status: z.looseObject({ created_at: timeSchema }),
+  deployment: z.looseObject({ id: z.number().int() }),
+  sender: senderSchema,
+});
+
+const workflowRunBodySchema = z.looseObject({
+  action: z.string().min(1),
+  workflow_run: z.looseObject({ id: z.number().int(), updated_at: timeSchema }),
+  sender: senderSchema,
+});
+
+const issueBodySchema = z.looseObject({
+  action: z.string().min(1),
+  issue: z.looseObject({ id: z.number().int(), updated_at: timeSchema }),
+  sender: senderSchema,
+});
+
+const reviewBodySchema = z.looseObject({
+  action: z.string().min(1),
+  review: z.looseObject({ id: z.number().int(), submitted_at: timeSchema.nullish() }),
+  pull_request: z.looseObject({ updated_at: timeSchema }),
+  sender: senderSchema,
+});
+
+// One rule per webhook event name. Any other event is ignored.
+export const githubEventRules: Readonly<Record<string, GithubEventRule>> = {
+  pull_request: {
+    family: pullRequestFamily,
+    read: (body) => {
+      const parsed = pullRequestBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      const pullRequest = parsed.data.pull_request;
+
+      return pickFields(pullRequestFamily, `github.pull_request.${parsed.data.action}`, {
+        entity: `github:pull_request:${pullRequest.id}`,
+        actor: parsed.data.sender,
+        time: pullRequest.updated_at,
+        locator: toAPIPath(pullRequest.url),
+      });
+    },
+  },
+  release: {
+    family: releaseFamily,
+    read: (body) => {
+      const parsed = releaseBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      const release = parsed.data.release;
+
+      return pickFields(releaseFamily, `github.release.${parsed.data.action}`, {
+        entity: `github:release:${release.id}`,
+        actor: parsed.data.sender,
+        time: release.updated_at ?? release.published_at ?? release.created_at,
+        locator: toAPIPath(release.url),
+      });
+    },
+  },
+  push: {
+    family: refFamily,
+    read: (body) => {
+      const parsed = pushBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      const push = parsed.data;
+      const isDefaultBranch = push.ref === `refs/heads/${push.repository.default_branch}`;
+
+      if (!isDefaultBranch && !push.ref.startsWith('refs/tags/')) {
+        return 'ignored';
+      }
+
+      // A push gives its time as seconds, or as text with an offset; the fact holds one form, so
+      // the times of two pushes compare as text.
+      const pushedAt = push.repository.pushed_at;
+
+      const timeValue =
+        typeof pushedAt === 'number'
+          ? pushedAt * 1000
+          : (pushedAt ?? push.head_commit?.timestamp ?? '');
+
+      const time = new Date(timeValue);
+
+      return pickFields(refFamily, 'github.ref.pushed', {
+        entity: `github:ref:${push.repository.id}:${push.ref}`,
+        actor: push.sender,
+        time: Number.isNaN(time.getTime()) ? undefined : time.toISOString(),
+        locator: undefined,
+      });
+    },
+  },
+  deployment: {
+    family: deploymentFamily,
+    read: (body) => {
+      const parsed = deploymentBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      return pickFields(deploymentFamily, 'github.deployment.created', {
+        entity: `github:deployment:${parsed.data.deployment.id}`,
+        actor: parsed.data.sender,
+        time: parsed.data.deployment.created_at,
+        locator: undefined,
+      });
+    },
+  },
+  deployment_status: {
+    family: deploymentFamily,
+    read: (body) => {
+      const parsed = deploymentStatusBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      return pickFields(deploymentFamily, 'github.deployment.status_created', {
+        entity: `github:deployment:${parsed.data.deployment.id}`,
+        actor: parsed.data.sender,
+        time: parsed.data.deployment_status.created_at,
+        locator: undefined,
+      });
+    },
+  },
+  workflow_run: {
+    family: workflowRunFamily,
+    read: (body) => {
+      const parsed = workflowRunBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      return pickFields(workflowRunFamily, `github.workflow_run.${parsed.data.action}`, {
+        entity: `github:workflow_run:${parsed.data.workflow_run.id}`,
+        actor: parsed.data.sender,
+        time: parsed.data.workflow_run.updated_at,
+        locator: undefined,
+      });
+    },
+  },
+  issues: {
+    family: issueFamily,
+    read: (body) => {
+      const parsed = issueBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      return pickFields(issueFamily, `github.issue.${parsed.data.action}`, {
+        entity: `github:issue:${parsed.data.issue.id}`,
+        actor: parsed.data.sender,
+        time: parsed.data.issue.updated_at,
+        locator: undefined,
+      });
+    },
+  },
+  pull_request_review: {
+    family: pullRequestReviewFamily,
+    read: (body) => {
+      const parsed = reviewBodySchema.safeParse(body);
+
+      if (!parsed.success) {
+        return 'invalid';
+      }
+
+      const review = parsed.data.review;
+
+      return pickFields(
+        pullRequestReviewFamily,
+        `github.pull_request_review.${parsed.data.action}`,
+        {
+          entity: `github:pull_request_review:${review.id}`,
+          actor: parsed.data.sender,
+          time: review.submitted_at ?? parsed.data.pull_request.updated_at,
+          locator: undefined,
+        },
+      );
+    },
+  },
+};
+
+interface RuleValues {
+  entity: string;
+  actor: { id: number } | undefined;
+  time: string | null | undefined;
+  locator: string | undefined;
+}
+
+// An action the family does not list is ignored, so GitHub records no failed delivery for it. A
+// time with an offset, as a push's commit gives, is written in UTC.
+function pickFields(
+  family: FactFamily,
+  type: string,
+  values: RuleValues,
+): GithubFactFields | 'ignored' | 'invalid' {
+  if (!Object.hasOwn(family.types, type)) {
+    return 'ignored';
+  }
+
+  const at = toFactTime(values.time);
+
+  if (at === undefined) {
+    return 'invalid';
+  }
+
+  return {
+    type,
+    entity: values.entity,
+    actor: values.actor === undefined ? undefined : `github:user:${values.actor.id}`,
+    at,
+    locator: values.locator,
+  };
+}
+
+const utcTimeSchema = z.iso.datetime();
+
+function toFactTime(time: string | null | undefined): string | undefined {
+  if (time === null || time === undefined) {
+    return undefined;
+  }
+
+  if (utcTimeSchema.safeParse(time).success) {
+    return time;
+  }
+
+  const date = new Date(time);
+
+  return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
+}
+
+// The resync pull reads only the path, from the API origin it knows.
+function toAPIPath(url: string | undefined): string | undefined {
+  if (url === undefined || !URL.canParse(url)) {
+    return undefined;
+  }
+
+  const path = new URL(url).pathname;
+
+  return path.startsWith('/repos/') ? path : undefined;
+}

package/src/ingest/read-github-delivery.ts

@@ -0,0 +1,41 @@
+import type { Delivery, Received } from '@pepys/kit';
+import { buildGithubFact } from './build-github-fact.ts';
+
+// The read of a checked webhook delivery: the event's rule makes one fact, and an event with no
+// rule is ignored. The delivery ID is required and goes in no fact.
+export function readGithubDelivery(delivery: Delivery): Received {
+  const event = delivery.headers.get('x-github-event') ?? '';
+  const deliveryID = delivery.headers.get('x-github-delivery');
+
+  if (deliveryID === null || deliveryID === '') {
+    return { kind: 'refused', status: 400, reason: 'no delivery ID' };
+  }
+
+  const body = parseJSON(new TextDecoder().decode(delivery.body));
+
+  if (!body.ok) {
+    return { kind: 'refused', status: 400, reason: 'the body is not JSON' };
+  }
+
+  const fact = buildGithubFact(event, body.value, 'delivery');
+
+  if (fact === 'ignored') {
+    return { kind: 'ignored' };
+  }
+
+  if (fact === 'invalid') {
+    return { kind: 'refused', status: 422, reason: `not a ${event} delivery` };
+  }
+
+  return { kind: 'batch', batch: { facts: [fact] } };
+}
+
+function parseJSON(text: string): { ok: true; value: unknown } | { ok: false } {
+  try {
+    const value: unknown = JSON.parse(text);
+
+    return { ok: true, value };
+  } catch {
+    return { ok: false };
+  }
+}

package/src/ingest/recovery/pull-failed-deliveries.ts

@@ -0,0 +1,100 @@
+import type { PendingFact, PullContext, Pulled } from '@pepys/kit';
+import { z } from 'zod';
+import { buildGithubFact } from '../build-github-fact.ts';
+import { findAppJWT } from '../github-app/find-app-jwt.ts';
+import { pickPullFailure } from '../github-app/pick-pull-failure.ts';
+import { requestGithubAPI } from '../github-app/request-github-api.ts';
+import type { GithubAnswer } from '../github-app/request-github-api.ts';
+
+const deliveryDetailSchema = z.looseObject({
+  event: z.string(),
+  request: z.looseObject({ payload: z.unknown() }),
+});
+
+// The recovery pull: each delivery of the App's webhook that got no 2xx answer, read from GitHub's
+// delivery log and built as the receive path builds it, so a delivery that did arrive is a copy.
+// The cursor is the highest delivery ID seen; GitHub keeps deliveries for 3 days.
+export async function pullFailedDeliveries(
+  cursor: string | null,
+  context: PullContext,
+): Promise<Pulled> {
+  const appJWT = findAppJWT(context.env, Date.now());
+
+  if (appJWT === undefined) {
+    return { kind: 'rejected', reason: 'the GitHub App private key cannot sign a token' };
+  }
+
+  const seenID = cursor === null ? 0 : Number(cursor);
+
+  const listed = await readDeliveriesAfter(seenID, appJWT);
+
+  if (listed.kind !== 'listed') {
+    return pickPullFailure(listed.status);
+  }
+
+  const facts: PendingFact[] = [];
+
+  for (const id of collectFailedIDs(listed.items)) {
+    const answer = await requestGithubAPI(`/app/hook/deliveries/${id}`, appJWT);
+
+    const detail = deliveryDetailSchema.safeParse(answer.body);
+
+    if (answer.status !== 200 || !detail.success) {
+      return pickPullFailure(answer.status);
+    }
+
+    const fact = buildGithubFact(detail.data.event, detail.data.request.payload, 'delivery');
+
+    if (fact !== 'ignored' && fact !== 'invalid') {
+      facts.push(fact);
+    }
+  }
+
+  const highestID = Math.max(seenID, ...listed.items.map((item) => item.id));
+
+  return { kind: 'batch', batch: { facts }, cursor: `${highestID}`, isDone: true };
+}
+
+const deliveryItemSchema = z.looseObject({
+  id: z.number().int(),
+  status_code: z.number().int(),
+  event: z.string(),
+});
+
+const deliveryListSchema = z.array(deliveryItemSchema);
+
+type DeliveryItem = z.infer<typeof deliveryItemSchema>;
+
+type DeliveryList = { kind: 'listed'; items: DeliveryItem[] } | { kind: 'failed'; status: number };
+
+// GitHub lists the newest delivery first, so the pages stop at the first one already seen.
+async function readDeliveriesAfter(seenID: number, appJWT: string): Promise<DeliveryList> {
+  const items: DeliveryItem[] = [];
+  let url: string | undefined = '/app/hook/deliveries?per_page=100';
+
+  while (url !== undefined) {
+    const answer: GithubAnswer = await requestGithubAPI(url, appJWT);
+
+    const page = deliveryListSchema.safeParse(answer.body);
+
+    if (answer.status !== 200 || !page.success) {
+      return { kind: 'failed', status: answer.status };
+    }
+
+    const unseen = page.data.filter((item) => item.id > seenID);
+
+    items.push(...unseen);
+
+    url = unseen.length < page.data.length ? undefined : answer.next;
+  }
+
+  return { kind: 'listed', items };
+}
+
+// oldest first, so the facts reach the log in the order GitHub sent them
+function collectFailedIDs(items: readonly DeliveryItem[]): number[] {
+  return items
+    .filter((item) => item.status_code < 200 || item.status_code >= 300)
+    .map((item) => item.id)
+    .toSorted((left, right) => left - right);
+}

package/src/ingest/resync/collect-resync-targets.ts

@@ -0,0 +1,33 @@
+import type { ResyncMark } from '@pepys/kit';
+
+// One marked object that the resync pull reads again: its repository, `owner/name`, and its API path.
+export interface ResyncTarget {
+  kind: 'pull_request' | 'release';
+  entity: string;
+  repository: string;
+  path: string;
+}
+
+const locatorPatterns = [
+  { kind: 'pull_request', pattern: /^\/repos\/(?<repository>[^/]+\/[^/]+)\/pulls\/\d+$/u },
+  { kind: 'release', pattern: /^\/repos\/(?<repository>[^/]+\/[^/]+)\/releases\/\d+$/u },
+] as const;
+
+// The marks that the pull can read. A mark of another family, or with no locator, has nothing to
+// read, so the pull leaves it out, and ingest removes it with the rest once the batch is stored.
+export function collectResyncTargets(marks: readonly ResyncMark[]): ResyncTarget[] {
+  return marks.flatMap((mark) => {
+    const locator = mark.locator ?? '';
+
+    for (const locatorPattern of locatorPatterns) {
+      const repository = locatorPattern.pattern.exec(locator)?.groups?.['repository'];
+      const kind = locatorPattern.kind;
+
+      if (repository !== undefined && mark.entity.startsWith(`github:${kind}:`)) {
+        return [{ kind, entity: mark.entity, repository, path: locator }];
+      }
+    }
+
+    return [];
+  });
+}