@pepys/source-github 0.0.1 → 0.1.1

package/src/ingest/resync/pull-marked-objects.ts

@@ -0,0 +1,130 @@
+import type { PendingFact, PullContext, Pulled } from '@pepys/kit';
+import { z } from 'zod';
+import { buildGithubFact } from '../build-github-fact.ts';
+import { findAppJWT } from '../github-app/find-app-jwt.ts';
+import { pickPullFailure } from '../github-app/pick-pull-failure.ts';
+import { readInstallationToken } from '../github-app/read-installation-token.ts';
+import type { InstallationToken } from '../github-app/read-installation-token.ts';
+import { requestGithubAPI } from '../github-app/request-github-api.ts';
+import { collectResyncTargets } from './collect-resync-targets.ts';
+import type { ResyncTarget } from './collect-resync-targets.ts';
+
+// The resync pull: each marked pull request and release, read again from the API as one fact in the
+// shape of its webhook body. The fact comes after the deliveries, so it wins a tie of times and sets
+// the row to GitHub's state.
+export async function pullMarkedObjects(
+  _cursor: string | null,
+  context: PullContext,
+): Promise<Pulled> {
+  const targets = collectResyncTargets(context.marks);
+
+  if (targets.length === 0) {
+    return { kind: 'batch', batch: { facts: [] }, cursor: 'resynced', isDone: true };
+  }
+
+  const appJWT = findAppJWT(context.env, Date.now());
+
+  if (appJWT === undefined) {
+    return { kind: 'rejected', reason: 'the GitHub App private key cannot sign a token' };
+  }
+
+  const tokens = new Map<string, InstallationToken>();
+
+  const facts: PendingFact[] = [];
+
+  for (const target of targets) {
+    const token = await readCachedToken(tokens, target.repository, appJWT);
+
+    if (token.kind === 'failed') {
+      return pickPullFailure(token.status);
+    }
+
+    if (token.kind === 'not-installed') {
+      continue;
+    }
+
+    const read = await readObject(target, token.token);
+
+    if (read.kind === 'failed') {
+      return pickPullFailure(read.status);
+    }
+
+    if (read.kind === 'fact') {
+      facts.push(read.fact);
+    }
+  }
+
+  return { kind: 'batch', batch: { facts }, cursor: 'resynced', isDone: true };
+}
+
+// one installation token per repository in one run
+async function readCachedToken(
+  tokens: Map<string, InstallationToken>,
+  repository: string,
+  appJWT: string,
+): Promise<InstallationToken> {
+  const known = tokens.get(repository);
+
+  if (known !== undefined) {
+    return known;
+  }
+
+  const token = await readInstallationToken(repository, appJWT);
+
+  tokens.set(repository, token);
+
+  return token;
+}
+
+const repositorySchema = z.looseObject({ id: z.number().int(), full_name: z.string() });
+
+type ReadObject =
+  | { kind: 'fact'; fact: PendingFact }
+  | { kind: 'none' }
+  | { kind: 'failed'; status: number };
+
+// An object the API answers 404 for is left alone. GitHub deletes no pull request, and shows a draft
+// release only to a token that can push, so a 404 cannot prove a delete. A deleted release's own
+// `deleted` delivery sets its tombstone.
+async function readObject(target: ResyncTarget, token: string): Promise<ReadObject> {
+  const answer = await requestGithubAPI(target.path, token);
+
+  if (answer.status === 404) {
+    return { kind: 'none' };
+  }
+
+  if (answer.status !== 200) {
+    return { kind: 'failed', status: answer.status };
+  }
+
+  if (target.kind === 'pull_request') {
+    return toReadObject(
+      buildGithubFact(
+        'pull_request',
+        { action: 'resynced', pull_request: answer.body },
+        'observation',
+      ),
+    );
+  }
+
+  const repository = await requestGithubAPI(`/repos/${target.repository}`, token);
+
+  const parsed = repositorySchema.safeParse(repository.body);
+
+  if (repository.status !== 200 || !parsed.success) {
+    return { kind: 'failed', status: repository.status };
+  }
+
+  // only the two fields the release consumer reads, so the hash moves with the release alone
+  const body = {
+    action: 'resynced',
+    release: answer.body,
+    repository: { id: parsed.data.id, full_name: parsed.data.full_name },
+  };
+
+  return toReadObject(buildGithubFact('release', body, 'observation'));
+}
+
+function toReadObject(fact: PendingFact | 'ignored' | 'invalid'): ReadObject {
+  return typeof fact === 'string' ? { kind: 'none' } : { kind: 'fact', fact };
+}

package/src/queries/github-delivery-consumers.ts

@@ -0,0 +1,13 @@
+import { githubDeployConsumer } from '../github-deploy-consumer.ts';
+import { githubPullRequestConsumer } from '../github-pull-request-consumer.ts';
+import { githubRefConsumer } from '../github-ref-consumer.ts';
+import { githubReleaseConsumer } from '../github-release-consumer.ts';
+
+// The consumers whose tables the delivery query reads. A query of another plug-in that calls the
+// query module passes these in its `reads`.
+export const githubDeliveryConsumers = [
+  githubPullRequestConsumer,
+  githubReleaseConsumer,
+  githubRefConsumer,
+  githubDeployConsumer,
+] as const;

package/src/queries/github-delivery.ts

@@ -0,0 +1,13 @@
+import { githubDeliveryConsumers } from './github-delivery-consumers.ts';
+import {
+  parsePullRequestDeliveries,
+  selectPullRequestDelivery,
+} from './select-pull-request-delivery.ts';
+
+// What a query of another plug-in needs to read deliveries: `consumers` for its `reads`, `select`
+// for the statement to run on its `db`, and `parse` for the rows of the result.
+export const githubDelivery = {
+  consumers: githubDeliveryConsumers,
+  select: selectPullRequestDelivery,
+  parse: parsePullRequestDeliveries,
+} as const;

package/src/queries/pull-request-delivery-query.ts

@@ -0,0 +1,49 @@
+import { defineQuery } from '@pepys/kit';
+import { z } from 'zod';
+import { deploySignals } from '../utils/deploy-signals.ts';
+import { githubDeliveryConsumers } from './github-delivery-consumers.ts';
+import {
+  parsePullRequestDeliveries,
+  selectPullRequestDelivery,
+} from './select-pull-request-delivery.ts';
+
+const pullRequestsSchema = z.array(z.string()).max(200);
+const nameSchema = z.string().min(1);
+const releaseSchema = z.object({ id: z.string(), tag: z.string(), at: z.string() });
+const signalSchema = z.enum(deploySignals);
+const deploySchema = z.object({ id: z.string(), at: z.string(), signal: signalSchema });
+
+const deliverySchema = z.object({
+  pullRequest: z.string(),
+  mergedAt: z.string().nullable(),
+  release: releaseSchema.nullable(),
+  deploy: deploySchema.nullable(),
+});
+
+export const pullRequestDeliveryQuery = defineQuery({
+  name: 'github_pull_request_delivery',
+  description:
+    'Returns, for each pull request, when it merged, the first release of its repository that contains it, and the first deploy to one environment of a commit that pins such a release. With a target, only a deploy of that target counts. A pull request with no row is left out.',
+  input: z.object({
+    pullRequests: pullRequestsSchema,
+    environment: nameSchema,
+    target: nameSchema.optional(),
+  }),
+  output: z.array(deliverySchema),
+  reads: githubDeliveryConsumers,
+  handler: async (call) => {
+    const [first, ...rest] = call.input.pullRequests;
+
+    if (first === undefined) {
+      return [];
+    }
+
+    const result = await selectPullRequestDelivery({
+      pullRequests: [first, ...rest],
+      environment: call.input.environment,
+      target: call.input.target,
+    }).execute(call.db);
+
+    return parsePullRequestDeliveries(result.rows);
+  },
+});

package/src/queries/select-pull-request-delivery.ts

@@ -0,0 +1,155 @@
+import { sql } from 'kysely';
+import type { RawBuilder } from 'kysely';
+import { z } from 'zod';
+import { deploySignals } from '../utils/deploy-signals.ts';
+import type { DeploySignal } from '../utils/deploy-signals.ts';
+
+interface DeliveryOptions {
+  // at least one
+  pullRequests: readonly [string, ...string[]];
+  environment: string;
+
+  // Absent: a deploy of any target, and of none, counts. Given: only a deploy of that target.
+  target?: string | undefined;
+}
+
+// The statement that ties pull requests to releases and deploys. The rules it holds are in the
+// "Delivery query" section of the plug-ins design doc. Run it with `.execute(db)`, and turn its rows
+// into deliveries with the parse function below.
+export function selectPullRequestDelivery(options: DeliveryOptions): RawBuilder<unknown> {
+  return sql`select
+    pull_request.id as "pullRequest",
+    pull_request.merged_at as "mergedAt",
+    release.id as "releaseID",
+    release.tag_name as "releaseTag",
+    release.published_at as "releaseAt",
+    deploy.id as "deployID",
+    deploy.deployed_at as "deployAt",
+    deploy.signal as "deploySignal"
+  from github_pull_request as pull_request
+  left join github_release as release on release.id = ${selectFirstReleaseID()}
+  left join github_deploy as deploy on deploy.id = ${selectFirstDeployID(options)}
+  where pull_request.id in (${sql.join(options.pullRequests)})
+  order by pull_request.id`;
+}
+
+function selectFirstReleaseID(): RawBuilder<unknown> {
+  const releaseTime = formatCommitTime(
+    'candidate.repository_id',
+    'candidate.target_commitish',
+    'candidate.created_at',
+  );
+
+  return sql`(select candidate.id from github_release as candidate
+    where ${formatContains('candidate')}
+    order by ${releaseTime}, candidate.id
+    limit 1)`;
+}
+
+// The commit time of a sha of a repository, or the fallback time. The three names are columns.
+function formatCommitTime(repository: string, sha: string, fallback: string): RawBuilder<unknown> {
+  return sql`julianday(coalesce((select commit_row.committed_at from github_commit as commit_row
+    where commit_row.repository_id = ${sql.ref(repository)}
+      and commit_row.sha = ${sql.ref(sha)}), ${sql.ref(fallback)}))`;
+}
+
+// The release under `alias` contains `pull_request`.
+function formatContains(alias: string): RawBuilder<unknown> {
+  const release = sql.table(alias);
+
+  const releaseTime = formatCommitTime(
+    `${alias}.repository_id`,
+    `${alias}.target_commitish`,
+    `${alias}.created_at`,
+  );
+
+  const mergeTime = formatCommitTime(
+    'pull_request.repository_id',
+    'pull_request.merge_commit_sha',
+    'pull_request.merged_at',
+  );
+
+  return sql`(pull_request.merged_at is not null
+    and ${release}.repository_id = pull_request.repository_id
+    and ${release}.published_at is not null
+    and ${release}.is_draft = 0
+    and ${release}.deleted_at is null
+    and ${releaseTime} >= ${mergeTime})`;
+}
+
+function selectFirstDeployID(options: DeliveryOptions): RawBuilder<unknown> {
+  const target = options.target ?? null;
+
+  const deployTime = formatCommitTime(
+    'carrier.repository_id',
+    'carrier.sha',
+    'carrier.deployed_at',
+  );
+
+  const pinnerMergeTime = formatCommitTime(
+    'pinner.repository_id',
+    'pinner.merge_commit_sha',
+    'pinner.merged_at',
+  );
+
+  return sql`(select carrier.id from github_deploy as carrier
+    join github_pull_request as pinner
+      on pinner.repository_id = carrier.repository_id and pinner.merged_at is not null
+    join github_release_bump as bump on bump.pull_request_id = pinner.id
+    join github_release as pinned
+      on pinned.repository_full_name = bump.release_repository
+      and pinned.tag_name = bump.release_tag
+    where ${formatContains('pinned')}
+      and carrier.environment = ${options.environment}
+      and (${target} is null or carrier.target = ${target})
+      and carrier.signal = (case when exists (
+        select 1 from github_deploy as tagged
+        where tagged.repository_id = carrier.repository_id
+          and tagged.environment = carrier.environment
+          and (${target} is null or tagged.target = ${target})
+          and tagged.signal = 'tag'
+      ) then 'tag' else 'deployment_status' end)
+      and ${deployTime} >= ${pinnerMergeTime}
+    order by julianday(carrier.deployed_at), carrier.id
+    limit 1)`;
+}
+
+export interface PullRequestDelivery {
+  pullRequest: string;
+  mergedAt: string | null;
+  release: { id: string; tag: string; at: string } | null;
+  deploy: { id: string; at: string; signal: DeploySignal } | null;
+}
+
+const textSchema = z.string().nullable();
+
+// One row per pull request, as the statement selects it.
+const deliveryRowSchema = z.object({
+  pullRequest: z.string(),
+  mergedAt: textSchema,
+  releaseID: textSchema,
+  releaseTag: textSchema,
+  releaseAt: textSchema,
+  deployID: textSchema,
+  deployAt: textSchema,
+  deploySignal: z.enum(deploySignals).nullable(),
+});
+
+const deliveryRowsSchema = z.array(deliveryRowSchema);
+
+// The rows of the statement as deliveries. A row of another shape is a bug in the statement, so the
+// parse throws.
+export function parsePullRequestDeliveries(rows: unknown): PullRequestDelivery[] {
+  return deliveryRowsSchema.parse(rows).map((row) => ({
+    pullRequest: row.pullRequest,
+    mergedAt: row.mergedAt,
+    release:
+      row.releaseID === null || row.releaseTag === null || row.releaseAt === null
+        ? null
+        : { id: row.releaseID, tag: row.releaseTag, at: row.releaseAt },
+    deploy:
+      row.deployID === null || row.deployAt === null || row.deploySignal === null
+        ? null
+        : { id: row.deployID, at: row.deployAt, signal: row.deploySignal },
+  }));
+}

package/src/utils/build-body-hash.ts

@@ -0,0 +1,34 @@
+import { createHash } from 'node:crypto';
+
+// The SHA-256 of the canonical JSON of a value: keys sorted at every depth, no white space. Two
+// bodies that parse to one JSON value give one hash, however each was spaced or ordered.
+export function buildBodyHash(value: unknown): string {
+  return createHash('sha256').update(formatCanonicalJSON(value)).digest('hex');
+}
+
+function formatCanonicalJSON(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `[${value.map((item: unknown) => formatCanonicalJSON(item)).join(',')}]`;
+  }
+
+  if (typeof value === 'object' && value !== null) {
+    // code-unit order, as JSON canonical forms sort keys, and never the locale's order
+    const entries = Object.entries(value)
+      .toSorted(([left], [right]) => compareCodeUnits(left, right))
+      .map(
+        ([key, item]: [string, unknown]) => `${JSON.stringify(key)}:${formatCanonicalJSON(item)}`,
+      );
+
+    return `{${entries.join(',')}}`;
+  }
+
+  return JSON.stringify(value);
+}
+
+function compareCodeUnits(left: string, right: string): number {
+  if (left === right) {
+    return 0;
+  }
+
+  return left < right ? -1 : 1;
+}

package/src/utils/deploy-signals.ts

@@ -0,0 +1,5 @@
+// The two signals that report a deploy, in the order a query trusts them: the tag rows of one
+// repository and environment when it has any, and its deployment status rows otherwise.
+export const deploySignals = ['tag', 'deployment_status'] as const;
+
+export type DeploySignal = (typeof deploySignals)[number];

package/src/utils/find-field.ts

@@ -0,0 +1,19 @@
+// The value at a path of keys inside a parsed JSON value, or undefined when a step is no object or
+// has no such key. A consumer reads a delivery body with it and no schema library.
+export function findField(value: unknown, ...path: readonly string[]): unknown {
+  let current = value;
+
+  for (const key of path) {
+    if (typeof current !== 'object' || current === null || Array.isArray(current)) {
+      return undefined;
+    }
+
+    if (!Object.hasOwn(current, key)) {
+      return undefined;
+    }
+
+    current = Reflect.get(current, key);
+  }
+
+  return current;
+}

package/src/ingest/build-pull-request-fact.ts

@@ -1,38 +0,0 @@
-import { buildWebhookFactID, parseFactEnvelope } from '@pepys/kit';
-import type { FactEnvelope, Result } from '@pepys/kit';
-import { z } from 'zod';
-import type { ZodError } from 'zod';
-
-interface PullRequestFactParameters {
-  deliveryID: string;
-  body: unknown;
-}
-
-const pullRequestBodySchema = z.looseObject({
-  action: z.string().min(1),
-  pull_request: z.looseObject({ id: z.number().int(), updated_at: z.string() }),
-  sender: z.looseObject({ id: z.number().int() }).optional(),
-});
-
-export function buildPullRequestFact(
-  parameters: PullRequestFactParameters,
-): Result<FactEnvelope<unknown>, ZodError> {
-  const parsed = pullRequestBodySchema.safeParse(parameters.body);
-
-  if (!parsed.success) {
-    return { ok: false, failure: parsed.error };
-  }
-
-  const body = parsed.data;
-
-  return parseFactEnvelope({
-    id: buildWebhookFactID({ source: 'github', deliveryID: parameters.deliveryID }),
-    type: `github.pull_request.${body.action}`,
-    v: 1,
-    api: 'github/2022-11-28',
-    entity: `github:pull_request:${body.pull_request.id}`,
-    ...(body.sender === undefined ? {} : { actor: `github:user:${body.sender.id}` }),
-    at: body.pull_request.updated_at,
-    payload: parameters.body,
-  });
-}

package/src/ingest/pull-request-stream.ts

@@ -1,4 +0,0 @@
-import type { StreamName } from '@pepys/kit';
-import { toTagged } from '@zgeoff/std';
-
-export const pullRequestStream = toTagged<StreamName>('facts.github.pull_request');

package/src/ingest/read-pull-request-delivery.ts

@@ -1,44 +0,0 @@
-import type { Delivery, Received } from '@pepys/kit';
-import { buildPullRequestFact } from './build-pull-request-fact.ts';
-import { pullRequestStream } from './pull-request-stream.ts';
-
-// The read of a checked webhook delivery: a pull request event becomes one fact, and any other event
-// is ignored.
-export function readPullRequestDelivery(delivery: Delivery): Received {
-  if (delivery.headers.get('x-github-event') !== 'pull_request') {
-    return { kind: 'ignored' };
-  }
-
-  const deliveryID = delivery.headers.get('x-github-delivery');
-
-  if (deliveryID === null || deliveryID === '') {
-    return { kind: 'refused', status: 400, reason: 'no delivery ID' };
-  }
-
-  const body = parseJSON(new TextDecoder().decode(delivery.body));
-
-  if (!body.ok) {
-    return { kind: 'refused', status: 400, reason: 'the body is not JSON' };
-  }
-
-  const fact = buildPullRequestFact({ deliveryID, body: body.value });
-
-  if (!fact.ok) {
-    return { kind: 'refused', status: 422, reason: 'not a pull request delivery' };
-  }
-
-  return {
-    kind: 'batch',
-    batch: { facts: [{ stream: pullRequestStream, fact: fact.value, blobs: [] }] },
-  };
-}
-
-function parseJSON(text: string): { ok: true; value: unknown } | { ok: false } {
-  try {
-    const value: unknown = JSON.parse(text);
-
-    return { ok: true, value };
-  } catch {
-    return { ok: false };
-  }
-}