@useairfoil/producer-github 0.0.1 → 0.0.3

package/src/manifest.ts

@@ -0,0 +1,45 @@
+import * as Manifest from "@useairfoil/connector-kit/manifest";
+
+export const GitHubConfigDef = Manifest.defineConfig({
+  appClientId: Manifest.string({
+    runtimeKey: "GITHUB_APP_CLIENT_ID",
+    description: "Client ID of the GitHub App, for example Iv23li...",
+  }),
+  appPrivateKey: Manifest.secret({
+    runtimeKey: "GITHUB_APP_PRIVATE_KEY",
+    description: "PEM private key of the GitHub App. Newlines may be written as \\n.",
+  }),
+  installationId: Manifest.number({
+    runtimeKey: "GITHUB_INSTALLATION_ID",
+    description: "ID of the GitHub App installation to sync.",
+    integer: true,
+    minimum: 1,
+  }),
+  webhookSecret: Manifest.secret({
+    runtimeKey: "GITHUB_WEBHOOK_SECRET",
+    description: "Webhook secret of the GitHub App.",
+  }),
+});
+
+export type GitHubConfig = Manifest.ConfigValuesOf<typeof GitHubConfigDef>;
+
+export const manifest = Manifest.define({
+  name: "producer-github",
+  title: "GitHub",
+  config: GitHubConfigDef.spec,
+  resources: [
+    { name: "repositories", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "issues", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "pull_requests", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "issue_comments", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "pull_request_reviews", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "pull_request_review_comments", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "pull_request_commits", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "commits", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "check_runs", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "commit_statuses", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "workflow_runs", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "workflow_jobs", capabilities: ["backfill", "changes", "webhook"] },
+    { name: "releases", capabilities: ["backfill", "changes", "webhook"] },
+  ],
+});

package/src/resources/check-runs.ts

@@ -0,0 +1,72 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { DateTime, Effect } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { CommitObject } from "../schemas/commits";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import {
+  type CheckRun,
+  type CheckRunObject,
+  CheckRunObjectSchema,
+  CheckRunSchema,
+} from "../schemas/check-runs";
+import { CheckRunEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import { commitSpec } from "./commits";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  childRows,
+  decodeEvent,
+  wrappedList,
+} from "./shared";
+
+const toRow = (checkRun: CheckRunObject, repository_id: bigint, version: Date): CheckRun => ({
+  ...checkRun,
+  version,
+  repository_id,
+});
+
+/** Check runs of default-branch commits. Webhooks also bring other branches. */
+const checkRunSpec = (client: GitHubClientService): RepoResourceSpec<CommitObject, CheckRun> => ({
+  ...commitSpec,
+  rows: childRows(client, {
+    path: (commit) => `commits/${commit.sha}/check-runs`,
+    pageSchema: wrappedList("check_runs", CheckRunObjectSchema),
+    // Without it, a re-run hides the earlier runs.
+    params: { filter: "all" },
+    toRow: (checkRun, _commit, repositoryId, readAt) => toRow(checkRun, repositoryId, readAt),
+  }),
+});
+
+export const makeCheckRuns = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) => {
+  const spec = checkRunSpec(client);
+  return Resource.entity({
+    name: "check_runs",
+    rowSchema: CheckRunSchema,
+    version: "version",
+
+    check: checkRepoResource(client, spec),
+    backfill: backfillRepoResource(client, repositories, spec),
+    changes: changesRepoResource(client, spec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      // No update time, so read the latest state instead of trusting the payload.
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const { repository, check_run } = yield* decodeEvent(CheckRunEventSchema, payload);
+          const readAt = yield* DateTime.nowAsDate;
+          const page = yield* client.get(
+            CheckRunObjectSchema,
+            `/repos/${repository.full_name}/check-runs/${check_run.id}`,
+          );
+          return [toRow(page.body, repository.id, readAt)];
+        }),
+    },
+  });
+};

package/src/resources/commit-statuses.ts

@@ -0,0 +1,84 @@
+import { ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Array as Arr, Effect, Schema } from "effect";
+
+import type { CommitObject } from "../schemas/commits";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import { type GitHubClientService, withParams } from "../client/client";
+import {
+  type CommitStatus,
+  type CommitStatusObject,
+  CommitStatusObjectSchema,
+  CommitStatusSchema,
+} from "../schemas/commit-statuses";
+import { StatusEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import { commitSpec } from "./commits";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  childRows,
+  decodeEvent,
+  pageSize,
+  readPages,
+} from "./shared";
+
+const toRow = (
+  { updated_at, ...rest }: CommitStatusObject,
+  sha: string,
+  repository_id: bigint,
+): CommitStatus => ({ ...rest, version: updated_at, repository_id, sha });
+
+/** Statuses of default-branch commits. Webhooks also bring other branches. */
+const commitStatusSpec = (
+  client: GitHubClientService,
+): RepoResourceSpec<CommitObject, CommitStatus> => ({
+  ...commitSpec,
+  rows: childRows(client, {
+    path: (commit) => `commits/${commit.sha}/statuses`,
+    pageSchema: Schema.Array(CommitStatusObjectSchema),
+    toRow: (status, commit, repositoryId) => toRow(status, commit.sha, repositoryId),
+  }),
+});
+
+export const makeCommitStatuses = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) => {
+  const spec = commitStatusSpec(client);
+  return Resource.entity({
+    name: "commit_statuses",
+    rowSchema: CommitStatusSchema,
+    version: "version",
+
+    check: checkRepoResource(client, spec),
+    backfill: backfillRepoResource(client, repositories, spec),
+    changes: changesRepoResource(client, spec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      // The payload has no node ID or creator, so read the status from the list.
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const { id, sha, repository } = yield* decodeEvent(StatusEventSchema, payload);
+          const isStatus = (status: CommitStatusObject) => status.id === id;
+          const status = yield* readPages(
+            client,
+            Schema.Array(CommitStatusObjectSchema),
+            withParams(`/repos/${repository.full_name}/commits/${sha}/statuses`, {
+              per_page: pageSize,
+            }),
+            (page) => !page.some(isStatus),
+          ).pipe(
+            Effect.flatMap((statuses) =>
+              Effect.fromOption(
+                Arr.findFirst(statuses, isStatus),
+                () => new ConnectorError({ message: `Commit status ${id} is not in the list` }),
+              ),
+            ),
+          );
+          return [toRow(status, sha, repository.id)];
+        }),
+    },
+  });
+};

package/src/resources/commits.ts

@@ -0,0 +1,101 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema } from "effect";
+
+import type { RepositoryRef } from "../schemas/repositories";
+
+import { type GitHubClientService, withParams } from "../client/client";
+import {
+  type Commit,
+  type CommitObject,
+  CommitObjectSchema,
+  CommitSchema,
+} from "../schemas/commits";
+import { PushEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  decodeEvent,
+  pageSize,
+  readPages,
+  rowsOf,
+  wrappedList,
+} from "./shared";
+
+const toRow = (
+  { sha, node_id, commit, author, committer, parents, html_url }: CommitObject,
+  repository_id: bigint,
+): Commit => ({
+  key: `${repository_id}:${sha}`,
+  version: commit.committer.date,
+  repository_id,
+  sha,
+  node_id,
+  message: commit.message,
+  author: commit.author,
+  committer: commit.committer,
+  author_user: author,
+  committer_user: committer,
+  parents: parents.map((parent) => parent.sha),
+  verified: commit.verification?.verified ?? false,
+  html_url,
+});
+
+const committedAt = (item: CommitObject) => item.commit.committer.date;
+
+/** The list only has the default branch. */
+export const commitSpec: RepoResourceSpec<CommitObject, Commit> = {
+  path: "commits",
+  pageSchema: Schema.Array(CommitObjectSchema),
+  params: {},
+  // `since` filters on the commit time, not the push time. Pushes of older
+  // commits come by webhook.
+  since: true,
+  order: "newest",
+  createdAt: committedAt,
+  updatedAt: committedAt,
+  keep: () => true,
+  enabled: (repository) => repository.size > 0,
+  rows: rowsOf(toRow),
+};
+
+const noCommit = /^0+$/;
+
+export const makeCommits = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) =>
+  Resource.entity({
+    name: "commits",
+    rowSchema: CommitSchema,
+    version: "version",
+
+    check: checkRepoResource(client, commitSpec),
+    backfill: backfillRepoResource(client, repositories, commitSpec),
+    changes: changesRepoResource(client, commitSpec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      // The push payload is not the API's commit, so the commits are read from
+      // the compare API. A new or deleted branch has nothing to compare.
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const { ref, before, after, repository } = yield* decodeEvent(PushEventSchema, payload);
+          if (
+            ref !== `refs/heads/${repository.default_branch}` ||
+            noCommit.test(before) ||
+            noCommit.test(after)
+          ) {
+            return [];
+          }
+          const commits = yield* readPages(
+            client,
+            wrappedList("commits", CommitObjectSchema),
+            withParams(`/repos/${repository.full_name}/compare/${before}...${after}`, {
+              per_page: pageSize,
+            }),
+          );
+          return commits.map((commit) => toRow(commit, repository.id));
+        }),
+    },
+  });

package/src/resources/issue-comments.ts

@@ -0,0 +1,75 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import {
+  IssueCommentDeletedEventSchema,
+  IssueCommentEventSchema,
+  WebhookPayloadSchema,
+} from "../schemas/events";
+import {
+  type IssueComment,
+  type IssueCommentObject,
+  IssueCommentObjectSchema,
+  IssueCommentSchema,
+} from "../schemas/issue-comments";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  decodeEvent,
+  deleteRow,
+  numberAtEnd,
+  rowsOf,
+  timestamps,
+} from "./shared";
+
+const toRow = (
+  { updated_at, issue_url, ...rest }: IssueCommentObject,
+  repository_id: bigint,
+): IssueComment => ({
+  ...rest,
+  version: updated_at,
+  repository_id,
+  issue_number: numberAtEnd(issue_url),
+});
+
+export const issueCommentSpec: RepoResourceSpec<IssueCommentObject, IssueComment> = {
+  path: "issues/comments",
+  pageSchema: Schema.Array(IssueCommentObjectSchema),
+  params: {},
+  since: true,
+  order: "sorted",
+  ...timestamps,
+  keep: () => true,
+  enabled: (repository) => repository.has_issues || repository.has_pull_requests,
+  rows: rowsOf(toRow),
+};
+
+export const makeIssueComments = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) =>
+  Resource.entity({
+    name: "issue_comments",
+    rowSchema: IssueCommentSchema,
+    version: "version",
+
+    check: checkRepoResource(client, issueCommentSpec),
+    backfill: backfillRepoResource(client, repositories, issueCommentSpec),
+    changes: changesRepoResource(client, issueCommentSpec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const event = yield* decodeEvent(IssueCommentEventSchema, payload);
+          if (Schema.is(IssueCommentDeletedEventSchema)(event)) {
+            return [yield* deleteRow(event.comment.id)];
+          }
+          return [toRow(event.comment, event.repository.id)];
+        }),
+    },
+  });

package/src/resources/issues.ts

@@ -0,0 +1,78 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import {
+  IssueRemovedEventSchema,
+  IssuesEventSchema,
+  WebhookPayloadSchema,
+} from "../schemas/events";
+import { type Issue, type IssueObject, IssueObjectSchema, IssueSchema } from "../schemas/issues";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  decodeEvent,
+  deleteRow,
+  rowsOf,
+  timestamps,
+} from "./shared";
+
+const toRow = (
+  { updated_at, labels, assignees, pull_request: _pullRequest, ...rest }: IssueObject,
+  repository_id: bigint,
+): Issue => ({
+  ...rest,
+  version: updated_at,
+  repository_id,
+  labels: labels ?? [],
+  assignees: assignees ?? [],
+});
+
+// A webhook can leave a list out, so read the issue again.
+const hasLists = ({ labels }: IssueObject) => labels !== undefined;
+
+export const issueSpec: RepoResourceSpec<IssueObject, Issue> = {
+  path: "issues",
+  pageSchema: Schema.Array(IssueObjectSchema),
+  params: { state: "all" },
+  since: true,
+  order: "sorted",
+  ...timestamps,
+  keep: (item) => item.pull_request === undefined,
+  enabled: (repository) => repository.has_issues,
+  rows: rowsOf(toRow),
+};
+
+export const makeIssues = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) =>
+  Resource.entity({
+    name: "issues",
+    rowSchema: IssueSchema,
+    version: "version",
+
+    check: checkRepoResource(client, issueSpec),
+    backfill: backfillRepoResource(client, repositories, issueSpec),
+    changes: changesRepoResource(client, issueSpec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const event = yield* decodeEvent(IssuesEventSchema, payload);
+          if (Schema.is(IssueRemovedEventSchema)(event)) return [yield* deleteRow(event.issue.id)];
+          const { issue, repository } = event;
+          const full = hasLists(issue)
+            ? issue
+            : (yield* client.get(
+                IssueObjectSchema,
+                `/repos/${repository.full_name}/issues/${issue.number}`,
+              )).body;
+          return [toRow(full, repository.id)];
+        }),
+    },
+  });

package/src/resources/pull-request-commits.ts

@@ -0,0 +1,91 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema } from "effect";
+
+import type { PullRequestObject } from "../schemas/pull-requests";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import { type GitHubClientService, withParams } from "../client/client";
+import { type CommitObject, CommitObjectSchema } from "../schemas/commits";
+import { PullRequestCommitsEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import { type PullRequestCommit, PullRequestCommitSchema } from "../schemas/pull-request-commits";
+import { pullRequestSpec } from "./pull-requests";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  childRows,
+  decodeEvent,
+  pageSize,
+  readPages,
+} from "./shared";
+
+const toRow = (
+  { sha, commit, author, committer, html_url }: CommitObject,
+  pullRequest: { readonly number: bigint; readonly updated_at: Date },
+  repository_id: bigint,
+): PullRequestCommit => ({
+  key: `${repository_id}:${pullRequest.number}:${sha}`,
+  version: pullRequest.updated_at,
+  repository_id,
+  pull_request_number: pullRequest.number,
+  sha,
+  message: commit.message,
+  author: commit.author,
+  committer: commit.committer,
+  author_user: author,
+  committer_user: committer,
+  html_url,
+});
+
+const commitsPath = (pullRequest: { readonly number: bigint }) =>
+  `pulls/${pullRequest.number}/commits`;
+
+const pullRequestCommitSpec = (
+  client: GitHubClientService,
+): RepoResourceSpec<PullRequestObject, PullRequestCommit> => ({
+  ...pullRequestSpec,
+  rows: childRows(client, {
+    path: (pullRequest) => commitsPath(pullRequest),
+    pageSchema: Schema.Array(CommitObjectSchema),
+    toRow,
+  }),
+});
+
+// Other actions don't change the commits. An edit does only when it changes the base.
+const commitActions = new Set(["opened", "reopened", "synchronize"]);
+
+export const makePullRequestCommits = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) => {
+  const spec = pullRequestCommitSpec(client);
+  return Resource.entity({
+    name: "pull_request_commits",
+    rowSchema: PullRequestCommitSchema,
+    version: "version",
+
+    check: checkRepoResource(client, spec),
+    backfill: backfillRepoResource(client, repositories, spec),
+    changes: changesRepoResource(client, spec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const { action, changes, repository, pull_request } = yield* decodeEvent(
+            PullRequestCommitsEventSchema,
+            payload,
+          );
+          if (!commitActions.has(action) && changes?.base === undefined) return [];
+          const commits = yield* readPages(
+            client,
+            Schema.Array(CommitObjectSchema),
+            withParams(`/repos/${repository.full_name}/${commitsPath(pull_request)}`, {
+              per_page: pageSize,
+            }),
+          );
+          return commits.map((commit) => toRow(commit, pull_request, repository.id));
+        }),
+    },
+  });
+};

package/src/resources/pull-request-review-comments.ts

@@ -0,0 +1,78 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import {
+  PullRequestReviewCommentDeletedEventSchema,
+  PullRequestReviewCommentEventSchema,
+  WebhookPayloadSchema,
+} from "../schemas/events";
+import {
+  type PullRequestReviewComment,
+  type PullRequestReviewCommentObject,
+  PullRequestReviewCommentObjectSchema,
+  PullRequestReviewCommentSchema,
+} from "../schemas/pull-request-review-comments";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  decodeEvent,
+  deleteRow,
+  numberAtEnd,
+  rowsOf,
+  timestamps,
+} from "./shared";
+
+const toRow = (
+  { updated_at, pull_request_url, ...rest }: PullRequestReviewCommentObject,
+  repository_id: bigint,
+): PullRequestReviewComment => ({
+  ...rest,
+  version: updated_at,
+  repository_id,
+  pull_request_number: numberAtEnd(pull_request_url),
+});
+
+export const pullRequestReviewCommentSpec: RepoResourceSpec<
+  PullRequestReviewCommentObject,
+  PullRequestReviewComment
+> = {
+  path: "pulls/comments",
+  pageSchema: Schema.Array(PullRequestReviewCommentObjectSchema),
+  params: {},
+  since: true,
+  order: "sorted",
+  ...timestamps,
+  keep: () => true,
+  enabled: (repository) => repository.has_pull_requests,
+  rows: rowsOf(toRow),
+};
+
+export const makePullRequestReviewComments = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) =>
+  Resource.entity({
+    name: "pull_request_review_comments",
+    rowSchema: PullRequestReviewCommentSchema,
+    version: "version",
+
+    check: checkRepoResource(client, pullRequestReviewCommentSpec),
+    backfill: backfillRepoResource(client, repositories, pullRequestReviewCommentSpec),
+    changes: changesRepoResource(client, pullRequestReviewCommentSpec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const event = yield* decodeEvent(PullRequestReviewCommentEventSchema, payload);
+          if (Schema.is(PullRequestReviewCommentDeletedEventSchema)(event)) {
+            return [yield* deleteRow(event.comment.id)];
+          }
+          return [toRow(event.comment, event.repository.id)];
+        }),
+    },
+  });