@useairfoil/producer-github 0.0.0-stage → 0.0.2

package/src/resources/pull-request-reviews.ts

@@ -0,0 +1,76 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { DateTime, Effect, Schema } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { PullRequestObject } from "../schemas/pull-requests";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import { PullRequestReviewEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import {
+  type PullRequestReview,
+  type PullRequestReviewObject,
+  PullRequestReviewObjectSchema,
+  PullRequestReviewSchema,
+} from "../schemas/pull-request-reviews";
+import { pullRequestSpec } from "./pull-requests";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  childRows,
+  decodeEvent,
+} from "./shared";
+
+const toRow = (
+  review: PullRequestReviewObject,
+  pull_request_number: bigint,
+  repository_id: bigint,
+  version: Date,
+): PullRequestReview => ({ ...review, version, repository_id, pull_request_number });
+
+/** A review bumps its pull request's update time, so changes follow pull requests. */
+const reviewSpec = (
+  client: GitHubClientService,
+): RepoResourceSpec<PullRequestObject, PullRequestReview> => ({
+  ...pullRequestSpec,
+  rows: childRows(client, {
+    path: (pullRequest) => `pulls/${pullRequest.number}/reviews`,
+    pageSchema: Schema.Array(PullRequestReviewObjectSchema),
+    toRow: (review, pullRequest, repositoryId, readAt) =>
+      toRow(review, pullRequest.number, repositoryId, readAt),
+  }),
+});
+
+export const makePullRequestReviews = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) => {
+  const spec = reviewSpec(client);
+  return Resource.entity({
+    name: "pull_request_reviews",
+    rowSchema: PullRequestReviewSchema,
+    version: "version",
+
+    check: checkRepoResource(client, spec),
+    backfill: backfillRepoResource(client, repositories, spec),
+    changes: changesRepoResource(client, spec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      // No update time, so read the latest state instead of trusting the payload.
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const { repository, pull_request, review } = yield* decodeEvent(
+            PullRequestReviewEventSchema,
+            payload,
+          );
+          const readAt = yield* DateTime.nowAsDate;
+          const page = yield* client.get(
+            PullRequestReviewObjectSchema,
+            `/repos/${repository.full_name}/pulls/${pull_request.number}/reviews/${review.id}`,
+          );
+          return [toRow(page.body, pull_request.number, repository.id, readAt)];
+        }),
+    },
+  });
+};

package/src/resources/pull-requests.ts

@@ -0,0 +1,79 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import { PullRequestEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import {
+  type PullRequest,
+  type PullRequestObject,
+  PullRequestObjectSchema,
+  PullRequestSchema,
+} from "../schemas/pull-requests";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  decodeEvent,
+  rowsOf,
+  timestamps,
+} from "./shared";
+
+const toRow = (
+  { updated_at, labels, assignees, requested_reviewers, ...rest }: PullRequestObject,
+  repository_id: bigint,
+): PullRequest => ({
+  ...rest,
+  version: updated_at,
+  repository_id,
+  labels: labels ?? [],
+  assignees: assignees ?? [],
+  requested_reviewers: requested_reviewers ?? [],
+});
+
+// A webhook can leave a list out, so read the pull request again.
+const hasLists = ({ assignees, requested_reviewers }: PullRequestObject) =>
+  assignees !== undefined && requested_reviewers !== undefined;
+
+export const pullRequestSpec: RepoResourceSpec<PullRequestObject, PullRequest> = {
+  path: "pulls",
+  pageSchema: Schema.Array(PullRequestObjectSchema),
+  params: { state: "all" },
+  // The pulls list has no `since`.
+  since: false,
+  order: "sorted",
+  ...timestamps,
+  keep: () => true,
+  enabled: (repository) => repository.has_pull_requests,
+  rows: rowsOf(toRow),
+};
+
+export const makePullRequests = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) =>
+  Resource.entity({
+    name: "pull_requests",
+    rowSchema: PullRequestSchema,
+    version: "version",
+
+    check: checkRepoResource(client, pullRequestSpec),
+    backfill: backfillRepoResource(client, repositories, pullRequestSpec),
+    changes: changesRepoResource(client, pullRequestSpec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const { pull_request, repository } = yield* decodeEvent(PullRequestEventSchema, payload);
+          const full = hasLists(pull_request)
+            ? pull_request
+            : (yield* client.get(
+                PullRequestObjectSchema,
+                `/repos/${repository.full_name}/pulls/${pull_request.number}`,
+              )).body;
+          return [toRow(full, repository.id)];
+        }),
+    },
+  });

package/src/resources/releases.ts

@@ -0,0 +1,74 @@
+import { type ConnectorError, Resource } from "@useairfoil/connector-kit";
+import { Effect, Schema, Struct } from "effect";
+
+import type { GitHubClientService } from "../client/client";
+import type { RepositoryRef } from "../schemas/repositories";
+
+import {
+  ReleaseDeletedEventSchema,
+  ReleaseEventSchema,
+  WebhookPayloadSchema,
+} from "../schemas/events";
+import {
+  type Release,
+  type ReleaseObject,
+  ReleaseObjectSchema,
+  ReleaseSchema,
+} from "../schemas/releases";
+import {
+  type RepoResourceSpec,
+  backfillRepoResource,
+  changesRepoResource,
+  checkRepoResource,
+  decodeEvent,
+  deleteRow,
+  rowsOf,
+} from "./shared";
+
+const updatedAt = (release: ReleaseObject) => release.updated_at ?? release.created_at;
+
+const toRow = (release: ReleaseObject, repository_id: bigint): Release => ({
+  ...Struct.omit(release, ["updated_at"]),
+  version: updatedAt(release),
+  repository_id,
+});
+
+export const releaseSpec: RepoResourceSpec<ReleaseObject, Release> = {
+  path: "releases",
+  pageSchema: Schema.Array(ReleaseObjectSchema),
+  params: {},
+  since: false,
+  // The list can't sort or filter, so changes read it all. Most repositories
+  // have few releases.
+  order: "newest",
+  createdAt: (release) => release.created_at,
+  updatedAt,
+  keep: () => true,
+  enabled: () => true,
+  rows: rowsOf(toRow),
+};
+
+export const makeReleases = (
+  client: GitHubClientService,
+  repositories: Effect.Effect<ReadonlyArray<RepositoryRef>, ConnectorError>,
+) =>
+  Resource.entity({
+    name: "releases",
+    rowSchema: ReleaseSchema,
+    version: "version",
+
+    check: checkRepoResource(client, releaseSpec),
+    backfill: backfillRepoResource(client, repositories, releaseSpec),
+    changes: changesRepoResource(client, releaseSpec),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const event = yield* decodeEvent(ReleaseEventSchema, payload);
+          if (Schema.is(ReleaseDeletedEventSchema)(event)) {
+            return [yield* deleteRow(event.release.id)];
+          }
+          return [toRow(event.release, event.repository.id)];
+        }),
+    },
+  });

package/src/resources/repositories.ts

@@ -0,0 +1,101 @@
+import { Cursor, Fetch, Resource } from "@useairfoil/connector-kit";
+import { Effect, Option, Schema } from "effect";
+
+import { type GitHubClientService, withParams } from "../client/client";
+import {
+  RepositoryDeletedEventSchema,
+  RepositoryEventSchema,
+  WebhookPayloadSchema,
+} from "../schemas/events";
+import {
+  type Repository,
+  type RepositoryObject,
+  RepositoryObjectSchema,
+  RepositorySchema,
+} from "../schemas/repositories";
+import {
+  changesInterval,
+  decodeEvent,
+  deleteRow,
+  installationRepositoriesPath,
+  isSince,
+  listRepositories,
+  pageSize,
+  parseCutoff,
+  readChanges,
+  repositoryPage,
+} from "./shared";
+
+const toRow = ({ updated_at, topics, ...rest }: RepositoryObject): Repository => ({
+  ...rest,
+  version: updated_at,
+  topics: topics ?? [],
+  _deleted: false,
+});
+
+export const makeRepositories = (client: GitHubClientService) =>
+  Resource.entity({
+    name: "repositories",
+    rowSchema: RepositorySchema,
+    version: "version",
+
+    check: client
+      .get(
+        repositoryPage(RepositoryObjectSchema),
+        withParams(installationRepositoriesPath, { per_page: "1" }),
+      )
+      .pipe(Effect.asVoid),
+    // The installation's list can't filter on time, so the cutoff is applied here.
+    backfill: Fetch.page({
+      pageCursor: Cursor.string(),
+      cutoff: Cursor.isoDateTime(),
+      fetch: ({ pageCursor, cutoff }) =>
+        Effect.gen(function* () {
+          const cutoffDate = parseCutoff(String(cutoff));
+          const page = yield* client.get(
+            repositoryPage(RepositoryObjectSchema),
+            typeof pageCursor === "string"
+              ? pageCursor
+              : withParams(installationRepositoriesPath, { per_page: pageSize }),
+          );
+          return {
+            rows: page.body.repositories
+              .filter((repository) => repository.created_at.getTime() <= cutoffDate.getTime())
+              .map(toRow),
+            nextPageCursor: Option.getOrUndefined(page.next),
+            hasMore: Option.isSome(page.next),
+          };
+        }),
+    }),
+    // New repositories are kept too, so one added later still gets a row.
+    changes: Fetch.changes({
+      cursor: Cursor.string(),
+      interval: changesInterval,
+      fetch: ({ cursor }) =>
+        readChanges(
+          String(cursor),
+          listRepositories(client, RepositoryObjectSchema),
+          (repositories) =>
+            repositories
+              .filter(({ repository, since }) => isSince(since, repository.updated_at))
+              .map(({ repository }) => toRow(repository)),
+        ),
+    }),
+    webhook: {
+      schema: WebhookPayloadSchema,
+      handler: ({ payload }) =>
+        Effect.gen(function* () {
+          const event = yield* decodeEvent(RepositoryEventSchema, payload);
+          if (Schema.is(RepositoryDeletedEventSchema)(event)) {
+            return [yield* deleteRow(event.repository.id)];
+          }
+          const { repository } = event;
+          // A webhook can leave the topics out, so read the repository again.
+          const full =
+            repository.topics !== undefined
+              ? repository
+              : (yield* client.get(RepositoryObjectSchema, `/repos/${repository.full_name}`)).body;
+          return [toRow(full)];
+        }),
+    },
+  });