@useairfoil/producer-github 0.0.1 → 0.0.2

package/src/schemas/repositories.ts

@@ -0,0 +1,53 @@
+import { Iceberg } from "@useairfoil/connector-kit";
+import { Effect, Schema, Struct } from "effect";
+
+import { Long, Text, Timestamp, deleted, field, list, optionalList, user, version } from "./shared";
+
+export const RepositorySchema = Schema.Struct({
+  id: Long.pipe(field(1, "Unique repository ID.")),
+  version: version(2, "repository"),
+  node_id: Schema.String.pipe(field(3, "GraphQL node ID of the repository.")),
+  name: Schema.String.pipe(field(4, "Repository name.")),
+  full_name: Schema.String.pipe(field(5, "Owner and name, such as octo-org/octo-repo.")),
+  owner: user(100).pipe(field(6, "Account that owns the repository.")),
+  private: Schema.Boolean.pipe(field(7, "Whether the repository is private.")),
+  visibility: Schema.optionalKey(Schema.String).pipe(
+    field(8, "Visibility: public, private, or internal."),
+  ),
+  description: Text.pipe(field(9, "Repository description. Empty when not set.")),
+  fork: Schema.Boolean.pipe(field(10, "Whether the repository is a fork.")),
+  archived: Schema.Boolean.pipe(field(11, "Whether the repository is archived.")),
+  disabled: Schema.Boolean.pipe(field(12, "Whether the repository is disabled.")),
+  default_branch: Schema.String.pipe(field(13, "Default branch name.")),
+  language: Schema.NullOr(Schema.String).pipe(field(14, "Main language GitHub detected.")),
+  topics: list(103, Schema.String).pipe(field(15, "Repository topics.")),
+  html_url: Schema.String.pipe(field(16, "Repository page on GitHub.")),
+  created_at: Timestamp.pipe(field(17, "Time the repository was created.")),
+  _deleted: deleted(18, "repository"),
+})
+  .annotate({
+    description: "Repositories the GitHub App installation can read.",
+  })
+  .pipe(Iceberg.identifier("id"));
+
+export type Repository = Schema.Schema.Type<typeof RepositorySchema>;
+
+export const RepositoryObjectSchema = RepositorySchema.mapFields((fields) => ({
+  ...Struct.omit(fields, ["version", "topics", "_deleted"]),
+  updated_at: Timestamp,
+  topics: optionalList(Schema.String),
+}));
+
+export type RepositoryObject = Schema.Schema.Type<typeof RepositoryObjectSchema>;
+
+export const RepositoryRefSchema = Schema.Struct({
+  id: Schema.Number,
+  full_name: Schema.String,
+  has_issues: Schema.Boolean,
+  // GitHub documents it as optional, defaulting to true.
+  has_pull_requests: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(true))),
+  // In KB. An empty repository has 0, and its git lists return 409.
+  size: Schema.Number,
+});
+
+export type RepositoryRef = Schema.Schema.Type<typeof RepositoryRefSchema>;

package/src/schemas/shared.ts

@@ -0,0 +1,86 @@
+import { Iceberg } from "@useairfoil/connector-kit";
+import { Schema, SchemaTransformation } from "effect";
+
+export const field = (fieldId: number, description: string) =>
+  Iceberg.field(fieldId, { description });
+
+/** IDs and counts. Connector Kit maps whole numbers to Iceberg longs. */
+export const Long = Schema.Int.pipe(
+  Schema.decodeTo(
+    Schema.BigInt,
+    SchemaTransformation.transform({
+      decode: (value) => BigInt(value),
+      encode: (value) => Number(value),
+    }),
+  ),
+);
+
+export const Timestamp = Schema.DateFromString;
+
+/** Text GitHub can clear, such as a body. */
+export const Text = Schema.NullOr(Schema.String).pipe(
+  Schema.decodeTo(
+    Schema.String,
+    SchemaTransformation.transform({
+      decode: (value) => value ?? "",
+      encode: (value) => value,
+    }),
+  ),
+);
+
+/** `null` becomes an empty list. A webhook can leave the list out. */
+export const optionalList = <S extends Schema.Top>(item: S) =>
+  Schema.optionalKey(
+    Schema.NullOr(Schema.Array(item)).pipe(
+      Schema.decodeTo(
+        Schema.Array(Schema.toType(item)),
+        SchemaTransformation.transform({
+          decode: (value) => value ?? [],
+          encode: (value) => value,
+        }),
+      ),
+    ),
+  );
+
+// Each helper takes the first field ID of its block.
+export const user = (baseId: number) =>
+  Schema.Struct({
+    id: Long.pipe(field(baseId, "GitHub user ID.")),
+    login: Schema.String.pipe(field(baseId + 1, "GitHub username.")),
+    type: Schema.String.pipe(
+      field(baseId + 2, "Account type, such as User, Bot, or Organization."),
+    ),
+  });
+
+export const label = (baseId: number) =>
+  Schema.Struct({
+    id: Long.pipe(field(baseId, "Label ID.")),
+    name: Schema.String.pipe(field(baseId + 1, "Label name.")),
+  });
+
+export const milestone = (baseId: number) =>
+  Schema.Struct({
+    id: Long.pipe(field(baseId, "Milestone ID.")),
+    number: Long.pipe(field(baseId + 1, "Milestone number in the repository.")),
+    title: Schema.String.pipe(field(baseId + 2, "Milestone title.")),
+  });
+
+/** A list column. The element ID is separate from the column's field ID. */
+export const list = <S extends Schema.Top>(elementId: number, item: S) =>
+  Schema.Array(item.annotate({ fieldId: elementId }));
+
+export const version = (fieldId: number, object: string) =>
+  Schema.Date.pipe(field(fieldId, `Time GitHub last updated the ${object}.`));
+
+/** For objects without an update time. A newer read wins. */
+export const readVersion = (fieldId: number, object: string) =>
+  Schema.Date.pipe(
+    field(fieldId, `Time the connector read the ${object}. GitHub has no update time for it.`),
+  );
+
+export const deleted = (fieldId: number, object: string) =>
+  Schema.optional(Schema.Boolean).pipe(
+    field(fieldId, `Whether the ${object} was deleted in GitHub.`),
+  );
+
+export const ObjectRefSchema = Schema.Struct({ id: Long });

package/src/schemas/workflow-jobs.ts

@@ -0,0 +1,54 @@
+import { Iceberg } from "@useairfoil/connector-kit";
+import { Schema, Struct } from "effect";
+
+import { Long, Timestamp, field, list, readVersion } from "./shared";
+
+const step = Schema.Struct({
+  number: Long.pipe(field(102, "Step number.")),
+  name: Schema.String.pipe(field(103, "Step name.")),
+  status: Schema.String.pipe(field(104, "Status, such as queued, in_progress, or completed.")),
+  conclusion: Schema.NullOr(Schema.String).pipe(field(105, "Result of a completed step.")),
+  started_at: Schema.optionalKey(Schema.NullOr(Timestamp)).pipe(
+    field(106, "Time the step started."),
+  ),
+  completed_at: Schema.optionalKey(Schema.NullOr(Timestamp)).pipe(
+    field(107, "Time the step completed."),
+  ),
+});
+
+export const WorkflowJobSchema = Schema.Struct({
+  id: Long.pipe(field(1, "Unique job ID.")),
+  version: readVersion(2, "job"),
+  node_id: Schema.String.pipe(field(3, "GraphQL node ID of the job.")),
+  repository_id: Long.pipe(field(4, "ID of the repository the job belongs to.")),
+  run_id: Long.pipe(field(5, "ID of the workflow run.")),
+  run_attempt: Schema.optionalKey(Long).pipe(field(6, "Attempt of the run the job belongs to.")),
+  workflow_name: Schema.NullOr(Schema.String).pipe(field(7, "Workflow name.")),
+  name: Schema.String.pipe(field(8, "Job name.")),
+  head_sha: Schema.String.pipe(field(9, "SHA of the commit the job ran on.")),
+  head_branch: Schema.NullOr(Schema.String).pipe(field(10, "Branch the job ran on.")),
+  status: Schema.String.pipe(field(11, "Status, such as queued, in_progress, or completed.")),
+  conclusion: Schema.NullOr(Schema.String).pipe(
+    field(12, "Result of a completed job, such as success or failure."),
+  ),
+  created_at: Timestamp.pipe(field(13, "Time the job was queued.")),
+  started_at: Timestamp.pipe(field(14, "Time the job started.")),
+  completed_at: Schema.NullOr(Timestamp).pipe(field(15, "Time the job completed.")),
+  runner_name: Schema.NullOr(Schema.String).pipe(field(16, "Name of the runner.")),
+  labels: list(100, Schema.String).pipe(field(17, "Runner labels the job asked for.")),
+  steps: list(101, step).pipe(field(18, "Steps of the job.")),
+  html_url: Schema.NullOr(Schema.String).pipe(field(19, "Job page on GitHub.")),
+})
+  .annotate({
+    description: "Jobs of GitHub Actions workflow runs. Each run attempt has its own jobs.",
+  })
+  .pipe(Iceberg.identifier("id"));
+
+export type WorkflowJob = Schema.Schema.Type<typeof WorkflowJobSchema>;
+
+export const WorkflowJobObjectSchema = WorkflowJobSchema.mapFields((fields) => ({
+  ...Struct.omit(fields, ["version", "repository_id", "steps"]),
+  steps: Schema.optionalKey(Schema.Array(step)),
+}));
+
+export type WorkflowJobObject = Schema.Schema.Type<typeof WorkflowJobObjectSchema>;

package/src/schemas/workflow-runs.ts

@@ -0,0 +1,55 @@
+import { Iceberg } from "@useairfoil/connector-kit";
+import { Schema, Struct } from "effect";
+
+import { Long, Timestamp, field, list, user, version } from "./shared";
+
+export const WorkflowRunSchema = Schema.Struct({
+  id: Long.pipe(field(1, "Unique workflow run ID.")),
+  version: version(2, "workflow run"),
+  node_id: Schema.String.pipe(field(3, "GraphQL node ID of the run.")),
+  repository_id: Long.pipe(field(4, "ID of the repository the run belongs to.")),
+  workflow_id: Long.pipe(field(5, "ID of the workflow.")),
+  name: Schema.NullOr(Schema.String).pipe(field(6, "Workflow name.")),
+  display_title: Schema.String.pipe(field(7, "Title of the run, such as the commit message.")),
+  path: Schema.String.pipe(field(8, "Path of the workflow file.")),
+  event: Schema.String.pipe(field(9, "Event that started the run, such as push.")),
+  status: Schema.NullOr(Schema.String).pipe(
+    field(10, "Status, such as queued, in_progress, or completed."),
+  ),
+  conclusion: Schema.NullOr(Schema.String).pipe(
+    field(11, "Result of a completed run, such as success or failure."),
+  ),
+  head_branch: Schema.NullOr(Schema.String).pipe(field(12, "Branch the run is for.")),
+  head_sha: Schema.String.pipe(field(13, "SHA of the commit the run is for.")),
+  run_number: Long.pipe(field(14, "Run number of the workflow.")),
+  run_attempt: Schema.optionalKey(Long).pipe(field(15, "Attempt number. Re-runs count up.")),
+  check_suite_id: Schema.optionalKey(Long).pipe(field(16, "ID of the run's check suite.")),
+  actor: Schema.optionalKey(user(100)).pipe(field(17, "Account that started the first attempt.")),
+  triggering_actor: Schema.optionalKey(user(103)).pipe(
+    field(18, "Account that started the latest attempt."),
+  ),
+  pull_requests: list(
+    106,
+    Schema.Struct({
+      id: Long.pipe(field(107, "Pull request ID.")),
+      number: Long.pipe(field(108, "Pull request number.")),
+    }),
+  ).pipe(field(19, "Pull requests the run is for. Empty for forks.")),
+  created_at: Timestamp.pipe(field(20, "Time the run was created.")),
+  run_started_at: Schema.optionalKey(Timestamp).pipe(field(21, "Time the latest attempt started.")),
+  html_url: Schema.String.pipe(field(22, "Run page on GitHub.")),
+})
+  .annotate({
+    description: "GitHub Actions workflow runs in the installation's repositories.",
+  })
+  .pipe(Iceberg.identifier("id"));
+
+export type WorkflowRun = Schema.Schema.Type<typeof WorkflowRunSchema>;
+
+export const WorkflowRunObjectSchema = WorkflowRunSchema.mapFields((fields) => ({
+  ...Struct.omit(fields, ["version", "repository_id", "pull_requests"]),
+  updated_at: Timestamp,
+  pull_requests: Schema.NullOr(fields.pull_requests),
+}));
+
+export type WorkflowRunObject = Schema.Schema.Type<typeof WorkflowRunObjectSchema>;

package/src/start.ts

@@ -0,0 +1,47 @@
+import { PgClient } from "@effect/sql-pg";
+import {
+  ConnectorApp,
+  Ingestor,
+  RuntimeConfig,
+  StateStore,
+  Telemetry,
+} from "@useairfoil/connector-kit";
+import { Config, Effect, Layer, Logger, Schema } from "effect";
+import { Command } from "effect/cli";
+
+import { GitHubConnector } from "./index";
+
+const ConnectorLayer = GitHubConnector.layerConfig(GitHubConnector.GitHubConfigDef.config);
+const TelemetryLayer = Telemetry.layerOtlp();
+const PostgresLayer = PgClient.layerConfig({
+  url: Config.schema(
+    Schema.Redacted(Schema.NonEmptyString),
+    RuntimeConfig.PlatformRuntimeKey.postgresConnectionString,
+  ),
+  maxConnections: Config.succeed(2),
+  connectTimeout: Config.succeed("5 seconds"),
+  applicationName: Config.succeed("airfoil-producer-github"),
+});
+const StateStoreLayer = StateStore.layerSql().pipe(Layer.provide(PostgresLayer));
+
+export const startCommand = Command.make("start", {}, () =>
+  Effect.gen(function* () {
+    const port = yield* RuntimeConfig.httpPort;
+    const entrypoint = yield* GitHubConnector.GitHubConnector;
+
+    return yield* ConnectorApp.start(entrypoint, { port }).pipe(
+      Effect.provide(Ingestor.layerWingsConfig(entrypoint)),
+    );
+  }).pipe(
+    Effect.annotateLogs({ component: "github" }),
+    Effect.provide(
+      Layer.mergeAll(
+        StateStoreLayer,
+        ConnectorLayer,
+        Logger.layer([Logger.consolePretty()]),
+        TelemetryLayer,
+      ),
+    ),
+    Effect.provide(RuntimeConfig.layerHosted()),
+  ),
+).pipe(Command.withDescription("Run the production connector against Wings"));

package/src/tables.ts

@@ -0,0 +1,31 @@
+import type { Schema } from "effect";
+
+import { CheckRunSchema } from "./schemas/check-runs";
+import { CommitStatusSchema } from "./schemas/commit-statuses";
+import { CommitSchema } from "./schemas/commits";
+import { IssueCommentSchema } from "./schemas/issue-comments";
+import { IssueSchema } from "./schemas/issues";
+import { PullRequestCommitSchema } from "./schemas/pull-request-commits";
+import { PullRequestReviewCommentSchema } from "./schemas/pull-request-review-comments";
+import { PullRequestReviewSchema } from "./schemas/pull-request-reviews";
+import { PullRequestSchema } from "./schemas/pull-requests";
+import { ReleaseSchema } from "./schemas/releases";
+import { RepositorySchema } from "./schemas/repositories";
+import { WorkflowJobSchema } from "./schemas/workflow-jobs";
+import { WorkflowRunSchema } from "./schemas/workflow-runs";
+
+export const tableSchemas: Readonly<Record<string, Schema.Top>> = {
+  repositories: RepositorySchema,
+  issues: IssueSchema,
+  pull_requests: PullRequestSchema,
+  issue_comments: IssueCommentSchema,
+  pull_request_reviews: PullRequestReviewSchema,
+  pull_request_review_comments: PullRequestReviewCommentSchema,
+  pull_request_commits: PullRequestCommitSchema,
+  commits: CommitSchema,
+  check_runs: CheckRunSchema,
+  commit_statuses: CommitStatusSchema,
+  workflow_runs: WorkflowRunSchema,
+  workflow_jobs: WorkflowJobSchema,
+  releases: ReleaseSchema,
+};

package/src/webhook/route.ts

@@ -0,0 +1,46 @@
+import { type ResourceDefinition, Webhook } from "@useairfoil/connector-kit";
+import { Effect, Option, Result, Schema } from "effect";
+import { HttpServerResponse } from "effect/http";
+
+import { InstallationEventSchema, WebhookPayloadSchema } from "../schemas/events";
+import { verifySignature } from "./signature";
+
+export const webhookPath = "/webhooks/github";
+
+export const makeWebhookRoute = (options: {
+  readonly secret: string;
+  readonly installationId: number;
+  /** Resources by `X-GitHub-Event` name. One event can feed several. */
+  readonly targets: ReadonlyMap<string, ReadonlyArray<ResourceDefinition>>;
+}) =>
+  Webhook.route({
+    path: webhookPath,
+    ackMode: "after-enqueue",
+    schema: WebhookPayloadSchema,
+    handler: ({ request, rawBody, payload, to }) =>
+      Effect.gen(function* () {
+        const verified = yield* verifySignature({
+          rawBody,
+          header: request.headers["x-hub-signature-256"],
+          secret: options.secret,
+        }).pipe(Effect.result);
+        if (Result.isFailure(verified)) {
+          return HttpServerResponse.jsonUnsafe(
+            { ok: false, error: verified.failure.message },
+            { status: 401 },
+          );
+        }
+
+        // One app can be installed on several accounts, and all of their events
+        // come to the same URL. Only ours are stored. `ping` has no installation.
+        const installationId = Schema.decodeUnknownOption(InstallationEventSchema)(payload).pipe(
+          Option.flatMap(({ installation }) => Option.fromNullishOr(installation?.id)),
+        );
+        if (Option.contains(installationId, options.installationId)) {
+          const resources = options.targets.get(request.headers["x-github-event"] ?? "") ?? [];
+          yield* Effect.forEach(resources, (resource) => to(resource, payload), { discard: true });
+        }
+
+        return HttpServerResponse.jsonUnsafe({ ok: true });
+      }),
+  });

package/src/webhook/signature.ts

@@ -0,0 +1,20 @@
+import { ConnectorError } from "@useairfoil/connector-kit";
+import { Effect } from "effect";
+import { createHmac, timingSafeEqual } from "node:crypto";
+
+const prefix = "sha256=";
+
+export const verifySignature = (options: {
+  readonly rawBody: Uint8Array;
+  readonly header: string | undefined;
+  readonly secret: string;
+}): Effect.Effect<void, ConnectorError> => {
+  const header = options.header ?? "";
+  const expected = createHmac("sha256", options.secret).update(options.rawBody).digest();
+  const provided = Buffer.from(header.slice(prefix.length), "hex");
+  return header.startsWith(prefix) &&
+    provided.length === expected.length &&
+    timingSafeEqual(provided, expected)
+    ? Effect.void
+    : Effect.fail(new ConnectorError({ message: "Invalid GitHub webhook signature" }));
+};