@alchemy.run/pkg 0.0.0-placeholder → 2.0.0-beta.78

package/src/Registry/Sweep.ts

@@ -0,0 +1,117 @@
+import * as Cloudflare from "alchemy/Cloudflare";
+import * as Clock from "effect/Clock";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Stream from "effect/Stream";
+import {
+  Bucket,
+  ORPHAN_GRACE,
+  RegistryConfig,
+  SWEEP_LOOKAHEAD,
+  tarballKey,
+} from "./Bindings.ts";
+import * as GitHub from "./GitHub.ts";
+import * as Tags from "./Tags.ts";
+
+/**
+ * Expiry sweep. Tags tied to pull requests are extended while any of those
+ * pull requests is open and pinned to the latest close time plus TTL once
+ * they all close. Expired tags go away, and tarballs nothing points at are
+ * deleted from R2.
+ */
+export const sweep = Effect.gen(function* () {
+  const { policy } = yield* RegistryConfig;
+  const r2 = yield* Cloudflare.R2.ReadWriteBucket(Bucket);
+  const github = yield* GitHub.GitHubApp;
+
+  const now = yield* Clock.currentTimeMillis;
+  const ttl = Duration.toMillis(policy.ttl);
+  const due = yield* Tags.dueLinked(now + Duration.toMillis(SWEEP_LOOKAHEAD));
+
+  // One lookup per pull request across every due row. `undefined` means the
+  // pull request is still open; a failed lookup is recorded as such.
+  const lookups = yield* Effect.forEach(
+    new Set(due.flatMap((row) => row.linked_prs)),
+    (ref) => {
+      const [repo, number] = ref.split("#");
+      return github.getPullRequest(repo!, Number(number)).pipe(
+        Effect.map((pr) =>
+          pr.state === "open"
+            ? undefined
+            : Date.parse(pr.merged_at ?? pr.closed_at ?? "") || now,
+        ),
+        Effect.map((closedAt) => [ref, { closedAt }] as const),
+        Effect.catch((e) =>
+          Effect.logWarning(
+            `pull request ${ref} lookup failed: ${GitHub.describe(e)}`,
+          ).pipe(Effect.as([ref, undefined] as const)),
+        ),
+      );
+    },
+    { concurrency: 4 },
+  );
+  const results = new Map(lookups);
+  const extended = yield* Effect.forEach(due, (row) => {
+    const known = row.linked_prs.flatMap((ref) => {
+      const result = results.get(ref);
+      return result === undefined ? [] : [result.closedAt];
+    });
+    // Open pull requests dominate; otherwise the latest close wins. A row
+    // whose pull requests could not all be checked is kept past the next
+    // sweep rather than left to expire: it is re-checked, never deleted,
+    // on GitHub's bad day.
+    const closes = known.filter((closedAt) => closedAt !== undefined);
+    const expiresAt =
+      closes.length < known.length
+        ? now + ttl
+        : known.length < row.linked_prs.length
+          ? Math.max(row.expires_at, now + Duration.toMillis(SWEEP_LOOKAHEAD))
+          : Math.max(...closes) + ttl;
+    return expiresAt === row.expires_at
+      ? Effect.succeed(false)
+      : Tags.setExpiry(row.package, row.tag, expiresAt).pipe(Effect.as(true));
+  }).pipe(Effect.map((changed) => changed.filter(Boolean).length));
+
+  const removed = yield* Tags.deleteExpired(now);
+  const referenced = yield* Tags.referencedTarballs();
+  const unreferenced = removed.filter(
+    ({ package: pkg, sha256 }) => !referenced.has(`${pkg}/${sha256}`),
+  );
+  yield* Effect.forEach(
+    unreferenced,
+    ({ package: pkg, sha256 }) => r2.delete(tarballKey(pkg, sha256)),
+    { discard: true },
+  );
+
+  // Uploads that never got tagged.
+  const orphans = yield* Stream.paginate(
+    undefined as string | undefined,
+    (cursor) =>
+      r2
+        .list({ cursor, limit: 500 })
+        .pipe(
+          Effect.map(
+            (page) =>
+              [
+                page.objects,
+                page.truncated ? Option.some(page.cursor) : Option.none(),
+              ] as const,
+          ),
+        ),
+  ).pipe(
+    Stream.filter((object) => {
+      const match = object.key.match(/^(.+)\/([a-f0-9]{64})\.tgz$/);
+      return (
+        match !== null &&
+        !referenced.has(`${decodeURIComponent(match[1]!)}/${match[2]}`) &&
+        object.uploaded.getTime() < now - Duration.toMillis(ORPHAN_GRACE)
+      );
+    }),
+    Stream.mapEffect((object) => r2.delete(object.key)),
+    Stream.runCount,
+  );
+  yield* Effect.logInfo(
+    `sweep: ${due.length} due, ${extended} extended, ${removed.length} tags expired, ${unreferenced.length + orphans} tarballs deleted`,
+  );
+});

package/src/Registry/Tags.ts

@@ -0,0 +1,117 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+/**
+ * The `tags` table: one row per `(package, tag)` pointing at a tarball hash.
+ * A tarball lives in R2 for as long as any row points at it. Every query
+ * runs against the ambient `SqlClient`, which the Worker provides from its
+ * D1 binding.
+ */
+
+const LinkedPrs = Schema.fromJsonString(Schema.Array(Schema.String));
+
+export const TagRow = Schema.Struct({
+  package: Schema.String,
+  tag: Schema.String,
+  sha256: Schema.String,
+  expires_at: Schema.Number,
+  /** Pull requests whose runs produced this tag, as `owner/repo#N`. */
+  linked_prs: LinkedPrs,
+});
+export type TagRow = typeof TagRow.Type;
+
+const TarballRow = Schema.Struct({
+  package: Schema.String,
+  sha256: Schema.String,
+});
+
+const decodeTags = Schema.decodeUnknownEffect(Schema.Array(TagRow));
+const decodeTarballs = Schema.decodeUnknownEffect(Schema.Array(TarballRow));
+
+export const get = Effect.fn("Tags.get")(function* (pkg: string, tag: string) {
+  const sql = yield* SqlClient.SqlClient;
+  const rows = yield* decodeTags(
+    yield* sql`SELECT * FROM tags WHERE package = ${pkg} AND tag = ${tag}`,
+  );
+  return rows[0];
+});
+
+/**
+ * Point `tag` at `sha256`, keeping the row alive for at least `expiresAt`
+ * and remembering every pull request that produced it. One statement, so
+ * concurrent publications of a shared tag merge rather than overwrite.
+ */
+export const upsert = Effect.fn("Tags.upsert")(function* (input: {
+  readonly package: string;
+  readonly tag: string;
+  readonly sha256: string;
+  readonly expiresAt: number;
+  readonly prs: ReadonlyArray<string>;
+}) {
+  const sql = yield* SqlClient.SqlClient;
+  const prs = yield* Schema.encodeEffect(LinkedPrs)(input.prs);
+  yield* sql`
+    INSERT INTO tags (package, tag, sha256, expires_at, linked_prs)
+    VALUES (${input.package}, ${input.tag}, ${input.sha256}, ${input.expiresAt}, ${prs})
+    ON CONFLICT (package, tag) DO UPDATE SET
+      sha256 = excluded.sha256,
+      expires_at = max(tags.expires_at, excluded.expires_at),
+      linked_prs = (
+        SELECT json_group_array(value) FROM (
+          SELECT value FROM json_each(tags.linked_prs)
+          UNION
+          SELECT value FROM json_each(excluded.linked_prs)
+        )
+      )
+  `;
+});
+
+export const resolve = (pkg: string, tag: string) =>
+  Effect.map(get(pkg, tag), (row) => row?.sha256);
+
+/** Rows tied to pull requests whose expiry falls before `before`. */
+export const dueLinked = Effect.fn("Tags.dueLinked")(function* (
+  before: number,
+) {
+  const sql = yield* SqlClient.SqlClient;
+  return yield* decodeTags(
+    yield* sql`
+      SELECT * FROM tags WHERE linked_prs != '[]' AND expires_at < ${before}
+    `,
+  );
+});
+
+export const setExpiry = Effect.fn("Tags.setExpiry")(function* (
+  pkg: string,
+  tag: string,
+  expiresAt: number,
+) {
+  const sql = yield* SqlClient.SqlClient;
+  yield* sql`
+    UPDATE tags SET expires_at = ${expiresAt} WHERE package = ${pkg} AND tag = ${tag}
+  `;
+});
+
+/** Delete every expired row and return the tarballs those rows pointed at. */
+export const deleteExpired = Effect.fn("Tags.deleteExpired")(function* (
+  now: number,
+) {
+  const sql = yield* SqlClient.SqlClient;
+  return yield* decodeTarballs(
+    yield* sql`
+      DELETE FROM tags WHERE expires_at < ${now} RETURNING package, sha256
+    `,
+  );
+});
+
+/** Every tarball some tag still points at, as `<package>/<sha256>`. */
+export const referencedTarballs = Effect.fn("Tags.referencedTarballs")(
+  function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const rows = yield* decodeTarballs(
+      yield* sql`SELECT DISTINCT package, sha256 FROM tags`,
+    );
+    return new Set(rows.map((row) => `${row.package}/${row.sha256}`));
+  },
+);

package/src/Registry.ts

@@ -0,0 +1,197 @@
+import * as Cloudflare from "alchemy/Cloudflare";
+import * as Namespace from "alchemy/Namespace";
+import type * as ByteSize from "effect/ByteSize";
+import * as Config from "effect/Config";
+import type * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Schema from "effect/Schema";
+import {
+  APP_ID_ENV,
+  DEFAULT_API_URL,
+  DEFAULT_CRON,
+  DEFAULT_TTL,
+  PRIVATE_KEY_ENV,
+  RegistryConfig,
+  SETTINGS_ENV,
+  Settings,
+  type GitHubCredentials,
+  type WorkerOverrides,
+} from "./Registry/Bindings.ts";
+import { handler } from "./Registry/Handler.ts";
+
+export {
+  Bucket,
+  Cache,
+  Database,
+  Policy,
+  RegistryConfig,
+  Settings,
+} from "./Registry/Bindings.ts";
+
+export interface RegistryOptions {
+  /**
+   * Worker props forwarded verbatim: `domain`, `name`, `compatibility`,
+   * `observability`, `access`, and so on. `main`, `env`, and `crons` are
+   * owned by the registry.
+   */
+  readonly worker?: WorkerOverrides;
+  /** GitHub App credentials, read at deploy time and bound to the Worker. */
+  readonly github: GitHubCredentials & {
+    /** @default "https://api.github.com" */
+    readonly apiUrl?: string;
+  };
+  readonly policy: {
+    /** Repositories allowed to publish, as `owner/name`. */
+    readonly repos: ReadonlyArray<string>;
+    /** @default Duration.weeks(1) */
+    readonly ttl?: Duration.Duration;
+    /** Upper bound on a single tarball. Absent means unlimited. */
+    readonly maxPackageSize?: ByteSize.ByteSize;
+  };
+  /**
+   * Hostname to package scope. Requests on an aliased host resolve an
+   * unscoped name under that scope, so `pkg.distilled.cloud/core/<sha>`
+   * serves `@distilled.cloud/core`.
+   */
+  readonly aliases?: Readonly<Record<string, string>>;
+  /**
+   * Cron expression for the expiry sweep.
+   * @default "0 * * * *"
+   */
+  readonly cron?: string;
+}
+
+/**
+ * The registry Worker. Its props are an Effect so the same declaration
+ * serves both sides of the bundle boundary: at plan time
+ * {@link RegistryConfig} is the caller's options, inside the isolate it is
+ * rebuilt from the Worker's bindings.
+ */
+export class Registry extends Cloudflare.Worker<Registry, {}>()("Worker") {}
+
+const RegistryLive = Registry.make(
+  Effect.gen(function* () {
+    const config = yield* RegistryConfig;
+    return {
+      ...config.worker,
+      main: import.meta.url,
+      env: {
+        [SETTINGS_ENV]: yield* Schema.encodeEffect(Settings)({
+          policy: config.policy,
+          aliases: config.aliases,
+          cron: config.cron,
+          github: { apiUrl: config.github.apiUrl },
+        }).pipe(Effect.orDie),
+        [APP_ID_ENV]: config.github.appId,
+        [PRIVATE_KEY_ENV]: config.github.privateKey,
+      },
+    };
+  }),
+  handler,
+);
+
+/**
+ * Runtime side of {@link RegistryConfig}: decode the settings binding and
+ * point the credentials at the bindings `RegistryLive` declared.
+ */
+const RegistryConfigFromEnv = Layer.effect(
+  RegistryConfig,
+  Effect.gen(function* () {
+    const env = yield* Cloudflare.Workers.WorkerEnvironment;
+    const settings = yield* Schema.decodeUnknownEffect(Settings)(
+      env[SETTINGS_ENV],
+    ).pipe(Effect.orDie);
+    return {
+      ...settings,
+      worker: {},
+      github: {
+        ...settings.github,
+        appId: Config.String(APP_ID_ENV),
+        privateKey: Config.Redacted(PRIVATE_KEY_ENV),
+      },
+    };
+  }),
+);
+
+/**
+ * The bundle entry. `RegistryLive` names this file as `main`, so the
+ * generated Worker entry imports this default export and builds it once per
+ * isolate. User code never imports it; use {@link PkgRegistry}.
+ */
+export default RegistryLive.pipe(Layer.provide(RegistryConfigFromEnv));
+
+/**
+ * A preview package registry: a Worker serving install URLs and the publish
+ * API, an R2 bucket of content-addressed tarballs, a D1 index of tags, and a
+ * KV namespace caching what the Worker learns from GitHub. They are declared
+ * under `id` as `<id>/Worker`, `<id>/Bucket`, `<id>/Database`, and
+ * `<id>/Cache`.
+ *
+ * ### Deploying a registry
+ * **Example:** Stack
+ * ```typescript
+ * // stacks/pkg.ts
+ * import { PkgRegistry } from "@alchemy.run/pkg/Registry";
+ * import * as Alchemy from "alchemy";
+ * import * as Cloudflare from "alchemy/Cloudflare";
+ * import * as ByteSize from "effect/ByteSize";
+ * import * as Config from "effect/Config";
+ * import * as Duration from "effect/Duration";
+ * import * as Effect from "effect/Effect";
+ *
+ * export default Alchemy.Stack(
+ *   "Pkg",
+ *   { providers: Cloudflare.providers(), state: Cloudflare.state() },
+ *   Effect.gen(function* () {
+ *     const registry = yield* PkgRegistry("Pkg", {
+ *       worker: { domain: "pkg.alchemy.run" },
+ *       github: {
+ *         appId: Config.String("GH_APP_ID"),
+ *         privateKey: Config.Redacted("GH_APP_PRIVATE_KEY"),
+ *       },
+ *       policy: {
+ *         repos: ["alchemy-run/alchemy", "alchemy-run/distilled"],
+ *         ttl: Duration.weeks(1),
+ *         maxPackageSize: ByteSize.megabytes(100),
+ *       },
+ *     });
+ *     return { url: registry.url.as<string>() };
+ *   }),
+ * );
+ * ```
+ *
+ * @resource
+ * @product Workers
+ * @category Workers & Compute
+ */
+export const PkgRegistry = <const Id extends string>(
+  id: Id,
+  options: RegistryOptions,
+) =>
+  Registry.pipe(
+    Effect.provide(
+      RegistryLive.pipe(
+        Layer.provide(
+          Layer.succeed(RegistryConfig, {
+            worker: options.worker ?? {},
+            policy: {
+              repos: options.policy.repos,
+              ttl: options.policy.ttl ?? DEFAULT_TTL,
+              ...(options.policy.maxPackageSize !== undefined
+                ? { maxPackageSize: options.policy.maxPackageSize }
+                : {}),
+            },
+            aliases: options.aliases ?? {},
+            cron: options.cron ?? DEFAULT_CRON,
+            github: {
+              apiUrl: options.github.apiUrl ?? DEFAULT_API_URL,
+              appId: options.github.appId,
+              privateKey: options.github.privateKey,
+            },
+          }),
+        ),
+      ),
+    ),
+    Namespace.push(id),
+  );

package/src/cli/bin.ts

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+import { PlatformServices, runMain } from "alchemy/Util/PlatformServices";
+import * as Config from "effect/Config";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import {
+  CliConfig,
+  CliError,
+  Command,
+  Flag,
+  GlobalFlag,
+} from "effect/unstable/cli";
+import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient";
+import pkg from "../../package.json" with { type: "json" };
+import { Group, pack } from "./pack.ts";
+import { publish } from "./publish.ts";
+
+const groupFlag = Flag.String("group").pipe(
+  Flag.withDescription(
+    "Display group and directory glob, as NAME=GLOB or NAME[Collapsed]=GLOB (repeatable), e.g. --group Alchemy=./packages/*",
+  ),
+  Flag.withSchema(Group),
+  Flag.atLeast(1),
+);
+
+const registryFlag = Flag.String("registry").pipe(
+  Flag.withDescription(
+    "Registry origin used for install URLs. Falls back to PKG_REGISTRY, then https://pkg.alchemy.run",
+  ),
+  Flag.withFallbackConfig(Config.String("PKG_REGISTRY")),
+  Flag.withDefault("https://pkg.alchemy.run"),
+);
+
+const outFlag = Flag.String("out").pipe(
+  Flag.withDescription("Directory to write tarballs and the manifest into"),
+  Flag.withDefault(".pkg"),
+);
+
+export const packCommand = Command.make(
+  "pack",
+  { group: groupFlag, registry: registryFlag, out: outFlag },
+  ({ group, registry, out }) =>
+    Effect.gen(function* () {
+      const cwd = yield* Effect.sync(() => process.cwd());
+      yield* pack({ cwd, groups: group, registry, out });
+    }),
+).pipe(
+  Command.withDescription(
+    "Pack workspace packages into reproducible tarballs with dependencies rewritten to registry URLs",
+  ),
+  Command.withExamples([
+    {
+      command:
+        "pkg pack --group 'alchemy=./packages/alchemy' --group '@alchemy.run[Collapsed]=./packages/{better-auth,pkg}' --group '@distilled.cloud[Collapsed]=./submodules/distilled/packages/*'",
+    },
+  ]),
+);
+
+const dirFlag = Flag.String("dir").pipe(
+  Flag.withDescription("Directory written by pkg pack"),
+  Flag.withDefault(".pkg"),
+);
+
+export const publishCommand = Command.make(
+  "publish",
+  { registry: registryFlag, dir: dirFlag },
+  ({ registry, dir }) =>
+    Effect.gen(function* () {
+      const cwd = yield* Effect.sync(() => process.cwd());
+      yield* publish({ cwd, dir, registry });
+    }),
+).pipe(
+  Command.withDescription(
+    "Publish a pkg pack directory from the current GitHub Actions job, after its manifest artifact has been uploaded",
+  ),
+  Command.withExamples([
+    { command: "pkg publish --registry https://pkg.alchemy.run" },
+  ]),
+);
+
+export const root = Command.make("pkg", {}, () =>
+  Effect.fail(new CliError.ShowHelp({ commandPath: ["pkg"], errors: [] })),
+).pipe(
+  Command.withDescription(
+    "Pack and publish preview packages for pull requests.",
+  ),
+  Command.withSubcommands([packCommand, publishCommand]),
+);
+
+Command.run(root, { version: pkg.version }).pipe(
+  Effect.provide(
+    Layer.mergeAll(
+      PlatformServices,
+      FetchHttpClient.layer,
+      CliConfig.layer({
+        builtIns: [GlobalFlag.Help, GlobalFlag.Version],
+      }),
+    ),
+  ),
+  runMain,
+);