@checkstack/script-packages-store-s3 0.2.0

package/CHANGELOG.md

@@ -0,0 +1,44 @@
+# @checkstack/script-packages-store-s3
+
+## 0.2.0
+
+### Minor Changes
+
+- 270ef29: Add the two built-in blob-store plugins for script packages.
+
+  - `script-packages-store-postgres` (default, zero extra infra): persists
+    content-addressed blobs in Postgres as base64 `text` (following the
+    existing repo convention for binary columns), registered as the
+    `postgres` backend via `blobStoreExtensionPoint`.
+  - `script-packages-store-s3` (preferred when configured): S3-compatible
+    store backed by Bun's native `S3Client` (no AWS SDK). Config from env
+    (`endpoint`, `bucket`, `region`, `accessKeyId`, `secretAccessKey`,
+    `forcePathStyle`); credentials never touch the DB. Registers the `s3`
+    backend only when configured, and reports a partial-config error
+    instead of silently falling back.
+
+### Patch Changes
+
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+  - @checkstack/backend-api@0.19.0
+  - @checkstack/script-packages-backend@0.2.0

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@checkstack/script-packages-store-s3",
+  "version": "0.2.0",
+  "license": "Elastic-2.0",
+  "type": "module",
+  "main": "src/index.ts",
+  "checkstack": {
+    "type": "backend"
+  },
+  "scripts": {
+    "typecheck": "tsgo -b",
+    "lint": "bun run lint:code",
+    "lint:code": "eslint . --max-warnings 0",
+    "test": "bun test"
+  },
+  "dependencies": {
+    "@checkstack/backend-api": "0.18.0",
+    "@checkstack/common": "0.12.0",
+    "@checkstack/script-packages-backend": "0.1.0",
+    "zod": "^4.0.0"
+  },
+  "devDependencies": {
+    "@checkstack/scripts": "0.3.4",
+    "@checkstack/tsconfig": "0.0.7",
+    "@types/bun": "^1.0.0",
+    "typescript": "^5.7.2"
+  }
+}

package/src/config.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, test } from "bun:test";
+import { loadS3ConfigFromEnv } from "./config";
+
+const full = {
+  CHECKSTACK_SCRIPT_PACKAGES_S3_ENDPOINT: "https://s3.example.com",
+  CHECKSTACK_SCRIPT_PACKAGES_S3_BUCKET: "checkstack",
+  CHECKSTACK_SCRIPT_PACKAGES_S3_REGION: "us-east-1",
+  CHECKSTACK_SCRIPT_PACKAGES_S3_ACCESS_KEY_ID: "AKIA",
+  CHECKSTACK_SCRIPT_PACKAGES_S3_SECRET_ACCESS_KEY: "secret",
+  CHECKSTACK_SCRIPT_PACKAGES_S3_FORCE_PATH_STYLE: "true",
+};
+
+describe("loadS3ConfigFromEnv", () => {
+  test("returns unconfigured when nothing is set", () => {
+    expect(loadS3ConfigFromEnv({})).toEqual({ ok: "unconfigured" });
+  });
+
+  test("parses a complete config", () => {
+    const res = loadS3ConfigFromEnv(full);
+    expect(res.ok).toBe(true);
+    if (res.ok !== true) throw new Error("expected ok");
+    expect(res.config.bucket).toBe("checkstack");
+    expect(res.config.forcePathStyle).toBe(true);
+    expect(res.config.endpoint).toBe("https://s3.example.com");
+  });
+
+  test("reports an error for a partial config instead of silently ignoring", () => {
+    const res = loadS3ConfigFromEnv({
+      CHECKSTACK_SCRIPT_PACKAGES_S3_BUCKET: "checkstack",
+      // missing credentials
+    });
+    expect(res.ok).toBe(false);
+    if (res.ok !== false) throw new Error("expected error");
+    expect(res.error).toMatch(/accessKeyId|secretAccessKey/);
+  });
+
+  test("forcePathStyle defaults to false when only other fields set", () => {
+    const { CHECKSTACK_SCRIPT_PACKAGES_S3_FORCE_PATH_STYLE: _omit, ...rest } =
+      full;
+    const res = loadS3ConfigFromEnv(rest);
+    expect(res.ok).toBe(true);
+    if (res.ok !== true) throw new Error("expected ok");
+    expect(res.config.forcePathStyle).toBe(false);
+  });
+});

package/src/config.ts

@@ -0,0 +1,77 @@
+import { z } from "zod";
+
+/**
+ * S3-compatible blob-store configuration.
+ *
+ * Credentials are object-store secrets and come from the environment (the
+ * standard for S3 creds) rather than the application DB - we never persist
+ * `secretAccessKey` in plaintext. The admin UI persists only the
+ * active-backend *selection* (`script_package_storage_config`); the
+ * connection details live in env so secret material stays out of the DB.
+ */
+export const S3StoreConfigSchema = z.object({
+  endpoint: z.string().url().optional(),
+  bucket: z.string().min(1),
+  region: z.string().optional(),
+  accessKeyId: z.string().min(1),
+  secretAccessKey: z.string().min(1),
+  /** Path-style addressing (needed by MinIO / non-AWS S3). */
+  forcePathStyle: z.boolean().default(false),
+});
+export type S3StoreConfig = z.infer<typeof S3StoreConfigSchema>;
+
+function parseBool(value: string | undefined): boolean | undefined {
+  if (value === undefined) return undefined;
+  return value === "true" || value === "1";
+}
+
+/**
+ * Load + validate the S3 config from env. Returns `undefined` when the
+ * backend isn't configured (no bucket / creds), so the platform falls back
+ * to the postgres default. Returns a `{ error }` when partially configured
+ * so the admin can see what's missing instead of silently falling back.
+ */
+export function loadS3ConfigFromEnv(
+  env: NodeJS.ProcessEnv = process.env,
+):
+  | { ok: true; config: S3StoreConfig }
+  | { ok: false; error: string }
+  | { ok: "unconfigured" } {
+  const raw = {
+    endpoint: env.CHECKSTACK_SCRIPT_PACKAGES_S3_ENDPOINT,
+    bucket: env.CHECKSTACK_SCRIPT_PACKAGES_S3_BUCKET,
+    region: env.CHECKSTACK_SCRIPT_PACKAGES_S3_REGION,
+    accessKeyId: env.CHECKSTACK_SCRIPT_PACKAGES_S3_ACCESS_KEY_ID,
+    secretAccessKey: env.CHECKSTACK_SCRIPT_PACKAGES_S3_SECRET_ACCESS_KEY,
+    forcePathStyle: parseBool(
+      env.CHECKSTACK_SCRIPT_PACKAGES_S3_FORCE_PATH_STYLE,
+    ),
+  };
+
+  const anySet =
+    raw.endpoint ||
+    raw.bucket ||
+    raw.region ||
+    raw.accessKeyId ||
+    raw.secretAccessKey ||
+    raw.forcePathStyle !== undefined;
+  if (!anySet) {
+    return { ok: "unconfigured" };
+  }
+
+  const parsed = S3StoreConfigSchema.safeParse({
+    ...raw,
+    // Drop undefined so zod defaults / optionals apply cleanly.
+    forcePathStyle: raw.forcePathStyle ?? false,
+  });
+  if (!parsed.success) {
+    const missing = parsed.error.issues
+      .map((issue) => issue.path.join("."))
+      .join(", ");
+    return {
+      ok: false,
+      error: `S3 blob store is partially configured; invalid/missing: ${missing}`,
+    };
+  }
+  return { ok: true, config: parsed.data };
+}

package/src/index.ts

@@ -0,0 +1,47 @@
+import { createBackendPlugin, coreServices } from "@checkstack/backend-api";
+import { blobStoreExtensionPoint } from "@checkstack/script-packages-backend";
+import { pluginMetadata } from "./plugin-metadata";
+import { loadS3ConfigFromEnv } from "./config";
+import { createS3BlobStore, S3_BLOB_STORE_ID } from "./store";
+
+/**
+ * S3-compatible blob-store plugin. Registers an `s3` {@link BlobStore} when
+ * S3 env vars are configured; otherwise stays dormant so the platform
+ * falls back to the postgres default. A partially-configured S3 (bucket
+ * set but creds missing) logs an error rather than silently registering a
+ * broken store.
+ */
+export default createBackendPlugin({
+  metadata: pluginMetadata,
+
+  register(env) {
+    env.registerInit({
+      deps: {
+        logger: coreServices.logger,
+      },
+      init: async ({ logger }) => {
+        const loaded = loadS3ConfigFromEnv();
+        if (loaded.ok === "unconfigured") {
+          logger.debug(
+            "📦 S3 script-package blob store not configured; skipping.",
+          );
+          return;
+        }
+        if (loaded.ok === false) {
+          logger.error(loaded.error);
+          return;
+        }
+        const store = createS3BlobStore(loaded.config);
+        env
+          .getExtensionPoint(blobStoreExtensionPoint)
+          .registerBlobStore(store, pluginMetadata);
+        logger.debug(
+          `📦 Registered "${S3_BLOB_STORE_ID}" script-package blob store (bucket "${loaded.config.bucket}").`,
+        );
+      },
+    });
+  },
+});
+
+export { createS3BlobStore, S3_BLOB_STORE_ID } from "./store";
+export { loadS3ConfigFromEnv, type S3StoreConfig } from "./config";

package/src/key.test.ts

@@ -0,0 +1,20 @@
+import { describe, expect, test } from "bun:test";
+import { integrityToKey, keyToIntegrity, S3_KEY_PREFIX } from "./key";
+
+describe("integrity <-> S3 key", () => {
+  test("round-trips an SRI integrity with special chars", () => {
+    const integrity = "sha512-AbC+/def==";
+    const key = integrityToKey(integrity);
+    expect(key.startsWith(S3_KEY_PREFIX)).toBe(true);
+    expect(key.slice(S3_KEY_PREFIX.length)).toMatch(/^[a-f0-9]+$/);
+    expect(keyToIntegrity(key)).toBe(integrity);
+  });
+
+  test("rejects keys outside the prefix", () => {
+    expect(keyToIntegrity("other/thing")).toBeUndefined();
+  });
+
+  test("rejects non-hex payloads", () => {
+    expect(keyToIntegrity(`${S3_KEY_PREFIX}zzz`)).toBeUndefined();
+  });
+});

package/src/key.ts

@@ -0,0 +1,24 @@
+/**
+ * Map a content-addressed integrity (SRI, e.g. `sha512-AbC+/d==`) to a
+ * safe, reversible S3 object key under a fixed prefix.
+ *
+ * SRI strings contain `+`, `/`, and `=`, which are legal-but-awkward in S3
+ * keys (and `/` would create spurious "folders"). We hex-encode the raw
+ * integrity bytes so the key is `[a-f0-9]` only, and prefix it so blobs are
+ * namespaced within a shared bucket.
+ */
+const KEY_PREFIX = "script-packages/blobs/";
+
+export function integrityToKey(integrity: string): string {
+  const hex = Buffer.from(integrity, "utf8").toString("hex");
+  return `${KEY_PREFIX}${hex}`;
+}
+
+export function keyToIntegrity(key: string): string | undefined {
+  if (!key.startsWith(KEY_PREFIX)) return undefined;
+  const hex = key.slice(KEY_PREFIX.length);
+  if (!/^[a-f0-9]*$/.test(hex) || hex.length % 2 !== 0) return undefined;
+  return Buffer.from(hex, "hex").toString("utf8");
+}
+
+export const S3_KEY_PREFIX = KEY_PREFIX;

package/src/plugin-metadata.ts

@@ -0,0 +1,5 @@
+import { definePluginMetadata } from "@checkstack/common";
+
+export const pluginMetadata = definePluginMetadata({
+  pluginId: "script-packages-store-s3",
+});

package/src/store.ts

@@ -0,0 +1,73 @@
+import { S3Client } from "bun";
+import type { BlobStore } from "@checkstack/script-packages-backend";
+import type { S3StoreConfig } from "./config";
+import { integrityToKey, keyToIntegrity, S3_KEY_PREFIX } from "./key";
+
+export const S3_BLOB_STORE_ID = "s3";
+
+interface S3ListResponse {
+  contents?: { key: string }[];
+  isTruncated?: boolean;
+  nextContinuationToken?: string;
+}
+
+/**
+ * S3-compatible {@link BlobStore} backed by Bun's native `S3Client` (no
+ * AWS SDK dependency). Content-addressed: blobs are immutable, so `put`
+ * unconditionally writes (overwriting with identical bytes is harmless).
+ *
+ * Credentials come from {@link S3StoreConfig} (loaded from env), never the
+ * application DB. `forcePathStyle` is honored via Bun's endpoint handling
+ * for MinIO / non-AWS S3.
+ */
+export function createS3BlobStore(config: S3StoreConfig): BlobStore {
+  const client = new S3Client({
+    accessKeyId: config.accessKeyId,
+    secretAccessKey: config.secretAccessKey,
+    bucket: config.bucket,
+    ...(config.region ? { region: config.region } : {}),
+    ...(config.endpoint ? { endpoint: config.endpoint } : {}),
+  });
+
+  return {
+    id: S3_BLOB_STORE_ID,
+
+    async put({ integrity, bytes }) {
+      await client.write(integrityToKey(integrity), bytes);
+    },
+
+    async get({ integrity }) {
+      const file = client.file(integrityToKey(integrity));
+      if (!(await file.exists())) return;
+      const buf = await file.arrayBuffer();
+      return new Uint8Array(buf);
+    },
+
+    async has({ integrity }) {
+      return client.file(integrityToKey(integrity)).exists();
+    },
+
+    async delete({ integrity }) {
+      await client.delete(integrityToKey(integrity));
+    },
+
+    async list() {
+      const integrities: string[] = [];
+      let continuationToken: string | undefined;
+      do {
+        const res: S3ListResponse = await client.list({
+          prefix: S3_KEY_PREFIX,
+          ...(continuationToken ? { continuationToken } : {}),
+        });
+        for (const item of res.contents ?? []) {
+          const integrity = keyToIntegrity(item.key);
+          if (integrity !== undefined) integrities.push(integrity);
+        }
+        continuationToken = res.isTruncated
+          ? res.nextContinuationToken
+          : undefined;
+      } while (continuationToken);
+      return integrities;
+    },
+  };
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "@checkstack/tsconfig/backend.json",
+  "include": [
+    "src"
+  ],
+  "references": [
+    {
+      "path": "../backend-api"
+    },
+    {
+      "path": "../common"
+    },
+    {
+      "path": "../script-packages-backend"
+    }
+  ]
+}