@supatype/cli 0.2.1 → 0.3.0

package/src/self-host-compose.ts

@@ -16,8 +16,10 @@ import {
 } from "./project-config.js"
 import { hasEngineOverride, hasStudioOverride, pinnedVersion, fetchLatestVersion, VERSION_PIN_LOCAL } from "./binary-cache.js"
 import { buildKongDeclarative } from "./kong-config.js"
+import { keyspaceInPostgres } from "./cache-provider.js"
 import { readEnvFile } from "./env-file.js"
 import { fieldMaskingTierFromProject, type FieldMaskingTier } from "./field-masking-tier.js"
+import { projectHasVersionedModels } from "./model-versioning.js"
 
 /** Env keys written when `versions` pins exist in supatype.config.ts. */
 export const COMPOSE_PINNED_IMAGE_ENV_KEYS = [
@@ -34,8 +36,31 @@ export const COMPOSE_IMAGE_ENV_KEYS = [
   "SUPATYPE_STUDIO_IMAGE",
   "SUPATYPE_STORAGE_IMAGE",
   "SUPATYPE_FUNCTIONS_WORKER_IMAGE",
+  "SUPATYPE_SEAWEEDFS_IMAGE",
 ] as const
 
+/** Pinned by digest, because the registry this replaced proved a tag can be taken away. */
+const SEAWEEDFS_IMAGE =
+  "chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62"
+
+/** Credentials the object store is configured with and the storage service is handed. */
+/**
+ * The prefix Kong's ACME plugin stores its certificates under.
+ *
+ * Not a guess and not configurable: the plugin prefixes every key it writes
+ * with this and enforces it in its own `reserved_words.lua`, so it is the one
+ * durable thing in a keyspace that is otherwise a cache. Verified on a live
+ * instance — both ACME keys survived `kill -9` with only this prefix named
+ * durable, and a response-cache key beside them did not.
+ */
+const ACME_KEY_PREFIX = "kong_acme:"
+
+const OBJECT_STORE_ACCESS_KEY = "supatype"
+const OBJECT_STORE_SECRET_KEY = "supatype-secret"
+
+/** Where the rendered SeaweedFS identity file sits, resolved from the project root like kong.yml. */
+const SEAWEED_CONFIG_MOUNT = ".supatype/self-host/s3.json"
+
 type DockerPinComponent = "engine" | "server" | "postgres"
 
 /** Map a config version pin to a Docker Hub image reference. */
@@ -142,11 +167,36 @@ export async function schemaEngineImageForPush(
   return dockerImageRef("engine", version)
 }
 
+/**
+ * SeaweedFS reads its identities from a file, so the stack's credentials are a generated artifact
+ * beside the compose file rather than two env vars inside it.
+ *
+ * There is deliberately no `anonymous` identity. Declaring one with `Read` would make every bucket
+ * world-readable and the per-bucket policy decorative, which is the opposite of what
+ * `applyPublicPolicy` is for: a bucket is public because its policy says so.
+ */
+export function renderSeaweedIdentities(): string {
+  const identities = {
+    identities: [
+      {
+        name: "supatype",
+        credentials: [
+          { accessKey: OBJECT_STORE_ACCESS_KEY, secretKey: OBJECT_STORE_SECRET_KEY },
+        ],
+        actions: ["Admin", "Read", "List", "Tagging", "Write"],
+      },
+    ],
+  }
+  return `${JSON.stringify(identities, null, 2)}\n`
+}
+
 export interface SelfHostComposePaths {
   dir: string
   composePath: string
   kongPath: string
   nginxPath: string
+  /** SeaweedFS identities, rendered beside the compose file the server reads them through. */
+  s3ConfigPath: string
 }
 
 export function selfHostComposePaths(cwd: string): SelfHostComposePaths {
@@ -156,6 +206,7 @@ export function selfHostComposePaths(cwd: string): SelfHostComposePaths {
     composePath: join(dir, "docker-compose.yml"),
     kongPath: join(dir, "kong.yml"),
     nginxPath: join(dir, "nginx.conf"),
+    s3ConfigPath: join(dir, "s3.json"),
   }
 }
 
@@ -286,6 +337,14 @@ export interface SelfHostComposeOptions {
    * rules and for every project on `supatype/postgres`.
    */
   fieldMaskingTier?: FieldMaskingTier
+  /**
+   * Whether any model declares `versions`, which puts the generated `draft` schema on the exposed
+   * list.
+   *
+   * Same reasoning as the tier above: only a caller that has loaded the schema can say, and left
+   * unset the list is today's, which is correct for every project with no versioned model.
+   */
+  drafts?: boolean
 }
 
 export function renderSelfHostCompose(
@@ -296,6 +355,12 @@ export function renderSelfHostCompose(
   const projectMount = projectMountPath(cwd)
   const kongMount = kongMountPath(cwd)
   const external = usesExternalDatabase(config)
+  // Both halves come from the caller because both need the schema loaded, which this renderer does
+  // not do. See `SelfHostComposeOptions`.
+  const restSchemas = apiSchemaList(config, {
+    ...(options?.fieldMaskingTier !== undefined && { tier: options.fieldMaskingTier }),
+    drafts: options?.drafts === true,
+  })
   const ownerUrl = ownerDatabaseUrl(config)
   // the auth driver wants the `postgres://` spelling; an external URL is used as given.
   const authUrl = external ? ownerUrl : ownerDatabaseUrl(config, "postgres")
@@ -345,12 +410,23 @@ ${studioService}
     : `    ports:
       - "9999:9999"
 `
-  const minioPorts = devLocal
+  const seaweedPorts = devLocal
     ? ""
     : `    ports:
-      - "9000:9000"
-      - "9001:9001"
+      - "8333:8333"
 `
+  // One source for the credentials: the server is configured with them and the storage service is
+  // handed them, and a mismatch does not fail at start, it fails at the first upload.
+  const objectStoreBlock = `  seaweedfs:
+    # Identities come from a config file rather than env vars, and the anonymous identity is
+    # deliberately absent: a bucket is public because of its policy, never because the server is
+    # open. Written beside this file by the same generator, so the two cannot drift.
+    image: \${SUPATYPE_SEAWEEDFS_IMAGE:-${SEAWEEDFS_IMAGE}}
+    command: server -dir=/data -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json
+    volumes:
+      - storage-data:/data
+      - ${SEAWEED_CONFIG_MOUNT}:/etc/seaweedfs/s3.json:ro
+${seaweedPorts}`
   const kongTlsEnv = tlsEnabled
     ? `      KONG_PROXY_LISTEN: "0.0.0.0:8000, 0.0.0.0:8443 ssl"
       KONG_LUA_SSL_TRUSTED_CERTIFICATE: system
@@ -361,7 +437,16 @@ ${studioService}
     ? `      - "80:8000"
       - "443:8443"`
     : `      - "\${SUPATYPE_KONG_PORT:-18473}:8000"`
-  const valkeyBlock = `
+  // The RESP server the response cache and Kong's ACME certificates use.
+  // Either a Valkey sidecar, or pg_keyspace inside the Postgres container --
+  // one stateful service instead of two.
+  const keyspaceInPg = keyspaceInPostgres(config)
+  const respHost = keyspaceInPg ? "db" : "valkey"
+  const valkeyBlock = keyspaceInPg
+    ? `  # No \`valkey\` service: cache.provider is "pg_keyspace", so the RESP keyspace is
+  # served by the \`db\` container on :6379. See extensions/pg_keyspace in supatype/postgres.
+`
+    : `
   valkey:
     image: \${SUPATYPE_VALKEY_IMAGE:-valkey/valkey:8-alpine}
     command: ["valkey-server", "--appendonly", "yes"]
@@ -370,7 +455,9 @@ ${studioService}
     volumes:
       - valkey-data:/data
 `
-  const kongValkeyDepends = "\n      - valkey"
+  // With the keyspace inside Postgres, waiting on `db` is already what every
+  // other service does, so Kong needs no extra dependency of its own.
+  const kongValkeyDepends = keyspaceInPg ? "" : "\n      - valkey"
   const tlsHintComment = tlsEnabled
     ? ""
     : `  # HTTPS is off. To enable automatic TLS (Let's Encrypt) for production, set in supatype.config.ts:
@@ -379,9 +466,8 @@ ${studioService}
 `
   // An external database is not ours to declare a volume for.
   const volumesBlock = `volumes:
-${external ? "" : "  db-data:\n"}  minio-data:
-  valkey-data:
-`
+${external ? "" : "  db-data:\n"}  storage-data:
+${keyspaceInPg ? "" : "  valkey-data:\n"}`
 
   // `depends_on` for the services that talk to Postgres. With an external database there is no
   // container to wait on, so the clause disappears entirely and each service retries on connect
@@ -426,6 +512,66 @@ ${dbDependency}`
     : ""
   const realtimeServerEnv = realtime ? "      SUPATYPE_REALTIME_URL: http://realtime:4000" : ""
 
+  // pg_keyspace registers background workers and requests shared memory at
+  // postmaster start, so it has to be in `shared_preload_libraries` -- there is
+  // no runtime toggle. The image turns it on from the environment
+  // (SUPATYPE_KEYSPACE_ENABLED) and writes the configuration itself, which is
+  // why this does not restate the preload list: the entrypoint appends
+  // pg_keyspace to the image's own list, in the right place (before
+  // supatype_mask, so the mask stays outermost and the image's
+  // pg_keyspace.require_mask = on is satisfied rather than refusing to serve).
+  // Restating it here would be a copy that goes stale the first time the image
+  // adds a library.
+  //
+  // Durability is `ephemeral` for everything, with ACME the one exception.
+  // A response cache that went through the WAL would put this stack's busiest
+  // write on disk to keep a copy of something that expires in seconds; a
+  // certificate that did not survive `docker compose restart db` would be
+  // re-issued against Let's Encrypt's rate limits, or fail. So the default is
+  // the cheap one and the certificates are named.
+  //
+  // `kong_acme:` is not a guess. Kong's ACME plugin prefixes every key it
+  // stores with it and enforces that prefix in its own reserved_words.lua;
+  // proven on a live instance, where both ACME keys survived `kill -9` under
+  // this exact configuration and the response-cache key did not.
+  //
+  // It is set whether or not TLS is on today. The override costs nothing when
+  // no key matches it, and a stack that turns TLS on later should not need a
+  // database restart before its certificates are safe to store.
+  const durablePrefixes = config.cache?.durablePrefixes ?? []
+  const durabilityOverrides = [ACME_KEY_PREFIX, ...durablePrefixes]
+    .map((prefix) => `${prefix}=durable`)
+    .join(", ")
+
+  // Sizing is shared memory, reserved at postmaster start whether or not the
+  // cache is used, so the defaults are a self-hosted stack's floor rather than
+  // the extension's own (1M keys x 512 B reserves ~689 MiB per worker before
+  // rings and row cache). These are measured on a live PG16 build, not derived:
+  // this profile reserves ~235 MiB, of which ~16 MiB is a fixed row-cache
+  // directory that rowcache_mb does not describe and cannot be declined.
+  const keyspaceDbEnv = keyspaceInPg
+    ? `      # The RESP keyspace, in place of a Valkey sidecar. The image writes
+      # /etc/postgresql-custom/pg_keyspace.conf from these before any server starts.
+      SUPATYPE_KEYSPACE_ENABLED: "1"
+      # Durable keys are rows in this database; without it they land in \`postgres\`.
+      SUPATYPE_KEYSPACE_DATABASE: \${POSTGRES_DB:-supatype}
+      SUPATYPE_KEYSPACE_DURABILITY: ephemeral
+      SUPATYPE_KEYSPACE_DURABILITY_OVERRIDES: "${durabilityOverrides}"
+      SUPATYPE_KEYSPACE_KEYS: "200000"
+      SUPATYPE_KEYSPACE_RING_MB: "16"
+      SUPATYPE_KEYSPACE_ROWCACHE_MB: "64"
+`
+    : ""
+
+  // Not published to the host: the keyspace is reachable to the compose network
+  // (server, Kong) and nothing else. A RESP port on a public interface is an
+  // unauthenticated read of every cached response.
+  const keyspaceExpose = keyspaceInPg
+    ? `    expose:
+      - "6379"
+`
+    : ""
+
   const dbServiceBlock = external
     ? `  # No \`db\` service: database.external points this stack at a Postgres it does not manage.
   # Every service reads \${DATABASE_URL} from .env, and \`supatype db check\` reports what that
@@ -439,7 +585,7 @@ ${dbDependency}`
       POSTGRES_DB: \${POSTGRES_DB:-supatype}
       # Read by the image's init to password the \`authenticator\` role PostgREST connects as.
       AUTHENTICATOR_PASSWORD: \${AUTHENTICATOR_PASSWORD:?AUTHENTICATOR_PASSWORD is missing from .env}
-${dbPorts}    volumes:
+${keyspaceDbEnv}${dbPorts}${keyspaceExpose}    volumes:
       - db-data:/var/lib/postgresql/data
     healthcheck:
       # -h 127.0.0.1 forces TCP. Without it \`pg_isready\` uses the Unix socket, which the
@@ -477,7 +623,7 @@ ${dbServiceBlock}  postgrest:
       # Derived from schema.pg_schema (or schema.api_schemas). Hardcoding this is why choosing a
       # non-public pg_schema used to give a correct push and an API that answered PGRST106 for
       # everything: the engine moved and PostgREST was never told.
-      PGRST_DB_SCHEMA: "${apiSchemaList(config, options?.fieldMaskingTier)}"
+      PGRST_DB_SCHEMA: "${restSchemas}"
       PGRST_DB_ANON_ROLE: anon
       PGRST_JWT_SECRET: \${JWT_SECRET:?JWT_SECRET is missing from .env}
       PGRST_DB_EXTRA_SEARCH_PATH: public,extensions
@@ -491,10 +637,10 @@ ${dbDependency}
       PORT: 5000
       DATABASE_URL: "${ownerUrl}"
       JWT_SECRET: \${JWT_SECRET:?JWT_SECRET is missing from .env}
-      S3_ENDPOINT: http://minio:9000
+      S3_ENDPOINT: http://seaweedfs:8333
       S3_REGION: us-east-1
-      S3_ACCESS_KEY: supatype
-      S3_SECRET_KEY: supatype-secret
+      S3_ACCESS_KEY: ${OBJECT_STORE_ACCESS_KEY}
+      S3_SECRET_KEY: ${OBJECT_STORE_SECRET_KEY}
       S3_FORCE_PATH_STYLE: "true"
 ${dbDependency}
   functions-worker:
@@ -510,6 +656,10 @@ ${dbDependency}
       # outside. One worker rather than two: a second container would cost a pod per project on
       # cloud, for isolation the route boundary already provides.
       SUPATYPE_HOOKS_ROOT: /project/hooks
+      # Every log line the worker writes carries this, and on cloud it is the label Loki indexes
+      # each project's logs under. Without it a line is collected and filed under no project, which
+      # is the same as losing it.
+      SUPATYPE_PROJECT_REF: ${JSON.stringify(config.project.name)}
       PORT: "8001"
       # In-compose loopback to Kong (not API_EXTERNAL_URL / localhost, unreachable from this container).
       SUPATYPE_URL: http://kong:8000
@@ -573,7 +723,7 @@ ${serverPorts}    volumes:
       SUPATYPE_FUNCTIONS_WORKER_URL: http://functions-worker:8001
 ${realtimeServerEnv}
       SUPATYPE_CONTROL_PLANE_URL: http://control-plane:8080
-      SUPATYPE_VALKEY_ADDR: valkey:6379
+      SUPATYPE_VALKEY_ADDR: ${respHost}:6379
 ${appEnv}
       SUPATYPE_API_HOST: 0.0.0.0
       SUPATYPE_API_PORT: 9999
@@ -606,9 +756,7 @@ ${appEnv}
       SUPATYPE_DISABLE_SIGNUP: \${DISABLE_SIGNUP:-false}
 ${devLocal ? "      STUDIO_OPEN_DEV: \"1\"\n" : ""}
     depends_on:
-${dbDependencyClause}      valkey:
-        condition: service_started
-      postgrest:
+${dbDependencyClause}${keyspaceInPg ? "" : "      valkey:\n        condition: service_started\n"}      postgrest:
         condition: service_started
       storage:
         condition: service_started
@@ -617,15 +765,7 @@ ${dbDependencyClause}      valkey:
 ${realtimeDependency}      control-plane:
         condition: service_started
 
-  minio:
-    image: minio/minio:RELEASE.2024-11-07T00-52-20Z
-    command: server /data --console-address ":9001"
-    environment:
-      MINIO_ROOT_USER: supatype
-      MINIO_ROOT_PASSWORD: supatype-secret
-${minioPorts}    volumes:
-      - minio-data:/data
-
+${objectStoreBlock}
   schema-engine:
     image: \${SUPATYPE_ENGINE_IMAGE:-supatype/schema-engine:latest}
     profiles: ["tools"]
@@ -642,6 +782,16 @@ ${dbDependency}${studioBlock}${valkeyBlock}${tlsHintComment}  kong:
       KONG_ADMIN_ACCESS_LOG: /dev/stdout
       KONG_PROXY_ERROR_LOG: /dev/stderr
       KONG_ADMIN_ERROR_LOG: /dev/stderr
+      # Cookies are not scoped by port, so every project a developer runs on localhost shares one
+      # cookie jar and every request through this gateway carries the lot, which overruns nginx's
+      # default buffers and answers "Request header or cookie too large" about cookies belonging to
+      # something else entirely.
+      #
+      # Kept equal to large_client_header_buffers in packages/studio/nginx.conf, where the reason
+      # for the two numbers is written down. Both hops need it and they need the same value: this
+      # one carries API calls, that one the page, and a request that clears one and fails the other
+      # is worse to diagnose than one that fails outright.
+      KONG_NGINX_HTTP_LARGE_CLIENT_HEADER_BUFFERS: "4 32k"
 ${kongTlsEnv}    volumes:
       - ${kongMount}:/etc/kong/kong.yml:ro
     ports:
@@ -821,6 +971,21 @@ function resolveFieldMaskingTier(
   return fieldMaskingTierFromProject(cwd, config)
 }
 
+/**
+ * Whether this project has a versioned model, unless the caller already knows.
+ *
+ * Same seam as the tier above, and the same reason: the renderer does not load the schema, so a
+ * caller that has can say, and `writeSelfHostCompose` answers for the callers that have not.
+ */
+function resolveDrafts(
+  cwd: string,
+  config: SupatypeProjectConfig,
+  options?: SelfHostComposeOptions,
+): boolean {
+  if (options?.drafts !== undefined) return options.drafts
+  return projectHasVersionedModels(cwd, config)
+}
+
 export function writeSelfHostCompose(
   cwd: string,
   config: SupatypeProjectConfig,
@@ -831,6 +996,7 @@ export function writeSelfHostCompose(
   const resolved: SelfHostComposeOptions = {
     ...options,
     ...(tier !== undefined && { fieldMaskingTier: tier }),
+    drafts: resolveDrafts(cwd, config, options),
   }
   assertExternalUrlReachableFromContainers(config)
   const paths = selfHostComposePaths(cwd)
@@ -838,6 +1004,7 @@ export function writeSelfHostCompose(
   ensureProjectFunctionsDir(cwd, config)
   ensureComposeManifest(cwd, config)
   writeFileSync(paths.composePath, renderSelfHostCompose(config, cwd, resolved), "utf8")
+  writeFileSync(paths.s3ConfigPath, renderSeaweedIdentities(), "utf8")
   const studioHostDev = options?.devLocal === true && hasStudioOverride(config)
   const tlsEnabled = selfHostTlsEnabled(config, options?.devLocal === true)
   const domain = config.server.domain?.trim()
@@ -850,8 +1017,18 @@ export function writeSelfHostCompose(
         studioServiceUrl: COMPOSE_STUDIO_HOST_URL,
         studioStripPath: false,
       }),
+      // Kong stores its certificates wherever the RESP server is. With
+      // pg_keyspace that is the database container, and they are durable
+      // there because the keyspace defaults to durable and nothing names
+      // Kong's keys as an exception.
       ...(tlsEnabled && domain && acmeEmail
-        ? { acme: { email: acmeEmail, domain, redisHost: "valkey" } }
+        ? {
+            acme: {
+              email: acmeEmail,
+              domain,
+              redisHost: keyspaceInPostgres(config) ? "db" : "valkey",
+            },
+          }
         : {}),
     }),
     "utf8",

package/src/studio-admin-roles.ts

@@ -9,8 +9,22 @@ export function studioAdminRoles(cfg: SupatypeProjectConfig): string[] {
   return [...DEFAULT_STUDIO_ADMIN_ROLES]
 }
 
-/** Merge `adminRoles` into engine admin-config JSON for Studio and supatype-server. */
+/**
+ * Merge the project's own settings into engine admin-config JSON for Studio and supatype-server.
+ *
+ * The engine derives everything it can from the schema. These two cannot be derived from it: which
+ * JWT roles may open Studio, and where the project's front end renders each model. Both are facts
+ * about a deployment rather than about a shape, so they live in `supatype.config.ts` and are folded
+ * in here, at the one place the file is written.
+ */
 export function withAdminRoles(admin: unknown, cfg: SupatypeProjectConfig): Record<string, unknown> {
   const base = typeof admin === "object" && admin !== null ? (admin as Record<string, unknown>) : {}
-  return { ...base, adminRoles: studioAdminRoles(cfg) }
+  const livePreview = cfg.admin?.livePreview
+  return {
+    ...base,
+    adminRoles: studioAdminRoles(cfg),
+    // Omitted rather than written empty: Studio treats an absent entry as "this project has not
+    // said", which is what drives the message asking for it.
+    ...(livePreview !== undefined && Object.keys(livePreview).length > 0 && { livePreview }),
+  }
 }