@hogsend/engine 0.20.0 → 0.21.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hogsend/engine",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -40,14 +40,14 @@
     "svix": "^1.95.1",
     "winston": "^3.19.0",
     "zod": "^4.4.3",
-    "@hogsend/core": "^0.20.0",
-    "@hogsend/db": "^0.20.0",
-    "@hogsend/email": "^0.20.0",
-    "@hogsend/plugin-posthog": "^0.20.0",
-    "@hogsend/plugin-resend": "^0.20.0"
+    "@hogsend/core": "^0.21.0",
+    "@hogsend/db": "^0.21.0",
+    "@hogsend/email": "^0.21.0",
+    "@hogsend/plugin-posthog": "^0.21.0",
+    "@hogsend/plugin-resend": "^0.21.0"
   },
   "optionalDependencies": {
-    "@hogsend/plugin-postmark": "^0.20.0"
+    "@hogsend/plugin-postmark": "^0.21.0"
   },
   "devDependencies": {
     "@types/node": "^22.15.3",

package/src/index.ts

@@ -247,15 +247,20 @@ export {
   type OutboundPayloads,
 } from "./lib/outbound.js";
 export { getPostHog } from "./lib/posthog.js";
+// --- PostHog OAuth scopes (front-loaded set; gap-detector source of truth) ---
+export { EXPECTED_POSTHOG_SCOPES } from "./lib/posthog-scopes.js";
 // --- Provider credentials (encrypted-at-rest OAuth token store) ---
 export {
   type CredentialKind,
   type DecryptedProviderCredential,
+  type DerivedCredentialPayload,
   deleteProviderCredential,
+  getDerivedCredential,
   getProviderCredential,
   type OAuthCredentialPayload,
   ProviderCredentialDecryptError,
   type ProviderCredentialMeta,
+  saveDerivedCredential,
   saveProviderCredential,
   toCredentialMeta,
 } from "./lib/provider-credentials.js";

package/src/lib/posthog-scopes.ts

@@ -0,0 +1,29 @@
+/**
+ * The PostHog OAuth scope set Hogsend requests during
+ * `hogsend connect posthog`. Front-loaded beyond the webhook loop's needs
+ * (which requires only `hog_function:write` plus the minted webhook secret)
+ * so future read/write features activate without forcing a reconnect. Every
+ * name is validated against PostHog's published `scopes_supported`.
+ *
+ * LOCKSTEP (3 places — no single importable source of truth, the CLI has no
+ * engine dependency): this constant, the CLI's `POSTHOG_SCOPES`
+ * (`packages/cli/src/lib/oauth.ts`), and the `scope` field of the hosted CIMD
+ * document (`apps/docs/public/.well-known/hogsend-posthog-client.json`).
+ * PostHog gates the scopes it will grant by the CIMD doc, so grep all three
+ * and keep them identical before changing any of them.
+ */
+export const EXPECTED_POSTHOG_SCOPES: string[] = [
+  "person:read",
+  "person:write",
+  "project:read",
+  "organization:read",
+  "hog_function:read",
+  "hog_function:write",
+  "feature_flag:read",
+  "cohort:read",
+  "cohort:write",
+  "query:read",
+  "insight:read",
+  "event_definition:read",
+  "property_definition:read",
+];

package/src/lib/provider-credentials.ts

@@ -16,8 +16,11 @@ import { env } from "../env.js";
  * don't, so the two stay independent).
  */
 
-/** Only "oauth" today; widen the union when an "api_key" kind lands. */
-export type CredentialKind = "oauth";
+/**
+ * "oauth" holds the token grant; "derived" holds server-derived config grabbed
+ * during connect. Widen the union further when an "api_key" kind lands.
+ */
+export type CredentialKind = "oauth" | "derived";
 
 /**
  * The decrypted shape stored for kind="oauth". Provider-neutral: nothing in
@@ -39,6 +42,20 @@ export interface OAuthCredentialPayload {
   scopedOrganizations: string[];
 }
 
+/**
+ * The decrypted shape stored for kind="derived": server-derived PostHog config
+ * grabbed during connect. `webhookSecret` is minted when env lacks one
+ * (load-bearing for the inbound loop — the posthog webhook source resolves it
+ * from the store at request time); `projectApiKey` (phc_) is grabbed
+ * opportunistically for the OPTIONAL outbound capture path. All fields optional.
+ */
+export interface DerivedCredentialPayload {
+  webhookSecret?: string;
+  projectApiKey?: string;
+  projectId?: string;
+  privateHost?: string;
+}
+
 /** Row metadata — everything EXCEPT token material. Safe to surface. */
 export interface ProviderCredentialMeta {
   providerId: string;
@@ -85,14 +102,16 @@ function deriveKey(secret: string): Buffer {
   return createHash("sha256").update(secret).digest();
 }
 
-function encryptPayload(
-  payload: OAuthCredentialPayload,
-  secret: string,
-): string {
+/**
+ * Payload-agnostic encrypt: JSON-stringify any value, AES-256-GCM encrypt,
+ * return base64url(iv || ciphertext || tag). No shape validation — the kind's
+ * own encrypt helper owns that.
+ */
+function encryptJson(value: unknown, secret: string): string {
   const iv = randomBytes(IV_LENGTH);
   const cipher = createCipheriv("aes-256-gcm", deriveKey(secret), iv);
   const ciphertext = Buffer.concat([
-    cipher.update(JSON.stringify(payload), "utf-8"),
+    cipher.update(JSON.stringify(value), "utf-8"),
     cipher.final(),
   ]);
   return Buffer.concat([iv, ciphertext, cipher.getAuthTag()]).toString(
@@ -100,11 +119,16 @@ function encryptPayload(
   );
 }
 
-function decryptPayload(
+/**
+ * Payload-agnostic decrypt: decode base64url, AES-256-GCM decrypt, JSON.parse.
+ * Throws `ProviderCredentialDecryptError` on ANY failure or if the parsed
+ * result is not a non-null object. Per-kind validation lives in the caller.
+ */
+function decryptJson(
   blob: string,
   secret: string,
   providerId: string,
-): OAuthCredentialPayload {
+): unknown {
   let raw: Buffer;
   try {
     raw = Buffer.from(blob, "base64url");
@@ -119,7 +143,7 @@ function decryptPayload(
   const ciphertext = raw.subarray(IV_LENGTH, raw.length - TAG_LENGTH);
   const tag = raw.subarray(raw.length - TAG_LENGTH);
 
-  let payload: OAuthCredentialPayload;
+  let value: unknown;
   try {
     const decipher = createDecipheriv("aes-256-gcm", deriveKey(secret), iv);
     decipher.setAuthTag(tag);
@@ -127,11 +151,35 @@ function decryptPayload(
       decipher.update(ciphertext),
       decipher.final(),
     ]).toString("utf-8");
-    payload = JSON.parse(plaintext);
+    value = JSON.parse(plaintext);
   } catch {
     throw new ProviderCredentialDecryptError(providerId);
   }
 
+  if (typeof value !== "object" || value === null) {
+    throw new ProviderCredentialDecryptError(providerId);
+  }
+  return value;
+}
+
+function encryptPayload(
+  payload: OAuthCredentialPayload,
+  secret: string,
+): string {
+  return encryptJson(payload, secret);
+}
+
+function decryptPayload(
+  blob: string,
+  secret: string,
+  providerId: string,
+): OAuthCredentialPayload {
+  const payload = decryptJson(
+    blob,
+    secret,
+    providerId,
+  ) as OAuthCredentialPayload;
+
   if (
     typeof payload.accessToken !== "string" ||
     typeof payload.tokenEndpoint !== "string"
@@ -248,3 +296,54 @@ export async function deleteProviderCredential(
 
   return deleted.length > 0;
 }
+
+/**
+ * Read + decrypt the kind="derived" config for a provider. `null` when none
+ * stored; throws `ProviderCredentialDecryptError` when a row exists but cannot
+ * be decrypted. Unlike the oauth path there's no required-field validation —
+ * every field on `DerivedCredentialPayload` is optional.
+ */
+export async function getDerivedCredential(
+  db: Database,
+  providerId: string,
+): Promise<DerivedCredentialPayload | null> {
+  const [row] = await db
+    .select()
+    .from(providerCredentials)
+    .where(
+      and(
+        eq(providerCredentials.providerId, providerId),
+        eq(providerCredentials.kind, "derived"),
+      ),
+    )
+    .limit(1);
+
+  if (!row) return null;
+
+  return decryptJson(
+    row.payload,
+    env.BETTER_AUTH_SECRET,
+    providerId,
+  ) as DerivedCredentialPayload;
+}
+
+/**
+ * Encrypt + UPSERT the kind="derived" config (full-payload overwrite, same dumb
+ * store as `saveProviderCredential`: merging old + new fields is the CALLER's
+ * job).
+ */
+export async function saveDerivedCredential(
+  db: Database,
+  providerId: string,
+  payload: DerivedCredentialPayload,
+): Promise<void> {
+  const encrypted = encryptJson(payload, env.BETTER_AUTH_SECRET);
+
+  await db
+    .insert(providerCredentials)
+    .values({ providerId, kind: "derived", payload: encrypted })
+    .onConflictDoUpdate({
+      target: [providerCredentials.providerId, providerCredentials.kind],
+      set: { payload: encrypted, updatedAt: new Date() },
+    });
+}

package/src/lib/provision-posthog-loop.ts

@@ -56,6 +56,12 @@ export interface ProvisionPostHogLoopResult {
   projectId: string;
   /** The URL the destination POSTs to: `${apiPublicUrl}/v1/webhooks/posthog`. */
   webhookUrl: string;
+  /**
+   * The phc_ public capture key, read from the project object's `api_token`
+   * field. Grabbed opportunistically for the OPTIONAL outbound capture path;
+   * NOT needed for the inbound webhook loop. `undefined` if absent.
+   */
+  projectApiKey?: string;
   /** Best-effort deep link (pattern unverified — cosmetic, confirm in e2e). */
   dashboardUrl: string;
 }
@@ -269,10 +275,11 @@ export async function provisionPostHogLoop(
     );
   }
 
-  const projectId =
-    opts.projectId !== undefined
-      ? String(opts.projectId)
-      : await discoverProjectId({ privateHost, accessToken });
+  const { id: projectId, apiToken: projectApiKey } = await fetchProject({
+    privateHost,
+    accessToken,
+    projectId: opts.projectId,
+  });
 
   const basePath = `/api/environments/${projectId}/hog_functions/`;
   const desired: DesiredLoop = { webhookUrl, webhookSecret };
@@ -328,6 +335,7 @@ export async function provisionPostHogLoop(
     functionId,
     projectId,
     webhookUrl,
+    projectApiKey,
     dashboardUrl:
       `${privateHost.replace(/\/+$/, "")}/project/${projectId}` +
       `/pipeline/destinations/hog-${functionId}/configuration`,
@@ -335,21 +343,31 @@ export async function provisionPostHogLoop(
 }
 
 /**
- * One-shot `@current` discovery, used only when `opts.projectId` is not
- * given. Deliberately uncached and NOT shared with plugin-posthog's
+ * One-shot project fetch — ALWAYS runs (even when `opts.projectId` is
+ * env-set) so the project's `api_token` (the phc_ public capture key) is
+ * read on the way through. GETs `/api/projects/${projectId}/` when the id
+ * is known, else `/api/projects/@current/` for discovery. Returns both the
+ * resolved numeric id (stringified) and the optional `api_token`.
+ *
+ * Deliberately uncached and NOT shared with plugin-posthog's
  * `resolveProjectId` — different process/cadence; provisioning is a
  * one-shot admin action that tolerates a stray re-discovery.
  */
-async function discoverProjectId(opts: {
+async function fetchProject(opts: {
   privateHost: string;
   accessToken: string;
-}): Promise<string> {
+  projectId?: string | number;
+}): Promise<{ id: string; apiToken?: string }> {
+  const path =
+    opts.projectId !== undefined
+      ? `/api/projects/${opts.projectId}/`
+      : "/api/projects/@current/";
   let body: unknown;
   try {
     body = await phFetch({
       privateHost: opts.privateHost,
       accessToken: opts.accessToken,
-      path: "/api/projects/@current/",
+      path,
     });
   } catch (err) {
     if (
@@ -370,11 +388,15 @@ async function discoverProjectId(opts: {
   if (typeof id !== "number" && typeof id !== "string") {
     throw new ProvisionPostHogLoopError({
       code: "project-discovery-failed",
-      message: "PostHog /api/projects/@current/ returned no project id.",
+      message: `PostHog ${path} returned no project id.`,
       remediation: PROJECT_DISCOVERY_REMEDIATION,
     });
   }
-  return String(id);
+  const apiToken =
+    isRecord(body) && typeof body.api_token === "string"
+      ? body.api_token
+      : undefined;
+  return { id: String(id), apiToken };
 }
 
 /** Bearer + JSON fetch with the documented error mapping. */

package/src/routes/admin/analytics.ts

@@ -1,7 +1,14 @@
+import { randomBytes } from "node:crypto";
 import { DEFAULT_HOST, derivePrivateHost } from "@hogsend/plugin-posthog";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import type { AppEnv } from "../../app.js";
 import { createTokenManager } from "../../lib/oauth-token-manager.js";
+import { EXPECTED_POSTHOG_SCOPES } from "../../lib/posthog-scopes.js";
+import {
+  getDerivedCredential,
+  getProviderCredential,
+  saveDerivedCredential,
+} from "../../lib/provider-credentials.js";
 import {
   ProvisionPostHogLoopError,
   provisionPostHogLoop,
@@ -30,8 +37,16 @@ export const connectInfoSchema = z.object({
   personalKeyConfigured: z.boolean(),
   webhookSecretConfigured: z.boolean(),
   apiPublicUrl: z.string(),
+  /**
+   * Expected PostHog OAuth scopes the stored credential is MISSING (the
+   * CLI surfaces these so the user can reconnect for the broader grant).
+   * `[]` when nothing is stored or the stored grant already covers them.
+   * Computed in the handler — NOT part of the pure `resolveConnectInfo`
+   * env projection (which `ConnectInfo` mirrors), so it omits this key.
+   */
+  scopeGap: z.array(z.string()),
 });
-export type ConnectInfo = z.infer<typeof connectInfoSchema>;
+export type ConnectInfo = Omit<z.infer<typeof connectInfoSchema>, "scopeGap">;
 
 /**
  * True when a public URL points at a loopback/unspecified address — PostHog
@@ -169,8 +184,35 @@ const provisionLoopRoute = createRoute({
 });
 
 export const analyticsAdminRouter = new OpenAPIHono<AppEnv>()
-  .openapi(connectInfoRoute, (c) => {
-    return c.json(resolveConnectInfo(c.get("container").env), 200);
+  .openapi(connectInfoRoute, async (c) => {
+    const { db, env } = c.get("container");
+
+    // The env projection is the pure base. The store can ALSO hold a minted
+    // webhook secret (provision-loop mints + persists one when env lacks it),
+    // so the configured-ness flag OR-s the two sources of truth.
+    const storedDerived = await getDerivedCredential(db, "posthog");
+    const webhookSecretConfigured =
+      Boolean(env.POSTHOG_WEBHOOK_SECRET) ||
+      Boolean(storedDerived?.webhookSecret);
+
+    // scopeGap = expected scopes the STORED oauth credential is missing. No
+    // credential ⇒ no gap to report (the connect flow will request the full
+    // set). A decrypt failure is non-fatal here — fall back to no gap.
+    let scopeGap: string[] = [];
+    try {
+      const oauth = await getProviderCredential(db, "posthog");
+      if (oauth) {
+        const granted = oauth.payload.scopes;
+        scopeGap = EXPECTED_POSTHOG_SCOPES.filter((s) => !granted.includes(s));
+      }
+    } catch {
+      scopeGap = [];
+    }
+
+    return c.json(
+      { ...resolveConnectInfo(env), webhookSecretConfigured, scopeGap },
+      200,
+    );
   })
   .openapi(provisionLoopRoute, async (c) => {
     const { db, env, logger } = c.get("container");
@@ -215,6 +257,24 @@ export const analyticsAdminRouter = new OpenAPIHono<AppEnv>()
       );
     }
 
+    // Resolve the webhook secret instead of requiring it: env wins, else a
+    // previously-minted stored secret, else mint a fresh one. When env has no
+    // secret, persist the resolved value so it survives AND the inbound
+    // posthog webhook source can resolve it from the store at request time
+    // (the source falls OPEN otherwise). The stored payload is merged so an
+    // existing phc_/projectId is preserved.
+    const storedDerived = await getDerivedCredential(db, "posthog");
+    const webhookSecret =
+      env.POSTHOG_WEBHOOK_SECRET ??
+      storedDerived?.webhookSecret ??
+      randomBytes(32).toString("hex");
+    if (env.POSTHOG_WEBHOOK_SECRET === undefined) {
+      await saveDerivedCredential(db, "posthog", {
+        ...(storedDerived ?? {}),
+        webhookSecret,
+      });
+    }
+
     try {
       const result = await provisionPostHogLoop({
         privateHost: info.privateHost,
@@ -225,10 +285,24 @@ export const analyticsAdminRouter = new OpenAPIHono<AppEnv>()
           ? { projectId: env.POSTHOG_PROJECT_ID }
           : {}),
         apiPublicUrl: info.apiPublicUrl,
-        webhookSecret: env.POSTHOG_WEBHOOK_SECRET,
+        webhookSecret,
         logger,
       });
 
+      // Opportunistically persist the phc_ (project api_token) the provisioner
+      // read on its way through — it powers the OPTIONAL outbound capture path
+      // and activates on the next deploy (no lazy boot-time seam). Re-read the
+      // stored payload to merge over the just-persisted webhook secret.
+      if (result.projectApiKey) {
+        const cur = (await getDerivedCredential(db, "posthog")) ?? {};
+        await saveDerivedCredential(db, "posthog", {
+          ...cur,
+          projectApiKey: result.projectApiKey,
+          projectId: result.projectId,
+          privateHost: info.privateHost,
+        });
+      }
+
       // M4 translation: the CLI's documented shape, with `action` riding
       // along for --json consumers.
       return c.json(

package/src/routes/admin/provider-credentials.ts

@@ -127,9 +127,13 @@ const deleteRoute = createRoute({
 });
 
 function serializeMeta(meta: ProviderCredentialMeta) {
+  // This admin surface is OAuth-only: PUT forces `kind: "oauth"` and GET reads
+  // the oauth credential, so the meta's kind is always "oauth" at runtime even
+  // though `ProviderCredentialMeta.kind` widened to the "oauth" | "derived"
+  // union when the derived store landed. Narrow it back to the schema literal.
   return {
     providerId: meta.providerId,
-    kind: meta.kind,
+    kind: "oauth" as const,
     scopes: meta.scopes,
     expiresAt: meta.expiresAt.toISOString(),
     scopedTeams: meta.scopedTeams,

package/src/routes/webhooks/sources.ts

@@ -1,10 +1,57 @@
+import type { Database } from "@hogsend/db";
 import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
 import type { AppEnv } from "../../app.js";
 import { headersToRecord } from "../../lib/headers.js";
 import { ingestEvent } from "../../lib/ingestion.js";
+import type { Logger } from "../../lib/logger.js";
+import { getDerivedCredential } from "../../lib/provider-credentials.js";
 import type { DefinedWebhookSource } from "../../webhook-sources/define-webhook-source.js";
 import { verifySignature } from "../../webhook-sources/verify.js";
 
+/** Negative-cache window for the stored PostHog secret (mirrors the token
+ * manager's ABSENT_RECHECK_MS) — caches present AND absent results so an
+ * inbound PostHog webhook POST does not hit the DB on every request. */
+const STORED_SECRET_RECHECK_MS = 30_000;
+
+let storedPosthogSecret: { value: string | undefined; at: number } | undefined;
+
+/**
+ * The minted PostHog webhook secret falls back to the `kind="derived"` store
+ * when `POSTHOG_WEBHOOK_SECRET` is unset — so an inbound event verifies WITHOUT
+ * a redeploy after `hogsend connect posthog`. Cached (present and absent) for
+ * `STORED_SECRET_RECHECK_MS` to keep the hot webhook path off the DB.
+ *
+ * A store read failure (e.g. DB blip, or a derived row that no longer decrypts)
+ * resolves to `undefined` rather than throwing — the inbound webhook path must
+ * not 500 on a degraded store. `undefined` keeps match-auth in its pre-feature
+ * posture (OPEN when no secret is configured anywhere), and the failure is
+ * logged so the misconfiguration is still visible.
+ */
+async function resolveStoredPosthogSecret(
+  db: Database,
+  logger: Logger,
+): Promise<string | undefined> {
+  const now = Date.now();
+  if (
+    storedPosthogSecret &&
+    now - storedPosthogSecret.at <= STORED_SECRET_RECHECK_MS
+  ) {
+    return storedPosthogSecret.value;
+  }
+
+  let value: string | undefined;
+  try {
+    value = (await getDerivedCredential(db, "posthog"))?.webhookSecret;
+  } catch (err) {
+    logger.warn("Failed to resolve stored PostHog webhook secret", {
+      error: err instanceof Error ? err.message : String(err),
+    });
+    value = undefined;
+  }
+  storedPosthogSecret = { value, at: now };
+  return value;
+}
+
 export function registerWebhookSourceRoutes(
   app: OpenAPIHono<AppEnv>,
   sources: DefinedWebhookSource[],
@@ -72,10 +119,22 @@ export function registerWebhookSourceRoutes(
     const rawBody = await c.req.text();
     const headers = headersToRecord(c.req.raw.headers);
 
-    const secret = env[source.auth.envKey as keyof typeof env] as
+    let secret = env[source.auth.envKey as keyof typeof env] as
       | string
       | undefined;
 
+    // For the inbound PostHog source, fall back to the secret minted by
+    // `hogsend connect` (kind="derived" store) when env has none — so an
+    // inbound event verifies WITHOUT a redeploy. Leaves match-auth OPEN when
+    // neither env nor the store has a secret (current behavior preserved).
+    if (
+      !secret &&
+      source.auth.type === "match" &&
+      source.auth.envKey === "POSTHOG_WEBHOOK_SECRET"
+    ) {
+      secret = await resolveStoredPosthogSecret(db, logger);
+    }
+
     if (source.auth.type === "signature") {
       // Signature sources FAIL CLOSED: an unset secret is a 401, never an open
       // pass-through (deliberate divergence from the "match" variant).