@hobocode/thought-layer 0.4.2 → 0.6.0

package/core/deploy-io.ts

@@ -25,6 +25,10 @@ import {
   buildFileDigests, uploadPath, sanitizeSiteName, parseCliDeployOutput, deployRecord,
   type FileMap, type DeployRecord,
 } from "./deploy.ts";
+import { normalizeBackendMeta, planEnvVars, type BackendMeta } from "./backend.ts";
+import {
+  pushEnvVarsApi, cliImportEnv, resolveDbUrl, provisionNeon, applySchema, type EnvPushResult,
+} from "./backend-io.ts";
 
 const NETLIFY_API = "https://api.netlify.com/api/v1";
 
@@ -34,6 +38,9 @@ export interface DeployRunOptions {
   anonymous?: boolean; // force the no-account CLI path even if a token is set
   siteName?: string;   // create the site under this name (else Netlify auto-names)
   siteId?: string;     // deploy to an existing site (re-deploy) instead of creating one
+  staticOnly?: boolean;   // ship only the front end even when build.json has a backend
+  provisionDb?: boolean;  // opt in: provision Neon with the user's own NEON_API_KEY
+  applySchema?: boolean;  // opt in: apply schema.sql with psql after the DB is reachable
 }
 
 // ---- locate + read the build manifest ----------------------------------------
@@ -241,6 +248,78 @@ function cliDeploy(publishDirAbs: string, opts: { siteName?: string; siteId?: st
   return { url: parsed.url, claimUrl: loggedIn ? null : parsed.claimUrl, owned: loggedIn, siteName, raw };
 }
 
+// ---- backend deploy helpers (functions ship via the user's Netlify CLI) ------
+
+// Resolve the functions dir from build.json (relative to the project root, then
+// cwd). Returns null when it is not on disk so the caller can ship the front end
+// only and say so, rather than failing.
+function resolveFunctionsDir(functionsDir: string, projectRoot: string): string | null {
+  const candidates = [resolve(projectRoot, functionsDir), resolve(process.cwd(), functionsDir)];
+  for (const c of candidates) if (existsSync(c)) return c;
+  return null;
+}
+
+// Count the top-level function entries (each file or directory is one function),
+// for the dry-run plan only.
+function countFunctionFiles(dir: string): number {
+  try {
+    return readdirSync(dir).filter((n) => !n.startsWith(".")).length;
+  } catch {
+    return 0;
+  }
+}
+
+// Create a site in the user's own account via the API (token path).
+async function createSiteApi(token: string, name?: string): Promise<{ id: string; slug: string; adminUrl: string; url: string }> {
+  const body = name ? JSON.stringify({ name: sanitizeSiteName(name) }) : JSON.stringify({});
+  const site = await netlifyJson(`${NETLIFY_API}/sites`, { method: "POST", headers: { "Content-Type": "application/json" }, body }, token);
+  return {
+    id: String(site["id"] || ""),
+    slug: String(site["account_slug"] || site["account_id"] || ""),
+    adminUrl: String(site["admin_url"] || ""),
+    url: String(site["ssl_url"] || site["url"] || ""),
+  };
+}
+
+// Create a site via the Netlify CLI's own stored auth (no-token path), so the
+// site id is known before env import and deploy.
+function createSiteCli(name?: string): { id: string } {
+  const payload = JSON.stringify(name ? { name: sanitizeSiteName(name) } : {});
+  const r = spawnSync("netlify", ["api", "createSite", "--data", payload], { encoding: "utf8", timeout: 60000 });
+  if (r.status !== 0) {
+    throw new Error(`netlify api createSite failed (exit ${r.status}): ${(r.stderr || r.stdout || "").slice(0, 300)}`);
+  }
+  let id = "";
+  try { id = String((JSON.parse(r.stdout || "{}") as Record<string, unknown>)["id"] || ""); } catch { /* unparseable */ }
+  if (!id) throw new Error("could not parse the new site id from the Netlify CLI");
+  return { id };
+}
+
+// Deploy the static dir plus the functions in one CLI invocation, targeting a
+// known site id. Drives functions explicitly with --functions (independent of
+// netlify.toml). --no-build keeps the CLI from re-running a framework build of
+// the already built static dir. NOTE (live-verify gate): if a real run shows
+// --no-build suppresses function bundling, drop it for backend deploys.
+function cliDeployWithFunctions(
+  publishDirAbs: string,
+  functionsDirAbs: string | null,
+  siteId: string,
+  token: string,
+): { url: string | null; raw: string } {
+  const args = [
+    "deploy", "--prod", "--dir", publishDirAbs, "--no-build",
+    ...(functionsDirAbs ? ["--functions", functionsDirAbs] : []),
+    "--site", siteId,
+  ];
+  const childEnv = token ? { ...process.env, NETLIFY_AUTH_TOKEN: token } : process.env;
+  const r = spawnSync("netlify", args, { encoding: "utf8", timeout: 300000, env: childEnv });
+  const raw = `${r.stdout || ""}\n${r.stderr || ""}`.trim();
+  if (r.status !== 0) {
+    throw new Error(`netlify ${args.join(" ")} failed (exit ${r.status}). Output:\n${raw.slice(0, 800)}`);
+  }
+  return { url: parseCliDeployOutput(raw).url, raw };
+}
+
 // ---- the orchestrator (mirrors runScaffold's result shape) -------------------
 
 export async function runDeploy(opts: DeployRunOptions, ctx: { deployedAt: string }): Promise<StateOpResult> {
@@ -257,9 +336,30 @@ export async function runDeploy(opts: DeployRunOptions, ctx: { deployedAt: strin
     return { ok: false, message: `Publish dir ${publishDirAbs} is empty - nothing to deploy.`, details: {} };
   }
 
-  const backendWarn = manifest.hasBackend
-    ? ` WARNING: build.json says hasBackend:true${manifest.backendNote ? ` (${manifest.backendNote})` : ""}; ` +
-      `this static deploy publishes only the front end - the server part needs serverless functions or a separate host.`
+  // build.json may predate the backend block, so normalize defensively.
+  const backend = normalizeBackendMeta(manifest.backend);
+  // Ship the backend automatically when the build declares a serverless one,
+  // unless the caller forced a static-only deploy. Static deploys with no
+  // backend are byte-for-byte unchanged (backend is null, shipBackend false).
+  const shipBackend = manifest.hasBackend && backend?.backendKind === "serverless" && !opts.staticOnly;
+
+  // The note shown only when there is a backend that this deploy will NOT ship
+  // (static-only by request, or a non-serverless backend we cannot automate).
+  const backendWarn = manifest.hasBackend && !shipBackend
+    ? (() => {
+        const guide = backend?.guide || "BACKEND.md";
+        const dbEnv = backend?.database?.envVar || "DATABASE_URL";
+        const names = (backend?.envVars || []).map((v) => v.name).filter(Boolean);
+        const others = names.filter((n) => n !== dbEnv);
+        const envList = others.length ? `${dbEnv} plus ${others.join(", ")}` : dbEnv;
+        const lead = opts.staticOnly
+          ? `\n\nStatic only: the front end is live, the backend was not shipped (you passed --static-only). Re-run without it to ship the backend.`
+          : `\n\nStatic deploy: the front end is live. This build also declares a backend that this deploy cannot ship automatically.`;
+        return (
+          `${lead} To run the backend, follow ${guide}: provision Neon Postgres, set ${envList} in your host environment, ` +
+          `then run netlify deploy with the functions present.`
+        );
+      })()
     : "";
 
   const token = process.env.NETLIFY_AUTH_TOKEN || process.env.NETLIFY_TOKEN || "";
@@ -274,15 +374,45 @@ export async function runDeploy(opts: DeployRunOptions, ctx: { deployedAt: strin
   // --- dry run: plan only, no network, no spawn ---
   if (opts.dryRun) {
     const { digests } = buildFileDigests(files);
+    let backendPlanMsg = backendWarn;
+    let backendPlan: Record<string, unknown> | null = null;
+    if (shipBackend && backend) {
+      // Names + counts only; never read or print an env value.
+      const fnDir = resolveFunctionsDir(backend.functionsDir, dirname(dirname(stateFile)));
+      const fnCount = fnDir ? countFunctionFiles(fnDir) : 0;
+      const plan = planEnvVars(backend);
+      const names = plan.map((p) => p.name);
+      const missing = names.filter((n) => !(typeof process.env[n] === "string" && process.env[n] !== ""));
+      const db = resolveDbUrl(process.env);
+      const path = token ? "Netlify CLI for functions plus the token API for env" : "Netlify CLI (logged in) for functions and env import";
+      backendPlanMsg =
+        `\n\nBackend plan: ship ${fnCount} function${fnCount === 1 ? "" : "s"} from ${backend.functionsDir} via the ${path}. ` +
+        `Env var names (${names.length}): ${names.join(", ") || "none"}.` +
+        `${missing.length ? ` Missing from this environment: ${missing.join(", ")}.` : ""}` +
+        `${db.name ? ` Database url from ${db.name}.` : " No database url found (set DATABASE_URL, or use --provision-db)."}` +
+        `${opts.provisionDb ? " Would provision Neon (--provision-db)." : ""}` +
+        `${opts.applySchema ? " Would apply schema.sql (--apply-schema)." : ""}`;
+      backendPlan = { functionsDir: backend.functionsDir, functionCount: fnCount, envVarNames: names, envVarsMissing: missing, dbUrlFrom: db.name, provisionDb: !!opts.provisionDb, applySchema: !!opts.applySchema };
+    }
     return {
       ok: true,
       message:
         `Dry run: would deploy ${fileCount} files from ${publishDirAbs} (entry ${manifest.entry}) to Netlify ` +
-        `via the ${token && !opts.anonymous ? "BYO-token digest" : "Netlify CLI (logged in -> a site in your account; logged out -> an anonymous claimable site)"} path.${backendWarn}`,
-      details: { dryRun: true, publishDir: publishDirAbs, entry: manifest.entry, fileCount, files: Object.keys(digests), hasBackend: manifest.hasBackend },
+        `via the ${token && !opts.anonymous ? "BYO-token digest" : "Netlify CLI (logged in -> a site in your account; logged out -> an anonymous claimable site)"} path.${backendPlanMsg}`,
+      details: { dryRun: true, publishDir: publishDirAbs, entry: manifest.entry, fileCount, files: Object.keys(digests), hasBackend: manifest.hasBackend, shipBackend, backendPlan },
     };
   }
 
+  // --- backend deploy: ship functions + env into the user's own account ---
+  if (shipBackend && backend) {
+    return runBackendDeploy(
+      { manifest, backend, publishDirAbs, stateFile, fileCount, files },
+      opts,
+      ctx,
+      { token, writeRecord },
+    );
+  }
+
   // --- CLI path: explicit (--anonymous), or the fallback when no env token ---
   const wantCli = opts.anonymous || !token;
   if (wantCli) {
@@ -318,7 +448,8 @@ export async function runDeploy(opts: DeployRunOptions, ctx: { deployedAt: strin
         deployRecord({
           deployedAt: ctx.deployedAt, mode: owned ? "cli" : "anonymous", publishDir: manifest.publishDir, fileCount,
           url, adminUrl: null, claimUrl, siteId: null, deployId: null,
-          hasBackend: manifest.hasBackend, backendNote: manifest.backendNote, buildProducer: manifest.producer, stateFile,
+          hasBackend: manifest.hasBackend, backendNote: manifest.backendNote, backendKind: backend?.backendKind ?? null,
+          buildProducer: manifest.producer, stateFile,
         }),
       );
       // If anonymity was explicitly asked for but the CLI is logged in, it went
@@ -349,7 +480,8 @@ export async function runDeploy(opts: DeployRunOptions, ctx: { deployedAt: strin
       deployRecord({
         deployedAt: ctx.deployedAt, mode: "token", publishDir: manifest.publishDir, fileCount,
         url: r.url || null, adminUrl: r.adminUrl || null, claimUrl: null, siteId: r.siteId, deployId: r.deployId,
-        hasBackend: manifest.hasBackend, backendNote: manifest.backendNote, buildProducer: manifest.producer, stateFile,
+        hasBackend: manifest.hasBackend, backendNote: manifest.backendNote, backendKind: backend?.backendKind ?? null,
+        buildProducer: manifest.producer, stateFile,
       }),
     );
     return {
@@ -365,3 +497,143 @@ export async function runDeploy(opts: DeployRunOptions, ctx: { deployedAt: strin
     return { ok: false, message: `Deploy failed: ${(e as Error).message}`, details: { mode: "token" } };
   }
 }
+
+// Ship the backend (functions + env vars) into the user's own account. Functions
+// are bundled and shipped by the user's Netlify CLI (the digest API cannot bundle
+// TypeScript), so this forces the CLI deploy mechanism even when a token is set;
+// the token, when present, is used for the secret-capable env API and to authorize
+// the CLI non-interactively (owned, no claim). Env var VALUES are read only from
+// process.env inside the backend-io helpers and are never recorded or printed.
+async function runBackendDeploy(
+  build: { manifest: BuildManifest; backend: BackendMeta; publishDirAbs: string; stateFile: string; fileCount: number; files: FileMap },
+  opts: DeployRunOptions,
+  ctx: { deployedAt: string },
+  io: { token: string; writeRecord: (rec: DeployRecord) => string },
+): Promise<StateOpResult> {
+  const { manifest, backend, publishDirAbs, stateFile, fileCount, files } = build;
+  const { token, writeRecord } = io;
+  const projectRoot = dirname(dirname(stateFile));
+  const plan = planEnvVars(backend);
+  const guide = backend.guide || "BACKEND.md";
+  const notes: string[] = [];
+
+  // Opt in: provision Neon with the user's own key, so the DB url is known.
+  let dbUrl = resolveDbUrl(process.env).value;
+  let dbProvisioned = false;
+  if (opts.provisionDb) {
+    const pr = await provisionNeon(process.env);
+    notes.push(pr.note);
+    if (pr.provisioned && pr.url) { dbUrl = pr.url; dbProvisioned = true; }
+  }
+
+  const functionsDirAbs = resolveFunctionsDir(backend.functionsDir, projectRoot);
+
+  // No CLI means functions cannot be bundled. Do not half-deploy: with a token,
+  // still take the front end live and push env via the API; without one, stop
+  // and guide. functionsShipped stays false either way.
+  if (!hasNetlifyCli()) {
+    if (!token) {
+      return {
+        ok: false,
+        message:
+          `This build has a backend, which needs the Netlify CLI to bundle and ship the functions, and the CLI is not installed. ` +
+          `Install it (npm i -g netlify-cli@latest) and re-run, or set NETLIFY_AUTH_TOKEN to at least take the front end live, or follow ${guide}.`,
+        details: { backendMode: "static-only-fallback", functionsShipped: false, needs: "netlify-cli" },
+      };
+    }
+    try {
+      const r = await digestDeploy(files, { token, siteName: opts.siteName, siteId: opts.siteId });
+      let env: EnvPushResult | null = null;
+      try { env = await pushEnvVarsApi(r.siteId, token, plan, process.env); }
+      catch (e) { notes.push(`env push failed: ${(e as Error).message}`); }
+      const recPath = writeRecord(deployRecord({
+        deployedAt: ctx.deployedAt, mode: "token", publishDir: manifest.publishDir, fileCount,
+        url: r.url || null, adminUrl: r.adminUrl || null, claimUrl: null, siteId: r.siteId, deployId: r.deployId,
+        hasBackend: true, backendNote: manifest.backendNote, backendKind: backend.backendKind,
+        backendMode: "static-only-fallback", functionsShipped: false, functionsDir: backend.functionsDir,
+        envVarsSet: env?.set ?? [], envVarsMissing: env?.missing ?? plan.map((p) => p.name),
+        dbProvisioned, schemaApplied: false, buildProducer: manifest.producer, stateFile,
+      }));
+      return {
+        ok: true,
+        message:
+          `Deployed the static front end to your Netlify account${r.url ? ` (live: ${r.url})` : ""}. ` +
+          `The functions were NOT shipped: the Netlify CLI bundles them and it is not installed. ` +
+          `Install it (npm i -g netlify-cli@latest) and re-run to ship the backend, or follow ${guide}.` +
+          (env?.set.length ? `\nSet env var names: ${env.set.join(", ")}.` : "") +
+          (env?.missing.length ? `\nDeclared but missing from this environment: ${env.missing.join(", ")}.` : "") +
+          `\nRecorded ${recPath}.${notes.length ? `\n${notes.join("\n")}` : ""}`,
+        details: { mode: "token", backendMode: "static-only-fallback", functionsShipped: false, url: r.url, siteId: r.siteId, envVarsSet: env?.set, envVarsMissing: env?.missing },
+      };
+    } catch (e) {
+      return { ok: false, message: `Static front end deploy failed: ${(e as Error).message}`, details: { backendMode: "static-only-fallback" } };
+    }
+  }
+
+  if (!functionsDirAbs) {
+    notes.push(`functions dir "${backend.functionsDir}" was not found on disk; shipping the front end only`);
+  }
+
+  // Resolve or create the target site so env can be set against a known id.
+  let siteId = opts.siteId || "";
+  let accountSlug: string | undefined;
+  try {
+    if (!siteId) {
+      if (token) { const s = await createSiteApi(token, opts.siteName); siteId = s.id; accountSlug = s.slug; }
+      else { siteId = createSiteCli(opts.siteName).id; }
+    }
+  } catch (e) {
+    return { ok: false, message: `Could not create the Netlify site for the backend deploy: ${(e as Error).message}`, details: {} };
+  }
+
+  // Push env vars (API when a token is present, else CLI import).
+  let env: EnvPushResult;
+  try {
+    env = token
+      ? await pushEnvVarsApi(siteId, token, plan, process.env, accountSlug)
+      : cliImportEnv(plan, process.env, siteId);
+  } catch (e) {
+    env = { method: token ? "api" : "cli", set: [], missing: plan.map((p) => p.name), note: `env push failed: ${(e as Error).message}` };
+    notes.push(env.note);
+  }
+
+  // Opt in: apply schema.sql now that the database url is known.
+  let schemaApplied = false;
+  if (opts.applySchema) {
+    const schemaPath = resolve(projectRoot, backend.database?.schemaFile || "schema.sql");
+    const sr = applySchema(schemaPath, dbUrl, process.env);
+    schemaApplied = sr.applied;
+    notes.push(sr.note);
+  }
+
+  // Deploy the static dir plus the functions in one CLI invocation.
+  let url: string | null = null;
+  try {
+    const d = cliDeployWithFunctions(publishDirAbs, functionsDirAbs, siteId, token);
+    url = d.url;
+  } catch (e) {
+    return { ok: false, message: `Backend deploy failed: ${(e as Error).message}${notes.length ? `\n${notes.join("\n")}` : ""}`, details: { siteId, envVarsSet: env.set } };
+  }
+  const functionsShipped = !!functionsDirAbs;
+
+  const recPath = writeRecord(deployRecord({
+    deployedAt: ctx.deployedAt, mode: "cli", publishDir: manifest.publishDir, fileCount,
+    url, adminUrl: null, claimUrl: null, siteId, deployId: null,
+    hasBackend: true, backendNote: manifest.backendNote, backendKind: backend.backendKind,
+    backendMode: "cli", functionsShipped, functionsDir: backend.functionsDir,
+    envVarsSet: env.set, envVarsMissing: env.missing, dbProvisioned, schemaApplied,
+    buildProducer: manifest.producer, stateFile,
+  }));
+  return {
+    ok: true,
+    message:
+      `Deployed your backend to your Netlify account via the CLI${url ? ` (live: ${url})` : ""}. ` +
+      `${functionsShipped ? `Functions shipped from ${backend.functionsDir}.` : `No functions were found on disk (${backend.functionsDir}); front end only.`} ` +
+      `It is owned by your account, no claim needed. Re-deploy to the same site with --site ${siteId}.` +
+      (env.set.length ? `\nSet env var names (${env.method}): ${env.set.join(", ")}.` : "") +
+      (env.missing.length ? `\nDeclared but missing from this environment (set them, then re-run): ${env.missing.join(", ")}.` : "") +
+      `\nRecorded ${recPath}.${notes.length ? `\n${notes.join("\n")}` : ""}` +
+      (!url ? `\n(Could not parse a live URL from the CLI output; re-run to confirm.)` : ""),
+    details: { mode: "cli", backendMode: "cli", url, siteId, functionsShipped, functionsDir: backend.functionsDir, envVarsSet: env.set, envVarsMissing: env.missing, dbProvisioned, schemaApplied },
+  };
+}

package/core/deploy.ts

@@ -10,6 +10,7 @@
 // import and it is pure (content in, hex out).
 
 import { createHash } from "node:crypto";
+import type { BackendKind } from "./backend.ts";
 
 export const sha1Hex = (data: Buffer | string): string =>
   createHash("sha1").update(data).digest("hex");
@@ -106,6 +107,18 @@ export interface DeployRecord {
   deployId: string | null;
   hasBackend: boolean;
   backendNote: string | null;
+  // Provenance of the backend payload from build.json (null for a static build).
+  backendKind?: BackendKind | null;
+  // Backend deploy provenance (all optional so static deploy.json records, and
+  // records written before this field set existed, round-trip unchanged). NEVER
+  // a secret value: env vars are recorded by NAME only.
+  backendMode?: "none" | "cli" | "static-only-fallback";
+  functionsShipped?: boolean;
+  functionsDir?: string | null;
+  envVarsSet?: string[]; // names only
+  envVarsMissing?: string[]; // names declared but absent from the deploy environment
+  dbProvisioned?: boolean;
+  schemaApplied?: boolean;
   buildProducer: "agent" | "scaffold" | null;
   stateFile: string | null;
 }

package/core/index.ts

@@ -15,6 +15,8 @@ export * from "./stages.ts";
 export * from "./stage-map.ts";
 export * from "./state-file.ts";
 export * from "./state-ops.ts";
+export * from "./backend.ts";
+export * from "./backend-io.ts";
 export * from "./scaffold.ts";
 export * from "./scaffold-io.ts";
 export * from "./deploy.ts";

package/core/scaffold.ts

@@ -11,6 +11,7 @@
 // so this emits plain HTML. Pure: no fs, no Date - the caller supplies builtAt.
 
 import type { ProgressState } from "./progress.ts";
+import type { BackendMeta } from "./backend.ts";
 
 export interface StarterSiteSpec {
   brandName: string;
@@ -45,6 +46,11 @@ export interface BuildManifest {
   stack: string;
   hasBackend: boolean;
   backendNote: string | null;
+  // The structured backend payload, present only when the build emitted a real
+  // backend (hasBackend true). Optional + nullable so existing static build.json
+  // files round-trip and the deploy reader tolerates its absence. The deploy
+  // automation follow-up consumes it; today only deploy messaging reads it.
+  backend?: BackendMeta | null;
   buildCommand: string | null;
   installCommand: string | null;
   nodeVersion: string;
@@ -264,6 +270,7 @@ export function scaffoldManifest(
     stack: "static",
     hasBackend: false,
     backendNote: null,
+    backend: null,
     buildCommand: null,
     installCommand: null,
     nodeVersion: "20",