@nextsparkjs/core 0.1.0-beta.19 → 0.1.0-beta.191

package/scripts/db/run-migrations.mjs

@@ -1,9 +1,7 @@
-import pg from "pg";
 import fs from "fs";
 import path from "path";
 import { fileURLToPath } from 'url';
-
-const { Client } = pg;
+import { migrationTimeLimit, migrationClient, ignoredParametersNotice, runMigrationSql, recordMigration, migrationFailure } from './migration-time-limit.mjs';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -17,47 +15,74 @@ const packageRoot = path.join(__dirname, '..', '..'); // scripts/db/ -> core/
 const isMonorepoMode = fs.existsSync(path.join(projectRoot, 'packages', 'core'));
 const rootDir = projectRoot; // For backward compatibility with code below
 
-// Read .env from the project root (where user runs the command)
+// Read environment variables: prefer .env file if it exists, fallback to process.env (e.g. Vercel/CI).
+// `--no-env-file` skips the .env file, so the connection and theme come only from the
+// environment this process was started with (db:verify-theme runs it that way).
 const envPath = path.join(projectRoot, '.env');
-const envContent = fs.readFileSync(envPath, 'utf8');
-const envLines = envContent.split('\n');
-
-let DATABASE_URL = null;
-let ACTIVE_THEME = null;
-
-envLines.forEach(line => {
-  if (line && !line.startsWith('#')) {
-    const [key, ...valueParts] = line.split('=');
-    const value = valueParts.join('=').replace(/^["']|["']$/g, '').trim();
-
-    if (key?.trim() === 'DATABASE_URL' && valueParts.length > 0) {
-      DATABASE_URL = value;
-    }
-    if (key?.trim() === 'NEXT_PUBLIC_ACTIVE_THEME' && valueParts.length > 0) {
-      ACTIVE_THEME = value;
+const readEnvFile = !process.argv.includes('--no-env-file');
+
+let DATABASE_URL = process.env.DATABASE_URL ?? null;
+// Migrations and seeds run as the table OWNER. After the runtime cutover the app
+// connects DATABASE_URL as the non-owner `nextspark_app` role, so the owner
+// credential for migrations is provided separately via MIGRATE_DATABASE_URL.
+// Falls back to DATABASE_URL when unset (pre-cutover: same owner connection).
+let MIGRATE_DATABASE_URL = process.env.MIGRATE_DATABASE_URL ?? null;
+let ACTIVE_THEME = process.env.NEXT_PUBLIC_ACTIVE_THEME ?? null;
+let MIGRATION_TIMEOUT_SECONDS = process.env.MIGRATION_TIMEOUT_SECONDS;
+
+if (readEnvFile && fs.existsSync(envPath)) {
+  const envContent = fs.readFileSync(envPath, 'utf8');
+  const envLines = envContent.split('\n');
+
+  envLines.forEach(line => {
+    if (line && !line.startsWith('#')) {
+      const [key, ...valueParts] = line.split('=');
+      const value = valueParts.join('=').replace(/^["']|["']$/g, '').trim();
+
+      if (key?.trim() === 'DATABASE_URL' && valueParts.length > 0) {
+        DATABASE_URL = value;
+      }
+      if (key?.trim() === 'MIGRATE_DATABASE_URL' && valueParts.length > 0) {
+        MIGRATE_DATABASE_URL = value;
+      }
+      if (key?.trim() === 'NEXT_PUBLIC_ACTIVE_THEME' && valueParts.length > 0) {
+        ACTIVE_THEME = value;
+      }
+      if (key?.trim() === 'MIGRATION_TIMEOUT_SECONDS' && valueParts.length > 0) {
+        MIGRATION_TIMEOUT_SECONDS = value;
+      }
     }
-  }
-});
+  });
+}
 
 if (!DATABASE_URL) {
   console.error("❌ DATABASE_URL not found in environment variables");
   process.exit(1);
 }
 
+// The URL used for the actual migration connection (owner credential).
+const MIGRATION_URL = MIGRATE_DATABASE_URL || DATABASE_URL;
+
 if (!ACTIVE_THEME) {
   console.error("❌ NEXT_PUBLIC_ACTIVE_THEME not found in environment variables");
   process.exit(1);
 }
 
+// How long each migration may run, when MIGRATION_TIMEOUT_SECONDS asks for a limit
+// (see migration-time-limit.mjs). Without it, a migration runs for as long as it takes.
+let TIME_LIMIT;
+try {
+  TIME_LIMIT = migrationTimeLimit({ MIGRATION_TIMEOUT_SECONDS });
+} catch (error) {
+  console.error(`❌ ${error.message}`);
+  process.exit(1);
+}
+
+const ignoredParameters = ignoredParametersNotice(MIGRATION_URL, TIME_LIMIT);
+if (ignoredParameters) console.log(`⚠️  ${ignoredParameters}\n`);
+
 async function runMigrations() {
-  const client = new Client({
-    connectionString: DATABASE_URL,
-    ssl: { 
-      rejectUnauthorized: false,
-      require: true
-    },
-    connectionTimeoutMillis: 10000
-  });
+  const client = migrationClient(MIGRATION_URL, TIME_LIMIT);
   
   try {
     await client.connect();
@@ -99,17 +124,14 @@ async function runMigrations() {
       const sql = fs.readFileSync(path.join(migrationsDir, file), 'utf8');
       
       try {
-        await client.query(sql);
+        await runMigrationSql(client, { sql, limit: TIME_LIMIT, connectionString: MIGRATION_URL });
         
         // Record successful migration
-        await client.query(
-          'INSERT INTO "_migrations" (filename) VALUES ($1)',
-          [file]
-        );
+        await recordMigration(client, { file, table: '_migrations', row: { filename: file }, limit: TIME_LIMIT });
         
         console.log(`✅ Successfully executed ${file}\n`);
       } catch (error) {
-        console.error(`❌ Failed to execute ${file}:`, error.message);
+        console.error(`❌ ${migrationFailure(file, error)}`);
         throw error;
       }
     }
@@ -314,11 +336,13 @@ async function executeContentMigration(client, migration) {
   console.log(`  🔄 ${filename}...`);
   const sql = fs.readFileSync(fullPath, 'utf8');
 
-  await client.query(sql);
-  await client.query(
-    'INSERT INTO "_content_migrations" (source_type, source_name, filename) VALUES ($1, $2, $3)',
-    [sourceType, sourceName, filename]
-  );
+  await runMigrationSql(client, { sql, limit: TIME_LIMIT, connectionString: MIGRATION_URL });
+  await recordMigration(client, {
+    file: filename,
+    table: '_content_migrations',
+    row: { source_type: sourceType, source_name: sourceName, filename },
+    limit: TIME_LIMIT,
+  });
 
   console.log(`  ✅ ${filename} executed successfully`);
   return true;
@@ -343,11 +367,13 @@ async function executeEntityMigration(client, migration) {
   console.log(`${indent}🔄 ${filename}...`);
   const sql = fs.readFileSync(fullPath, 'utf8');
 
-  await client.query(sql);
-  await client.query(
-    'INSERT INTO "_entity_migrations" (entity_name, source_type, source_name, filename) VALUES ($1, $2, $3, $4)',
-    [entityName, sourceType, sourceName, filename]
-  );
+  await runMigrationSql(client, { sql, limit: TIME_LIMIT, connectionString: MIGRATION_URL });
+  await recordMigration(client, {
+    file: filename,
+    table: '_entity_migrations',
+    row: { entity_name: entityName, source_type: sourceType, source_name: sourceName, filename },
+    limit: TIME_LIMIT,
+  });
 
   console.log(`${indent}✅ ${filename} executed successfully`);
   return true;
@@ -355,14 +381,7 @@ async function executeEntityMigration(client, migration) {
 
 // Entity migrations runner - WordPress-like architecture with sample_data deferred execution
 async function runEntityMigrations() {
-  const client = new Client({
-    connectionString: DATABASE_URL,
-    ssl: {
-      rejectUnauthorized: false,
-      require: true
-    },
-    connectionTimeoutMillis: 10000
-  });
+  const client = migrationClient(MIGRATION_URL, TIME_LIMIT);
 
   try {
     await client.connect();
@@ -454,6 +473,19 @@ async function runEntityMigrations() {
     const themeEntitiesDir = path.join(contentsDir, 'themes', ACTIVE_THEME, 'entities');
     allEntityMigrations.push(...collectEntityMigrations(themeEntitiesDir, 'theme', ACTIVE_THEME));
 
+    // Collect theme settings migrations (settings/<area>/migrations/)
+    const themeSettingsDir = path.join(contentsDir, 'themes', ACTIVE_THEME, 'settings');
+    if (fs.existsSync(themeSettingsDir)) {
+      const settingAreas = fs.readdirSync(themeSettingsDir, { withFileTypes: true })
+        .filter(e => e.isDirectory()).map(e => e.name);
+      for (const area of settingAreas) {
+        const settingsMigrationsDir = path.join(themeSettingsDir, area, 'migrations');
+        allContentMigrations.push(...collectContentMigrations(
+          settingsMigrationsDir, 'theme-settings', `${ACTIVE_THEME}/settings/${area}`
+        ));
+      }
+    }
+
     // Collect plugin migrations
     const pluginsDir = path.join(contentsDir, 'plugins');
     if (fs.existsSync(pluginsDir) && activePlugins.length > 0) {
@@ -467,6 +499,19 @@ async function runEntityMigrations() {
         // Plugin entity migrations
         const pluginEntitiesDir = path.join(pluginDir, 'entities');
         allEntityMigrations.push(...collectEntityMigrations(pluginEntitiesDir, 'plugin', pluginName));
+
+        // Plugin settings migrations (settings/<area>/migrations/)
+        const pluginSettingsDir = path.join(pluginDir, 'settings');
+        if (fs.existsSync(pluginSettingsDir)) {
+          const settingAreas = fs.readdirSync(pluginSettingsDir, { withFileTypes: true })
+            .filter(e => e.isDirectory()).map(e => e.name);
+          for (const area of settingAreas) {
+            const settingsMigrationsDir = path.join(pluginSettingsDir, area, 'migrations');
+            allContentMigrations.push(...collectContentMigrations(
+              settingsMigrationsDir, 'plugin-settings', `${pluginName}/settings/${area}`
+            ));
+          }
+        }
       }
     }
 
@@ -505,7 +550,7 @@ async function runEntityMigrations() {
           const executed = await executeContentMigration(client, migration);
           if (executed) totalContentMigrations++;
         } catch (error) {
-          console.error(`  ❌ Failed to execute ${migration.filename}:`, error.message);
+          console.error(`  ❌ ${migrationFailure(migration.filename, error)}`);
           throw error;
         }
       }
@@ -526,7 +571,7 @@ async function runEntityMigrations() {
           const executed = await executeEntityMigration(client, migration);
           if (executed) totalEntityMigrations++;
         } catch (error) {
-          console.error(`  ❌ Failed to execute ${migration.filename}:`, error.message);
+          console.error(`  ❌ ${migrationFailure(migration.filename, error)}`);
           throw error;
         }
       }
@@ -546,7 +591,7 @@ async function runEntityMigrations() {
           const executed = await executeContentMigration(client, migration);
           if (executed) totalContentMigrations++;
         } catch (error) {
-          console.error(`  ❌ Failed to execute ${migration.filename}:`, error.message);
+          console.error(`  ❌ ${migrationFailure(migration.filename, error)}`);
           throw error;
         }
       }
@@ -567,7 +612,7 @@ async function runEntityMigrations() {
           const executed = await executeEntityMigration(client, migration);
           if (executed) totalEntityMigrations++;
         } catch (error) {
-          console.error(`  ❌ Failed to execute ${migration.filename}:`, error.message);
+          console.error(`  ❌ ${migrationFailure(migration.filename, error)}`);
           throw error;
         }
       }

package/scripts/db/ssl-config.mjs

@@ -0,0 +1,194 @@
+import pg from 'pg'
+import { EventEmitter } from 'node:events'
+
+/**
+ * SSL handling for the database maintenance scripts.
+ *
+ * This deliberately mirrors `src/lib/db.ts` without importing application
+ * runtime code: scripts run as standalone Node ESM programs while the runtime
+ * remains bundled separately until its next release.
+ */
+
+/**
+ * Strip SSL-related parameters so pg cannot override the explicit `ssl` option.
+ * pg v8+ otherwise resolves `sslmode=require` as certificate validation.
+ */
+export function stripSSLParams(databaseUrl) {
+  if (!databaseUrl) return databaseUrl
+  try {
+    const url = new URL(databaseUrl)
+    url.searchParams.delete('sslmode')
+    url.searchParams.delete('uselibpqcompat')
+    return url.toString()
+  } catch {
+    return databaseUrl
+      .replace(/[?&]sslmode=[^&]*/g, '')
+      .replace(/[?&]uselibpqcompat=[^&]*/g, '')
+      .replace(/\?&/, '?')
+      .replace(/\?$/, '')
+  }
+}
+
+/**
+ * Parse the SSL policy used by database scripts.
+ *
+ * Explicit sslmode takes precedence. With no recognized mode, only production
+ * uses validated TLS; all other environments use no TLS.
+ */
+export function parseSSLConfig(databaseUrl) {
+  const isProduction = process.env.NODE_ENV === 'production'
+
+  if (!databaseUrl) return isProduction ? { rejectUnauthorized: true } : false
+
+  try {
+    const url = new URL(databaseUrl)
+    const sslmode = url.searchParams.get('sslmode')?.trim().toLowerCase()
+
+    if (sslmode) {
+      switch (sslmode) {
+        case 'disable':
+          if (isProduction) console.warn('[DB] WARNING: SSL disabled in production environment. This is insecure!')
+          return false
+        case 'require':
+        case 'prefer':
+        case 'allow':
+          return { rejectUnauthorized: false }
+        case 'verify-ca':
+        case 'verify-full':
+          return { rejectUnauthorized: true }
+        default:
+          console.warn(`[DB] Unknown sslmode value: "${sslmode}". Using environment defaults.`)
+      }
+    }
+
+    return isProduction ? { rejectUnauthorized: true } : false
+  } catch {
+    if (databaseUrl.includes('sslmode=disable')) return false
+    return isProduction ? { rejectUnauthorized: true } : false
+  }
+}
+
+/** True only for a syntactically valid URL which does not state an sslmode. */
+export function prefersSSL(databaseUrl) {
+  if (!databaseUrl) return false
+  try {
+    return !new URL(databaseUrl).searchParams.has('sslmode')
+  } catch {
+    // Keep malformed URLs on the runtime's conservative, documented policy.
+    return false
+  }
+}
+
+/**
+ * The initial configuration for a script connection.
+ *
+ * A URL without sslmode has libpq's `prefer` semantics: first ask for TLS
+ * without validating the certificate, and fall back only when Postgres says it
+ * has no TLS support. `parseSSLConfig` intentionally remains the runtime's
+ * policy; the distinct initial value here is what makes the scripts differ on
+ * that one, documented case.
+ */
+export function scriptConnectionOptions(databaseUrl, options = {}) {
+  return {
+    ...options,
+    connectionString: stripSSLParams(databaseUrl),
+    ssl: prefersSSL(databaseUrl) ? { rejectUnauthorized: false } : parseSSLConfig(databaseUrl),
+  }
+}
+
+const NO_SSL_SUPPORT = 'The server does not support SSL connections'
+
+function isNoSSLSupport(error) {
+  return error?.message === NO_SSL_SUPPORT
+}
+
+/**
+ * A small Client facade for the only case where scripts deliberately differ
+ * from the runtime: a URL with no sslmode. It preserves pg's Client surface
+ * while swapping the first TLS-only connection for one without TLS only after
+ * pg's precise "server does not support SSL" error. Every other failure is
+ * returned unchanged.
+ */
+class SSLPreferClient extends EventEmitter {
+  constructor(options) {
+    super()
+    this.options = options
+    this.client = new pg.Client(options)
+    this.eventsForwarded = false
+  }
+
+  get connection() { return this.client.connection }
+  get connectionParameters() { return this.client.connectionParameters }
+  get database() { return this.client.database }
+  set database(value) { this.client.database = value }
+  get processID() { return this.client.processID }
+  get _queryable() { return this.client._queryable }
+  get _ending() { return this.client._ending }
+
+  forwardEvents() {
+    if (this.eventsForwarded) return
+    this.eventsForwarded = true
+    // Pool relies on these two; the scripts use the underlying connection for
+    // protocol events. Forwarding also keeps the normal pg error/end contract.
+    for (const event of ['error', 'end', 'notice', 'notification']) {
+      this.client.on(event, (...args) => this.emit(event, ...args))
+    }
+  }
+
+  async connectPromise() {
+    const startedAt = performance.now()
+    try {
+      await this.client.connect()
+    } catch (error) {
+      if (!isNoSSLSupport(error)) throw error
+
+      // A retry shares, rather than resets, the connection time budget. This
+      // matters for timeLimitedClient(), whose connectMs is a hard bound.
+      const timeout = Number(this.options.connectionTimeoutMillis) || 0
+      const elapsed = performance.now() - startedAt
+      if (timeout > 0 && elapsed >= timeout) throw new Error('timeout expired')
+
+      // The facade exposes pg's live connection parameters. Callers such as
+      // timeLimitedClient() deliberately update those after construction so a
+      // connection string cannot disable their limits. Build the plaintext
+      // retry from that live, resolved state (rather than re-parsing the URL),
+      // or it would silently discard every such update.
+      const { connectionString: _connectionString, ...options } = this.options
+      this.client = new pg.Client({
+        ...options,
+        ...this.client.connectionParameters,
+        password: this.client.password,
+        ssl: false,
+        ...(timeout > 0 ? { connectionTimeoutMillis: Math.max(1, Math.ceil(timeout - elapsed)) } : {}),
+      })
+      await this.client.connect()
+    }
+    this.forwardEvents()
+  }
+
+  connect(callback) {
+    const promise = this.connectPromise()
+    if (!callback) return promise
+    promise.then(() => callback(null, this), callback)
+  }
+
+  query(...args) { return this.client.query(...args) }
+  end(...args) { return this.client.end(...args) }
+  ref() { return this.client.ref() }
+  unref() { return this.client.unref() }
+}
+
+/** A Client with explicit modes unchanged, or libpq-style `prefer` otherwise. */
+export function scriptClient(databaseUrl, options = {}) {
+  const config = scriptConnectionOptions(databaseUrl, options)
+  return prefersSSL(databaseUrl) ? new SSLPreferClient(config) : new pg.Client(config)
+}
+
+/** A Pool whose clients have the same `prefer` behavior as direct clients. */
+export function scriptPool(databaseUrl, options = {}) {
+  const config = scriptConnectionOptions(databaseUrl, options)
+  return new pg.Pool({
+    ...config,
+    ...(prefersSSL(databaseUrl) ? { Client: SSLPreferClient } : {}),
+  })
+}

package/scripts/db/theme-location.mjs

@@ -0,0 +1,69 @@
+// Where db:verify-theme finds a theme, and the directory run-migrations.mjs has
+// to run in to migrate it.
+//
+// run-migrations.mjs reads the active theme from `<cwd>/contents/themes/<theme>`.
+// The monorepo's themes reach it through apps/dev/contents, which links to
+// themes/. The starter a generated project ships is not among them: it lives in
+// packages/core/templates/contents/themes/starter, and themes/starter holds only
+// its tests. packages/core/templates has the same contents/ layout a generated
+// project has, so the runner reads the starter from there.
+
+import fs from 'fs';
+import path from 'path';
+
+/** The directories the runner can run in, each with its own contents/themes. */
+export const THEME_PROJECTS = ['apps/dev', 'packages/core/templates'];
+
+// The runner reads a theme's plugins from one of these; a directory with
+// neither is not a theme, and migrating it migrates core alone.
+const THEME_CONFIGS = ['config/theme.config.ts', 'theme.config.ts'];
+
+const SKIP_DIRS = new Set(['node_modules', '.next', 'dist']);
+
+/** Every .sql file inside a `migrations` directory under `dir`, at any depth. */
+export function migrationFiles(dir, insideMigrations = false, found = []) {
+  if (!fs.existsSync(dir)) return found;
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (SKIP_DIRS.has(entry.name)) continue;
+    const entryPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      migrationFiles(entryPath, insideMigrations || entry.name === 'migrations', found);
+    } else if (insideMigrations && entry.name.endsWith('.sql')) {
+      found.push(entryPath);
+    }
+  }
+  return found;
+}
+
+/**
+ * The theme called `theme`: its directory, the directory to run the runner in,
+ * and the migration files it holds. `{ error }` when there is no such theme,
+ * when the name is a theme in more than one place, or when it has no
+ * migrations, since a run over none of them proves nothing about it.
+ */
+export function findTheme(repoRoot, theme) {
+  const candidates = THEME_PROJECTS.map(project => ({
+    projectDir: path.join(repoRoot, project),
+    themeDir: path.join(repoRoot, project, 'contents', 'themes', theme),
+  }));
+  // A theme reached through a link is shown where it really lives: themes/blog, not apps/dev/contents/themes/blog.
+  const shown = dir =>
+    fs.existsSync(dir) ? path.relative(fs.realpathSync(repoRoot), fs.realpathSync(dir)) : path.relative(repoRoot, dir);
+  const found = candidates.filter(({ themeDir }) => THEME_CONFIGS.some(config => fs.existsSync(path.join(themeDir, config))));
+
+  if (found.length === 0) {
+    return {
+      error: `no theme "${theme}": none of ${candidates.map(({ themeDir }) => shown(themeDir)).join(', ')} has a theme config`,
+    };
+  }
+  if (found.length > 1) {
+    return { error: `"${theme}" is a theme in more than one place: ${found.map(({ themeDir }) => shown(themeDir)).join(', ')}` };
+  }
+
+  const [{ projectDir, themeDir }] = found;
+  const migrations = migrationFiles(themeDir);
+  if (migrations.length === 0) {
+    return { error: `theme "${theme}" at ${shown(themeDir)} has no migrations to verify` };
+  }
+  return { projectDir, themeDir, shownDir: shown(themeDir), migrations };
+}

package/scripts/db/verify-tables.mjs

@@ -1,9 +1,7 @@
-import pg from "pg";
 import fs from "fs";
 import path from "path";
 import { fileURLToPath } from 'url';
-
-const { Client } = pg;
+import { scriptClient } from './ssl-config.mjs';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -24,10 +22,7 @@ envLines.forEach(line => {
 });
 
 async function verifyTables() {
-  const client = new Client({
-    connectionString: DATABASE_URL,
-    ssl: { rejectUnauthorized: false }
-  });
+  const client = scriptClient(DATABASE_URL);
   
   try {
     await client.connect();
@@ -139,4 +134,4 @@ async function verifyTables() {
   }
 }
 
-verifyTables();
+verifyTables();

package/scripts/db/verify-teams.mjs

@@ -5,17 +5,13 @@
  * Run with: node core/scripts/db/verify-teams.mjs
  */
 
-import pg from 'pg'
 import dotenv from 'dotenv'
+import { scriptPool } from './ssl-config.mjs'
 
 dotenv.config()
 
-const { Pool } = pg
-
 async function verifyTeamsSetup() {
-  const pool = new Pool({
-    connectionString: process.env.DATABASE_URL,
-  })
+  const pool = scriptPool(process.env.DATABASE_URL)
 
   try {
     console.log('🔍 Verifying Teams Core Setup...\n')