d1s 0.2.13 → 0.3.0

package/dist/app/api/migrations/list/route.js

@@ -0,0 +1,137 @@
+// api/migrations/list/route.js
+// Returns the list of local migration files cross-referenced with the local d1_migrations table.
+
+import { NextResponse } from 'next/server';
+import { findWranglerConfig } from '@/lib/find-wrangler-config.js';
+import { promisify } from 'util';
+import { exec } from 'child_process';
+import { readdir } from 'fs/promises';
+import path from 'path';
+import logger from '@/lib/logger';
+import { ensureChecksumColumn } from '@/lib/migrations';
+import { executeLocalSQL } from '@/lib/local-db.js';
+
+const execAsync = promisify(exec);
+
+async function getAppliedMigrations(projectDir, dbConfig) {
+  const databaseName = dbConfig.database_name;
+
+  // Fast path: use better-sqlite3 directly — no wrangler subprocess needed
+  // Handle missing checksum column gracefully (wrangler creates d1_migrations without it)
+  try {
+    let localOutput = executeLocalSQL(
+      projectDir, dbConfig,
+      'SELECT name, checksum, applied_at FROM d1_migrations ORDER BY applied_at ASC;'
+    );
+    if (localOutput !== null) {
+      const parsed = JSON.parse(localOutput);
+      return parsed[0]?.results || [];
+    }
+  } catch (err) {
+    const msg = err?.message || '';
+    if (msg.includes('no such column')) {
+      // Table exists but was created by wrangler without checksum column — query without it
+      try {
+        const localOutput = executeLocalSQL(
+          projectDir, dbConfig,
+          'SELECT name, applied_at FROM d1_migrations ORDER BY applied_at ASC;'
+        );
+        if (localOutput !== null) {
+          const parsed = JSON.parse(localOutput);
+          return (parsed[0]?.results || []).map(r => ({ ...r, checksum: null }));
+        }
+      } catch {
+        // fall through to wrangler CLI
+      }
+    }
+    // Any other error (e.g. no such table) — fall through to wrangler CLI
+  }
+
+  // Slow path: wrangler CLI fallback (used when better-sqlite3 can't find the SQLite file)
+  try {
+    await ensureChecksumColumn(projectDir, databaseName);
+    const { stdout } = await execAsync(
+      `wrangler d1 execute ${databaseName} --local --command="SELECT name, checksum, applied_at FROM d1_migrations ORDER BY applied_at ASC;" --json`,
+      { cwd: projectDir, timeout: 15000 }
+    );
+    const result = JSON.parse(stdout);
+    return result[0]?.results || [];
+  } catch {
+    // d1_migrations table may not exist yet
+    return [];
+  }
+}
+
+export async function GET(request) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const dbParam = searchParams.get('db');
+
+    const projectDir = process.env.D1_PROJECT_DIR;
+    if (!projectDir) throw new Error('Project directory not configured');
+
+    const wranglerResult = await findWranglerConfig(projectDir);
+    if (!wranglerResult?.config) throw new Error('No Wrangler configuration found');
+    const config = wranglerResult.config;
+    if (!config.d1_databases?.length) throw new Error('No D1 databases configured');
+
+    const dbConfig = dbParam
+      ? config.d1_databases.find(d => d.database_name === dbParam)
+      : config.d1_databases[0];
+    if (!dbConfig) throw new Error(`Database not found: ${dbParam}`);
+    const databaseName = dbConfig.database_name;
+
+    // Read migration files from disk (respect per-database migrations_dir if configured)
+    const migrationsDir = dbConfig.migrations_dir
+      ? path.join(projectDir, dbConfig.migrations_dir)
+      : path.join(projectDir, 'migrations');
+    let files = [];
+    try {
+      const entries = await readdir(migrationsDir);
+      files = entries
+        .filter(f => f.endsWith('.sql'))
+        .sort(); // timestamp prefix ensures chronological order
+    } catch {
+      // migrations/ folder doesn't exist yet — that's fine
+    }
+
+    // Get applied migrations from local DB
+    const applied = await getAppliedMigrations(projectDir, dbConfig);
+    const appliedMap = new Map(applied.map(m => [m.name, m]));
+
+    const migrations = files.map(name => {
+      const record = appliedMap.get(name);
+      return {
+        name,
+        applied: !!record,
+        applied_at: record?.applied_at || null,
+        checksum: record?.checksum || null,
+      };
+    });
+
+    // Also include any DB records whose files are missing (orphaned)
+    applied.forEach(record => {
+      if (!files.includes(record.name)) {
+        migrations.push({
+          name: record.name,
+          applied: true,
+          applied_at: record.applied_at,
+          checksum: record.checksum,
+          orphaned: true,
+        });
+      }
+    });
+
+    return NextResponse.json({ success: true, migrations, migrationsDir });
+  } catch (error) {
+    logger.error('Migrations list error:', error);
+    return NextResponse.json(
+      { success: false, error: error.message },
+      { status: 500 }
+    );
+  }
+}
+
+export async function OPTIONS() {
+  return new NextResponse(null, { status: 204 });
+}

package/dist/app/api/migrations/sync-files/route.js

@@ -6,7 +6,7 @@ import { readFile, readdir } from 'fs/promises';
 import path from 'path';
 import { promisify } from 'util';
 import { exec } from 'child_process';
-import { createChecksum } from '@/lib/migrations';
+import { createChecksum, ensureChecksumColumn } from '@/lib/migrations';
 
 const execAsync = promisify(exec);
 
@@ -15,6 +15,7 @@ const execAsync = promisify(exec);
  */
 async function getDatabaseMigrations(projectDir, databaseName) {
   try {
+    await ensureChecksumColumn(projectDir, databaseName);
     const { stdout } = await execAsync(
       `wrangler d1 execute ${databaseName} --local --command="SELECT name, checksum FROM d1_migrations;" --json`,
       { cwd: projectDir }

package/dist/app/api/pro/route.js

@@ -0,0 +1,7 @@
+// api/pro/route.js
+// Returns whether Pro features are enabled for the current session.
+// D1_IS_PRO is set by scripts/studio.js at startup after license validation.
+
+export async function GET() {
+  return Response.json({ isPro: process.env.D1_IS_PRO === 'true' });
+}

package/dist/app/api/pull-remote/confirm/route.js

@@ -1,54 +1,46 @@
 // api/pull-remote/confirm/route.js
 
 import { NextResponse } from 'next/server';
-import { writeFile, mkdir, readdir, unlink } from 'fs/promises';
+import { writeFile, readFile, mkdir, readdir, unlink } from 'fs/promises';
+import crypto from 'crypto';
 import path from 'path';
+import { promisify } from 'util';
+import { exec } from 'child_process';
 import { findWranglerConfig } from '@/lib/find-wrangler-config.js';
-import { 
+import {
   executeSQL,
   ErrorTypes,
-  formatSQLStatement,
-  getMigrationFileContent
 } from '@/lib/sql-operations';
 import { getCurrentSchema, analyzeDependencies } from '@/lib/schema-operations';
 import { initMigrationsTable, recordMigration, getRemoteMigrations } from '@/lib/migrations.js';
+import { getLocalDb, invalidateLocalDb, executeLocalSQL } from '@/lib/local-db.js';
 import logger from '@/lib/logger';
 
+const execAsync = promisify(exec);
+
 /**
  * Applies the pull migrations by:
  * 1. Fetching remote migration metadata.
- * 2. For each remote migration, fetching the migration file content from remote.
- * 3. Staging the migration files.
- * 4. Performing a destructive update on local objects.
- * 5. Resetting the local migrations table.
- * 6. Applying staged migrations sequentially (writing files, recording and executing them).
- * 7. Replacing the local migrations folder with the staged files.
- *
- * This completely replaces the local state with the remote state.
- *
- * @param {string} projectDir - The project directory.
- * @param {string} accountId - Cloudflare account ID.
- * @param {string} apiToken - Cloudflare API token.
- * @param {string} databaseName - The target D1 database name.
- * @returns {object} Details of applied migrations.
+ * 2. Reading each migration file from the local migrations folder.
+ * 3. Performing a destructive update on local objects via better-sqlite3 (FK-safe).
+ * 4. Resetting the local migrations table.
+ * 5. Applying staged migrations sequentially.
+ * 6. Replacing the local migrations folder with only the remote-matched files.
  */
-async function applyPullMigrations(projectDir, accountId, apiToken, databaseName) {
-  const env = {
-    ...process.env,
-    CLOUDFLARE_API_TOKEN: apiToken,
-    CLOUDFLARE_ACCOUNT_ID: accountId
-  };
+async function applyPullMigrations(projectDir, dbConfig) {
+  const databaseName = dbConfig.database_name;
+  const env = process.env;
 
   try {
-    // Define paths for the main migrations folder and a temporary staging folder.
-    const migrationsDir = path.join(projectDir, 'migrations');
+    const migrationsDir = dbConfig.migrations_dir
+      ? path.join(projectDir, dbConfig.migrations_dir)
+      : path.join(projectDir, 'migrations');
     const stagingDir = path.join(projectDir, 'migrations-temp');
 
-    // Ensure both directories exist.
     await mkdir(migrationsDir, { recursive: true });
     await mkdir(stagingDir, { recursive: true });
 
-    // Clear out any existing files in the staging folder.
+    // Clear staging folder
     const stagedFilesList = await readdir(stagingDir).catch(() => []);
     await Promise.all(stagedFilesList.map(file => unlink(path.join(stagingDir, file))));
 
@@ -57,123 +49,134 @@ async function applyPullMigrations(projectDir, accountId, apiToken, databaseName
     const remoteMigrations = await getRemoteMigrations(projectDir, databaseName, env);
     logger.debug('Remote migration metadata:', remoteMigrations);
 
-    // --- STEP 2: For each remote migration, fetch its file content ---
-    logger.debug('Fetching migration file content from remote...');
+    // --- STEP 2: Read migration content from local files ---
+    logger.debug('Reading migration files from local migrations folder...');
     const stagedFiles = new Map();
     for (const migration of remoteMigrations) {
+      const migrationFileName = migration.name;
+      const localFilePath = path.join(migrationsDir, migrationFileName);
       try {
-        const migrationFileName = migration.name; // Use the remote migration's name.
-        const content = await getMigrationFileContent(projectDir, databaseName, migrationFileName, env);
+        const content = await readFile(localFilePath, 'utf8');
         stagedFiles.set(migrationFileName, content);
-        logger.debug(`Staged migration file: ${migrationFileName}`);
         await writeFile(path.join(stagingDir, migrationFileName), content, 'utf8');
+        logger.debug(`Staged migration file: ${migrationFileName}`);
       } catch (fileError) {
-        logger.error(`Error fetching content for migration ${migration.name}:`, fileError);
-        throw new Error(`Failed to fetch migration file content for ${migration.name}`);
+        logger.error(`Migration file not found locally: ${migrationFileName}`, fileError.message);
+        throw new Error(`Migration file not found locally: ${migrationFileName}`);
       }
     }
 
-    // --- STEP 3: Fetch local schema for destructive update ---
-    logger.debug('Fetching local schema for destructive update...');
+    // --- STEP 3: Get local schema for drop order ---
+    logger.debug('Fetching local schema...');
     const currentSchema = await getCurrentSchema(projectDir, databaseName, env, false);
     const tableSchema = currentSchema.filter(obj =>
       obj.type === 'table' && obj.name && obj.name.toLowerCase() !== 'if'
     );
-    logger.debug('Local table schema:', tableSchema);
     const dropOrder = tableSchema.length > 0
-      ? analyzeDependencies(
-          tableSchema.map(obj => ({
-            tableName: obj.name,
-            sql: obj.sql,
-            type: obj.type
-          }))
-        )
+      ? analyzeDependencies(tableSchema.map(obj => ({ tableName: obj.name, sql: obj.sql, type: obj.type })))
       : [];
-    logger.debug('Computed drop order:', dropOrder);
 
-    // --- STEP 4: Destructive update (all-or-nothing) ---
+    // --- STEP 4: Destructive update ---
+    // Filter out Cloudflare/SQLite system objects — these are created by the runtime,
+    // not by migrations, and wrangler blocks dropping them with SQLITE_AUTH.
+    const SYSTEM_OBJECT = /^(_cf_|sqlite_)/i;
+    const triggers = currentSchema.filter(obj => obj.type === 'trigger' && !SYSTEM_OBJECT.test(obj.name));
+    const indexes = currentSchema.filter(obj => obj.type === 'index' && !SYSTEM_OBJECT.test(obj.name));
+    const reversedDropOrder = [...dropOrder].filter(t => !SYSTEM_OBJECT.test(t)).reverse();
+
     logger.debug('Beginning destructive update...');
-    try {
-      logger.debug('Disabling foreign keys...');
-      await executeSQL(projectDir, databaseName, 'PRAGMA foreign_keys=OFF;', env, false);
-      
-      // Drop triggers.
-      const triggers = currentSchema.filter(obj => obj.type === 'trigger');
-      for (const trigger of triggers) {
-        logger.debug(`Dropping trigger: ${trigger.name}`);
-        await executeSQL(
-          projectDir,
-          databaseName,
-          formatSQLStatement(`DROP TRIGGER IF EXISTS ${trigger.name}`),
-          env,
-          false
-        );
-      }
-      // Drop indexes.
-      const indexes = currentSchema.filter(obj => obj.type === 'index');
-      for (const index of indexes) {
-        logger.debug(`Dropping index: ${index.name}`);
-        await executeSQL(
-          projectDir,
-          databaseName,
-          formatSQLStatement(`DROP INDEX IF EXISTS ${index.name}`),
-          env,
-          false
-        );
+    const db = getLocalDb(projectDir, dbConfig);
+
+    if (db) {
+      // better-sqlite3: persistent connection so PRAGMA FK=OFF persists across all drops
+      try {
+        db.pragma('foreign_keys = OFF');
+        for (const trigger of triggers) {
+          logger.debug(`Dropping trigger: ${trigger.name}`);
+          db.prepare(`DROP TRIGGER IF EXISTS "${trigger.name}"`).run();
+        }
+        for (const index of indexes) {
+          logger.debug(`Dropping index: ${index.name}`);
+          db.prepare(`DROP INDEX IF EXISTS "${index.name}"`).run();
+        }
+        for (const tableName of reversedDropOrder) {
+          logger.debug(`Dropping table: ${tableName}`);
+          db.prepare(`DROP TABLE IF EXISTS "${tableName}"`).run();
+        }
+      } finally {
+        db.pragma('foreign_keys = ON');
       }
-      // Drop tables in reverse dependency order.
-      for (const tableName of dropOrder.reverse()) {
-        logger.debug(`Dropping table: ${tableName}`);
-        await executeSQL(
-          projectDir,
-          databaseName,
-          formatSQLStatement(`DROP TABLE IF EXISTS ${tableName}`),
-          env,
-          false
-        );
+    } else {
+      // Wrangler CLI fallback: drop in correct order (child-first) so FK constraints
+      // are satisfied without needing PRAGMA — PRAGMA is blocked by SQLITE_AUTH anyway.
+      logger.debug('better-sqlite3 unavailable, falling back to wrangler CLI for destructive update');
+      const stmts = [
+        ...triggers.map(t => `DROP TRIGGER IF EXISTS "${t.name}"`),
+        ...indexes.map(i => `DROP INDEX IF EXISTS "${i.name}"`),
+        ...reversedDropOrder.map(t => `DROP TABLE IF EXISTS "${t}"`),
+      ];
+      if (stmts.length > 0) {
+        await executeSQL(projectDir, databaseName, stmts.join(';\n') + ';', env, false, dbConfig);
       }
-      logger.debug('Re-enabling foreign keys...');
-      await executeSQL(projectDir, databaseName, 'PRAGMA foreign_keys=ON;', env, false);
-    } catch (dropError) {
-      throw new Error(`Error dropping existing objects: ${dropError.message}`);
     }
 
     // --- STEP 5: Reset the local migrations table ---
+    logger.debug('Resetting local migrations table...');
     try {
-      logger.debug('Resetting local migrations table...');
       await initMigrationsTable(projectDir, databaseName, 'local', true);
     } catch (initError) {
       throw new Error(`Error resetting migrations table: ${initError.message}`);
     }
 
+    // Invalidate cached DB instance so next access reflects cleared schema
+    invalidateLocalDb(projectDir, databaseName);
+
     // --- STEP 6: Apply staged migrations ---
-    // (Transactions removed because D1 operates in auto-commit mode)
+    // Use better-sqlite3 directly (handles multi-statement SQL, no shell escaping issues).
+    // Fall back to `wrangler --file` if the SQLite file can't be located — never use
+    // `--command` for migration files because it collapses newlines and breaks SQL comments.
+    const isWindows = process.platform === 'win32';
+    const npxCommand = isWindows ? 'npx.cmd' : 'npx';
+
     const appliedPreview = { preview: [] };
     let currentFilename = '';
     try {
       for (const [filename, content] of stagedFiles.entries()) {
         currentFilename = filename;
         logger.debug(`Applying migration: ${filename}`);
-        // Write the migration file to the main migrations folder.
-        await writeFile(path.join(migrationsDir, filename), content, 'utf8');
 
-        // Compute a checksum from the content.
-        const checksum = require('crypto')
-          .createHash('sha256')
-          .update(content)
-          .digest('hex');
+        const filePath = path.join(migrationsDir, filename);
+        await writeFile(filePath, content, 'utf8');
 
-        logger.debug(`Recording migration: ${filename} with checksum ${checksum}`);
-        await recordMigration(
-          projectDir,
-          databaseName,
-          { name: filename, checksum },
-          env,
-          false
-        );
+        const checksum = crypto.createHash('sha256').update(content).digest('hex');
+        await recordMigration(projectDir, databaseName, { name: filename, checksum }, env, false);
 
-        // Execute the migration SQL on the local database.
-        await executeSQL(projectDir, databaseName, content, env, false);
+        // Try better-sqlite3 first; it handles multi-statement SQL in a single transaction
+        let usedLocal = false;
+        try {
+          const localResult = executeLocalSQL(projectDir, dbConfig, content);
+          if (localResult !== null) usedLocal = true;
+        } catch (localErr) {
+          // SQL error (not "file not found") — propagate immediately
+          throw localErr;
+        }
+
+        if (!usedLocal) {
+          // Wrangler CLI fallback: --file preserves SQL as-is (no whitespace collapse, no escaping)
+          logger.debug(`Applying ${filename} via wrangler --file`);
+          const { stdout } = await execAsync(
+            `${npxCommand} wrangler d1 execute ${databaseName} --local --file="${filePath}" --json`,
+            { cwd: projectDir, env: { ...process.env, ...env }, maxBuffer: 1024 * 1024 * 10 }
+          );
+          // Success output is a JSON array: [{results, success, meta}]
+          // Error output is a JSON object: {"error": {"text": "..."}}
+          // Only parse when it's an object (error case) — array means success.
+          const trimmed = stdout?.trim();
+          if (trimmed?.startsWith('{')) {
+            const parsed = JSON.parse(trimmed);
+            if (parsed?.error) throw new Error(parsed.error.text || JSON.stringify(parsed.error));
+          }
+        }
 
         appliedPreview.preview.push({ migration: filename, action: 'applied' });
       }
@@ -181,17 +184,13 @@ async function applyPullMigrations(projectDir, accountId, apiToken, databaseName
       throw new Error(`Error applying migration ${currentFilename}: ${applyError.message}`);
     }
 
-    // --- STEP 7: Replace main migrations folder with staged files ---
-    logger.debug('Replacing main migrations folder with staged files...');
+    // --- STEP 7: Replace migrations folder with only staged (remote) files ---
+    logger.debug('Replacing migrations folder with staged files...');
     const mainFiles = await readdir(migrationsDir).catch(() => []);
-    if (mainFiles.length) {
-      for (const file of mainFiles) {
-        logger.debug(`Deleting local migration file: ${file}`);
-        await unlink(path.join(migrationsDir, file));
-      }
+    for (const file of mainFiles) {
+      await unlink(path.join(migrationsDir, file));
     }
     for (const [filename, content] of stagedFiles.entries()) {
-      logger.debug(`Copying staged file ${filename} to main migrations folder`);
       await writeFile(path.join(migrationsDir, filename), content, 'utf8');
     }
 
@@ -206,46 +205,48 @@ async function applyPullMigrations(projectDir, accountId, apiToken, databaseName
   }
 }
 
-/**
- * POST handler for pull-remote confirm.
- * Reads the Wrangler configuration (via findWranglerConfig) to obtain the D1 database name,
- * then applies the pull migrations.
- */
 export async function POST(req) {
+  if (process.env.D1_IS_PRO !== 'true') {
+    return NextResponse.json({ success: false, error: { type: 'PRO_REQUIRED', message: 'Pro license required. Run: d1s --license YOUR-KEY' } }, { status: 402 });
+  }
   try {
-    const { accountId, apiToken } = await req.json();
     const projectDir = process.env.D1_PROJECT_DIR;
     if (!projectDir) {
       return NextResponse.json({
         success: false,
-        error: { 
-          type: ErrorTypes.VALIDATION_ERROR,
-          message: 'Project directory not configured'
-        }
+        error: { type: ErrorTypes.VALIDATION_ERROR, message: 'Project directory not configured' }
       }, { status: 500 });
     }
+
     const wranglerResult = await findWranglerConfig(projectDir);
-    if (!wranglerResult || !wranglerResult.config) {
+    if (!wranglerResult?.config) {
       return NextResponse.json({
         success: false,
-        error: { 
-          type: ErrorTypes.VALIDATION_ERROR,
-          message: 'Could not find Wrangler configuration'
-        }
+        error: { type: ErrorTypes.VALIDATION_ERROR, message: 'Could not find Wrangler configuration' }
       }, { status: 400 });
     }
+
     const config = wranglerResult.config;
     if (!config.d1_databases?.length) {
       return NextResponse.json({
         success: false,
-        error: { 
-          type: ErrorTypes.VALIDATION_ERROR,
-          message: 'No D1 databases configured'
-        }
+        error: { type: ErrorTypes.VALIDATION_ERROR, message: 'No D1 databases configured' }
       }, { status: 400 });
     }
-    const databaseName = config.d1_databases[0].database_name;
-    const result = await applyPullMigrations(projectDir, accountId, apiToken, databaseName);
+
+    const body = await req.json().catch(() => ({}));
+    const dbParam = body?.db;
+    const dbConfig = dbParam
+      ? config.d1_databases.find(d => d.database_name === dbParam)
+      : config.d1_databases[0];
+    if (!dbConfig) {
+      return NextResponse.json({
+        success: false,
+        error: { type: ErrorTypes.VALIDATION_ERROR, message: `Database not found: ${dbParam}` }
+      }, { status: 400 });
+    }
+    const result = await applyPullMigrations(projectDir, dbConfig);
+
     return NextResponse.json({
       success: true,
       message: 'Pull migrations applied successfully',
@@ -258,21 +259,12 @@ export async function POST(req) {
       error: {
         type: error.type || ErrorTypes.EXECUTION_ERROR,
         message: error.message || 'Failed to apply pull migrations',
-        details: error.details || {
-          suggestions: [
-            'Check your Cloudflare credentials',
-            'Verify database permissions',
-            'Ensure all migration files are accessible'
-          ]
-        }
+        details: error.details || {}
       }
     }, { status: 500 });
   }
 }
 
-/**
- * OPTIONS handler for pre-flight requests.
- */
 export async function OPTIONS() {
   return new NextResponse(null, { status: 204 });
 }