d1s 0.2.8 → 0.2.9

package/dist/lib/logger.js

@@ -4,14 +4,21 @@ const { combine, timestamp, printf, colorize } = winston.format;
 
 const isDevelopment = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === undefined;
 const isProduction = process.env.NODE_ENV === 'production';
+// Check for DEBUG env var which is set by --debug flag
 const isDebug = process.env.DEBUG === 'true';
 
-const logFormat = printf(({ level, message, timestamp }) => {
-  return `${timestamp} ${level}: ${message}`;
+const logFormat = printf(({ level, message, timestamp, ...metadata }) => {
+  let msg = `${timestamp} ${level}: ${message}`;
+  // Add metadata if present
+  if (Object.keys(metadata).length > 0) {
+    msg += ` ${JSON.stringify(metadata)}`;
+  }
+  return msg;
 });
 
 const logger = winston.createLogger({
-  level: isProduction ? 'warn' : (isDebug ? 'debug' : 'info'),
+  // Set log level based on environment and debug flag
+  level: isDebug ? 'debug' : (isProduction ? 'warn' : 'info'),
   format: combine(
     timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
     colorize(),
@@ -19,10 +26,33 @@ const logger = winston.createLogger({
   ),
   transports: [
     new winston.transports.Console({
-      level: isProduction ? 'warn' : (isDebug ? 'debug' : 'info'),
+      level: isDebug ? 'debug' : (isProduction ? 'warn' : 'info'),
       silent: process.env.NODE_ENV === 'test'
     })
   ]
 });
 
-export default logger;
+// Add helper methods
+logger.logRequest = (method, path, params = {}) => {
+  logger.debug(`Request: ${method} ${path}`, params);
+};
+
+logger.logError = (error, source = 'unknown') => {
+  logger.error(`Error in ${source}: ${error.message}`, {
+    error: error.stack || error.message,
+    source
+  });
+};
+
+logger.shouldLog = (level) => {
+  const currentLevel = logger.level;
+  const levels = {
+    error: 0,
+    warn: 1,
+    info: 2,
+    debug: 3
+  };
+  return levels[level] <= levels[currentLevel];
+};
+
+export default logger;

package/dist/lib/sql-operations.js

@@ -342,6 +342,83 @@ export async function executeSQL(projectDir, databaseName, sql, env = {}, remote
   }
 }
 
+/**
+ * Fetches the latest `limit` records for a given table from the remote database.
+ *
+ * This implementation uses the executeSQL function with the remote flag enabled.
+ * It orders the results based on the provided ordering column. If no ordering column is provided,
+ * it falls back to the default "id" column.
+ *
+ * @param {string} projectDir - The project directory.
+ * @param {string} databaseName - The D1 database name.
+ * @param {string} tableName - The table to fetch records for.
+ * @param {object} env - Environment variables (including credentials).
+ * @param {number} limit - The number of records to fetch.
+ * @param {string} [orderingColumn="id"] - The column to order by.
+ * @returns {Promise<Array<object>>} Array of row objects.
+ */
+export async function fetchRemoteTableData(
+  projectDir,
+  databaseName,
+  tableName,
+  env,
+  limit,
+  orderingColumn = "id"
+) {
+  // Use the provided ordering column; if not provided, default to "id".
+  const query = `SELECT * FROM "${tableName}" ORDER BY "${orderingColumn}" DESC LIMIT ${limit};`;
+  try {
+    // Execute the SQL command on the remote database.
+    const result = await executeSQL(projectDir, databaseName, query, env, true);
+    
+    // ExecuteSQL returns stdout that should contain a JSON array starting from a '['.
+    const jsonStart = result.indexOf('[');
+    if (jsonStart === -1) {
+      throw new Error(`Unexpected result format: ${result}`);
+    }
+    const jsonStr = result.substring(jsonStart);
+    const parsed = JSON.parse(jsonStr);
+    // Assuming the structure: parsed[0].results contains the rows.
+    const rows = parsed[0]?.results || [];
+    return rows;
+  } catch (error) {
+    logger.error(`Error fetching remote data for table ${tableName}:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Generates a bulk insert SQL statement for the specified table and rows.
+ *
+ * This function constructs a single INSERT statement that inserts multiple rows.
+ * It escapes single quotes in string values. You may need to extend this for
+ * additional data types (e.g., numbers, dates, booleans) or more complex structures.
+ *
+ * @param {string} tableName - The table to insert into.
+ * @param {Array<object>} rows - An array of objects, each representing a row.
+ * @returns {string} The bulk insert SQL statement.
+ */
+export function generateBulkInsertSQL(tableName, rows) {
+  if (!rows || rows.length === 0) return '';
+  
+  // Get the list of columns from the first row object.
+  const columns = Object.keys(rows[0]);
+  const columnList = columns.map(col => `"${col}"`).join(', ');
+
+  // Build the values list for each row.
+  const valuesList = rows.map(row => {
+    const values = columns.map(col => {
+      const value = row[col];
+      if (value === null || value === undefined) return 'NULL';
+      // For simplicity, treat the value as a string.
+      return `'${String(value).replace(/'/g, "''")}'`;
+    }).join(', ');
+    return `(${values})`;
+  }).join(', ');
+
+  return `INSERT INTO "${tableName}" (${columnList}) VALUES ${valuesList};`;
+}
+
 // ---------------------------------------------------------------------
 // SQL Formatting and Helper Functions
 // ---------------------------------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "d1s",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "type": "module",
   "description": "D1 Studio – a native studio for managing Cloudflare D1 databases.",
   "main": "./dist/d1-studio.js",

package/scripts/studio.js

@@ -13,7 +13,14 @@ import next from 'next';
 import { findWranglerConfig } from '../src/lib/find-wrangler-config.js';
 import logger from '../src/lib/logger.js';
 import updateNotifier from 'update-notifier';
-import pkg from '../package.json' assert { type: 'json' };
+import pkg from '../package.json' with { type: 'json' };
+
+// Check for debug flag
+const hasDebugFlag = process.argv.includes('--debug');
+if (hasDebugFlag) {
+  process.env.DEBUG = 'true';
+  logger.debug('Debug mode enabled');
+}
 
 const notifier = updateNotifier({ pkg, updateCheckInterval: 1000 * 60 * 60 * 24 });
 notifier.notify({ defer: false });

package/src/app/api/pull-data/confirm/route.js

@@ -0,0 +1,230 @@
+// api/pull-data/confirm/route.js
+
+import { NextResponse } from 'next/server';
+import { executeSQL } from '@/lib/sql-operations';
+import { getCurrentSchema } from '@/lib/schema-operations';
+import { findWranglerConfig } from '@/lib/find-wrangler-config.js';
+import logger from '@/lib/logger';
+import {
+  fetchRemoteTableData as fetchRemoteDataHelper,
+  generateBulkInsertSQL
+} from '@/lib/sql-operations';
+
+/**
+ * Helper to extract the ordering column (primary key) from a table's schema.
+ * Assumes that each table’s schema object contains a "columns" array with an "isPrimaryKey" flag.
+ *
+ * @param {object} tableSchema - Schema info for the table.
+ * @returns {string} The column name to use for ordering.
+ */
+function getOrderingColumn(tableSchema) {
+  const pkColumn = tableSchema.columns?.find(col => col.isPrimaryKey);
+  const orderingCol = pkColumn ? pkColumn.name : 'rowid';
+  logger.debug(`Determined ordering column for table ${tableSchema.name || 'unknown'}: ${orderingCol}`);
+  return orderingCol;
+}
+
+/**
+ * Computes an insertion order for tables based on their foreign key dependencies.
+ * The returned array is sorted so that if table A is referenced by table B,
+ * then A appears earlier in the list.
+ *
+ * @param {Array<object>} tableSchemas - Array of table schema objects.
+ * @returns {Array<string>} Ordered array of table names.
+ */
+function computeInsertionOrder(tableSchemas) {
+  // Build a dependency graph: key = table name, value = set of tables that this table depends on.
+  const graph = {};
+  tableSchemas.forEach(table => {
+    // Initialize graph for this table.
+    graph[table.name] = new Set();
+    // Use a regex to find FOREIGN KEY references in the table's SQL.
+    // This regex assumes the foreign key clause looks like:
+    // FOREIGN KEY (...) REFERENCES referencedTable(...)
+    const fkRegex = /FOREIGN KEY\s*\([^)]*\)\s+REFERENCES\s+(\w+)/gi;
+    let match;
+    while ((match = fkRegex.exec(table.sql)) !== null) {
+      const referencedTable = match[1];
+      graph[table.name].add(referencedTable);
+    }
+  });
+  logger.debug('Computed dependency graph:', graph);
+  
+  // Topological sort (depth-first search) to compute insertion order.
+  const sorted = [];
+  const visited = {}; // 'temp' = visiting, 'perm' = done
+
+  function visit(tableName) {
+    if (visited[tableName] === 'temp') {
+      throw new Error(`Cycle detected in dependencies at table ${tableName}`);
+    }
+    if (!visited[tableName]) {
+      visited[tableName] = 'temp';
+      // Visit all tables that this table depends on.
+      graph[tableName].forEach(dep => {
+        // Only visit if the dependency is in our graph (i.e. among our tables)
+        if (graph[dep]) {
+          visit(dep);
+        }
+      });
+      visited[tableName] = 'perm';
+      sorted.push(tableName);
+    }
+  }
+
+  Object.keys(graph).forEach(tableName => {
+    if (!visited[tableName]) {
+      visit(tableName);
+    }
+  });
+
+  logger.debug('Computed insertion order (parent tables first):', sorted);
+  return sorted;
+}
+
+/**
+ * Applies the data pull by:
+ * 1. Fetching the local schema to determine table names and primary keys.
+ * 2. Computing the insertion order so that parent tables are inserted before child tables.
+ * 3. For each table (in insertion order), fetching the latest 100 records from the remote database.
+ * 4. Generating a bulk insert statement and executing it.
+ *
+ * This function aborts the entire process if any insert fails.
+ *
+ * @param {string} projectDir
+ * @param {string} accountId
+ * @param {string} apiToken
+ * @param {string} databaseName
+ * @returns {object} Details of data pulled.
+ */
+async function applyDataPull(projectDir, accountId, apiToken, databaseName) {
+  logger.debug('Starting data pull process...');
+  const env = {
+    ...process.env,
+    CLOUDFLARE_API_TOKEN: apiToken,
+    CLOUDFLARE_ACCOUNT_ID: accountId
+  };
+
+  logger.debug('Fetching current schema for data pull...');
+  const currentSchema = await getCurrentSchema(projectDir, databaseName, env, false);
+  // Filter for table objects that have a name.
+  const tableSchemas = currentSchema.filter(obj => obj.type === 'table' && obj.name);
+  logger.debug('Local table schema for data pull:', tableSchemas.map(t => t.name));
+
+  // Compute insertion order based on foreign key dependencies.
+  const insertionOrder = computeInsertionOrder(tableSchemas);
+  logger.debug('Final computed insertion order:', insertionOrder);
+
+  // Process each table in the computed insertion order.
+  for (const tableName of insertionOrder) {
+    const table = tableSchemas.find(t => t.name === tableName);
+    if (!table) {
+      logger.debug(`Skipping table ${tableName} as it is not in the current schema list.`);
+      continue;
+    }
+    logger.debug(`Processing table: ${tableName}`);
+    const orderingColumn = getOrderingColumn(table);
+    logger.debug(`Fetching data for table: ${tableName} ordered by ${orderingColumn}`);
+
+    // Fetch remote data.
+    const rows = await fetchRemoteDataHelper(
+      projectDir,
+      databaseName,
+      tableName,
+      env,
+      75,
+      orderingColumn
+    );
+    logger.debug(`Fetched ${rows.length} records for table: ${tableName}`);
+
+    if (rows.length > 0) {
+      const insertSQL = generateBulkInsertSQL(tableName, rows);
+      logger.debug(`Generated bulk insert SQL for ${tableName}: ${insertSQL}`);
+
+      try {
+        // Execute the insert SQL directly.
+        await executeSQL(projectDir, databaseName, insertSQL, env, false);
+        logger.debug(`Bulk inserted ${rows.length} records into table: ${tableName}`);
+      } catch (insertError) {
+        logger.error(`Error inserting data into table ${tableName}:`, insertError);
+        // Abort the entire process to preserve data integrity.
+        throw new Error(`Data pull failed for table ${tableName}: ${insertError.message}`);
+      }
+    } else {
+      logger.debug(`No data fetched for table: ${tableName}`);
+    }
+  }
+  logger.debug('Data pull process completed successfully.');
+  return { message: 'All data pulled successfully.' };
+}
+
+export async function POST(req) {
+  try {
+    logger.debug('Received request for data pull.');
+    const { accountId, apiToken } = await req.json();
+    const projectDir = process.env.D1_PROJECT_DIR;
+    if (!projectDir) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: {
+            type: 'VALIDATION_ERROR',
+            message: 'Project directory not configured'
+          }
+        },
+        { status: 500 }
+      );
+    }
+    const wranglerResult = await findWranglerConfig(projectDir);
+    if (!wranglerResult || !wranglerResult.config) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: {
+            type: 'VALIDATION_ERROR',
+            message: 'Could not find Wrangler configuration'
+          }
+        },
+        { status: 400 }
+      );
+    }
+    const config = wranglerResult.config;
+    if (!config.d1_databases?.length) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: {
+            type: 'VALIDATION_ERROR',
+            message: 'No D1 databases configured'
+          }
+        },
+        { status: 400 }
+      );
+    }
+    const databaseName = config.d1_databases[0].database_name;
+    logger.debug(`Initiating data pull for database: ${databaseName}`);
+    const result = await applyDataPull(projectDir, accountId, apiToken, databaseName);
+    logger.debug('Data pull result:', result);
+    return NextResponse.json({
+      success: true,
+      message: result.message,
+      details: result
+    });
+  } catch (error) {
+    logger.error('Pull data confirm error:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: {
+          type: error.type || 'EXECUTION_ERROR',
+          message: error.message || 'Failed to apply data pull'
+        }
+      },
+      { status: 500 }
+    );
+  }
+}
+
+export async function OPTIONS() {
+  return new NextResponse(null, { status: 204 });
+}

package/src/app/api/update/route.js

@@ -4,8 +4,33 @@ import updateNotifier from 'update-notifier';
 import pkg from '../../../../package.json' assert { type: 'json' };
 
 export async function GET() {
-  // Check for updates (you might want to cache this result to avoid too many requests)
-  const notifier = updateNotifier({ pkg, updateCheckInterval: 1000 * 60 * 60 * 24 });
-  const update = notifier.update;
-  return NextResponse.json({ update });
-}
+  try {
+    // Create notifier instance with package info
+    const notifier = updateNotifier({ 
+      pkg,
+      updateCheckInterval: 1000 * 60 * 60 * 24 // Check once per day
+    });
+
+    // If there's an update, notifier.update will contain the update info
+    if (notifier.update) {
+      const { latest, current } = notifier.update;
+      
+      // Only return update info if versions are different
+      if (latest !== current) {
+        return NextResponse.json({ 
+          update: {
+            latest,
+            current
+          }
+        });
+      }
+    }
+
+    // No update available
+    return NextResponse.json({ update: null });
+  } catch (error) {
+    console.error('Error checking for updates:', error);
+    // Return null on error to prevent banner from showing
+    return NextResponse.json({ update: null });
+  }
+}

package/src/app/page.jsx

@@ -7,7 +7,6 @@ import { SQLEditor } from '../components/sql-editor';
 import { TableView } from '../components/table-view';
 import UpdateBanner from '@/components/update-banner';
 
-
 export default function StudioPage() {
   const [currentView, setCurrentView] = useState('sql-editor');
   const [selectedTable, setSelectedTable] = useState(null);
@@ -18,23 +17,28 @@ export default function StudioPage() {
   };
 
   return (
-    <div className="min-h-screen bg-background">
-      {/* notification banner at the top of the page */}
+    <div className="flex flex-col min-h-screen bg-background">
+      {/* Banner */}
       <UpdateBanner />
-      {/* sidebar */}
-      <AppSidebar 
-        className="w-64 border-r" 
-        currentView={currentView}
-        onNavigate={setCurrentView}
-        onTableSelect={handleTableSelect}
-      />
-      {/* main content */}
-      <main className="ml-64 h-screen">
-        {currentView === 'sql-editor' && <SQLEditor />}
-        {currentView === 'table-view' && selectedTable && (
-          <TableView tableName={selectedTable} />
-        )}
-      </main>
+      
+      {/* Main layout */}
+      <div className="flex flex-1">
+        {/* Sidebar */}
+        <AppSidebar 
+          className="w-64 border-r fixed h-[calc(100vh-var(--banner-height,0px))]"
+          currentView={currentView}
+          onNavigate={setCurrentView}
+          onTableSelect={handleTableSelect}
+        />
+        
+        {/* Main content */}
+        <main className="ml-64 flex-1 h-[calc(100vh-var(--banner-height,0px))]">
+          {currentView === 'sql-editor' && <SQLEditor />}
+          {currentView === 'table-view' && selectedTable && (
+            <TableView tableName={selectedTable} />
+          )}
+        </main>
+      </div>
     </div>
   );
 }

package/src/components/environment-handler.jsx

@@ -18,11 +18,14 @@ const ErrorTypes = {
 export function useEnvironmentController() {
   const [isConnecting, setIsConnecting] = useState(false);
   const [isPushing, setIsPushing] = useState(false);
+  // New state for data pull status
+  const [isDataPulling, setIsDataPulling] = useState(false);
   const [showPreview, setShowPreview] = useState(false);
   const [previewData, setPreviewData] = useState(null);
   const [previewError, setPreviewError] = useState(null);
-  const [previewType, setPreviewType] = useState(null); // "push" or "pull"
+  const [previewType, setPreviewType] = useState(null);
   const [previewSuccess, setPreviewSuccess] = useState(null);
+  const [environment, setEnvironment] = useState('local');
   const { toast } = useToast();
   const { refreshTables } = useDatabase();
 
@@ -93,7 +96,7 @@ export function useEnvironmentController() {
     }
   };
 
-  // Confirm push
+  // Update the confirmPush function:
   const confirmPush = async () => {
     try {
       setIsPushing(true);
@@ -113,7 +116,7 @@ export function useEnvironmentController() {
       if (!data.success) {
         throw new Error(data.error?.message || 'Failed to confirm push');
       }
-
+  
       // Update the preview data to show applied status
       setPreviewData(prev => ({
         ...prev,
@@ -122,12 +125,16 @@ export function useEnvironmentController() {
           action: item.action === 'will be applied' ? 'applied' : item.action
         }))
       }));
-
-      // Set success message instead of showing toast
+  
       setPreviewSuccess(`Applied ${data.details.applied} migration(s), skipped ${data.details.skipped}.`);
+      
+      // Reset environment after success
+      setEnvironment('local');
     } catch (error) {
       console.error("Push to remote confirm failed:", error);
       setPreviewError(error);
+      // Reset environment after error
+      setEnvironment('local');
     } finally {
       setIsPushing(false);
     }
@@ -169,11 +176,13 @@ export function useEnvironmentController() {
     }
   };
 
-  // Confirm pull
+  // In confirmPull function, add environment reset:
   const confirmPull = async () => {
     try {
       setIsPushing(true);
+      setEnvironment('pull'); // Ensure correct state during pull
       setPreviewError(null);
+      setPreviewSuccess(null);
       
       const credentials = await validateStoredCredentials();
       if (!credentials) throw new Error("No valid credentials found");
@@ -189,25 +198,61 @@ export function useEnvironmentController() {
         throw new Error(data.error?.message || 'Failed to confirm pull');
       }
   
-      clearPreviewState();
-      toast({
-        title: "Migration Success",
-        description: `Pull migrations applied successfully.`,
-      });
+      setPreviewSuccess(`Pull migrations applied successfully.`);
+      
+      setPreviewData(prev => ({
+        ...prev,
+        preview: prev.preview.map(item => ({
+          ...item,
+          action: item.action === 'will be applied' ? 'applied' : item.action
+        }))
+      }));
+  
       await refreshTables();
+      
     } catch (error) {
       console.error("Pull to remote confirm failed:", error);
       setPreviewError(error);
-      toast({
-        title: "Migration Failed",
-        description: error.message || "Failed to pull migrations from remote",
-        variant: "destructive",
-      });
     } finally {
       setIsPushing(false);
+      setEnvironment('local'); // Reset to local regardless of success/failure
     }
   };
 
+    // In confirmDataPull function, add syncing state:
+    const confirmDataPull = async () => {
+      try {
+        setIsDataPulling(true);
+        setEnvironment('syncing'); // Set environment to syncing during data pull
+        setPreviewError(null);
+        setPreviewSuccess(null);
+        
+        const credentials = await validateStoredCredentials();
+        if (!credentials) throw new Error("No valid credentials found");
+    
+        const response = await fetch("/api/pull-data/confirm", {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(credentials)
+        });
+        
+        const data = await response.json();
+        if (!data.success) {
+          throw new Error(data.error?.message || 'Failed to confirm data pull');
+        }
+    
+        setPreviewSuccess("Remote data synced successfully.");
+        await refreshTables();
+        
+      } catch (error) {
+        console.error("Data pull confirm failed:", error);
+        setPreviewError(error);
+      } finally {
+        setIsDataPulling(false);
+        setEnvironment('local'); // Reset to local regardless of success/failure
+      }
+    };
+
   // Connect to Cloudflare
   async function connectToCloudflare(credentials) {
     try {
@@ -240,14 +285,18 @@ export function useEnvironmentController() {
     }
   }
 
+  // Update PreviewUI to pass onDataPull and isDataPulling for pull previews.
   const PreviewUI = showPreview && (
     previewType === "pull" ? (
       <PullPreviewToast
         preview={previewData?.preview || []}
         onConfirm={confirmPull}
+        onDataPull={confirmDataPull} // pass the new data pull handler
         onCancel={clearPreviewState}
         isConfirming={isPushing}
+        isDataPulling={isDataPulling}
         error={previewError}
+        success={previewSuccess}
         isLoading={isPushing && !previewData?.preview?.length}
       />
     ) : (
@@ -266,13 +315,18 @@ export function useEnvironmentController() {
   return {
     isConnecting,
     isPushing,
+    isDataPulling,
     showPreview,
     previewError,
+    environment, // Add this
+    setEnvironment, // Add this if needed
     pushToRemote,
     pullToRemotePreview,
     confirmPull,
+    confirmDataPull, // export the new function
     connectToCloudflare,
     validateStoredCredentials,
     PreviewUI
   };
 }
+