d1s 0.2.8 → 0.2.10

package/dist/lib/sql-operations.js

@@ -112,26 +112,18 @@ export function parseStatement(sql) {
     // Extract the index name.
     const indexMatch = cleanSql.match(/INDEX\s+(?:IF\s+NOT\s+EXISTS\s+)?([\w$]+)/i);
     const indexName = indexMatch ? indexMatch[1] : '';
-    // Extract the columns from the ON clause.
-    const onMatch = cleanSql.match(/ON\s+[\w$]+\s*\(([^)]+)\)/i);
-    const columns = onMatch ? onMatch[1].trim() : '';
-    // Remove spaces and replace commas with hyphens.
-    const formattedColumns = columns.replace(/\s+/g, '').replace(/,/g, '-').toLowerCase();
     
-    // Build a description that uses the index name,
-    // and appends the columns if they are not already included.
-    let description = indexName;
-    if (formattedColumns && !description.toLowerCase().includes(formattedColumns)) {
-      description += `-${formattedColumns}`;
-    }
+    // For index migrations, simply use the index name as the description.
+    const description = indexName;
     
     return {
       type: Operations.CREATE_INDEX,
       tableName,
       sql: cleanSql,
-      description // e.g. "idx_movies_name" or "idx_movies_name-name"
+      description // now simply the index name, e.g. "idx_technology_timestamp"
     };
-  }  
+  }
+    
 
   if (upperSql.startsWith('DROP INDEX')) {
     const tableName = extractTableNameFromIndex(cleanSql);
@@ -342,6 +334,83 @@ export async function executeSQL(projectDir, databaseName, sql, env = {}, remote
   }
 }
 
+/**
+ * Fetches the latest `limit` records for a given table from the remote database.
+ *
+ * This implementation uses the executeSQL function with the remote flag enabled.
+ * It orders the results based on the provided ordering column. If no ordering column is provided,
+ * it falls back to the default "id" column.
+ *
+ * @param {string} projectDir - The project directory.
+ * @param {string} databaseName - The D1 database name.
+ * @param {string} tableName - The table to fetch records for.
+ * @param {object} env - Environment variables (including credentials).
+ * @param {number} limit - The number of records to fetch.
+ * @param {string} [orderingColumn="id"] - The column to order by.
+ * @returns {Promise<Array<object>>} Array of row objects.
+ */
+export async function fetchRemoteTableData(
+  projectDir,
+  databaseName,
+  tableName,
+  env,
+  limit,
+  orderingColumn = "id"
+) {
+  // Use the provided ordering column; if not provided, default to "id".
+  const query = `SELECT * FROM "${tableName}" ORDER BY "${orderingColumn}" DESC LIMIT ${limit};`;
+  try {
+    // Execute the SQL command on the remote database.
+    const result = await executeSQL(projectDir, databaseName, query, env, true);
+    
+    // ExecuteSQL returns stdout that should contain a JSON array starting from a '['.
+    const jsonStart = result.indexOf('[');
+    if (jsonStart === -1) {
+      throw new Error(`Unexpected result format: ${result}`);
+    }
+    const jsonStr = result.substring(jsonStart);
+    const parsed = JSON.parse(jsonStr);
+    // Assuming the structure: parsed[0].results contains the rows.
+    const rows = parsed[0]?.results || [];
+    return rows;
+  } catch (error) {
+    logger.error(`Error fetching remote data for table ${tableName}:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Generates a bulk insert SQL statement for the specified table and rows.
+ *
+ * This function constructs a single INSERT statement that inserts multiple rows.
+ * It escapes single quotes in string values. You may need to extend this for
+ * additional data types (e.g., numbers, dates, booleans) or more complex structures.
+ *
+ * @param {string} tableName - The table to insert into.
+ * @param {Array<object>} rows - An array of objects, each representing a row.
+ * @returns {string} The bulk insert SQL statement.
+ */
+export function generateBulkInsertSQL(tableName, rows) {
+  if (!rows || rows.length === 0) return '';
+  
+  // Get the list of columns from the first row object.
+  const columns = Object.keys(rows[0]);
+  const columnList = columns.map(col => `"${col}"`).join(', ');
+
+  // Build the values list for each row.
+  const valuesList = rows.map(row => {
+    const values = columns.map(col => {
+      const value = row[col];
+      if (value === null || value === undefined) return 'NULL';
+      // For simplicity, treat the value as a string.
+      return `'${String(value).replace(/'/g, "''")}'`;
+    }).join(', ');
+    return `(${values})`;
+  }).join(', ');
+
+  return `INSERT INTO "${tableName}" (${columnList}) VALUES ${valuesList};`;
+}
+
 // ---------------------------------------------------------------------
 // SQL Formatting and Helper Functions
 // ---------------------------------------------------------------------
@@ -482,14 +551,14 @@ export function formatMigrationContent(sql) {
  * @returns {Promise<string>} The migration file content.
  */
 
- export async function getMigrationFileContent(projectDir, databaseName, migrationFile, env) {
+export async function getMigrationFileContent(projectDir, databaseName, migrationFile, env) {
   try {
     // Extract table name using improved helper
     const tableName = await extractTableNameFromMigration(migrationFile);
     let sql;
 
-    if (tableName) {
-      // Try getting SQL from sqlite_master for table
+    // For non-index migrations, try getting SQL from sqlite_master for table
+    if (tableName && !migrationFile.includes('create-index')) {
       const result = await executeSQL(
         projectDir,
         databaseName,
@@ -506,24 +575,38 @@ export function formatMigrationContent(sql) {
       }
     }
 
-    // If we couldn't get the SQL from table, try index
+    // If we couldn't get the SQL from table and it's an index migration, try fetching via index name.
     if (!sql && migrationFile.includes('create-index')) {
-      const indexMatch = migrationFile.match(/create-index[-_]([^_.]+)/);
+      const indexMatch = migrationFile.match(/create-index[_-](.+)\.sql$/i);
       if (indexMatch) {
-        const result = await executeSQL(
+        let indexName = indexMatch[1]; // e.g. "idx_technology_timestamp-technology-timestamp"
+        let result = await executeSQL(
           projectDir,
           databaseName,
-          `SELECT sql FROM sqlite_master WHERE type='index' AND name='${indexMatch[1]}';`,
+          `SELECT sql FROM sqlite_master WHERE type='index' AND name='${indexName}' COLLATE NOCASE;`,
           env,
           true
         );
-        const parsed = JSON.parse(result);
+        let parsed = JSON.parse(result);
         sql = parsed[0]?.results?.[0]?.sql;
+        // Fallback: if no result, try using only the part before the first hyphen.
+        if (!sql) {
+          const fallbackIndexName = indexName.split('-')[0]; // "idx_technology_timestamp"
+          result = await executeSQL(
+            projectDir,
+            databaseName,
+            `SELECT sql FROM sqlite_master WHERE type='index' AND name='${fallbackIndexName}' COLLATE NOCASE;`,
+            env,
+            true
+          );
+          parsed = JSON.parse(result);
+          sql = parsed[0]?.results?.[0]?.sql;
+        }
       }
-    }
+    }    
 
-    // If still no SQL, try fetching any object matching the name pattern
-    if (!sql && tableName) {
+    // If still no SQL and it's not an index migration, try a fallback query
+    if (!sql && tableName && !migrationFile.includes('create-index')) {
       const result = await executeSQL(
         projectDir,
         databaseName,

package/dist/lib/version-utils.js

@@ -0,0 +1,34 @@
+// lib/version-utils.js
+
+/**
+ * Compares semantic versions to determine if v2 is newer than v1
+ * @param {string} v1 - First version (e.g. "1.0.0")
+ * @param {string} v2 - Second version (e.g. "1.0.1")
+ * @returns {boolean} True if v2 is newer than v1
+ */
+export function isNewerVersion(v1, v2) {
+    if (!v1 || !v2) return false;
+    
+    // Split versions by dots
+    const v1Parts = v1.split('.').map(Number);
+    const v2Parts = v2.split('.').map(Number);
+    
+    // Compare major, minor, and patch versions
+    for (let i = 0; i < 3; i++) {
+      if (v2Parts[i] > v1Parts[i]) return true;
+      if (v2Parts[i] < v1Parts[i]) return false;
+    }
+    
+    // If we reach here, versions are equal
+    return false;
+  }
+  
+  /**
+   * Formats version strings for display by removing leading 'v' if present
+   * @param {string} version - The version string (e.g. "v1.0.0" or "1.0.0")
+   * @returns {string} Formatted version (e.g. "1.0.0")
+   */
+  export function formatVersion(version) {
+    if (!version) return '';
+    return version.startsWith('v') ? version.substring(1) : version;
+  }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "d1s",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "type": "module",
   "description": "D1 Studio – a native studio for managing Cloudflare D1 databases.",
   "main": "./dist/d1-studio.js",
@@ -97,4 +97,4 @@
     "tailwindcss": "^3.4.1",
     "wrangler": "^3.107.3"
   }
-}
+}

package/scripts/studio.js

@@ -3,17 +3,20 @@ import chalk from 'chalk';
 import open from 'open';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
-import { createHash } from 'crypto';
-import { homedir } from 'os';
-import os from 'os';
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
 import express from 'express';
 import { createServer } from 'http';
 import next from 'next';
 import { findWranglerConfig } from '../src/lib/find-wrangler-config.js';
 import logger from '../src/lib/logger.js';
 import updateNotifier from 'update-notifier';
-import pkg from '../package.json' assert { type: 'json' };
+import pkg from '../package.json' with { type: 'json' };
+
+// Check for debug flag
+const hasDebugFlag = process.argv.includes('--debug');
+if (hasDebugFlag) {
+  process.env.DEBUG = 'true';
+  logger.debug('Debug mode enabled');
+}
 
 const notifier = updateNotifier({ pkg, updateCheckInterval: 1000 * 60 * 60 * 24 });
 notifier.notify({ defer: false });
@@ -22,133 +25,6 @@ notifier.notify({ defer: false });
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
-// License validation utilities
-class LicenseError extends Error {
-  constructor(message, type = 'VALIDATION_ERROR') {
-    super(message);
-    this.name = 'LicenseError';
-    this.type = type;
-  }
-}
-
-class StudioLicenseManager {
-  constructor() {
-    this.licenseDir = join(homedir(), '.d1studio');
-    this.licensePath = join(this.licenseDir, 'license.json');
-  }
-
-  async checkLicenseFile() {
-    try {
-      const licenseData = JSON.parse(await readFile(this.licensePath, 'utf8'));
-      
-      // If no end_date or subscription_status, the license is valid (perpetual license)
-      if (!licenseData.end_date || !licenseData.subscription_status) {
-        return {
-          licenseData,
-          isValid: true,
-          isExpiringSoon: false
-        };
-      }
-
-      // Check subscription status first
-      if (licenseData.subscription_status !== 'active') {
-        return {
-          licenseData,
-          isValid: false,
-          isExpiringSoon: false
-        };
-      }
-
-      const endDate = new Date(licenseData.end_date);
-      const now = new Date();
-      
-      // Check if subscription has ended
-      if (endDate < now) {
-        return {
-          licenseData,
-          isValid: false,
-          isExpiringSoon: false
-        };
-      }
-
-      // Check if subscription ends in next 7 days
-      const sevenDaysFromNow = new Date(now.getTime() + (7 * 24 * 60 * 60 * 1000));
-      const isExpiringSoon = endDate <= sevenDaysFromNow;
-
-      return {
-        licenseData,
-        isValid: true,
-        isExpiringSoon
-      };
-    } catch (error) {
-      return null;
-    }
-  }
-
-  async saveLicenseFile(licenseData) {
-    try {
-      await mkdir(this.licenseDir, { recursive: true });
-      await writeFile(this.licensePath, JSON.stringify(licenseData, null, 2));
-    } catch (error) {
-      throw new LicenseError('Failed to save license information', 'SAVE_ERROR');
-    }
-  }
-}
-
-async function generateHardwareId() {
-  const osInfo = {
-    platform: process.platform,
-    arch: process.arch,
-    cpus: os.cpus().length,
-    hostname: os.hostname(),
-    username: os.userInfo().username,
-  };
-  const systemString = JSON.stringify(osInfo);
-  return createHash('sha256').update(systemString).digest('hex');
-}
-
-async function validateLicenseRemote(licenseKey, email, hardwareId) {
-  const baseUrl =
-    process.env.NODE_ENV === 'development'
-      ? 'http://localhost:4000'
-      : 'https://d1-studio.com';
-
-  try {
-    const response = await fetch(`${baseUrl}/api/validate-license`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ licenseKey, email, hardwareId }),
-    });
-
-    const result = await response.json();
-
-    if (!response.ok) {
-      throw new LicenseError(
-        result.message || 'License validation failed',
-        result.type || 'VALIDATION_ERROR'
-      );
-    }
-
-    return result;
-  } catch (error) {
-    if (error instanceof LicenseError) {
-      throw error;
-    }
-    if (error.name === 'TypeError' && error.message.includes('fetch')) {
-      throw new LicenseError(
-        process.env.NODE_ENV === 'development'
-          ? 'Unable to connect to local license validation service. Make sure the development server is running.'
-          : 'Unable to connect to license server. Please check your internet connection.',
-        'CONNECTION_ERROR'
-      );
-    }
-    throw new LicenseError(
-      'An unexpected error occurred during license validation',
-      'VALIDATION_ERROR'
-    );
-  }
-}
-
 async function startServer(port, projectDirectory) {
   return new Promise((resolvePromise, rejectPromise) => {
     // Determine Next.js app directory based on environment:
@@ -203,69 +79,6 @@ async function startStudio() {
   try {
     logger.info(chalk.green('Initializing D1 Studio...'));
 
-    const licenseManager = new StudioLicenseManager();
-
-    // Check existing license
-    const licenseCheck = await licenseManager.checkLicenseFile();
-    
-    // Handle license validation
-    if (!licenseCheck || !licenseCheck.isValid) {
-      // Prompt for license information if needed
-      const { default: inquirer } = await import('inquirer');
-      logger.info(chalk.yellow('\nPlease enter your D1 Studio license information:'));
-
-      const answers = await inquirer.prompt([
-        {
-          type: 'input',
-          name: 'licenseKey',
-          message: 'License Key:',
-          validate: (input) => input.length > 0 || 'License key is required',
-        },
-        {
-          type: 'input',
-          name: 'email',
-          message: 'Email:',
-          validate: (input) => {
-            const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
-            return emailRegex.test(input) || 'Please enter a valid email';
-          },
-        },
-      ]);
-
-      const hardwareId = await generateHardwareId();
-
-      try {
-        const validation = await validateLicenseRemote(
-          answers.licenseKey,
-          answers.email,
-          hardwareId
-        );
-
-        // Save the complete validation response
-        await licenseManager.saveLicenseFile({
-          licenseKey: answers.licenseKey,
-          email: answers.email,
-          hardwareId,
-          end_date: validation.end_date,
-          subscription_status: validation.subscription_status,
-          type: validation.licenseType,
-          activatedAt: new Date().toISOString()
-        });
-
-        logger.info(chalk.green('\nLicense activated successfully!'));
-      } catch (error) {
-        logger.error(chalk.red(`\nLicense validation failed: ${error.message}`));
-        process.exit(1);
-      }
-    } else if (licenseCheck.isExpiringSoon) {
-      // Show warning about expiring subscription
-      const endDate = new Date(licenseCheck.licenseData.end_date).toLocaleDateString();
-      logger.warn(chalk.yellow(`
-        ⚠️  Your subscription will expire on ${endDate}
-        Please renew your subscription at https://d1-studio.com to continue using D1 Studio.
-      `));
-    }
-
     // Get wrangler configuration
     try {
       const userProjectPath = process.cwd();

package/src/app/api/pull-data/confirm/route.js

@@ -0,0 +1,230 @@
+// api/pull-data/confirm/route.js
+
+import { NextResponse } from 'next/server';
+import { executeSQL } from '@/lib/sql-operations';
+import { getCurrentSchema } from '@/lib/schema-operations';
+import { findWranglerConfig } from '@/lib/find-wrangler-config.js';
+import logger from '@/lib/logger';
+import {
+  fetchRemoteTableData as fetchRemoteDataHelper,
+  generateBulkInsertSQL
+} from '@/lib/sql-operations';
+
+/**
+ * Helper to extract the ordering column (primary key) from a table's schema.
+ * Assumes that each table’s schema object contains a "columns" array with an "isPrimaryKey" flag.
+ *
+ * @param {object} tableSchema - Schema info for the table.
+ * @returns {string} The column name to use for ordering.
+ */
+function getOrderingColumn(tableSchema) {
+  const pkColumn = tableSchema.columns?.find(col => col.isPrimaryKey);
+  const orderingCol = pkColumn ? pkColumn.name : 'rowid';
+  logger.debug(`Determined ordering column for table ${tableSchema.name || 'unknown'}: ${orderingCol}`);
+  return orderingCol;
+}
+
+/**
+ * Computes an insertion order for tables based on their foreign key dependencies.
+ * The returned array is sorted so that if table A is referenced by table B,
+ * then A appears earlier in the list.
+ *
+ * @param {Array<object>} tableSchemas - Array of table schema objects.
+ * @returns {Array<string>} Ordered array of table names.
+ */
+function computeInsertionOrder(tableSchemas) {
+  // Build a dependency graph: key = table name, value = set of tables that this table depends on.
+  const graph = {};
+  tableSchemas.forEach(table => {
+    // Initialize graph for this table.
+    graph[table.name] = new Set();
+    // Use a regex to find FOREIGN KEY references in the table's SQL.
+    // This regex assumes the foreign key clause looks like:
+    // FOREIGN KEY (...) REFERENCES referencedTable(...)
+    const fkRegex = /FOREIGN KEY\s*\([^)]*\)\s+REFERENCES\s+(\w+)/gi;
+    let match;
+    while ((match = fkRegex.exec(table.sql)) !== null) {
+      const referencedTable = match[1];
+      graph[table.name].add(referencedTable);
+    }
+  });
+  logger.debug('Computed dependency graph:', graph);
+  
+  // Topological sort (depth-first search) to compute insertion order.
+  const sorted = [];
+  const visited = {}; // 'temp' = visiting, 'perm' = done
+
+  function visit(tableName) {
+    if (visited[tableName] === 'temp') {
+      throw new Error(`Cycle detected in dependencies at table ${tableName}`);
+    }
+    if (!visited[tableName]) {
+      visited[tableName] = 'temp';
+      // Visit all tables that this table depends on.
+      graph[tableName].forEach(dep => {
+        // Only visit if the dependency is in our graph (i.e. among our tables)
+        if (graph[dep]) {
+          visit(dep);
+        }
+      });
+      visited[tableName] = 'perm';
+      sorted.push(tableName);
+    }
+  }
+
+  Object.keys(graph).forEach(tableName => {
+    if (!visited[tableName]) {
+      visit(tableName);
+    }
+  });
+
+  logger.debug('Computed insertion order (parent tables first):', sorted);
+  return sorted;
+}
+
+/**
+ * Applies the data pull by:
+ * 1. Fetching the local schema to determine table names and primary keys.
+ * 2. Computing the insertion order so that parent tables are inserted before child tables.
+ * 3. For each table (in insertion order), fetching the latest 100 records from the remote database.
+ * 4. Generating a bulk insert statement and executing it.
+ *
+ * This function aborts the entire process if any insert fails.
+ *
+ * @param {string} projectDir
+ * @param {string} accountId
+ * @param {string} apiToken
+ * @param {string} databaseName
+ * @returns {object} Details of data pulled.
+ */
+async function applyDataPull(projectDir, accountId, apiToken, databaseName) {
+  logger.debug('Starting data pull process...');
+  const env = {
+    ...process.env,
+    CLOUDFLARE_API_TOKEN: apiToken,
+    CLOUDFLARE_ACCOUNT_ID: accountId
+  };
+
+  logger.debug('Fetching current schema for data pull...');
+  const currentSchema = await getCurrentSchema(projectDir, databaseName, env, false);
+  // Filter for table objects that have a name.
+  const tableSchemas = currentSchema.filter(obj => obj.type === 'table' && obj.name);
+  logger.debug('Local table schema for data pull:', tableSchemas.map(t => t.name));
+
+  // Compute insertion order based on foreign key dependencies.
+  const insertionOrder = computeInsertionOrder(tableSchemas);
+  logger.debug('Final computed insertion order:', insertionOrder);
+
+  // Process each table in the computed insertion order.
+  for (const tableName of insertionOrder) {
+    const table = tableSchemas.find(t => t.name === tableName);
+    if (!table) {
+      logger.debug(`Skipping table ${tableName} as it is not in the current schema list.`);
+      continue;
+    }
+    logger.debug(`Processing table: ${tableName}`);
+    const orderingColumn = getOrderingColumn(table);
+    logger.debug(`Fetching data for table: ${tableName} ordered by ${orderingColumn}`);
+
+    // Fetch remote data.
+    const rows = await fetchRemoteDataHelper(
+      projectDir,
+      databaseName,
+      tableName,
+      env,
+      75,
+      orderingColumn
+    );
+    logger.debug(`Fetched ${rows.length} records for table: ${tableName}`);
+
+    if (rows.length > 0) {
+      const insertSQL = generateBulkInsertSQL(tableName, rows);
+      logger.debug(`Generated bulk insert SQL for ${tableName}: ${insertSQL}`);
+
+      try {
+        // Execute the insert SQL directly.
+        await executeSQL(projectDir, databaseName, insertSQL, env, false);
+        logger.debug(`Bulk inserted ${rows.length} records into table: ${tableName}`);
+      } catch (insertError) {
+        logger.error(`Error inserting data into table ${tableName}:`, insertError);
+        // Abort the entire process to preserve data integrity.
+        throw new Error(`Data pull failed for table ${tableName}: ${insertError.message}`);
+      }
+    } else {
+      logger.debug(`No data fetched for table: ${tableName}`);
+    }
+  }
+  logger.debug('Data pull process completed successfully.');
+  return { message: 'All data pulled successfully.' };
+}
+
+export async function POST(req) {
+  try {
+    logger.debug('Received request for data pull.');
+    const { accountId, apiToken } = await req.json();
+    const projectDir = process.env.D1_PROJECT_DIR;
+    if (!projectDir) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: {
+            type: 'VALIDATION_ERROR',
+            message: 'Project directory not configured'
+          }
+        },
+        { status: 500 }
+      );
+    }
+    const wranglerResult = await findWranglerConfig(projectDir);
+    if (!wranglerResult || !wranglerResult.config) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: {
+            type: 'VALIDATION_ERROR',
+            message: 'Could not find Wrangler configuration'
+          }
+        },
+        { status: 400 }
+      );
+    }
+    const config = wranglerResult.config;
+    if (!config.d1_databases?.length) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: {
+            type: 'VALIDATION_ERROR',
+            message: 'No D1 databases configured'
+          }
+        },
+        { status: 400 }
+      );
+    }
+    const databaseName = config.d1_databases[0].database_name;
+    logger.debug(`Initiating data pull for database: ${databaseName}`);
+    const result = await applyDataPull(projectDir, accountId, apiToken, databaseName);
+    logger.debug('Data pull result:', result);
+    return NextResponse.json({
+      success: true,
+      message: result.message,
+      details: result
+    });
+  } catch (error) {
+    logger.error('Pull data confirm error:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: {
+          type: error.type || 'EXECUTION_ERROR',
+          message: error.message || 'Failed to apply data pull'
+        }
+      },
+      { status: 500 }
+    );
+  }
+}
+
+export async function OPTIONS() {
+  return new NextResponse(null, { status: 204 });
+}