crushdataai 1.2.6 → 1.2.8

package/dist/connectors/cloud/index.d.ts

@@ -1,16 +1,21 @@
-import { Connector, Table, TableData } from '../index';
+import { Connector, Table, TableData, ColumnInfo } from '../index';
 import { Connection } from '../../connections';
 export declare class BigQueryConnector implements Connector {
     type: string;
+    private createClient;
     test(connection: Connection): Promise<boolean>;
     getTables(connection: Connection): Promise<Table[]>;
-    getData(connection: Connection, table: string, page: number, limit: number): Promise<TableData>;
+    getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }
 export declare class SnowflakeConnector implements Connector {
     type: string;
+    private createConnection;
+    private executeQuery;
     test(connection: Connection): Promise<boolean>;
     getTables(connection: Connection): Promise<Table[]>;
-    getData(connection: Connection, table: string, page: number, limit: number): Promise<TableData>;
+    getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }

package/dist/connectors/cloud/index.js

@@ -1,21 +1,172 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SnowflakeConnector = exports.BigQueryConnector = void 0;
+const bigquery_1 = require("@google-cloud/bigquery");
+const snowflake = __importStar(require("snowflake-sdk"));
 class BigQueryConnector {
     constructor() {
         this.type = 'bigquery';
     }
+    createClient(connection) {
+        return new bigquery_1.BigQuery({
+            projectId: connection.projectId,
+            keyFilename: connection.keyFile
+        });
+    }
     async test(connection) {
-        return true;
+        console.log(`[BigQuery] Testing connection for ${connection.name} (Project: ${connection.projectId})`);
+        if (!connection.projectId || !connection.keyFile) {
+            throw new Error('Project ID and Key File path are required');
+        }
+        // Check if key file exists
+        const fs = await Promise.resolve().then(() => __importStar(require('fs')));
+        if (!fs.existsSync(connection.keyFile)) {
+            throw new Error(`Key file not found: ${connection.keyFile}`);
+        }
+        try {
+            const bigquery = this.createClient(connection);
+            await bigquery.getDatasets({ maxResults: 1 });
+            console.log(`[BigQuery] Connection test successful for ${connection.name}`);
+            return true;
+        }
+        catch (error) {
+            console.error(`[BigQuery] Connection test failed:`, error.message);
+            throw new Error(`BigQuery connection failed: ${error.message}`);
+        }
     }
     async getTables(connection) {
-        return [];
+        console.log(`[BigQuery] getTables called for ${connection.name}`);
+        try {
+            const bigquery = this.createClient(connection);
+            const [datasets] = await bigquery.getDatasets();
+            const tables = [];
+            for (const dataset of datasets) {
+                const [datasetTables] = await dataset.getTables();
+                for (const table of datasetTables) {
+                    const [metadata] = await table.getMetadata();
+                    tables.push({
+                        name: `${dataset.id}.${table.id}`,
+                        type: metadata.type || 'TABLE',
+                        rowCount: parseInt(metadata.numRows || '0', 10)
+                    });
+                }
+            }
+            return tables;
+        }
+        catch (error) {
+            console.error(`[BigQuery] getTables failed:`, error.message);
+            throw new Error(`Failed to fetch tables: ${error.message}`);
+        }
+    }
+    async getData(connection, tableName, page, limit) {
+        console.log(`[BigQuery] getData called for ${connection.name}, table: ${tableName}, page: ${page}`);
+        try {
+            const bigquery = this.createClient(connection);
+            const offset = (page - 1) * limit;
+            const countQuery = `SELECT COUNT(*) as total FROM \`${tableName}\``;
+            const [countJob] = await bigquery.createQueryJob({ query: countQuery });
+            const [countRows] = await countJob.getQueryResults();
+            const totalRows = parseInt(countRows[0]?.total || '0', 10);
+            const dataQuery = `SELECT * FROM \`${tableName}\` LIMIT ${limit} OFFSET ${offset}`;
+            const [dataJob] = await bigquery.createQueryJob({ query: dataQuery });
+            const [rows] = await dataJob.getQueryResults();
+            const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
+            const totalPages = Math.ceil(totalRows / limit) || 1;
+            return {
+                columns,
+                rows: rows,
+                pagination: {
+                    page,
+                    limit,
+                    totalRows,
+                    totalPages,
+                    startIdx: offset + 1,
+                    endIdx: offset + rows.length
+                }
+            };
+        }
+        catch (error) {
+            console.error(`[BigQuery] getData failed:`, error.message);
+            throw new Error(`Failed to fetch data: ${error.message}`);
+        }
     }
-    async getData(connection, table, page, limit) {
-        return { columns: [], rows: [], pagination: { page, limit, totalRows: 0, totalPages: 0, startIdx: 0, endIdx: 0 } };
+    async getSchema(connection, tableName) {
+        console.log(`[BigQuery] getSchema called for ${connection.name}, table: ${tableName}`);
+        try {
+            const bigquery = this.createClient(connection);
+            const [dataset, table] = tableName.split('.');
+            const tableRef = bigquery.dataset(dataset).table(table);
+            const [metadata] = await tableRef.getMetadata();
+            return (metadata.schema?.fields || []).map((field) => ({
+                name: field.name,
+                type: field.type,
+                nullable: field.mode !== 'REQUIRED'
+            }));
+        }
+        catch (error) {
+            console.error(`[BigQuery] getSchema failed:`, error.message);
+            throw new Error(`Failed to fetch schema: ${error.message}`);
+        }
     }
     getSnippet(connection, lang) {
-        return `# BigQuery snippet generation not implemented yet`;
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
+        if (lang === 'python') {
+            return `import os
+from google.cloud import bigquery
+import pandas as pd
+
+# Connection: ${connection.name}
+# Type: bigquery
+# Credentials loaded from environment variables (set in .env file)
+client = bigquery.Client.from_service_account_json(os.environ["${prefix}_KEY_FILE"])
+
+# Example: Query a table
+project_id = os.environ["${prefix}_PROJECT_ID"]
+query = f"""
+    SELECT * 
+    FROM \`{project_id}.your_dataset.your_table\`
+    LIMIT 100
+"""
+
+df = client.query(query).to_dataframe()
+print(f"Successfully loaded {len(df)} rows from ${connection.name}")
+print(df.head())
+`;
+        }
+        return `# Language ${lang} not supported for BigQuery connector yet.`;
     }
 }
 exports.BigQueryConnector = BigQueryConnector;
@@ -23,17 +174,177 @@ class SnowflakeConnector {
     constructor() {
         this.type = 'snowflake';
     }
+    async createConnection(connection) {
+        return new Promise((resolve, reject) => {
+            const conn = snowflake.createConnection({
+                account: connection.account || '',
+                username: connection.user || '',
+                password: connection.password || '',
+                warehouse: connection.warehouse,
+                database: connection.database
+            });
+            conn.connect((err, conn) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(conn);
+                }
+            });
+        });
+    }
+    executeQuery(conn, query) {
+        return new Promise((resolve, reject) => {
+            conn.execute({
+                sqlText: query,
+                complete: (err, stmt, rows) => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve(rows || []);
+                    }
+                }
+            });
+        });
+    }
     async test(connection) {
-        return true;
+        console.log(`[Snowflake] Testing connection for ${connection.name} (Account: ${connection.account})`);
+        // Validate required fields
+        const account = connection.account?.trim();
+        const user = connection.user?.trim();
+        const password = connection.password?.trim();
+        if (!account) {
+            throw new Error('Snowflake account is required');
+        }
+        if (!user) {
+            throw new Error('Snowflake username is required');
+        }
+        if (!password) {
+            throw new Error('Snowflake password is required');
+        }
+        let conn = null;
+        try {
+            conn = await this.createConnection(connection);
+            await this.executeQuery(conn, 'SELECT CURRENT_VERSION()');
+            console.log(`[Snowflake] Connection test successful for ${connection.name}`);
+            return true;
+        }
+        catch (error) {
+            console.error(`[Snowflake] Connection test failed:`, error.message);
+            throw new Error(`Snowflake connection failed: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                conn.destroy(() => { });
+        }
     }
     async getTables(connection) {
-        return [];
+        console.log(`[Snowflake] getTables called for ${connection.name}`);
+        let conn = null;
+        try {
+            conn = await this.createConnection(connection);
+            const rows = await this.executeQuery(conn, 'SHOW TABLES');
+            return rows.map((row) => ({
+                name: row.name || row.TABLE_NAME,
+                type: 'table',
+                rowCount: row.rows || null
+            }));
+        }
+        catch (error) {
+            console.error(`[Snowflake] getTables failed:`, error.message);
+            throw new Error(`Failed to fetch tables: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                conn.destroy(() => { });
+        }
+    }
+    async getData(connection, tableName, page, limit) {
+        console.log(`[Snowflake] getData called for ${connection.name}, table: ${tableName}, page: ${page}`);
+        let conn = null;
+        try {
+            conn = await this.createConnection(connection);
+            const offset = (page - 1) * limit;
+            const countRows = await this.executeQuery(conn, `SELECT COUNT(*) as TOTAL FROM "${tableName}"`);
+            const totalRows = countRows[0]?.TOTAL || 0;
+            const rows = await this.executeQuery(conn, `SELECT * FROM "${tableName}" LIMIT ${limit} OFFSET ${offset}`);
+            const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
+            const totalPages = Math.ceil(totalRows / limit) || 1;
+            return {
+                columns,
+                rows,
+                pagination: {
+                    page,
+                    limit,
+                    totalRows,
+                    totalPages,
+                    startIdx: offset + 1,
+                    endIdx: offset + rows.length
+                }
+            };
+        }
+        catch (error) {
+            console.error(`[Snowflake] getData failed:`, error.message);
+            throw new Error(`Failed to fetch data: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                conn.destroy(() => { });
+        }
     }
-    async getData(connection, table, page, limit) {
-        return { columns: [], rows: [], pagination: { page, limit, totalRows: 0, totalPages: 0, startIdx: 0, endIdx: 0 } };
+    async getSchema(connection, tableName) {
+        console.log(`[Snowflake] getSchema called for ${connection.name}, table: ${tableName}`);
+        let conn = null;
+        try {
+            conn = await this.createConnection(connection);
+            const rows = await this.executeQuery(conn, `DESCRIBE TABLE "${tableName}"`);
+            return rows.map((row) => ({
+                name: row.name || row.COLUMN_NAME,
+                type: row.type || row.DATA_TYPE,
+                nullable: (row.null || row.IS_NULLABLE) === 'Y'
+            }));
+        }
+        catch (error) {
+            console.error(`[Snowflake] getSchema failed:`, error.message);
+            throw new Error(`Failed to fetch schema: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                conn.destroy(() => { });
+        }
     }
     getSnippet(connection, lang) {
-        return `# Snowflake snippet generation not implemented yet`;
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
+        if (lang === 'python') {
+            return `import os
+import snowflake.connector
+import pandas as pd
+
+# Connection: ${connection.name}
+# Type: snowflake
+# Credentials loaded from environment variables (set in .env file)
+conn = snowflake.connector.connect(
+    account=os.environ["${prefix}_ACCOUNT"],
+    user=os.environ["${prefix}_USER"],
+    password=os.environ["${prefix}_PASSWORD"],
+    warehouse=os.environ.get("${prefix}_WAREHOUSE", "COMPUTE_WH"),
+    database=os.environ.get("${prefix}_DATABASE", "")
+)
+
+try:
+    # Example: Query a table
+    query = "SELECT * FROM your_table LIMIT 100"
+    cursor = conn.cursor()
+    cursor.execute(query)
+    df = cursor.fetch_pandas_all()
+    print(f"Successfully loaded {len(df)} rows from ${connection.name}")
+    print(df.head())
+finally:
+    conn.close()
+`;
+        }
+        return `# Language ${lang} not supported for Snowflake connector yet.`;
     }
 }
 exports.SnowflakeConnector = SnowflakeConnector;

package/dist/connectors/csv/index.d.ts

@@ -5,5 +5,6 @@ export declare class CSVConnector implements Connector {
     test(connection: Connection): Promise<boolean>;
     getTables(connection: Connection): Promise<Table[]>;
     getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }

package/dist/connectors/csv/index.js

@@ -107,19 +107,39 @@ class CSVConnector {
             }
         };
     }
-    getSnippet(connection, lang) {
-        let filePath = connection.filePath || '';
-        if (filePath.startsWith('"') && filePath.endsWith('"')) {
+    async getSchema(connection, tableName) {
+        let filePath = connection.filePath;
+        if (filePath && filePath.startsWith('"') && filePath.endsWith('"')) {
             filePath = filePath.slice(1, -1);
         }
-        // Escape backslashes for string literals
-        const escapedPath = filePath.replace(/\\/g, '\\\\');
+        if (!filePath || !fs.existsSync(filePath)) {
+            throw new Error('CSV file not found');
+        }
+        // Read first row to get headers and infer types from first data row
+        const fileContent = fs.readFileSync(filePath, 'utf8');
+        const result = papaparse_1.default.parse(fileContent, {
+            header: true,
+            skipEmptyLines: true,
+            preview: 2 // Only parse header + 1 data row for type inference
+        });
+        const columns = result.meta.fields || [];
+        const firstRow = result.data[0] || {};
+        return columns.map(col => ({
+            name: col,
+            type: typeof firstRow[col] === 'number' ? 'number' : 'string',
+            nullable: true
+        }));
+    }
+    getSnippet(connection, lang) {
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
         if (lang === 'python') {
-            return `import pandas as pd
+            return `import os
+import pandas as pd
 
 # Connection: ${connection.name}
 # Type: csv
-file_path = "${escapedPath}"
+# File path loaded from environment variable (set in .env file)
+file_path = os.environ["${prefix}_FILE_PATH"]
 
 try:
     df = pd.read_csv(file_path)

package/dist/connectors/custom/index.d.ts

@@ -0,0 +1,10 @@
+import { Connector, Table, TableData, ColumnInfo } from '../index';
+import { Connection } from '../../connections';
+export declare class CustomConnector implements Connector {
+    type: string;
+    test(connection: Connection): Promise<boolean>;
+    getTables(connection: Connection): Promise<Table[]>;
+    getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
+    getSnippet(connection: Connection, lang: string): string;
+}

package/dist/connectors/custom/index.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CustomConnector = void 0;
+class CustomConnector {
+    constructor() {
+        this.type = 'custom';
+    }
+    async test(connection) {
+        console.log(`[Custom] Testing connection for ${connection.name}`);
+        if (!connection.connectionString?.trim()) {
+            throw new Error('Connection string is required');
+        }
+        // For custom connections, we just validate the string exists
+        // We cannot actually test without knowing the database type
+        console.log(`[Custom] Connection string provided (${connection.connectionString.length} chars)`);
+        console.log(`[Custom] Note: Custom connections cannot be fully tested - only validating string is present`);
+        return true;
+    }
+    async getTables(connection) {
+        console.log(`[Custom] getTables called for ${connection.name}`);
+        // Custom connections don't support table listing through our UI
+        // Return empty - user will use their own queries
+        return [];
+    }
+    async getData(connection, tableName, page, limit) {
+        console.log(`[Custom] getData called for ${connection.name}`);
+        // Custom connections don't support data fetching through our UI
+        throw new Error('Custom connections do not support data preview. Use the snippet to query directly.');
+    }
+    async getSchema(connection, tableName) {
+        console.log(`[Custom] getSchema called for ${connection.name}`);
+        // Custom connections don't support schema discovery
+        return [];
+    }
+    getSnippet(connection, lang) {
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
+        if (lang === 'python') {
+            return `import os
+from sqlalchemy import create_engine
+import pandas as pd
+
+# Connection: ${connection.name}
+# Type: custom
+# Connection string loaded from environment variable (set in .env file)
+connection_string = os.environ["${prefix}_CONNECTION_STRING"]
+engine = create_engine(connection_string)
+
+try:
+    # Example: Query a table
+    query = "SELECT * FROM your_table LIMIT 100"
+    df = pd.read_sql(query, engine)
+    print(f"Successfully loaded {len(df)} rows from ${connection.name}")
+    print(df.head())
+finally:
+    engine.dispose()
+`;
+        }
+        return `# Language ${lang} not supported for Custom connector yet.`;
+    }
+}
+exports.CustomConnector = CustomConnector;

package/dist/connectors/index.d.ts

@@ -4,6 +4,11 @@ export interface Table {
     type?: string;
     rowCount?: number | null;
 }
+export interface ColumnInfo {
+    name: string;
+    type: string;
+    nullable: boolean;
+}
 export interface TableData {
     columns: string[];
     rows: any[];
@@ -21,6 +26,7 @@ export interface Connector {
     test(connection: Connection): Promise<boolean>;
     getTables(connection: Connection): Promise<Table[]>;
     getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }
 export declare class ConnectorRegistry {

package/dist/connectors/mysql/index.d.ts

@@ -5,5 +5,6 @@ export declare class MySQLConnector implements Connector {
     test(connection: Connection): Promise<boolean>;
     getTables(connection: Connection): Promise<Table[]>;
     getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }