crushdataai 1.2.7 → 1.2.8

package/dist/connectors/mysql/index.js

@@ -1,29 +1,182 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MySQLConnector = void 0;
+const promise_1 = __importDefault(require("mysql2/promise"));
 class MySQLConnector {
     constructor() {
         this.type = 'mysql';
     }
     async test(connection) {
-        // Stub implementation
+        console.log(`[MySQL] Testing connection for ${connection.name} (Host: ${connection.host})`);
         if (!connection.host || !connection.user || !connection.database) {
             throw new Error('Host, user, and database are required');
         }
-        return true;
+        let conn = null;
+        try {
+            conn = await promise_1.default.createConnection({
+                host: connection.host,
+                port: connection.port || 3306,
+                user: connection.user,
+                password: connection.password || '',
+                database: connection.database,
+                connectTimeout: 10000
+            });
+            await conn.execute('SELECT 1');
+            console.log(`[MySQL] Connection test successful for ${connection.name}`);
+            return true;
+        }
+        catch (error) {
+            console.error(`[MySQL] Connection test failed:`, error.message);
+            throw new Error(`MySQL connection failed: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                await conn.end();
+        }
     }
     async getTables(connection) {
-        return [];
+        console.log(`[MySQL] getTables called for ${connection.name}`);
+        let conn = null;
+        try {
+            conn = await promise_1.default.createConnection({
+                host: connection.host,
+                port: connection.port || 3306,
+                user: connection.user,
+                password: connection.password || '',
+                database: connection.database
+            });
+            const [rows] = await conn.execute(`
+                SELECT 
+                    TABLE_NAME as name,
+                    TABLE_ROWS as rowCount
+                FROM information_schema.TABLES 
+                WHERE TABLE_SCHEMA = ?
+                ORDER BY TABLE_NAME
+            `, [connection.database]);
+            return rows.map(row => ({
+                name: row.name,
+                type: 'table',
+                rowCount: row.rowCount
+            }));
+        }
+        catch (error) {
+            console.error(`[MySQL] getTables failed:`, error.message);
+            throw new Error(`Failed to fetch tables: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                await conn.end();
+        }
     }
     async getData(connection, tableName, page, limit) {
-        return {
-            columns: [],
-            rows: [],
-            pagination: { page, limit, totalRows: 0, totalPages: 0, startIdx: 0, endIdx: 0 }
-        };
+        console.log(`[MySQL] getData called for ${connection.name}, table: ${tableName}, page: ${page}`);
+        let conn = null;
+        try {
+            conn = await promise_1.default.createConnection({
+                host: connection.host,
+                port: connection.port || 3306,
+                user: connection.user,
+                password: connection.password || '',
+                database: connection.database
+            });
+            // Get total row count
+            const [countResult] = await conn.execute(`SELECT COUNT(*) as total FROM \`${tableName}\``);
+            const totalRows = countResult[0]?.total || 0;
+            // Get paginated data
+            const offset = (page - 1) * limit;
+            const [rows] = await conn.execute(`SELECT * FROM \`${tableName}\` LIMIT ? OFFSET ?`, [limit, offset]);
+            const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
+            const totalPages = Math.ceil(totalRows / limit) || 1;
+            return {
+                columns,
+                rows: rows,
+                pagination: {
+                    page,
+                    limit,
+                    totalRows,
+                    totalPages,
+                    startIdx: offset + 1,
+                    endIdx: offset + rows.length
+                }
+            };
+        }
+        catch (error) {
+            console.error(`[MySQL] getData failed:`, error.message);
+            throw new Error(`Failed to fetch data: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                await conn.end();
+        }
+    }
+    async getSchema(connection, tableName) {
+        console.log(`[MySQL] getSchema called for ${connection.name}, table: ${tableName}`);
+        let conn = null;
+        try {
+            conn = await promise_1.default.createConnection({
+                host: connection.host,
+                port: connection.port || 3306,
+                user: connection.user,
+                password: connection.password || '',
+                database: connection.database
+            });
+            const [rows] = await conn.execute(`
+                SELECT 
+                    COLUMN_NAME as name,
+                    DATA_TYPE as type,
+                    IS_NULLABLE as nullable
+                FROM information_schema.COLUMNS 
+                WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
+                ORDER BY ORDINAL_POSITION
+            `, [connection.database, tableName]);
+            return rows.map(row => ({
+                name: row.name,
+                type: row.type,
+                nullable: row.nullable === 'YES'
+            }));
+        }
+        catch (error) {
+            console.error(`[MySQL] getSchema failed:`, error.message);
+            throw new Error(`Failed to fetch schema: ${error.message}`);
+        }
+        finally {
+            if (conn)
+                await conn.end();
+        }
     }
     getSnippet(connection, lang) {
-        return `# MySQL snippet generation not implemented yet`;
+        // Generate env var names
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
+        if (lang === 'python') {
+            return `import os
+import pymysql
+import pandas as pd
+
+# Connection: ${connection.name}
+# Type: mysql
+# Credentials loaded from environment variables (set in .env file)
+connection = pymysql.connect(
+    host=os.environ["${prefix}_HOST"],
+    port=int(os.environ.get("${prefix}_PORT", "3306")),
+    user=os.environ["${prefix}_USER"],
+    password=os.environ["${prefix}_PASSWORD"],
+    database=os.environ["${prefix}_DATABASE"]
+)
+
+try:
+    # Example: Fetch data from a table
+    query = "SELECT * FROM your_table LIMIT 100"
+    df = pd.read_sql(query, connection)
+    print(f"Successfully loaded {len(df)} rows from ${connection.name}")
+    print(df.head())
+finally:
+    connection.close()
+`;
+        }
+        return `# Language ${lang} not supported for MySQL connector yet.`;
     }
 }
 exports.MySQLConnector = MySQLConnector;

package/dist/connectors/postgresql/index.d.ts

@@ -2,8 +2,10 @@ import { Connector, Table, TableData } from '../index';
 import { Connection } from '../../connections';
 export declare class PostgreSQLConnector implements Connector {
     type: string;
+    private createClient;
     test(connection: Connection): Promise<boolean>;
     getTables(connection: Connection): Promise<Table[]>;
     getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }

package/dist/connectors/postgresql/index.js

@@ -1,28 +1,160 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgreSQLConnector = void 0;
+const pg_1 = require("pg");
 class PostgreSQLConnector {
     constructor() {
         this.type = 'postgresql';
     }
+    createClient(connection) {
+        return new pg_1.Client({
+            host: connection.host,
+            port: connection.port || 5432,
+            user: connection.user,
+            password: connection.password || '',
+            database: connection.database,
+            connectionTimeoutMillis: 10000
+        });
+    }
     async test(connection) {
+        console.log(`[PostgreSQL] Testing connection for ${connection.name} (Host: ${connection.host})`);
         if (!connection.host || !connection.user || !connection.database) {
             throw new Error('Host, user, and database are required');
         }
-        return true;
+        const client = this.createClient(connection);
+        try {
+            await client.connect();
+            await client.query('SELECT NOW()');
+            console.log(`[PostgreSQL] Connection test successful for ${connection.name}`);
+            return true;
+        }
+        catch (error) {
+            console.error(`[PostgreSQL] Connection test failed:`, error.message);
+            throw new Error(`PostgreSQL connection failed: ${error.message}`);
+        }
+        finally {
+            await client.end();
+        }
     }
     async getTables(connection) {
-        return [];
+        console.log(`[PostgreSQL] getTables called for ${connection.name}`);
+        const client = this.createClient(connection);
+        try {
+            await client.connect();
+            const result = await client.query(`
+                SELECT 
+                    t.table_name as name,
+                    (SELECT reltuples::bigint FROM pg_class WHERE relname = t.table_name) as row_count
+                FROM information_schema.tables t
+                WHERE t.table_schema = 'public'
+                AND t.table_type = 'BASE TABLE'
+                ORDER BY t.table_name
+            `);
+            return result.rows.map(row => ({
+                name: row.name,
+                type: 'table',
+                rowCount: row.row_count
+            }));
+        }
+        catch (error) {
+            console.error(`[PostgreSQL] getTables failed:`, error.message);
+            throw new Error(`Failed to fetch tables: ${error.message}`);
+        }
+        finally {
+            await client.end();
+        }
     }
     async getData(connection, tableName, page, limit) {
-        return {
-            columns: [],
-            rows: [],
-            pagination: { page, limit, totalRows: 0, totalPages: 0, startIdx: 0, endIdx: 0 }
-        };
+        console.log(`[PostgreSQL] getData called for ${connection.name}, table: ${tableName}, page: ${page}`);
+        const client = this.createClient(connection);
+        try {
+            await client.connect();
+            // Get total row count
+            const countResult = await client.query(`SELECT COUNT(*) as total FROM "${tableName}"`);
+            const totalRows = parseInt(countResult.rows[0]?.total || '0', 10);
+            // Get paginated data
+            const offset = (page - 1) * limit;
+            const dataResult = await client.query(`SELECT * FROM "${tableName}" LIMIT $1 OFFSET $2`, [limit, offset]);
+            const columns = dataResult.fields.map(f => f.name);
+            const totalPages = Math.ceil(totalRows / limit) || 1;
+            return {
+                columns,
+                rows: dataResult.rows,
+                pagination: {
+                    page,
+                    limit,
+                    totalRows,
+                    totalPages,
+                    startIdx: offset + 1,
+                    endIdx: offset + dataResult.rows.length
+                }
+            };
+        }
+        catch (error) {
+            console.error(`[PostgreSQL] getData failed:`, error.message);
+            throw new Error(`Failed to fetch data: ${error.message}`);
+        }
+        finally {
+            await client.end();
+        }
+    }
+    async getSchema(connection, tableName) {
+        console.log(`[PostgreSQL] getSchema called for ${connection.name}, table: ${tableName}`);
+        const client = this.createClient(connection);
+        try {
+            await client.connect();
+            const result = await client.query(`
+                SELECT 
+                    column_name as name,
+                    data_type as type,
+                    is_nullable as nullable
+                FROM information_schema.columns 
+                WHERE table_schema = 'public' AND table_name = $1
+                ORDER BY ordinal_position
+            `, [tableName]);
+            return result.rows.map(row => ({
+                name: row.name,
+                type: row.type,
+                nullable: row.nullable === 'YES'
+            }));
+        }
+        catch (error) {
+            console.error(`[PostgreSQL] getSchema failed:`, error.message);
+            throw new Error(`Failed to fetch schema: ${error.message}`);
+        }
+        finally {
+            await client.end();
+        }
     }
     getSnippet(connection, lang) {
-        return `# PostgreSQL snippet generation not implemented yet`;
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
+        if (lang === 'python') {
+            return `import os
+import psycopg2
+import pandas as pd
+
+# Connection: ${connection.name}
+# Type: postgresql
+# Credentials loaded from environment variables (set in .env file)
+connection = psycopg2.connect(
+    host=os.environ["${prefix}_HOST"],
+    port=int(os.environ.get("${prefix}_PORT", "5432")),
+    user=os.environ["${prefix}_USER"],
+    password=os.environ["${prefix}_PASSWORD"],
+    dbname=os.environ["${prefix}_DATABASE"]
+)
+
+try:
+    # Example: Fetch data from a table
+    query = "SELECT * FROM your_table LIMIT 100"
+    df = pd.read_sql(query, connection)
+    print(f"Successfully loaded {len(df)} rows from ${connection.name}")
+    print(df.head())
+finally:
+    connection.close()
+`;
+        }
+        return `# Language ${lang} not supported for PostgreSQL connector yet.`;
     }
 }
 exports.PostgreSQLConnector = PostgreSQLConnector;

package/dist/connectors/shopify/index.d.ts

@@ -6,5 +6,6 @@ export declare class ShopifyConnector implements Connector {
     getTables(connection: Connection): Promise<Table[]>;
     private fetchTotalCount;
     getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
     getSnippet(connection: Connection, lang: string): string;
 }

package/dist/connectors/shopify/index.js

@@ -133,17 +133,56 @@ class ShopifyConnector {
             throw new Error(`Failed to fetch data: ${error.message}`);
         }
     }
+    async getSchema(connection, tableName) {
+        // Shopify schemas are predefined - return common fields for each table type
+        const schemas = {
+            orders: [
+                { name: 'id', type: 'integer', nullable: false },
+                { name: 'email', type: 'string', nullable: true },
+                { name: 'created_at', type: 'datetime', nullable: false },
+                { name: 'total_price', type: 'decimal', nullable: false },
+                { name: 'currency', type: 'string', nullable: false },
+                { name: 'financial_status', type: 'string', nullable: true },
+                { name: 'fulfillment_status', type: 'string', nullable: true },
+                { name: 'customer', type: 'json', nullable: true },
+                { name: 'line_items', type: 'json', nullable: false }
+            ],
+            products: [
+                { name: 'id', type: 'integer', nullable: false },
+                { name: 'title', type: 'string', nullable: false },
+                { name: 'body_html', type: 'string', nullable: true },
+                { name: 'vendor', type: 'string', nullable: true },
+                { name: 'product_type', type: 'string', nullable: true },
+                { name: 'created_at', type: 'datetime', nullable: false },
+                { name: 'handle', type: 'string', nullable: false },
+                { name: 'status', type: 'string', nullable: false },
+                { name: 'variants', type: 'json', nullable: false }
+            ],
+            customers: [
+                { name: 'id', type: 'integer', nullable: false },
+                { name: 'email', type: 'string', nullable: true },
+                { name: 'first_name', type: 'string', nullable: true },
+                { name: 'last_name', type: 'string', nullable: true },
+                { name: 'created_at', type: 'datetime', nullable: false },
+                { name: 'orders_count', type: 'integer', nullable: false },
+                { name: 'total_spent', type: 'decimal', nullable: false },
+                { name: 'addresses', type: 'json', nullable: true }
+            ]
+        };
+        return schemas[tableName] || [];
+    }
     getSnippet(connection, lang) {
-        const storeUrl = connection.store?.replace(/\/$/, '') || 'your-shop.myshopify.com';
-        const apiKey = connection.apiKey || 'shpat_...';
+        const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
         if (lang === 'python') {
-            return `import requests
+            return `import os
+import requests
 import pandas as pd
 
 # Connection: ${connection.name}
 # Type: shopify
-shop_url = "${storeUrl}"
-access_token = "${apiKey}"
+# Credentials loaded from environment variables (set in .env file)
+shop_url = os.environ["${prefix}_STORE"]
+access_token = os.environ["${prefix}_API_KEY"]
 api_version = "2024-04"
 
 def fetch_shopify_data(endpoint):

package/dist/index.js

@@ -118,4 +118,11 @@ program
     const { snippet } = await Promise.resolve().then(() => __importStar(require('./commands/snippet')));
     await snippet(name, options.lang);
 });
+program
+    .command('schema <connection> [table]')
+    .description('Show table schema (columns) for a connection')
+    .action(async (connection, table) => {
+    const { schema } = await Promise.resolve().then(() => __importStar(require('./commands/schema')));
+    await schema(connection, table);
+});
 program.parse();

package/dist/routes/connections.js

@@ -45,7 +45,9 @@ router.post('/', (req, res) => {
             res.status(400).json({ success: false, error: 'Name and type are required' });
             return;
         }
+        // Save to encrypted config AND export to .env
         (0, connections_1.saveConnection)(connection);
+        (0, connections_1.writeToEnvFile)(connection);
         res.json({ success: true, message: 'Connection saved' });
     }
     catch (error) {

package/dist/server.js

@@ -45,6 +45,8 @@ const mysql_1 = require("./connectors/mysql");
 const postgresql_1 = require("./connectors/postgresql");
 const shopify_1 = require("./connectors/shopify");
 const cloud_1 = require("./connectors/cloud");
+const custom_1 = require("./connectors/custom");
+const additional_1 = require("./connectors/additional");
 const connections_1 = __importDefault(require("./routes/connections"));
 const app = (0, express_1.default)();
 app.use(express_1.default.json());
@@ -60,6 +62,12 @@ connectors_1.ConnectorRegistry.register(new postgresql_1.PostgreSQLConnector());
 connectors_1.ConnectorRegistry.register(new shopify_1.ShopifyConnector());
 connectors_1.ConnectorRegistry.register(new cloud_1.BigQueryConnector());
 connectors_1.ConnectorRegistry.register(new cloud_1.SnowflakeConnector());
+connectors_1.ConnectorRegistry.register(new custom_1.CustomConnector());
+connectors_1.ConnectorRegistry.register(new additional_1.SQLServerConnector());
+connectors_1.ConnectorRegistry.register(new additional_1.RedshiftConnector());
+connectors_1.ConnectorRegistry.register(new additional_1.DatabricksConnector());
+connectors_1.ConnectorRegistry.register(new additional_1.ClickHouseConnector());
+connectors_1.ConnectorRegistry.register(new additional_1.MongoDBConnector());
 // Serve static files from ui directory
 const uiPath = path.join(__dirname, '..', 'ui');
 app.use(express_1.default.static(uiPath));

package/package.json

@@ -1,13 +1,13 @@
 {
     "name": "crushdataai",
-    "version": "1.2.7",
+    "version": "1.2.8",
     "description": "CLI to install CrushData AI data analyst skill for AI coding assistants",
     "main": "dist/index.js",
     "bin": {
         "crushdataai": "./dist/index.js"
     },
     "scripts": {
-        "build": "tsc",
+        "build": "tsc && cd ui-react && npm install && npm run build",
         "dev": "ts-node src/index.ts",
         "prepublishOnly": "npm run build"
     },
@@ -27,14 +27,19 @@
         "@types/fs-extra": "^11.0.4",
         "@types/node": "^20.10.0",
         "@types/papaparse": "^5.5.2",
+        "@types/pg": "^8.16.0",
         "typescript": "^5.3.0"
     },
     "dependencies": {
+        "@google-cloud/bigquery": "^8.1.1",
         "commander": "^11.1.0",
         "express": "^4.18.2",
         "fs-extra": "^11.2.0",
+        "mysql2": "^3.16.0",
         "open": "^10.0.0",
-        "papaparse": "^5.5.3"
+        "papaparse": "^5.5.3",
+        "pg": "^8.16.3",
+        "snowflake-sdk": "^2.3.3"
     },
     "engines": {
         "node": ">=18.0.0"
@@ -48,4 +53,4 @@
         "type": "git",
         "url": "git+https://github.com/SankaiAI/crushdataai-agent.git"
     }
-}
+}