crushdataai 1.2.17 → 1.2.18

package/README.md

@@ -4,7 +4,7 @@
 
 An AI skill that provides structured, professional data analysis workflows with built-in validation - helping AI coding assistants perform data analysis like a careful human analyst.
 
-![CrushData AI Landing Page](assets/images/crushdataai-landing-page.png)
+![CrushData AI Landing Page](https://github.com/SankaiAI/crushdataai-agent/raw/main/assets/crushdataai-landing-page.png)
 
 ## 🎯 What It Does
 
@@ -83,7 +83,7 @@ crushdataai connect
 - **Supported Types**: CSV, MySQL, PostgreSQL, Shopify, BigQuery, Snowflake
 - **Private & Secure**: Credentials are stored **locally** on your machine (`~/.crushdataai/connections.json`). They are **never** uploaded to any server or included in the npm package.
 
-![CrushData AI Connection Manager](assets/images/crushdataai-data-connection-ui.png)
+![CrushData AI Connection Manager](https://github.com/SankaiAI/crushdataai-agent/raw/main/assets/crushdataai-data-connection-ui.png)
 
 > [!NOTE]
 > **Persistence**: Once you add a connection, you can **close the UI** (Ctrl+C). The AI IDE reads the saved connection details directly from your local config file, so the server does NOT need to keep running.
@@ -108,10 +108,10 @@ crushdataai dashboard
 npx crushdataai dashboard
 ```
 
-![Advanced Dashboard](assets/images/crushdataai-dashboard.png)
+![Advanced Dashboard](https://github.com/SankaiAI/crushdataai-agent/raw/main/assets/crushdataai-dashboard.png)
 *Advanced charts visualization (Funnel, Gauge, Radar, etc.)*
 
-![Simple Dashboard](assets/images/crushdataai-dashboard-simple-charts.png)
+![Simple Dashboard](https://github.com/SankaiAI/crushdataai-agent/raw/main/assets/crushdataai-dashboard-simple-charts.png)
 *Standard charts visualization (Line, Bar, Pie, etc.)*
 
 ### 2. Features

package/assets/.agent/workflows/data-analyst.md

@@ -130,10 +130,38 @@ dashboard = {
     "charts": [{"id": "chart-1", "type": "line", "title": "Trend", "data": {"labels": ["Jan","Feb"], "datasets": [{"label": "Revenue", "values": [10000,20000]}]}}]
 }
 
-with open("reports/dashboards/dashboard.json", "w") as f:
+    with open("reports/dashboards/dashboard.json", "w") as f:
     json.dump(dashboard, f, indent=2)
 ```
-Tell user: "Run `npx crushdataai dashboard` to view."
+
+### 5b. Making Charts Refreshable (Recommended)
+
+To allow the user to refresh data directly from the dashboard:
+1. Include a `query` object in the chart definition.
+2. Run `npx crushdataai connections` to list available connection names (secure - no passwords shown).
+3. Set `connection` to one of the listed names.
+4. Set `sql` to the query used to generate the data.
+
+> **SECURITY**: Never read `.env` directly to find connection names. Always use `npx crushdataai connections`.
+
+```json
+"query": {
+    "connection": "my_postgres_db",
+    "sql": "SELECT date, revenue FROM sales WHERE..."
+}
+```
+
+**Database Specifics:**
+- **SQL/Databases**: Provide the full SQL query.
+- **Shopify**: Provide the resource name (e.g. `orders`).
+- **CSV**: Provide the connection name. `sql` is ignored but required (set to "default").
+- **MongoDB**: Provide the collection name in the `sql` field.
+
+**Script-Based Refresh (for Python-aggregated charts):**
+```json
+"query": { "script": "analysis/my_dashboard_script.py" }
+```
+Use `script` for Shopify/API charts that need aggregation. CLI re-runs the script on Refresh.
 
 ---
 

package/assets/.claude/skills/data-analyst/SKILL.md

@@ -213,6 +213,43 @@ FROM table;
        json.dump(dashboard, f, indent=2)
    ```
 
+### 5b. Making Charts Refreshable (Recommended)
+
+To allow the user to refresh data directly from the dashboard:
+1. Include a `query` object in the chart definition.
+2. Run `npx crushdataai connections` to list available connection names (this is secure - no passwords shown).
+3. Set `connection` to one of the listed names.
+4. Set `sql` to the query used to generate the data.
+
+> **SECURITY**: Never read `.env` directly to find connection names. Always use `npx crushdataai connections`.
+
+```json
+"query": {
+    "connection": "my_postgres_db",
+    "sql": "SELECT date, revenue FROM sales WHERE..."
+}
+```
+
+**Database Specifics:**
+- **SQL/Databases**: Provide the full SQL query.
+- **Shopify**: Provide the resource name (e.g. `orders`).
+- **CSV**: Provide the connection name. `sql` is ignored but required (set to "default").
+- **MongoDB**: Provide the collection name in the `sql` field.
+
+**Script-Based Refresh (for Python-aggregated charts):**
+
+If your chart requires Python aggregation (e.g., grouping, custom calculations), use `script` instead of `connection`:
+
+```json
+"query": {
+    "script": "analysis/my_dashboard_script.py"
+}
+```
+
+When the user clicks Refresh, the CLI will **re-run your Python script**. The script should update the dashboard JSON file with fresh aggregated data.
+
+> **TIP**: Use `script` for Shopify/API charts that need aggregation. Use `connection` + `sql` only for SQL databases where the query returns pre-formatted chart data.
+
 3. **Tell user:**
    > "Dashboard ready! Run `npx crushdataai dashboard` to view."
 

package/assets/.cursor/commands/data-analyst.md

@@ -75,10 +75,38 @@ dashboard = {
     "charts": [{"id": "chart-1", "type": "line", "title": "Trend", "data": {"labels": ["Jan","Feb"], "datasets": [{"label": "Revenue", "values": [10000,20000]}]}}]
 }
 
-with open("reports/dashboards/dashboard.json", "w") as f:
+    with open("reports/dashboards/dashboard.json", "w") as f:
     json.dump(dashboard, f, indent=2)
 ```
-Tell user: "Run `npx crushdataai dashboard` to view."
+
+### 5b. Making Charts Refreshable (Recommended)
+
+To allow the user to refresh data directly from the dashboard:
+1. Include a `query` object in the chart definition.
+2. Run `npx crushdataai connections` to list available connection names (secure - no passwords shown).
+3. Set `connection` to one of the listed names.
+4. Set `sql` to the query used to generate the data.
+
+> **SECURITY**: Never read `.env` directly to find connection names. Always use `npx crushdataai connections`.
+
+```json
+"query": {
+    "connection": "my_postgres_db",
+    "sql": "SELECT date, revenue FROM sales WHERE..."
+}
+```
+
+**Database Specifics:**
+- **SQL/Databases**: Provide the full SQL query.
+- **Shopify**: Provide the resource name (e.g. `orders`).
+- **CSV**: Provide the connection name. `sql` is ignored but required (set to "default").
+- **MongoDB**: Provide the collection name in the `sql` field.
+
+**Script-Based Refresh (for Python-aggregated charts):**
+```json
+"query": { "script": "analysis/my_dashboard_script.py" }
+```
+Use `script` for Shopify/API charts that need aggregation. CLI re-runs the script on Refresh.
 
 ---
 

package/assets/.github/prompts/data-analyst.prompt.md

@@ -72,7 +72,35 @@ dashboard = {
     "kpis": [{"id": "kpi-1", "label": "Total", "value": "$50K", "trend": "+12%"}],
     "charts": [{"id": "chart-1", "type": "line", "title": "Trend", "data": {"labels": ["Jan","Feb"], "datasets": [{"label": "Revenue", "values": [10000,20000]}]}}]
 }
-with open("reports/dashboards/dashboard.json", "w") as f:
+    with open("reports/dashboards/dashboard.json", "w") as f:
     json.dump(dashboard, f, indent=2)
 ```
-Tell user: "Run `npx crushdataai dashboard` to view."
+
+### 5b. Making Charts Refreshable (Recommended)
+
+To allow the user to refresh data directly from the dashboard:
+1. Include a `query` object in the chart definition.
+2. Run `npx crushdataai connections` to list available connection names (secure - no passwords shown).
+3. Set `connection` to one of the listed names.
+4. Set `sql` to the query used to generate the data.
+
+> **SECURITY**: Never read `.env` directly to find connection names. Always use `npx crushdataai connections`.
+
+```json
+"query": {
+    "connection": "my_postgres_db",
+    "sql": "SELECT date, revenue FROM sales WHERE..."
+}
+```
+
+**Database Specifics:**
+- **SQL/Databases**: Provide the full SQL query.
+- **Shopify**: Provide the resource name (e.g. `orders`).
+- **CSV**: Provide the connection name. `sql` is ignored but required (set to "default").
+- **MongoDB**: Provide the collection name in the `sql` field.
+
+**Script-Based Refresh (for Python-aggregated charts):**
+```json
+"query": { "script": "analysis/my_dashboard_script.py" }
+```
+Use `script` for Shopify/API charts that need aggregation. CLI re-runs the script on Refresh.

package/assets/.kiro/steering/data-analyst.md

@@ -74,7 +74,35 @@ dashboard = {
     "kpis": [{"id": "kpi-1", "label": "Total", "value": "$50K", "trend": "+12%"}],
     "charts": [{"id": "chart-1", "type": "line", "title": "Trend", "data": {"labels": ["Jan","Feb"], "datasets": [{"label": "Revenue", "values": [10000,20000]}]}}]
 }
-with open("reports/dashboards/dashboard.json", "w") as f:
+    with open("reports/dashboards/dashboard.json", "w") as f:
     json.dump(dashboard, f, indent=2)
 ```
-Tell user: "Run `npx crushdataai dashboard` to view."
+
+### 5b. Making Charts Refreshable (Recommended)
+
+To allow the user to refresh data directly from the dashboard:
+1. Include a `query` object in the chart definition.
+2. Run `npx crushdataai connections` to list available connection names (secure - no passwords shown).
+3. Set `connection` to one of the listed names.
+4. Set `sql` to the query used to generate the data.
+
+> **SECURITY**: Never read `.env` directly to find connection names. Always use `npx crushdataai connections`.
+
+```json
+"query": {
+    "connection": "my_postgres_db",
+    "sql": "SELECT date, revenue FROM sales WHERE..."
+}
+```
+
+**Database Specifics:**
+- **SQL/Databases**: Provide the full SQL query.
+- **Shopify**: Provide the resource name (e.g. `orders`).
+- **CSV**: Provide the connection name. `sql` is ignored but required (set to "default").
+- **MongoDB**: Provide the collection name in the `sql` field.
+
+**Script-Based Refresh (for Python-aggregated charts):**
+```json
+"query": { "script": "analysis/my_dashboard_script.py" }
+```
+Use `script` for Shopify/API charts that need aggregation. CLI re-runs the script on Refresh.

package/assets/.windsurf/workflows/data-analyst.md

@@ -72,7 +72,35 @@ dashboard = {
     "kpis": [{"id": "kpi-1", "label": "Total", "value": "$50K", "trend": "+12%"}],
     "charts": [{"id": "chart-1", "type": "line", "title": "Trend", "data": {"labels": ["Jan","Feb"], "datasets": [{"label": "Revenue", "values": [10000,20000]}]}}]
 }
-with open("reports/dashboards/dashboard.json", "w") as f:
+    with open("reports/dashboards/dashboard.json", "w") as f:
     json.dump(dashboard, f, indent=2)
 ```
-Tell user: "Run `npx crushdataai dashboard` to view."
+
+### 5b. Making Charts Refreshable (Recommended)
+
+To allow the user to refresh data directly from the dashboard:
+1. Include a `query` object in the chart definition.
+2. Run `npx crushdataai connections` to list available connection names (secure - no passwords shown).
+3. Set `connection` to one of the listed names.
+4. Set `sql` to the query used to generate the data.
+
+> **SECURITY**: Never read `.env` directly to find connection names. Always use `npx crushdataai connections`.
+
+```json
+"query": {
+    "connection": "my_postgres_db",
+    "sql": "SELECT date, revenue FROM sales WHERE..."
+}
+```
+
+**Database Specifics:**
+- **SQL/Databases**: Provide the full SQL query.
+- **Shopify**: Provide the resource name (e.g. `orders`).
+- **CSV**: Provide the connection name. `sql` is ignored but required (set to "default").
+- **MongoDB**: Provide the collection name in the `sql` field.
+
+**Script-Based Refresh (for Python-aggregated charts):**
+```json
+"query": { "script": "analysis/my_dashboard_script.py" }
+```
+Use `script` for Shopify/API charts that need aggregation. CLI re-runs the script on Refresh.

package/dist/connectors/clickhouse/index.d.ts

@@ -0,0 +1,12 @@
+import { Connector, Table, TableData } from '../index';
+import { Connection } from '../../connections';
+export declare class ClickHouseConnector implements Connector {
+    type: string;
+    private createClient;
+    test(connection: Connection): Promise<boolean>;
+    getTables(connection: Connection): Promise<Table[]>;
+    getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
+    getSnippet(connection: Connection, lang: string): string;
+    executeQuery(connection: Connection, query: string): Promise<any[]>;
+}

package/dist/connectors/clickhouse/index.js

@@ -0,0 +1,140 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClickHouseConnector = void 0;
+const client_1 = require("@clickhouse/client");
+class ClickHouseConnector {
+    constructor() {
+        this.type = 'clickhouse';
+    }
+    createClient(connection) {
+        let url = connection.host || 'http://localhost:8123';
+        if (!url.startsWith('http')) {
+            url = `http://${url}`;
+        }
+        return (0, client_1.createClient)({
+            url,
+            username: connection.user || 'default',
+            password: connection.password || '',
+            database: connection.database || 'default',
+            request_timeout: 10000,
+        });
+    }
+    async test(connection) {
+        console.log(`[ClickHouse] Testing connection for ${connection.name}`);
+        try {
+            const client = this.createClient(connection);
+            await client.query({ query: 'SELECT 1' });
+            console.log(`[ClickHouse] Connection test successful`);
+            await client.close();
+            return true;
+        }
+        catch (error) {
+            console.error(`[ClickHouse] Connection test failed:`, error.message);
+            throw new Error(`ClickHouse connection failed: ${error.message}`);
+        }
+    }
+    async getTables(connection) {
+        console.log(`[ClickHouse] getTables called for ${connection.name}`);
+        const client = this.createClient(connection);
+        try {
+            const resultSet = await client.query({
+                query: `
+                    SELECT name, 'table' as type 
+                    FROM system.tables 
+                    WHERE database = '${connection.database || 'default'}'
+                `,
+                format: 'JSONEachRow'
+            });
+            const rows = await resultSet.json();
+            return rows.map((row) => ({
+                name: row.name,
+                type: 'table',
+                rowCount: null
+            }));
+        }
+        catch (error) {
+            throw new Error(`Failed to fetch tables: ${error.message}`);
+        }
+        finally {
+            await client.close();
+        }
+    }
+    async getData(connection, tableName, page, limit) {
+        console.log(`[ClickHouse] getData called for ${connection.name}`);
+        const client = this.createClient(connection);
+        try {
+            const offset = (page - 1) * limit;
+            // Get total count
+            const countResult = await client.query({
+                query: `SELECT count() as total FROM "${tableName}"`,
+                format: 'JSONEachRow'
+            });
+            const countRows = await countResult.json();
+            const totalRows = parseInt(countRows[0]?.total || '0', 10);
+            // Get data
+            const resultSet = await client.query({
+                query: `SELECT * FROM "${tableName}" LIMIT ${limit} OFFSET ${offset}`,
+                format: 'JSONEachRow'
+            });
+            const rows = await resultSet.json();
+            const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
+            const totalPages = Math.ceil(totalRows / limit) || 1;
+            return {
+                columns,
+                rows,
+                pagination: {
+                    page, limit, totalRows, totalPages,
+                    startIdx: offset + 1, endIdx: offset + rows.length
+                }
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to fetch data: ${error.message}`);
+        }
+        finally {
+            await client.close();
+        }
+    }
+    async getSchema(connection, tableName) {
+        const client = this.createClient(connection);
+        try {
+            const resultSet = await client.query({
+                query: `DESCRIBE "${tableName}"`,
+                format: 'JSONEachRow'
+            });
+            const rows = await resultSet.json();
+            return rows.map((row) => ({
+                name: row.name,
+                type: row.type,
+                nullable: false // ClickHouse nullable is explicit type wrapper usually
+            }));
+        }
+        catch (error) {
+            throw new Error(`Failed to fetch schema: ${error.message}`);
+        }
+        finally {
+            await client.close();
+        }
+    }
+    getSnippet(connection, lang) {
+        // ... ClickHouse python snippet ...
+        return '';
+    }
+    async executeQuery(connection, query) {
+        const client = this.createClient(connection);
+        try {
+            const resultSet = await client.query({
+                query: query,
+                format: 'JSONEachRow'
+            });
+            return await resultSet.json();
+        }
+        catch (error) {
+            throw new Error(`Failed to execute query: ${error.message}`);
+        }
+        finally {
+            await client.close();
+        }
+    }
+}
+exports.ClickHouseConnector = ClickHouseConnector;

package/dist/connectors/databricks/index.d.ts

@@ -0,0 +1,12 @@
+import { Connector, Table, TableData } from '../index';
+import { Connection } from '../../connections';
+export declare class DatabricksConnector implements Connector {
+    type: string;
+    private createSession;
+    test(connection: Connection): Promise<boolean>;
+    getTables(connection: Connection): Promise<Table[]>;
+    getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
+    getSnippet(connection: Connection, lang: string): string;
+    executeQuery(connection: Connection, query: string): Promise<any[]>;
+}

package/dist/connectors/databricks/index.js

@@ -0,0 +1,105 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatabricksConnector = void 0;
+const sql_1 = require("@databricks/sql");
+class DatabricksConnector {
+    constructor() {
+        this.type = 'databricks';
+    }
+    async createSession(connection) {
+        const client = new sql_1.DBSQLClient();
+        const token = connection.apiKey || connection.password;
+        const host = connection.host;
+        const path = connection.connectionString; // mapped from HTTP Path
+        if (!token || !host || !path) {
+            throw new Error('Host, Token (API Key), and HTTP Path (Connection String) are required for Databricks');
+        }
+        await client.connect({ token, host, path });
+        const session = await client.openSession();
+        return { client, session };
+    }
+    async test(connection) {
+        console.log(`[Databricks] Testing connection for ${connection.name}`);
+        try {
+            const { client, session } = await this.createSession(connection);
+            const op = await session.executeStatement('SELECT 1');
+            await op.close();
+            await session.close();
+            await client.close();
+            return true;
+        }
+        catch (error) {
+            throw new Error(`Databricks connection failed: ${error.message}`);
+        }
+    }
+    async getTables(connection) {
+        console.log(`[Databricks] getTables called for ${connection.name}`);
+        try {
+            const { client, session } = await this.createSession(connection);
+            const op = await session.executeStatement('SHOW TABLES');
+            const rows = await op.fetchAll();
+            await op.close();
+            await session.close();
+            await client.close();
+            return rows.map((row) => ({
+                name: row.tableName || row.name,
+                type: 'table',
+                rowCount: null
+            }));
+        }
+        catch (error) {
+            throw new Error(`Failed to fetch tables: ${error.message}`);
+        }
+    }
+    async getData(connection, tableName, page, limit) {
+        console.log(`[Databricks] getData called for ${connection.name}`);
+        try {
+            const { client, session } = await this.createSession(connection);
+            // Count
+            const countOp = await session.executeStatement(`SELECT COUNT(*) as total FROM ${tableName}`);
+            const countRows = await countOp.fetchAll();
+            await countOp.close();
+            const totalRows = parseInt(countRows[0]?.total || '0', 10);
+            // Data
+            const dataOp = await session.executeStatement(`SELECT * FROM ${tableName} LIMIT ${limit}`);
+            const rows = await dataOp.fetchAll();
+            await dataOp.close();
+            await session.close();
+            await client.close();
+            const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
+            const totalPages = Math.ceil(totalRows / limit) || 1;
+            return {
+                columns,
+                rows,
+                pagination: {
+                    page, limit, totalRows, totalPages,
+                    startIdx: 1, endIdx: rows.length
+                }
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to fetch data: ${error.message}`);
+        }
+    }
+    async getSchema(connection, tableName) {
+        return [];
+    }
+    getSnippet(connection, lang) {
+        return '';
+    }
+    async executeQuery(connection, query) {
+        try {
+            const { client, session } = await this.createSession(connection);
+            const op = await session.executeStatement(query);
+            const rows = await op.fetchAll();
+            await op.close();
+            await session.close();
+            await client.close();
+            return rows;
+        }
+        catch (error) {
+            throw new Error(`Failed to execute query: ${error.message}`);
+        }
+    }
+}
+exports.DatabricksConnector = DatabricksConnector;

package/dist/connectors/mongodb/index.d.ts

@@ -0,0 +1,12 @@
+import { Connector, Table, TableData } from '../index';
+import { Connection } from '../../connections';
+export declare class MongoDBConnector implements Connector {
+    type: string;
+    private getUrl;
+    test(connection: Connection): Promise<boolean>;
+    getTables(connection: Connection): Promise<Table[]>;
+    getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
+    getSchema(connection: Connection, tableName: string): Promise<import('../index').ColumnInfo[]>;
+    getSnippet(connection: Connection, lang: string): string;
+    executeQuery(connection: Connection, query: string): Promise<any[]>;
+}