crushdataai 1.2.7 → 1.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/antigravity/data-analyst.md +8 -4
- package/assets/claude/SKILL.md +7 -2
- package/assets/copilot/data-analyst.prompt.md +8 -4
- package/assets/cursor/data-analyst.md +8 -4
- package/assets/kiro/data-analyst.md +8 -4
- package/assets/windsurf/data-analyst.md +8 -4
- package/dist/commands/schema.d.ts +1 -0
- package/dist/commands/schema.js +54 -0
- package/dist/connections.d.ts +3 -1
- package/dist/connections.js +125 -0
- package/dist/connectors/additional/index.d.ts +42 -0
- package/dist/connectors/additional/index.js +268 -0
- package/dist/connectors/cloud/index.d.ts +8 -3
- package/dist/connectors/cloud/index.js +321 -10
- package/dist/connectors/csv/index.d.ts +1 -0
- package/dist/connectors/csv/index.js +27 -7
- package/dist/connectors/custom/index.d.ts +10 -0
- package/dist/connectors/custom/index.js +61 -0
- package/dist/connectors/index.d.ts +6 -0
- package/dist/connectors/mysql/index.d.ts +1 -0
- package/dist/connectors/mysql/index.js +162 -9
- package/dist/connectors/postgresql/index.d.ts +2 -0
- package/dist/connectors/postgresql/index.js +140 -8
- package/dist/connectors/shopify/index.d.ts +1 -0
- package/dist/connectors/shopify/index.js +44 -5
- package/dist/index.js +7 -0
- package/dist/routes/connections.js +2 -0
- package/dist/server.js +8 -0
- package/package.json +9 -4
- package/ui/assets/index-DK2tLINh.js +40 -0
- package/ui/assets/index-rlcHFDJB.css +1 -0
- package/ui/favicon.svg +4 -18
- package/ui/index.html +7 -324
- package/ui/favicon-32x32.png +0 -0
- package/ui/main.js +0 -541
- package/ui/styles.css +0 -718
|
@@ -30,11 +30,15 @@ Before writing any code, ask the user:
|
|
|
30
30
|
6. **Reports**: Save all validation/profiling outputs to `reports/` folder. Create if missing.
|
|
31
31
|
|
|
32
32
|
### 2. Secure Data Access
|
|
33
|
+
|
|
34
|
+
> **Credentials are stored in `.env`** - never hardcoded.
|
|
35
|
+
|
|
33
36
|
- **Check Connections**: Run `npx crushdataai connections` first.
|
|
34
|
-
- **Missing Data?**:
|
|
35
|
-
|
|
36
|
-
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>`
|
|
37
|
-
- **
|
|
37
|
+
- **Missing Data?**: Run `npx crushdataai connect` to add.
|
|
38
|
+
- **Discover Schema**: `npx crushdataai schema <connection> [table]`
|
|
39
|
+
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>` (uses env vars).
|
|
40
|
+
- **Load .env**: Ensure `python-dotenv` loads `.env`. Scripts use `os.environ["VAR"]`.
|
|
41
|
+
- **Security**: Treat connected data as read-only.
|
|
38
42
|
|
|
39
43
|
---
|
|
40
44
|
|
package/assets/claude/SKILL.md
CHANGED
|
@@ -45,10 +45,15 @@ When user requests data analysis work (analyze, query, dashboard, metrics, EDA,
|
|
|
45
45
|
```
|
|
46
46
|
|
|
47
47
|
### 1b. Secure Data Access
|
|
48
|
+
|
|
49
|
+
> **Credentials are stored in the project's `.env` file** - never hardcoded.
|
|
50
|
+
|
|
48
51
|
- **Check Connections**: Run `npx crushdataai connections` first.
|
|
49
|
-
- **Missing Data?**: If the data source is not listed
|
|
52
|
+
- **Missing Data?**: If the data source is not listed, **INSTRUCT** the user to run:
|
|
50
53
|
`npx crushdataai connect`
|
|
51
|
-
- **
|
|
54
|
+
- **Discover Schema**: Use `npx crushdataai schema <connection> [table]` to see available tables and columns.
|
|
55
|
+
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>` to get loading code (uses env vars).
|
|
56
|
+
- **Load .env**: Scripts use `os.environ["VAR"]`. Ensure `python-dotenv` is installed and `.env` is loaded.
|
|
52
57
|
- **Security**: **DO NOT** ask user to copy/move files to `data/`. Treat connected data as read-only.
|
|
53
58
|
|
|
54
59
|
### Step 2: Search Relevant Domains
|
|
@@ -23,11 +23,15 @@ Before coding, ask:
|
|
|
23
23
|
- **Reports**: Save all outputs to `reports/` folder (profiling, validation).
|
|
24
24
|
|
|
25
25
|
### 2. Secure Data Access
|
|
26
|
+
|
|
27
|
+
> **Credentials are stored in `.env`** - never hardcoded.
|
|
28
|
+
|
|
26
29
|
- **Check Connections**: Run `npx crushdataai connections` first.
|
|
27
|
-
- **Missing Data?**:
|
|
28
|
-
|
|
29
|
-
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>`
|
|
30
|
-
- **
|
|
30
|
+
- **Missing Data?**: Run `npx crushdataai connect` to add.
|
|
31
|
+
- **Discover Schema**: `npx crushdataai schema <connection> [table]`
|
|
32
|
+
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>` (uses env vars).
|
|
33
|
+
- **Load .env**: Ensure `python-dotenv` loads `.env`. Scripts use `os.environ["VAR"]`.
|
|
34
|
+
- **Security**: Treat connected data as read-only.
|
|
31
35
|
|
|
32
36
|
### 3. Search Knowledge Base
|
|
33
37
|
```bash
|
|
@@ -20,11 +20,15 @@ Before coding, ask:
|
|
|
20
20
|
- **Reports**: Save all outputs to `reports/` folder (profiling, validation).
|
|
21
21
|
|
|
22
22
|
### 2. Secure Data Access
|
|
23
|
+
|
|
24
|
+
> **Credentials are stored in `.env`** - never hardcoded.
|
|
25
|
+
|
|
23
26
|
- **Check Connections**: Run `npx crushdataai connections` first.
|
|
24
|
-
- **Missing Data?**:
|
|
25
|
-
|
|
26
|
-
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>`
|
|
27
|
-
- **
|
|
27
|
+
- **Missing Data?**: Run `npx crushdataai connect` to add.
|
|
28
|
+
- **Discover Schema**: `npx crushdataai schema <connection> [table]`
|
|
29
|
+
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>` (uses env vars).
|
|
30
|
+
- **Load .env**: Ensure `python-dotenv` loads `.env`. Scripts use `os.environ["VAR"]`.
|
|
31
|
+
- **Security**: Treat connected data as read-only.
|
|
28
32
|
|
|
29
33
|
### 2. Search Knowledge Base
|
|
30
34
|
```bash
|
|
@@ -19,11 +19,15 @@ Before writing code, gather:
|
|
|
19
19
|
- **Reports**: Save all outputs to `reports/` folder (profiling, validation).
|
|
20
20
|
|
|
21
21
|
### 2. Secure Data Access
|
|
22
|
+
|
|
23
|
+
> **Credentials are stored in `.env`** - never hardcoded.
|
|
24
|
+
|
|
22
25
|
- **Check Connections**: Run `npx crushdataai connections` first.
|
|
23
|
-
- **Missing Data?**:
|
|
24
|
-
|
|
25
|
-
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>`
|
|
26
|
-
- **
|
|
26
|
+
- **Missing Data?**: Run `npx crushdataai connect` to add.
|
|
27
|
+
- **Discover Schema**: `npx crushdataai schema <connection> [table]`
|
|
28
|
+
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>` (uses env vars).
|
|
29
|
+
- **Load .env**: Ensure `python-dotenv` loads `.env`. Scripts use `os.environ["VAR"]`.
|
|
30
|
+
- **Security**: Treat connected data as read-only.
|
|
27
31
|
|
|
28
32
|
### 3. Search Before Implementing
|
|
29
33
|
```bash
|
|
@@ -17,11 +17,15 @@ Ask before coding:
|
|
|
17
17
|
- **Reports**: Save all outputs to `reports/` folder (profiling, validation).
|
|
18
18
|
|
|
19
19
|
### 2. Secure Data Access
|
|
20
|
+
|
|
21
|
+
> **Credentials are stored in `.env`** - never hardcoded.
|
|
22
|
+
|
|
20
23
|
- **Check Connections**: Run `npx crushdataai connections` first.
|
|
21
|
-
- **Missing Data?**:
|
|
22
|
-
|
|
23
|
-
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>`
|
|
24
|
-
- **
|
|
24
|
+
- **Missing Data?**: Run `npx crushdataai connect` to add.
|
|
25
|
+
- **Discover Schema**: `npx crushdataai schema <connection> [table]`
|
|
26
|
+
- **Get Code**: **ALWAYS** use `npx crushdataai snippet <name>` (uses env vars).
|
|
27
|
+
- **Load .env**: Ensure `python-dotenv` loads `.env`. Scripts use `os.environ["VAR"]`.
|
|
28
|
+
- **Security**: Treat connected data as read-only.
|
|
25
29
|
|
|
26
30
|
### 3. Search Knowledge
|
|
27
31
|
```bash
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function schema(connectionName: string, tableName?: string): Promise<void>;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.schema = schema;
|
|
4
|
+
const connections_1 = require("../connections");
|
|
5
|
+
const connectors_1 = require("../connectors");
|
|
6
|
+
async function schema(connectionName, tableName) {
|
|
7
|
+
const connection = (0, connections_1.getConnection)(connectionName);
|
|
8
|
+
if (!connection) {
|
|
9
|
+
console.error(`❌ Connection '${connectionName}' not found.`);
|
|
10
|
+
console.error(` Run 'crushdataai connections' to see available connections.`);
|
|
11
|
+
process.exit(1);
|
|
12
|
+
}
|
|
13
|
+
try {
|
|
14
|
+
const connector = connectors_1.ConnectorRegistry.get(connection.type);
|
|
15
|
+
if (!connector) {
|
|
16
|
+
console.error(`❌ No connector found for type: ${connection.type}`);
|
|
17
|
+
process.exit(1);
|
|
18
|
+
}
|
|
19
|
+
// If no table specified, list all tables
|
|
20
|
+
if (!tableName) {
|
|
21
|
+
console.log(`\n📊 Tables in '${connectionName}':\n`);
|
|
22
|
+
const tables = await connector.getTables(connection);
|
|
23
|
+
if (tables.length === 0) {
|
|
24
|
+
console.log(' No tables found.');
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
tables.forEach(table => {
|
|
28
|
+
const rowInfo = table.rowCount != null ? ` (~${table.rowCount.toLocaleString()} rows)` : '';
|
|
29
|
+
console.log(` └─ ${table.name}${rowInfo}`);
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
console.log(`\n Run 'crushdataai schema ${connectionName} <table>' for column details.\n`);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
// Get schema for specific table
|
|
36
|
+
console.log(`\n📊 Schema for '${tableName}' in '${connectionName}':\n`);
|
|
37
|
+
const columns = await connector.getSchema(connection, tableName);
|
|
38
|
+
if (columns.length === 0) {
|
|
39
|
+
console.log(' No columns found.');
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
columns.forEach((col, idx) => {
|
|
43
|
+
const prefix = idx === columns.length - 1 ? '└─' : '├─';
|
|
44
|
+
const nullInfo = col.nullable ? 'NULL' : 'NOT NULL';
|
|
45
|
+
console.log(` ${prefix} ${col.name} (${col.type}, ${nullInfo})`);
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
console.log('');
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
console.error(`❌ Error fetching schema: ${error.message}`);
|
|
52
|
+
process.exit(1);
|
|
53
|
+
}
|
|
54
|
+
}
|
package/dist/connections.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type ConnectionType = 'mysql' | 'postgresql' | 'bigquery' | 'snowflake' | 'shopify' | 'csv';
|
|
1
|
+
export type ConnectionType = 'mysql' | 'postgresql' | 'bigquery' | 'snowflake' | 'shopify' | 'csv' | 'custom' | 'sqlserver' | 'redshift' | 'databricks' | 'clickhouse' | 'mongodb';
|
|
2
2
|
export interface Connection {
|
|
3
3
|
name: string;
|
|
4
4
|
type: ConnectionType;
|
|
@@ -15,6 +15,7 @@ export interface Connection {
|
|
|
15
15
|
apiKey?: string;
|
|
16
16
|
apiSecret?: string;
|
|
17
17
|
filePath?: string;
|
|
18
|
+
connectionString?: string;
|
|
18
19
|
createdAt: string;
|
|
19
20
|
updatedAt: string;
|
|
20
21
|
}
|
|
@@ -26,6 +27,7 @@ export declare function encrypt(text: string): string;
|
|
|
26
27
|
export declare function decrypt(encryptedText: string): string;
|
|
27
28
|
export declare function loadConnections(): ConnectionsConfig;
|
|
28
29
|
export declare function saveConnections(config: ConnectionsConfig): void;
|
|
30
|
+
export declare function writeToEnvFile(connection: Connection): void;
|
|
29
31
|
export declare function saveConnection(connection: Connection): void;
|
|
30
32
|
export declare function getConnection(name: string): Connection | null;
|
|
31
33
|
export declare function deleteConnection(name: string): boolean;
|
package/dist/connections.js
CHANGED
|
@@ -37,6 +37,7 @@ exports.encrypt = encrypt;
|
|
|
37
37
|
exports.decrypt = decrypt;
|
|
38
38
|
exports.loadConnections = loadConnections;
|
|
39
39
|
exports.saveConnections = saveConnections;
|
|
40
|
+
exports.writeToEnvFile = writeToEnvFile;
|
|
40
41
|
exports.saveConnection = saveConnection;
|
|
41
42
|
exports.getConnection = getConnection;
|
|
42
43
|
exports.deleteConnection = deleteConnection;
|
|
@@ -110,6 +111,124 @@ function saveConnections(config) {
|
|
|
110
111
|
fs.chmodSync(configPath, 0o600);
|
|
111
112
|
}
|
|
112
113
|
}
|
|
114
|
+
// Get the current working directory (project root)
|
|
115
|
+
function getProjectDir() {
|
|
116
|
+
return process.cwd();
|
|
117
|
+
}
|
|
118
|
+
// Create environment variable name from connection name
|
|
119
|
+
function toEnvVarName(connectionName, field) {
|
|
120
|
+
const sanitized = connectionName.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
121
|
+
return `${sanitized}_${field.toUpperCase()}`;
|
|
122
|
+
}
|
|
123
|
+
// Write credentials to .env file in project directory
|
|
124
|
+
function writeToEnvFile(connection) {
|
|
125
|
+
const projectDir = getProjectDir();
|
|
126
|
+
const envPath = path.join(projectDir, '.env');
|
|
127
|
+
const gitignorePath = path.join(projectDir, '.gitignore');
|
|
128
|
+
// Read existing .env or create empty
|
|
129
|
+
let envContent = '';
|
|
130
|
+
if (fs.existsSync(envPath)) {
|
|
131
|
+
envContent = fs.readFileSync(envPath, 'utf8');
|
|
132
|
+
}
|
|
133
|
+
// Build env vars based on connection type
|
|
134
|
+
const envVars = {};
|
|
135
|
+
const prefix = connection.name;
|
|
136
|
+
switch (connection.type) {
|
|
137
|
+
case 'mysql':
|
|
138
|
+
case 'postgresql':
|
|
139
|
+
case 'sqlserver':
|
|
140
|
+
case 'redshift':
|
|
141
|
+
case 'clickhouse':
|
|
142
|
+
if (connection.host)
|
|
143
|
+
envVars[toEnvVarName(prefix, 'HOST')] = connection.host;
|
|
144
|
+
if (connection.port)
|
|
145
|
+
envVars[toEnvVarName(prefix, 'PORT')] = String(connection.port);
|
|
146
|
+
if (connection.user)
|
|
147
|
+
envVars[toEnvVarName(prefix, 'USER')] = connection.user;
|
|
148
|
+
if (connection.password)
|
|
149
|
+
envVars[toEnvVarName(prefix, 'PASSWORD')] = connection.password;
|
|
150
|
+
if (connection.database)
|
|
151
|
+
envVars[toEnvVarName(prefix, 'DATABASE')] = connection.database;
|
|
152
|
+
break;
|
|
153
|
+
case 'bigquery':
|
|
154
|
+
if (connection.projectId)
|
|
155
|
+
envVars[toEnvVarName(prefix, 'PROJECT_ID')] = connection.projectId;
|
|
156
|
+
if (connection.keyFile)
|
|
157
|
+
envVars[toEnvVarName(prefix, 'KEY_FILE')] = connection.keyFile;
|
|
158
|
+
break;
|
|
159
|
+
case 'snowflake':
|
|
160
|
+
if (connection.account)
|
|
161
|
+
envVars[toEnvVarName(prefix, 'ACCOUNT')] = connection.account;
|
|
162
|
+
if (connection.user)
|
|
163
|
+
envVars[toEnvVarName(prefix, 'USER')] = connection.user;
|
|
164
|
+
if (connection.password)
|
|
165
|
+
envVars[toEnvVarName(prefix, 'PASSWORD')] = connection.password;
|
|
166
|
+
if (connection.warehouse)
|
|
167
|
+
envVars[toEnvVarName(prefix, 'WAREHOUSE')] = connection.warehouse;
|
|
168
|
+
if (connection.database)
|
|
169
|
+
envVars[toEnvVarName(prefix, 'DATABASE')] = connection.database;
|
|
170
|
+
break;
|
|
171
|
+
case 'databricks':
|
|
172
|
+
if (connection.host)
|
|
173
|
+
envVars[toEnvVarName(prefix, 'HOST')] = connection.host;
|
|
174
|
+
if (connection.connectionString)
|
|
175
|
+
envVars[toEnvVarName(prefix, 'HTTP_PATH')] = connection.connectionString;
|
|
176
|
+
if (connection.apiKey)
|
|
177
|
+
envVars[toEnvVarName(prefix, 'TOKEN')] = connection.apiKey;
|
|
178
|
+
break;
|
|
179
|
+
case 'mongodb':
|
|
180
|
+
if (connection.host)
|
|
181
|
+
envVars[toEnvVarName(prefix, 'HOST')] = connection.host;
|
|
182
|
+
if (connection.user)
|
|
183
|
+
envVars[toEnvVarName(prefix, 'USER')] = connection.user;
|
|
184
|
+
if (connection.password)
|
|
185
|
+
envVars[toEnvVarName(prefix, 'PASSWORD')] = connection.password;
|
|
186
|
+
if (connection.database)
|
|
187
|
+
envVars[toEnvVarName(prefix, 'DATABASE')] = connection.database;
|
|
188
|
+
if (connection.connectionString)
|
|
189
|
+
envVars[toEnvVarName(prefix, 'CONNECTION_STRING')] = connection.connectionString;
|
|
190
|
+
break;
|
|
191
|
+
case 'shopify':
|
|
192
|
+
if (connection.store)
|
|
193
|
+
envVars[toEnvVarName(prefix, 'STORE')] = connection.store;
|
|
194
|
+
if (connection.apiKey)
|
|
195
|
+
envVars[toEnvVarName(prefix, 'API_KEY')] = connection.apiKey;
|
|
196
|
+
if (connection.apiSecret)
|
|
197
|
+
envVars[toEnvVarName(prefix, 'API_SECRET')] = connection.apiSecret;
|
|
198
|
+
break;
|
|
199
|
+
case 'csv':
|
|
200
|
+
if (connection.filePath)
|
|
201
|
+
envVars[toEnvVarName(prefix, 'FILE_PATH')] = connection.filePath;
|
|
202
|
+
break;
|
|
203
|
+
case 'custom':
|
|
204
|
+
if (connection.connectionString)
|
|
205
|
+
envVars[toEnvVarName(prefix, 'CONNECTION_STRING')] = connection.connectionString;
|
|
206
|
+
break;
|
|
207
|
+
}
|
|
208
|
+
// Update or append each env var
|
|
209
|
+
for (const [key, value] of Object.entries(envVars)) {
|
|
210
|
+
const regex = new RegExp(`^${key}=.*$`, 'm');
|
|
211
|
+
const newLine = `${key}="${value}"`;
|
|
212
|
+
if (regex.test(envContent)) {
|
|
213
|
+
envContent = envContent.replace(regex, newLine);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
envContent = envContent.trimEnd() + (envContent ? '\n' : '') + newLine + '\n';
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
fs.writeFileSync(envPath, envContent, 'utf8');
|
|
220
|
+
console.log(`✅ Credentials saved to .env file`);
|
|
221
|
+
// Update .gitignore if needed
|
|
222
|
+
let gitignoreContent = '';
|
|
223
|
+
if (fs.existsSync(gitignorePath)) {
|
|
224
|
+
gitignoreContent = fs.readFileSync(gitignorePath, 'utf8');
|
|
225
|
+
}
|
|
226
|
+
if (!gitignoreContent.includes('.env')) {
|
|
227
|
+
gitignoreContent = gitignoreContent.trimEnd() + (gitignoreContent ? '\n' : '') + '\n# Environment variables\n.env\n';
|
|
228
|
+
fs.writeFileSync(gitignorePath, gitignoreContent, 'utf8');
|
|
229
|
+
console.log(`✅ Added .env to .gitignore`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
113
232
|
// Add or update a connection
|
|
114
233
|
function saveConnection(connection) {
|
|
115
234
|
const config = loadConnections();
|
|
@@ -124,6 +243,9 @@ function saveConnection(connection) {
|
|
|
124
243
|
if (secureConnection.apiSecret) {
|
|
125
244
|
secureConnection.apiSecret = encrypt(secureConnection.apiSecret);
|
|
126
245
|
}
|
|
246
|
+
if (secureConnection.connectionString) {
|
|
247
|
+
secureConnection.connectionString = encrypt(secureConnection.connectionString);
|
|
248
|
+
}
|
|
127
249
|
// Update or add
|
|
128
250
|
const existingIndex = config.connections.findIndex(c => c.name === connection.name);
|
|
129
251
|
if (existingIndex >= 0) {
|
|
@@ -151,6 +273,9 @@ function getConnection(name) {
|
|
|
151
273
|
if (decrypted.apiSecret) {
|
|
152
274
|
decrypted.apiSecret = decrypt(decrypted.apiSecret);
|
|
153
275
|
}
|
|
276
|
+
if (decrypted.connectionString) {
|
|
277
|
+
decrypted.connectionString = decrypt(decrypted.connectionString);
|
|
278
|
+
}
|
|
154
279
|
return decrypted;
|
|
155
280
|
}
|
|
156
281
|
// Delete a connection
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { Connector, Table, TableData, ColumnInfo } from '../index';
|
|
2
|
+
import { Connection } from '../../connections';
|
|
3
|
+
export declare class SQLServerConnector implements Connector {
|
|
4
|
+
type: string;
|
|
5
|
+
test(connection: Connection): Promise<boolean>;
|
|
6
|
+
getTables(connection: Connection): Promise<Table[]>;
|
|
7
|
+
getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
|
|
8
|
+
getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
|
|
9
|
+
getSnippet(connection: Connection, lang: string): string;
|
|
10
|
+
}
|
|
11
|
+
export declare class RedshiftConnector implements Connector {
|
|
12
|
+
type: string;
|
|
13
|
+
test(connection: Connection): Promise<boolean>;
|
|
14
|
+
getTables(connection: Connection): Promise<Table[]>;
|
|
15
|
+
getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
|
|
16
|
+
getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
|
|
17
|
+
getSnippet(connection: Connection, lang: string): string;
|
|
18
|
+
}
|
|
19
|
+
export declare class DatabricksConnector implements Connector {
|
|
20
|
+
type: string;
|
|
21
|
+
test(connection: Connection): Promise<boolean>;
|
|
22
|
+
getTables(connection: Connection): Promise<Table[]>;
|
|
23
|
+
getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
|
|
24
|
+
getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
|
|
25
|
+
getSnippet(connection: Connection, lang: string): string;
|
|
26
|
+
}
|
|
27
|
+
export declare class ClickHouseConnector implements Connector {
|
|
28
|
+
type: string;
|
|
29
|
+
test(connection: Connection): Promise<boolean>;
|
|
30
|
+
getTables(connection: Connection): Promise<Table[]>;
|
|
31
|
+
getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
|
|
32
|
+
getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
|
|
33
|
+
getSnippet(connection: Connection, lang: string): string;
|
|
34
|
+
}
|
|
35
|
+
export declare class MongoDBConnector implements Connector {
|
|
36
|
+
type: string;
|
|
37
|
+
test(connection: Connection): Promise<boolean>;
|
|
38
|
+
getTables(connection: Connection): Promise<Table[]>;
|
|
39
|
+
getData(connection: Connection, tableName: string, page: number, limit: number): Promise<TableData>;
|
|
40
|
+
getSchema(connection: Connection, tableName: string): Promise<ColumnInfo[]>;
|
|
41
|
+
getSnippet(connection: Connection, lang: string): string;
|
|
42
|
+
}
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MongoDBConnector = exports.ClickHouseConnector = exports.DatabricksConnector = exports.RedshiftConnector = exports.SQLServerConnector = void 0;
|
|
4
|
+
// SQL Server Connector - uses mssql package or Custom connection string
|
|
5
|
+
class SQLServerConnector {
|
|
6
|
+
constructor() {
|
|
7
|
+
this.type = 'sqlserver';
|
|
8
|
+
}
|
|
9
|
+
async test(connection) {
|
|
10
|
+
console.log(`[SQLServer] Testing connection for ${connection.name}`);
|
|
11
|
+
if (!connection.host?.trim() || !connection.user?.trim() || !connection.database?.trim()) {
|
|
12
|
+
throw new Error('Host, username, and database are required');
|
|
13
|
+
}
|
|
14
|
+
// For now, we validate inputs and trust the user
|
|
15
|
+
// Full implementation would use mssql package
|
|
16
|
+
console.log(`[SQLServer] Connection parameters validated for ${connection.name}`);
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
async getTables(connection) {
|
|
20
|
+
console.log(`[SQLServer] getTables called for ${connection.name}`);
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
async getData(connection, tableName, page, limit) {
|
|
24
|
+
throw new Error('Use the snippet to query SQL Server directly.');
|
|
25
|
+
}
|
|
26
|
+
async getSchema(connection, tableName) {
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
getSnippet(connection, lang) {
|
|
30
|
+
const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
31
|
+
if (lang === 'python') {
|
|
32
|
+
return `import os
|
|
33
|
+
import pyodbc
|
|
34
|
+
import pandas as pd
|
|
35
|
+
|
|
36
|
+
# Connection: ${connection.name}
|
|
37
|
+
# Type: sqlserver
|
|
38
|
+
# Credentials loaded from environment variables (set in .env file)
|
|
39
|
+
conn_str = (
|
|
40
|
+
f"DRIVER={{ODBC Driver 17 for SQL Server}};"
|
|
41
|
+
f"SERVER={os.environ['${prefix}_HOST']};"
|
|
42
|
+
f"DATABASE={os.environ['${prefix}_DATABASE']};"
|
|
43
|
+
f"UID={os.environ['${prefix}_USER']};"
|
|
44
|
+
f"PWD={os.environ['${prefix}_PASSWORD']}"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
connection = pyodbc.connect(conn_str)
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
query = "SELECT TOP 100 * FROM your_table"
|
|
51
|
+
df = pd.read_sql(query, connection)
|
|
52
|
+
print(f"Successfully loaded {len(df)} rows from ${connection.name}")
|
|
53
|
+
print(df.head())
|
|
54
|
+
finally:
|
|
55
|
+
connection.close()
|
|
56
|
+
`;
|
|
57
|
+
}
|
|
58
|
+
return `# Language ${lang} not supported for SQL Server connector yet.`;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
exports.SQLServerConnector = SQLServerConnector;
|
|
62
|
+
// Redshift Connector - PostgreSQL compatible with redshift_connector
|
|
63
|
+
class RedshiftConnector {
|
|
64
|
+
constructor() {
|
|
65
|
+
this.type = 'redshift';
|
|
66
|
+
}
|
|
67
|
+
async test(connection) {
|
|
68
|
+
console.log(`[Redshift] Testing connection for ${connection.name}`);
|
|
69
|
+
if (!connection.host?.trim() || !connection.user?.trim() || !connection.database?.trim()) {
|
|
70
|
+
throw new Error('Host, username, and database are required');
|
|
71
|
+
}
|
|
72
|
+
console.log(`[Redshift] Connection parameters validated for ${connection.name}`);
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
async getTables(connection) {
|
|
76
|
+
return [];
|
|
77
|
+
}
|
|
78
|
+
async getData(connection, tableName, page, limit) {
|
|
79
|
+
throw new Error('Use the snippet to query Redshift directly.');
|
|
80
|
+
}
|
|
81
|
+
async getSchema(connection, tableName) {
|
|
82
|
+
return [];
|
|
83
|
+
}
|
|
84
|
+
getSnippet(connection, lang) {
|
|
85
|
+
const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
86
|
+
if (lang === 'python') {
|
|
87
|
+
return `import os
|
|
88
|
+
import redshift_connector
|
|
89
|
+
import pandas as pd
|
|
90
|
+
|
|
91
|
+
# Connection: ${connection.name}
|
|
92
|
+
# Type: redshift
|
|
93
|
+
# Credentials loaded from environment variables (set in .env file)
|
|
94
|
+
conn = redshift_connector.connect(
|
|
95
|
+
host=os.environ["${prefix}_HOST"],
|
|
96
|
+
database=os.environ["${prefix}_DATABASE"],
|
|
97
|
+
user=os.environ["${prefix}_USER"],
|
|
98
|
+
password=os.environ["${prefix}_PASSWORD"],
|
|
99
|
+
port=int(os.environ.get("${prefix}_PORT", "5439"))
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
query = "SELECT * FROM your_table LIMIT 100"
|
|
104
|
+
df = pd.read_sql(query, conn)
|
|
105
|
+
print(f"Successfully loaded {len(df)} rows from ${connection.name}")
|
|
106
|
+
print(df.head())
|
|
107
|
+
finally:
|
|
108
|
+
conn.close()
|
|
109
|
+
`;
|
|
110
|
+
}
|
|
111
|
+
return `# Language ${lang} not supported for Redshift connector yet.`;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
exports.RedshiftConnector = RedshiftConnector;
|
|
115
|
+
// Databricks Connector - uses databricks-sql-connector
|
|
116
|
+
class DatabricksConnector {
|
|
117
|
+
constructor() {
|
|
118
|
+
this.type = 'databricks';
|
|
119
|
+
}
|
|
120
|
+
async test(connection) {
|
|
121
|
+
console.log(`[Databricks] Testing connection for ${connection.name}`);
|
|
122
|
+
if (!connection.host?.trim()) {
|
|
123
|
+
throw new Error('Databricks host (workspace URL) is required');
|
|
124
|
+
}
|
|
125
|
+
console.log(`[Databricks] Connection parameters validated for ${connection.name}`);
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
async getTables(connection) {
|
|
129
|
+
return [];
|
|
130
|
+
}
|
|
131
|
+
async getData(connection, tableName, page, limit) {
|
|
132
|
+
throw new Error('Use the snippet to query Databricks directly.');
|
|
133
|
+
}
|
|
134
|
+
async getSchema(connection, tableName) {
|
|
135
|
+
return [];
|
|
136
|
+
}
|
|
137
|
+
getSnippet(connection, lang) {
|
|
138
|
+
const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
139
|
+
if (lang === 'python') {
|
|
140
|
+
return `import os
|
|
141
|
+
from databricks import sql
|
|
142
|
+
import pandas as pd
|
|
143
|
+
|
|
144
|
+
# Connection: ${connection.name}
|
|
145
|
+
# Type: databricks
|
|
146
|
+
# Credentials loaded from environment variables (set in .env file)
|
|
147
|
+
connection = sql.connect(
|
|
148
|
+
server_hostname=os.environ["${prefix}_HOST"],
|
|
149
|
+
http_path=os.environ["${prefix}_HTTP_PATH"],
|
|
150
|
+
access_token=os.environ["${prefix}_TOKEN"]
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
try:
|
|
154
|
+
cursor = connection.cursor()
|
|
155
|
+
cursor.execute("SELECT * FROM your_table LIMIT 100")
|
|
156
|
+
df = pd.DataFrame(cursor.fetchall(), columns=[desc[0] for desc in cursor.description])
|
|
157
|
+
print(f"Successfully loaded {len(df)} rows from ${connection.name}")
|
|
158
|
+
print(df.head())
|
|
159
|
+
finally:
|
|
160
|
+
connection.close()
|
|
161
|
+
`;
|
|
162
|
+
}
|
|
163
|
+
return `# Language ${lang} not supported for Databricks connector yet.`;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
exports.DatabricksConnector = DatabricksConnector;
|
|
167
|
+
// ClickHouse Connector - uses clickhouse-connect
|
|
168
|
+
class ClickHouseConnector {
|
|
169
|
+
constructor() {
|
|
170
|
+
this.type = 'clickhouse';
|
|
171
|
+
}
|
|
172
|
+
async test(connection) {
|
|
173
|
+
console.log(`[ClickHouse] Testing connection for ${connection.name}`);
|
|
174
|
+
if (!connection.host?.trim()) {
|
|
175
|
+
throw new Error('ClickHouse host is required');
|
|
176
|
+
}
|
|
177
|
+
console.log(`[ClickHouse] Connection parameters validated for ${connection.name}`);
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
async getTables(connection) {
|
|
181
|
+
return [];
|
|
182
|
+
}
|
|
183
|
+
async getData(connection, tableName, page, limit) {
|
|
184
|
+
throw new Error('Use the snippet to query ClickHouse directly.');
|
|
185
|
+
}
|
|
186
|
+
async getSchema(connection, tableName) {
|
|
187
|
+
return [];
|
|
188
|
+
}
|
|
189
|
+
getSnippet(connection, lang) {
|
|
190
|
+
const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
191
|
+
if (lang === 'python') {
|
|
192
|
+
return `import os
|
|
193
|
+
import clickhouse_connect
|
|
194
|
+
import pandas as pd
|
|
195
|
+
|
|
196
|
+
# Connection: ${connection.name}
|
|
197
|
+
# Type: clickhouse
|
|
198
|
+
# Credentials loaded from environment variables (set in .env file)
|
|
199
|
+
client = clickhouse_connect.get_client(
|
|
200
|
+
host=os.environ["${prefix}_HOST"],
|
|
201
|
+
port=int(os.environ.get("${prefix}_PORT", "8123")),
|
|
202
|
+
username=os.environ.get("${prefix}_USER", "default"),
|
|
203
|
+
password=os.environ.get("${prefix}_PASSWORD", "")
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
query = "SELECT * FROM your_table LIMIT 100"
|
|
207
|
+
df = client.query_df(query)
|
|
208
|
+
print(f"Successfully loaded {len(df)} rows from ${connection.name}")
|
|
209
|
+
print(df.head())
|
|
210
|
+
`;
|
|
211
|
+
}
|
|
212
|
+
return `# Language ${lang} not supported for ClickHouse connector yet.`;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
exports.ClickHouseConnector = ClickHouseConnector;
|
|
216
|
+
// MongoDB Connector - uses pymongo
|
|
217
|
+
class MongoDBConnector {
|
|
218
|
+
constructor() {
|
|
219
|
+
this.type = 'mongodb';
|
|
220
|
+
}
|
|
221
|
+
async test(connection) {
|
|
222
|
+
console.log(`[MongoDB] Testing connection for ${connection.name}`);
|
|
223
|
+
if (!connection.host?.trim()) {
|
|
224
|
+
throw new Error('MongoDB connection string or host is required');
|
|
225
|
+
}
|
|
226
|
+
console.log(`[MongoDB] Connection parameters validated for ${connection.name}`);
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
async getTables(connection) {
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
232
|
+
async getData(connection, tableName, page, limit) {
|
|
233
|
+
throw new Error('Use the snippet to query MongoDB directly.');
|
|
234
|
+
}
|
|
235
|
+
async getSchema(connection, tableName) {
|
|
236
|
+
return [];
|
|
237
|
+
}
|
|
238
|
+
getSnippet(connection, lang) {
|
|
239
|
+
const prefix = connection.name.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
240
|
+
if (lang === 'python') {
|
|
241
|
+
return `import os
|
|
242
|
+
from pymongo import MongoClient
|
|
243
|
+
import pandas as pd
|
|
244
|
+
|
|
245
|
+
# Connection: ${connection.name}
|
|
246
|
+
# Type: mongodb
|
|
247
|
+
# Credentials loaded from environment variables (set in .env file)
|
|
248
|
+
# Use either a full connection string or host/user/password
|
|
249
|
+
connection_string = os.environ.get("${prefix}_CONNECTION_STRING",
|
|
250
|
+
f"mongodb://{os.environ.get('${prefix}_USER', '')}:{os.environ.get('${prefix}_PASSWORD', '')}@{os.environ['${prefix}_HOST']}")
|
|
251
|
+
|
|
252
|
+
client = MongoClient(connection_string)
|
|
253
|
+
db = client[os.environ.get("${prefix}_DATABASE", "test")]
|
|
254
|
+
|
|
255
|
+
# Example: Fetch documents from a collection
|
|
256
|
+
collection = db["your_collection"]
|
|
257
|
+
docs = list(collection.find().limit(100))
|
|
258
|
+
df = pd.DataFrame(docs)
|
|
259
|
+
print(f"Successfully loaded {len(df)} documents from ${connection.name}")
|
|
260
|
+
print(df.head())
|
|
261
|
+
|
|
262
|
+
client.close()
|
|
263
|
+
`;
|
|
264
|
+
}
|
|
265
|
+
return `# Language ${lang} not supported for MongoDB connector yet.`;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
exports.MongoDBConnector = MongoDBConnector;
|