@dataramen/cli 0.0.9 → 0.0.10

package/dist/code/services/pgConnector/index.js

@@ -0,0 +1,220 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PGSqlConnector = void 0;
+const pg_1 = __importDefault(require("pg"));
+const httpError_1 = require("../../utils/httpError");
+const getConnection = async ({ dbDatabase, dbPassword, dbUser, dbUrl, dbPort }) => {
+    const client = new pg_1.default.Client({
+        host: dbUrl,
+        user: dbUser,
+        database: dbDatabase,
+        password: dbPassword,
+        port: dbPort,
+        query_timeout: 10_000, // 10 seconds
+    });
+    await client.connect();
+    return client;
+};
+const extractPrimaryKeys = async (client) => {
+    const query = `
+      SELECT
+          kcu.table_name,
+          kcu.column_name,
+          kcu.ordinal_position
+      FROM
+        information_schema.table_constraints tc
+          JOIN information_schema.key_column_usage kcu
+             ON tc.constraint_name = kcu.constraint_name
+                 AND tc.table_schema = kcu.table_schema
+      WHERE
+          tc.constraint_type = 'PRIMARY KEY'
+      ORDER BY
+          kcu.table_name, kcu.ordinal_position;
+  `;
+    const result = await client.query(query);
+    const primaryKeysMap = {};
+    result.rows.forEach(row => {
+        const tableName = row.table_name;
+        const columnName = row.column_name;
+        if (!primaryKeysMap[tableName]) {
+            primaryKeysMap[tableName] = [];
+        }
+        primaryKeysMap[tableName].push(columnName);
+    });
+    return primaryKeysMap;
+};
+const getReferences = async (connection) => {
+    const query = `
+    SELECT
+      tc.table_name AS table_name,
+      kcu.column_name AS field,
+      ccu.table_name AS referenced_table,
+      ccu.column_name AS referenced_field
+    FROM
+      information_schema.table_constraints AS tc
+      JOIN information_schema.key_column_usage AS kcu
+        ON tc.constraint_name = kcu.constraint_name
+        AND tc.table_schema = kcu.table_schema
+      JOIN information_schema.constraint_column_usage AS ccu
+        ON ccu.constraint_name = tc.constraint_name
+        AND ccu.table_schema = tc.table_schema
+    WHERE tc.constraint_type = 'FOREIGN KEY';
+  `;
+    const res = await connection.query(query);
+    const result = {};
+    res.rows.forEach(row => {
+        if (!result[row.table_name]) {
+            result[row.table_name] = {};
+        }
+        result[row.table_name][row.field] = {
+            refTable: row.referenced_table,
+            refField: row.referenced_field
+        };
+    });
+    return result;
+};
+const inspectSchema = async (dataSource, connection) => {
+    const tableQuery = `SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = '${dataSource.dbSchema}'`;
+    const result = await connection.query(tableQuery);
+    const tables = result.rows;
+    const refs = await getReferences(connection);
+    const primaryKeys = await extractPrimaryKeys(connection);
+    const rows = tables.map(async (table) => {
+        const tableName = Object.values(table)[0];
+        const pgQuery = `
+        SELECT column_name, data_type
+        FROM information_schema.columns
+        WHERE
+            table_name = '${tableName}' and
+            table_schema = '${dataSource.dbSchema}'
+    `;
+        const { rows } = await connection.query(pgQuery);
+        const ref = refs[tableName];
+        return {
+            columns: rows
+                .map((column) => ({
+                name: column.column_name,
+                type: column.data_type,
+                isPrimary: primaryKeys[tableName]?.includes(column.column_name),
+                ref: ref?.[column.column_name] ? {
+                    table: ref[column.column_name].refTable,
+                    field: ref[column.column_name].refField,
+                } : undefined,
+            }))
+                .sort((col1, col2) => {
+                if (col1.isPrimary && col2.isPrimary) {
+                    return col1.name.localeCompare(col2.name);
+                }
+                return col1.isPrimary ? -1 : 1;
+            }),
+            createdAt: new Date(),
+            tableName,
+            updatedAt: new Date(),
+        };
+    });
+    return Promise.all(rows);
+};
+const extractTableNames = async (columnIds, connection) => {
+    const query = `select relname, attname, concat(pg_class.oid, '-', attnum) as row_key
+   from pg_attribute
+      left join pg_class on pg_attribute.attrelid = pg_class.oid
+   where
+     concat(pg_class.oid, '-', attnum) IN (${columnIds.join(", ")})
+     limit 25;`;
+    const result = await connection.query(query);
+    return result.rows.reduce((acc, row) => {
+        acc[row.row_key] = { table: row.relname, column: row.attname };
+        return acc;
+    }, {});
+};
+const executeQuery = async (query, connection, opts) => {
+    try {
+        console.log(`[PG CONN] Query: ${query}`);
+        const { rows, fields, command, rowCount } = await connection.query({
+            text: query,
+            rowMode: "array",
+        });
+        if (command === "UPDATE" || command === "INSERT" || command === "DELETE") {
+            if (rowCount != null && rowCount > 3 && opts.allowBulkUpdate !== true) {
+                throw new Error(`[PG CONN] Bulk update performed without permission.`);
+            }
+            return {
+                columns: [{ column: "affectedRows", alias: "Affected rows", full: "affectedRows" }],
+                rows: [[rowCount]],
+                query,
+            };
+        }
+        if (command === "SELECT") {
+            const cols = fields.map((column) => `'${column.tableID}-${column.columnID}'`);
+            const headerOG = await extractTableNames(cols, connection);
+            return {
+                columns: fields.map((column) => {
+                    const ogCol = headerOG[`${column.tableID}-${column.columnID}`];
+                    return {
+                        column: ogCol?.column || column.name,
+                        alias: column.name,
+                        table: ogCol?.table || '',
+                        full: ogCol ? ogCol.table + "." + ogCol.column : column.name,
+                    };
+                }),
+                rows: rows,
+                query,
+            };
+        }
+        throw new Error(`[PG CONN] Unsupported command: ${command}`);
+    }
+    catch (e) {
+        if (e instanceof httpError_1.HttpError) {
+            throw e;
+        }
+        throw new httpError_1.HttpError(400, e.message);
+    }
+};
+const withTransaction = async (client, fn) => {
+    await client.query("BEGIN");
+    try {
+        const result = await fn();
+        await client.query("COMMIT");
+        console.log(`[PG CONN] Commit`);
+        return result;
+    }
+    catch (e) {
+        await client.query("ROLLBACK");
+        console.log(`[PG CONN] Rollback`);
+        throw e;
+    }
+};
+const PGSqlConnector = async (dataSource) => {
+    const client = await getConnection(dataSource);
+    let _isClosed = false;
+    let isPathSet = false;
+    const withPathSet = async (fn) => {
+        if (!isPathSet) {
+            await client.query(`SET search_path TO ${dataSource.dbSchema}`);
+        }
+        return fn();
+    };
+    return {
+        dbType: 'postgres',
+        dataSource,
+        inspectSchema: () => inspectSchema(dataSource, client),
+        executeQuery: (query, opts) => withPathSet(() => {
+            if (opts.type === "SELECT") {
+                return executeQuery(query, client, opts);
+            }
+            return withTransaction(client, () => executeQuery(query, client, opts));
+        }),
+        checkConnection: async () => { },
+        isClosed: () => _isClosed,
+        close: async () => {
+            if (_isClosed)
+                return;
+            _isClosed = true;
+            return client.end();
+        },
+    };
+};
+exports.PGSqlConnector = PGSqlConnector;

package/dist/code/services/userSqlPromptRunner/index.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runRawSqlQuery = exports.runUserSqlQuery = exports.runUserSqlPrompt = exports.generateSqlFromPrompt = void 0;
+const httpError_1 = require("../../utils/httpError");
+const openai_1 = require("../openai");
+const connectorManager_1 = require("../connectorManager");
+const prompts_1 = require("../../utils/prompts");
+const db_1 = require("../../repository/db");
+const sql_builder_1 = require("sql-builder");
+const typeorm_1 = require("typeorm");
+const rawSql_1 = require("../../utils/rawSql");
+const sanitizeQuery = (query) => {
+    // transform query from ```sql\nSELECT COUNT(*) FROM users;\n``` to SELECT COUNT(*) FROM users;
+    return query.replace(/```sql\n/g, "").replace(/\n```/g, "");
+};
+function getSize(size) {
+    return size || 20;
+}
+function getPage(page) {
+    return page || 0;
+}
+const generateSqlFromPrompt = async (datasourceId, userId, userPrompt) => {
+    const [dataSource, userSettings] = await Promise.all([
+        db_1.DataSourceRepository.findOne({
+            where: {
+                id: datasourceId,
+            },
+            relations: {
+                inspections: true,
+            }
+        }),
+        db_1.UserSettingsRepository.findOneBy({
+            user: {
+                id: userId,
+            }
+        })
+    ]);
+    if (!dataSource) {
+        throw new httpError_1.HttpError(404, "Data source not found");
+    }
+    const openAiToken = userSettings?.openAiToken;
+    if (!openAiToken) {
+        throw new httpError_1.HttpError(404, "API key not found. Please set API key in settings");
+    }
+    const model = userSettings?.model;
+    if (!model) {
+        throw new httpError_1.HttpError(404, "Model not found. Please set model in settings");
+    }
+    const dbString = (0, prompts_1.buildDbString)(dataSource.inspections);
+    const result = await (0, openai_1.executePrompt)({
+        model,
+        temperature: 0.5,
+        messages: [
+            {
+                role: "system",
+                content: `Act as a ${dataSource.dbType} query writer. You can only answer with plain query text. Do not write explanations. Always limit queries to max 100 rows. Here is db schema:\n${dbString}`,
+            },
+            {
+                role: "user",
+                content: userPrompt,
+            },
+        ],
+        openaiApiKey: openAiToken,
+    });
+    const query = sanitizeQuery(result.choices[0].message.content);
+    return {
+        sql: query,
+        dataSource,
+    };
+};
+exports.generateSqlFromPrompt = generateSqlFromPrompt;
+const runUserSqlPrompt = async (req, datasourceId, userId, userPrompt) => {
+    const { dataSource, sql } = await (0, exports.generateSqlFromPrompt)(datasourceId, userId, userPrompt);
+    const dbConnectionManager = await (0, connectorManager_1.getDynamicConnection)(dataSource, req);
+    const queryBuilder = new sql_builder_1.SQLManipulator(dataSource.dbType, sql);
+    queryBuilder
+        .setLimit(20);
+    return dbConnectionManager.executeQuery(queryBuilder.toExecutableSQL(), {
+        type: queryBuilder.getParsed().type,
+        allowBulkUpdate: false,
+    });
+};
+exports.runUserSqlPrompt = runUserSqlPrompt;
+function handleAlias(value) {
+    if (value.includes(" ") && !value.startsWith("'")) {
+        return `\`${value}\``;
+    }
+    return value;
+}
+const runUserSqlQuery = async (req, { table, variables, datasourceId, filters, joins, orderBy, size, page, columns, groupBy }) => {
+    const dataSource = await db_1.DataSourceRepository.findOneBy({
+        id: datasourceId,
+    });
+    const tables = [];
+    const allColumns = [];
+    if (!dataSource) {
+        throw new httpError_1.HttpError(404, "Data source not found");
+    }
+    const queryBuilder = new sql_builder_1.SQLManipulator(dataSource.dbType, "SELECT");
+    queryBuilder.setTable(table);
+    if (variables) {
+        queryBuilder.setParameters(variables);
+    }
+    if (queryBuilder.getParsed().type !== "SELECT") {
+        throw new httpError_1.HttpError(400, "Only SELECT queries are allowed in this endpoint");
+    }
+    queryBuilder.setLimit(size || 20);
+    queryBuilder.setOffset(size * page);
+    filters?.forEach((w) => {
+        queryBuilder.addWhere(processWhereCondition(w));
+    });
+    if (joins) {
+        queryBuilder.addJoin(...joins);
+    }
+    if (orderBy) {
+        queryBuilder.addOrderBy(...orderBy.map((o) => ({
+            ...o,
+            column: handleAlias(o.column),
+        })));
+    }
+    if (columns && columns.length > 0) {
+        queryBuilder.selectColumns(columns);
+    }
+    if (groupBy && groupBy.length > 0) {
+        groupBy.forEach((g) => queryBuilder.addGroupBy(g));
+    }
+    const parsed = queryBuilder.getParsed();
+    if (parsed.table) {
+        tables.push(parsed.table);
+    }
+    if (parsed.joins && parsed.joins.length > 0) {
+        parsed.joins.forEach((join) => {
+            tables.push(join.table);
+        });
+    }
+    const info = await db_1.DatabaseInspectionRepository.find({
+        where: {
+            tableName: (0, typeorm_1.In)(tables),
+            datasource: {
+                id: datasourceId,
+            },
+        },
+    });
+    for (const table of info) {
+        if (!table.columns)
+            continue;
+        for (const column of table.columns) {
+            allColumns.push({ column: column.name, table: table.tableName || '' });
+        }
+    }
+    const dbConnectionManager = await (0, connectorManager_1.getDynamicConnection)(dataSource, req);
+    const result = await dbConnectionManager.executeQuery(queryBuilder.toExecutableSQL(), {
+        type: queryBuilder.getParsed().type,
+        allowBulkUpdate: false,
+    });
+    return {
+        ...result,
+        tables,
+        allColumns,
+    };
+};
+exports.runUserSqlQuery = runUserSqlQuery;
+const runRawSqlQuery = async (req, { sql, variables, datasourceId, size, page }) => {
+    const dataSource = await db_1.DataSourceRepository.findOneBy({
+        id: datasourceId,
+    });
+    if (!dataSource) {
+        throw new httpError_1.HttpError(404, "Data source not found");
+    }
+    const queryType = (0, rawSql_1.detectQueryType)(sql);
+    let query = sql;
+    if (queryType === "SELECT") {
+        if (query.match(/LIMIT|OFFSET/ig)) {
+            throw new httpError_1.HttpError(400, "Queries are automatically paginated, do not use LIMIT and/or OFFSET");
+        }
+        const finalSize = getSize(size);
+        query = (0, rawSql_1.applyRawPagination)(query, finalSize, finalSize * getPage(page));
+    }
+    if (variables) {
+        query = (0, rawSql_1.replaceSqlVariables)(query, variables);
+    }
+    const dbConnectionManager = await (0, connectorManager_1.getDynamicConnection)(dataSource, req);
+    return dbConnectionManager.executeQuery(query, {
+        type: queryType,
+        allowBulkUpdate: false,
+    });
+};
+exports.runRawSqlQuery = runRawSqlQuery;
+function processWhereCondition(where) {
+    const w = { ...where };
+    switch (w.operator) {
+        case "IS NULL":
+        case "IS NOT NULL":
+            w.value = undefined;
+            break;
+        case "LIKE":
+        case "ILIKE":
+        case "NOT LIKE":
+        case "NOT ILIKE":
+            w.value = w.value?.map((v) => ({
+                isColumn: v?.isColumn,
+                value: `%${v?.value}%`
+            }));
+            break;
+    }
+    return w;
+}

package/dist/code/types/connectors.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/code/utils/createRouter.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRouter = void 0;
+const createRouter = (fn) => {
+    return (instance, _, next) => {
+        fn(instance);
+        next();
+    };
+};
+exports.createRouter = createRouter;

package/dist/code/utils/httpError.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpError = void 0;
+class HttpError extends Error {
+    status;
+    message;
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+        this.message = message;
+    }
+}
+exports.HttpError = HttpError;

package/dist/code/utils/prompts.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildDbString = void 0;
+const buildDbString = (dbInspection) => {
+    return dbInspection.map((inspection) => {
+        // todo: FK
+        const columns = inspection.columns?.map((column) => `${column.name}: ${column.type}`).join(", ");
+        return `${inspection.tableName}: (${columns})`;
+    }).join("\n");
+};
+exports.buildDbString = buildDbString;

package/dist/code/utils/queryUtils.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseOrderQueryParam = parseOrderQueryParam;
+function parseOrderQueryParam(orderBy, defaultOrder = undefined) {
+    try {
+        if (orderBy) {
+            const parts = orderBy.split("&");
+            const order = {};
+            for (const part of parts) {
+                const split = part.split(":");
+                order[split[0]] = split[1];
+            }
+            return order;
+        }
+    }
+    catch (e) { }
+    return defaultOrder;
+}

package/dist/code/utils/rawSql.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyRawPagination = applyRawPagination;
+exports.replaceSqlVariables = replaceSqlVariables;
+exports.detectQueryType = detectQueryType;
+const httpError_1 = require("./httpError");
+function applyRawPagination(sql, limit, offset) {
+    const sqlWithoutSemicolon = sql.replace(/;\s*$/, '');
+    return `${sqlWithoutSemicolon.trim()} LIMIT ${limit} OFFSET ${offset}`;
+}
+function replaceSqlVariables(sql, values) {
+    return sql.replace(/:([a-zA-Z_][a-zA-Z0-9_]*)/g, (match, key) => {
+        if (!(key in values)) {
+            throw new Error(`Missing value for SQL variable: ${key}`);
+        }
+        return String(values[key]); // Raw replacement (no escaping/quoting)
+    });
+}
+// todo: improve
+function detectQueryType(sql) {
+    // Remove leading whitespace and comments
+    const cleanedSql = sql
+        .replace(/--.*$/gm, '') // Remove single-line comments
+        .replace(/\/\*[\s\S]*?\*\//g, '') // Remove block comments
+        .trim();
+    // Match the first keyword after any optional opening parentheses (e.g., CTEs)
+    const match = cleanedSql.match(/^\s*(\(?\s*)*([a-zA-Z]+)/);
+    if (!match) {
+        throw new httpError_1.HttpError(400, "Failed to detect query type");
+    }
+    return match[2].toUpperCase();
+}

package/dist/code/utils/request.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRequestUserId = exports.getRequestUserToken = exports.getRequestParams = exports.getRequestQuery = exports.getRequestPayload = void 0;
+const getRequestPayload = (request, validator) => {
+    const payload = request.body;
+    if (validator) {
+        validator(payload);
+    }
+    return payload;
+};
+exports.getRequestPayload = getRequestPayload;
+const getRequestQuery = (request, validator) => {
+    const query = request.query;
+    if (validator) {
+        validator(query);
+    }
+    return query;
+};
+exports.getRequestQuery = getRequestQuery;
+const getRequestParams = (request, validator) => {
+    const params = request.params;
+    if (validator) {
+        validator(params);
+    }
+    return params;
+};
+exports.getRequestParams = getRequestParams;
+const getRequestUserToken = (request) => {
+    return request.headers["phoenix-user-token"];
+};
+exports.getRequestUserToken = getRequestUserToken;
+const getRequestUserId = (request) => {
+    return request.headers["phoenix-user-id"];
+};
+exports.getRequestUserId = getRequestUserId;

package/dist/code/utils/token.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeOpenAIToken = void 0;
+const PLACEHOLDER = "********************************************";
+const sanitizeOpenAIToken = (token) => {
+    return token.slice(0, 4) + PLACEHOLDER + token.slice(token.length - 4);
+};
+exports.sanitizeOpenAIToken = sanitizeOpenAIToken;

package/dist/package.json

@@ -1 +1 @@
-{"name":"@dataramen/local-server","version":"0.0.34","license":"MIT","main":"code/proxy.js","dependencies":{"@fastify/cors":"^11.0.1","dotenv":"^16.5.0","fast-glob":"^3.3.3","fastify":"^5.3.2","mysql2":"^3.14.1","pg":"^8.15.6","sqlite3":"^5.1.7","typeorm":"^0.3.23"},"devDependencies":{"fs-extra":"^11.3.0","yargs":"^18.0.0"}}
+{"name":"@dataramen/local-server","version":"0.0.35","license":"MIT","main":"code/proxy.js","dependencies":{"@fastify/cors":"^11.0.1","dotenv":"^16.5.0","fast-glob":"^3.3.3","fastify":"^5.3.2","mysql2":"^3.14.1","pg":"^8.15.6","sqlite3":"^5.1.7","typeorm":"^0.3.23"},"devDependencies":{"fs-extra":"^11.3.0","yargs":"^18.0.0"}}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dataramen/cli",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "license": "MIT",
   "bin": {
     "dataramen": "bin/run.js"
@@ -14,5 +14,25 @@
   },
   "files": [
     "dist/"
+  ],
+  "keywords": [
+    "cli",
+    "sql",
+    "postgresql",
+    "mysql",
+    "database",
+    "schema",
+    "query-builder",
+    "data-browser",
+    "db-explorer",
+    "crud",
+    "data-inspector",
+    "visual-sql",
+    "no-code-sql",
+    "dataramen",
+    "nodejs",
+    "pm2",
+    "local-server",
+    "interactive-sql"
   ]
 }