dbopt-engine 0.2.0

package/.env.example

@@ -0,0 +1,19 @@
+# One or more connection strings, comma-separated: one process can watch
+# several databases at once (mixed engines are fine).
+# Schemes: postgres:// mysql:// sqlserver://
+# Single database:
+DATABASE_URL=postgres://postgres:changeme@localhost:5432/mydb
+# Multiple databases (uncomment; commas in passwords must be encoded as %2C):
+# DATABASE_URL=postgres://user:pass@host1:5432/appdb,mysql://user:pass@host2:3306/shopdb
+
+PORT=8000
+SCAN_INTERVAL_MINUTES=15
+MIN_ROWS_FOR_SEQ_SCAN_FLAG=1000
+MIN_IMPROVEMENT_PCT_TO_RECOMMEND=30
+AUTO_CREATE_INDEXES=true
+AUTO_DROP_UNUSED_INDEXES=false
+INDEX_NAME_PREFIX=opt
+UNUSED_INDEX_GRACE_MINUTES=60
+REAPPLY_COOLDOWN_MINUTES=60
+STORAGE_LIMIT_MB=0
+STATE_DB_PATH=optimizer_state.db

package/README.md

@@ -0,0 +1,143 @@
+# dbopt-engine
+
+Self-optimizing database engine for **PostgreSQL**, **MySQL 8** and
+**Microsoft SQL Server** — the Node.js edition. One process can watch a
+single database or several at once (mixed engines are fine).
+
+It watches the queries your application actually runs, finds the ones wasting
+time on full table scans, **verifies** that an index would genuinely help
+*before* creating anything (HypoPG hypothetical indexes on Postgres,
+INVISIBLE-index trials on MySQL, trial builds on SQL Server), then acts on its
+own: it creates the winning indexes and can drop unused ones. Every action is
+recorded with rollback SQL. Ships with a live dashboard and an interactive
+schema-topology explorer with animated query flow.
+
+## Quick start
+
+```bash
+npm install -g dbopt-engine     # installs the `dbopt` command
+```
+
+Create a `.env` file in the directory you'll run it from, defining **all** of
+the variables below (copy `.env.example` from the package as a template):
+
+```bash
+# one database — or several, comma-separated (see "Multiple databases")
+DATABASE_URL=mysql://user:password@host:3306/mydb
+PORT=8000
+SCAN_INTERVAL_MINUTES=15
+MIN_ROWS_FOR_SEQ_SCAN_FLAG=1000
+MIN_IMPROVEMENT_PCT_TO_RECOMMEND=30
+AUTO_CREATE_INDEXES=true
+AUTO_DROP_UNUSED_INDEXES=false
+INDEX_NAME_PREFIX=opt
+UNUSED_INDEX_GRACE_MINUTES=60
+REAPPLY_COOLDOWN_MINUTES=60
+STORAGE_LIMIT_MB=0
+STATE_DB_PATH=optimizer_state.db
+```
+
+Then start it:
+
+```bash
+dbopt
+# dashboard: http://localhost:8000/dashboard
+# topology:  http://localhost:8000/topology
+```
+
+Or per-project:
+
+```bash
+npm install dbopt-engine
+cp node_modules/dbopt-engine/.env.example .env   # then edit DATABASE_URL etc.
+npx dbopt-engine
+```
+
+If any variable is missing, startup fails with a list of what's not set.
+
+## Connection strings
+
+The scheme picks the engine; the port is optional (engine default is used):
+
+```
+postgres://user:password@host:5432/mydb
+mysql://user:password@host:3306/mydb
+sqlserver://user:password@host:1433/mydb
+sqlserver://host:1433;database=mydb;user=sa;password=secret   (ADO style works too)
+```
+
+URL-encode special password characters: `!`→`%21` `@`→`%40` `%`→`%25` `#`→`%23` `,`→`%2C`.
+
+## Multiple databases
+
+Put several comma-separated URLs in `DATABASE_URL` and one process watches
+them all — each gets its own analysis cycle, recommendations, and history:
+
+```
+DATABASE_URL=postgres://user:pass@host1:5432/appdb,mysql://user:pass@host2:3306/shopdb
+```
+
+Both web pages grow a database selector in the header, and every API
+endpoint accepts `?db=<name>` to pick the database (without it, the first
+one is used). `GET /targets` lists the configured databases and their
+names — normally just the database name, disambiguated with `@host` when
+two targets share it.
+
+## Options
+
+All variables are **required**; set them in your `.env` (a CLI flag can
+override the ones that have one — flags win).
+
+| Flag | Env var | Suggested value | Meaning |
+|---|---|---|---|
+| `--database-url` | `DATABASE_URL` | — | one or more comma-separated connection strings |
+| `--port` | `PORT` | 8000 | HTTP port for dashboard/API |
+| `--scan-interval` | `SCAN_INTERVAL_MINUTES` | 15 | minutes between analysis cycles |
+| `--auto-create` | `AUTO_CREATE_INDEXES` | true | create verified indexes automatically |
+| `--auto-drop` | `AUTO_DROP_UNUSED_INDEXES` | false | drop unused indexes automatically |
+| `--min-improvement` | `MIN_IMPROVEMENT_PCT_TO_RECOMMEND` | 30 | required planner-cost drop (%) |
+| `--storage-limit-mb` | `STORAGE_LIMIT_MB` | 0 (off) | capacity for out-of-storage alerts |
+| `--state-db` | `STATE_DB_PATH` | optimizer_state.db | SQLite action-history file |
+| | `MIN_ROWS_FOR_SEQ_SCAN_FLAG` | 1000 | ignore full scans smaller than this |
+| | `INDEX_NAME_PREFIX` | opt | its indexes are named `opt_idx_…` |
+| | `UNUSED_INDEX_GRACE_MINUTES` | 60 | own new indexes can't be "unused" yet |
+| | `REAPPLY_COOLDOWN_MINUTES` | 60 | never repeat the same action within this |
+
+A `.env` file in the working directory is loaded automatically.
+
+## How the cycle works
+
+Every scan interval (and once at startup): **collect** slow queries from the
+engine's own statistics (`pg_stat_statements` / `performance_schema` /
+`sys.dm_exec_query_stats`) → **analyze** plans for large full scans and
+extract the filtered columns (composite `a AND b` supported; partial-index
+predicates on Postgres) → **verify** with a trial index against the exact
+slow query → **execute** only wins above the improvement bar, using
+non-blocking DDL (`CONCURRENTLY` / online DDL), recording rollback SQL.
+
+Safety: indexes it creates live in the `opt_` namespace so they never
+collide with ORM/migration-managed names; primary keys, unique constraints,
+(MySQL) FK-required and FULLTEXT indexes are never drop candidates; grace and
+cooldown windows prevent create/delete loops; one state file serves any
+number of targets with rows scoped per database.
+
+## HTTP API
+
+`/targets` (the configured databases), `/health`, `/queries/slow`,
+`/tables/unused-indexes`, `/tables/growth`,
+`/recommendations` (list · `POST /:id/apply` · `POST /:id/rollback` ·
+`/:id/reject`…), `/analytics/*` (overview, config, actions, wasted-storage),
+`/topology/schema`, `/topology/table/:name`, `POST /topology/analyze-query`,
+`/topology/alerts` — same API surface as the Python edition, so the bundled
+dashboard and topology pages work identically. Every endpoint accepts
+`?db=<name>` to pick one of the configured databases (default: the first).
+
+## Notes
+
+- Node.js ≥ 18 (better-sqlite3 native module installs prebuilt on common
+  platforms).
+- Oracle and MongoDB are available in the Python edition; they're not in
+  this package yet.
+- Server prerequisites are checked at startup and reported via `/health`
+  and the dashboard banner (e.g. `shared_preload_libraries` on Postgres,
+  `GRANT VIEW SERVER STATE` on SQL Server). MySQL needs nothing.

package/bin/cli.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+"use strict";
+
+/* Simple CLI: flags map onto env vars before config loads.
+ *   dbopt --database-url mysql://u:p@host:3306/db --port 8000
+ */
+const args = process.argv.slice(2);
+const FLAG_TO_ENV = {
+  "--database-url": "DATABASE_URL",
+  "--port": "PORT",
+  "--scan-interval": "SCAN_INTERVAL_MINUTES",
+  "--auto-create": "AUTO_CREATE_INDEXES",
+  "--auto-drop": "AUTO_DROP_UNUSED_INDEXES",
+  "--min-improvement": "MIN_IMPROVEMENT_PCT_TO_RECOMMEND",
+  "--storage-limit-mb": "STORAGE_LIMIT_MB",
+  "--state-db": "STATE_DB_PATH",
+};
+
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === "--help" || args[i] === "-h") {
+    console.log(`dbopt — self-optimizing database engine
+
+Usage:
+  dbopt --database-url <engine://user:pass@host:port/db> [options]
+
+Options:
+  --database-url <urls>   postgres:// | mysql:// | sqlserver:// connection string;
+                          comma-separate several to watch multiple databases
+  --port <n>              HTTP port for dashboard/API (default 8000)
+  --scan-interval <min>   analysis cycle interval in minutes (default 15)
+  --auto-create <bool>    create verified indexes automatically (default true)
+  --auto-drop <bool>      drop unused indexes automatically (default false)
+  --min-improvement <pct> required cost improvement to act (default 30)
+  --storage-limit-mb <n>  capacity for out-of-storage alerts (default off)
+  --state-db <path>       SQLite history file (default optimizer_state.db)
+
+Environment variables (or a .env file) work too; flags win.
+Dashboard: http://localhost:<port>/dashboard · Topology: /topology`);
+    process.exit(0);
+  }
+  const env = FLAG_TO_ENV[args[i]];
+  if (env) {
+    process.env[env] = args[i + 1];
+    i++;
+  }
+}
+
+require("../src/index").start().catch((e) => {
+  console.error("Failed to start:", e.message);
+  process.exit(1);
+});

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "dbopt-engine",
+  "version": "0.2.0",
+  "description": "Self-optimizing database engine: watches real query traffic, verifies missing indexes before creating them, drops unused ones, with a live dashboard and schema topology explorer. PostgreSQL, MySQL 8, SQL Server.",
+  "license": "MIT",
+  "author": "Evvo Technology <dev@evvotechnology.com>",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Infantraj123/db_optimizer.git"
+  },
+  "keywords": [
+    "database",
+    "index",
+    "optimizer",
+    "postgres",
+    "mysql",
+    "sqlserver",
+    "performance",
+    "autonomous"
+  ],
+  "main": "src/index.js",
+  "bin": {
+    "dbopt": "bin/cli.js"
+  },
+  "files": [
+    "bin",
+    "src",
+    "README.md",
+    ".env.example"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "start": "node bin/cli.js"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.11.1",
+    "dotenv": "^16.4.5",
+    "express": "^4.21.0",
+    "fast-xml-parser": "^4.5.0",
+    "mssql": "^11.0.1",
+    "mysql2": "^3.11.3",
+    "pg": "^8.13.0"
+  }
+}

package/src/adapters/index.js

@@ -0,0 +1,20 @@
+"use strict";
+
+/** Engine name -> adapter module. Adapters are stateless: connection
+ *  parameters come from the target object passed to connect(). */
+function adapterFor(engine) {
+  switch (engine) {
+    case "postgres": return require("./postgres");
+    case "mysql": return require("./mysql");
+    case "mssql": return require("./mssql");
+    case "oracle":
+    case "mongodb":
+      throw new Error(
+        `Engine '${engine}' is not yet supported in the Node.js package ` +
+        "(available in the Python edition). Supported here: postgres, mysql, mssql.");
+    default:
+      throw new Error(`Unsupported engine '${engine}' (use postgres, mysql or mssql)`);
+  }
+}
+
+module.exports = { adapterFor };

package/src/adapters/mssql.js

@@ -0,0 +1,236 @@
+"use strict";
+/** SQL Server 2017+ adapter: DMVs + SHOWPLAN_XML + real trial-index builds.
+ *  Uses a max-1 connection pool per cycle so session settings (SHOWPLAN)
+ *  stick across sequential statements. */
+const sql = require("mssql");
+const { XMLParser } = require("fast-xml-parser");
+const { calcImprovementPct } = require("../planParser");
+
+const engine = "mssql";
+const supportsPartialIndexes = false;
+const xml = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
+
+async function connect(target) {
+  const pool = new sql.ConnectionPool({
+    server: target.host, port: target.port, database: target.database,
+    user: target.user, password: target.password,
+    pool: { max: 1, min: 0 },
+    options: { encrypt: true, trustServerCertificate: true },
+    requestTimeout: 120000,
+  });
+  await pool.connect();
+  return pool;
+}
+const close = (p) => p.close().catch(() => {});
+const ping = async (p) => { await p.request().query("SELECT 1"); };
+
+async function ensurePrerequisites(target) {
+  const status = { checked: true, dmv_access: false, actions_needed: [] };
+  let p;
+  try {
+    p = await connect(target);
+    try {
+      await p.request().query("SELECT TOP 1 execution_count FROM sys.dm_exec_query_stats;");
+      status.dmv_access = true;
+    } catch {
+      status.actions_needed.push(
+        `The login cannot read performance DMVs. Run as an admin (in master): ` +
+        `GRANT VIEW SERVER STATE TO [${target.user}]; ` +
+        `(on Azure SQL DB: GRANT VIEW DATABASE STATE instead).`);
+    }
+  } catch (e) {
+    status.actions_needed.push(`Could not connect to the target database: ${e.message}`);
+  } finally { if (p) close(p); }
+  Object.assign(target.prerequisites, status);
+  return status;
+}
+
+async function getSlowQueries(p, minCalls = 1, limit = 30) {
+  try {
+    const r = await p.request()
+      .input("lim", sql.Int, limit).input("minc", sql.Int, minCalls)
+      .query(`
+        SELECT TOP (@lim)
+               CONVERT(varchar(32), qs.query_hash, 2) AS queryid,
+               SUBSTRING(t.text, (qs.statement_start_offset / 2) + 1,
+                 ((CASE WHEN qs.statement_end_offset = -1 THEN DATALENGTH(t.text)
+                        ELSE qs.statement_end_offset END - qs.statement_start_offset) / 2) + 1) AS query,
+               qs.execution_count AS calls,
+               qs.total_elapsed_time / 1000.0 / qs.execution_count AS mean_exec_time,
+               qs.total_elapsed_time / 1000.0 AS total_exec_time,
+               qs.total_rows AS rows
+        FROM sys.dm_exec_query_stats qs
+        CROSS APPLY sys.dm_exec_sql_text(qs.sql_handle) t
+        WHERE t.dbid = DB_ID() AND qs.execution_count >= @minc
+          AND t.text NOT LIKE 'SET SHOWPLAN%' AND t.text NOT LIKE '%dm_exec_query_stats%'
+          AND t.text NOT LIKE 'CREATE%' AND t.text NOT LIKE 'DROP%'
+          AND t.text NOT LIKE 'ALTER%' AND t.text NOT LIKE 'INSERT%'
+        ORDER BY mean_exec_time DESC;`);
+    return r.recordset;
+  } catch (e) {
+    console.warn("[mssql] cannot read query stats (VIEW SERVER STATE missing?):", e.message.slice(0, 100));
+    return [];
+  }
+}
+
+async function getUnusedIndexes(p, target) {
+  try {
+    const r = await p.request().query(`
+      SELECT o.name AS table_name, i.name AS index_name,
+             COALESCE(u.user_seeks + u.user_scans + u.user_lookups, 0) AS idx_scan,
+             COALESCE((SELECT SUM(ps.used_page_count) * 8 * 1024 FROM sys.dm_db_partition_stats ps
+                       WHERE ps.object_id = i.object_id AND ps.index_id = i.index_id), 0) AS size_bytes,
+             (SELECT STRING_AGG(c.name, ',') WITHIN GROUP (ORDER BY ic.key_ordinal)
+              FROM sys.index_columns ic
+              JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
+              WHERE ic.object_id = i.object_id AND ic.index_id = i.index_id
+                AND ic.is_included_column = 0) AS columns_list
+      FROM sys.indexes i
+      JOIN sys.objects o ON o.object_id = i.object_id AND o.type = 'U'
+      LEFT JOIN sys.dm_db_index_usage_stats u
+        ON u.object_id = i.object_id AND u.index_id = i.index_id AND u.database_id = DB_ID()
+      WHERE i.type = 2 AND i.is_primary_key = 0 AND i.is_unique = 0
+        AND i.is_unique_constraint = 0 AND i.is_hypothetical = 0 AND i.is_disabled = 0
+        AND COALESCE(u.user_seeks + u.user_scans + u.user_lookups, 0) = 0;`);
+    return r.recordset.map(x => ({
+      schemaname: target.database, table_name: x.table_name, index_name: x.index_name,
+      idx_scan: Number(x.idx_scan), size_bytes: Number(x.size_bytes || 0),
+      indexdef: `CREATE INDEX ${x.index_name} ON ${x.table_name} (${x.columns_list})`,
+    }));
+  } catch (e) {
+    console.warn("[mssql] cannot read index usage stats:", e.message.slice(0, 100));
+    return [];
+  }
+}
+
+async function getAllIndexesForTable(p, table) {
+  const r = await p.request().input("t", sql.NVarChar, table).query(`
+    SELECT i.name AS n,
+           (SELECT STRING_AGG(c.name, ',') WITHIN GROUP (ORDER BY ic.key_ordinal)
+            FROM sys.index_columns ic
+            JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
+            WHERE ic.object_id = i.object_id AND ic.index_id = i.index_id) AS cols
+    FROM sys.indexes i WHERE i.object_id = OBJECT_ID(@t) AND i.name IS NOT NULL;`);
+  return r.recordset.map(x => ({ indexname: x.n, indexdef: `INDEX ${x.n} ON ${table} (${x.cols})` }));
+}
+
+function prettySize(b) {
+  for (const u of ["bytes", "kB", "MB", "GB"]) {
+    if (b < 1024) return `${Math.round(b)} ${u}`;
+    b /= 1024;
+  }
+  return `${Math.round(b)} TB`;
+}
+
+async function getTableGrowthStats(p) {
+  const r = await p.request().query(`
+    SELECT t.name AS table_name,
+           SUM(CASE WHEN ps.index_id IN (0, 1) THEN ps.row_count ELSE 0 END) AS live_rows,
+           SUM(ps.reserved_page_count) * 8 * 1024 AS total_size_bytes
+    FROM sys.tables t
+    JOIN sys.dm_db_partition_stats ps ON ps.object_id = t.object_id
+    GROUP BY t.name ORDER BY total_size_bytes DESC;`);
+  return r.recordset.map(x => ({
+    table_name: x.table_name, live_rows: Number(x.live_rows || 0), dead_rows: 0,
+    total_size: prettySize(Number(x.total_size_bytes || 0)),
+    total_size_bytes: Number(x.total_size_bytes || 0), seq_scan: 0, idx_scan: 0,
+  }));
+}
+
+const substitute = (q, asString) =>
+  q.replace(/^\s*\(@[^)]*\)\s*/, "").replace(/@\w+/g, asString ? "'1'" : "1");
+
+async function estimatedPlan(p, safeQuery) {
+  // max:1 pool -> sequential requests share the session, so SHOWPLAN sticks
+  await p.request().batch("SET SHOWPLAN_XML ON;");
+  try {
+    const r = await p.request().batch(safeQuery);
+    const row = r.recordset && r.recordset[0];
+    return xml.parse(row[Object.keys(row)[0]]);
+  } finally {
+    await p.request().batch("SET SHOWPLAN_XML OFF;").catch(() => {});
+  }
+}
+
+function collectRelOps(node, out) {
+  if (node && typeof node === "object") {
+    if (node["@_PhysicalOp"]) out.push(node);
+    for (const [k, v] of Object.entries(node)) {
+      if (k.startsWith("@_")) continue;
+      if (Array.isArray(v)) v.forEach(x => collectRelOps(x, out));
+      else collectRelOps(v, out);
+    }
+  }
+  return out;
+}
+function findFirst(node, key) {
+  if (!node || typeof node !== "object") return null;
+  if (node[key] !== undefined) return node[key];
+  for (const [k, v] of Object.entries(node)) {
+    if (k.startsWith("@_")) continue;
+    const hit = Array.isArray(v) ? v.map(x => findFirst(x, key)).find(Boolean) : findFirst(v, key);
+    if (hit) return hit;
+  }
+  return null;
+}
+
+const cleanPredicate = (s) =>
+  s.replace(/[\[\]]/g, "").replace(/\b\w+\.(\w+\.)*/g, "");
+
+async function analyzeQuery(p, queryText, minRows) {
+  for (const asString of [false, true]) {
+    const safeQuery = substitute(queryText, asString);
+    if (!safeQuery.trim()) continue;
+    let plan;
+    try { plan = await estimatedPlan(p, safeQuery); } catch { continue; }
+    const stmt = findFirst(plan, "StmtSimple");
+    const stmtNode = Array.isArray(stmt) ? stmt[0] : stmt;
+    const oldCost = stmtNode ? parseFloat(stmtNode["@_StatementSubTreeCost"]) || null : null;
+    const findings = [];
+    for (const rel of collectRelOps(plan, [])) {
+      if (!["Table Scan", "Clustered Index Scan"].includes(rel["@_PhysicalOp"])) continue;
+      const rows = parseFloat(rel["@_EstimatedRowsRead"] || 0)
+        || parseFloat(rel["@_TableCardinality"] || 0)
+        || parseFloat(rel["@_EstimateRows"] || 0);
+      if (rows <= minRows) continue;
+      const obj = findFirst(rel, "Object");
+      const objNode = Array.isArray(obj) ? obj[0] : obj;
+      const table = objNode ? String(objNode["@_Table"] || "").replace(/[\[\]]/g, "") : null;
+      const pred = findFirst(rel, "Predicate");
+      const scalar = pred ? findFirst(pred, "ScalarOperator") : null;
+      const scalarNode = Array.isArray(scalar) ? scalar[0] : scalar;
+      const filter = scalarNode && scalarNode["@_ScalarString"]
+        ? cleanPredicate(scalarNode["@_ScalarString"]) : null;
+      findings.push({ table, filter, rows_scanned: Math.round(rows), cost: null });
+    }
+    return { oldCost, findings, safeQuery };
+  }
+  return { oldCost: null, findings: [], safeQuery: null };
+}
+
+async function verifyIndexBenefit(p, safeQuery, table, columns, predicate, oldCost) {
+  const { indexNameFor, validateIdentifier } = require("../executor");
+  if (predicate) throw new Error("Filtered-index candidates are not wired up for SQL Server yet");
+  validateIdentifier(table);
+  columns.forEach(validateIdentifier);
+  const indexName = indexNameFor(engine, table, columns);
+  let created = false;
+  try {
+    await p.request().batch(`CREATE INDEX ${indexName} ON ${table} (${columns.join(", ")})`);
+    created = true;
+    const plan = await estimatedPlan(p, safeQuery);
+    const stmt = findFirst(plan, "StmtSimple");
+    const stmtNode = Array.isArray(stmt) ? stmt[0] : stmt;
+    const newCost = stmtNode ? parseFloat(stmtNode["@_StatementSubTreeCost"]) || oldCost : oldCost;
+    return { improvement: calcImprovementPct(oldCost, newCost), newCost };
+  } finally {
+    if (created) await p.request().batch(`DROP INDEX ${indexName} ON ${table}`).catch(() => {});
+  }
+}
+
+module.exports = {
+  engine, supportsPartialIndexes,
+  connect, close, ping, ensurePrerequisites,
+  getSlowQueries, getUnusedIndexes, getAllIndexesForTable, getTableGrowthStats,
+  analyzeQuery, verifyIndexBenefit,
+};