dbopt-engine 0.2.0

package/src/routes.js

@@ -0,0 +1,160 @@
+"use strict";
+const express = require("express");
+const { settings } = require("./config");
+const { targets } = require("./targets");
+const executor = require("./executor");
+
+const router = express.Router();
+
+/** run fn with a fresh connection to the request's target, always closed */
+async function withConn(target, fn) {
+  const conn = await target.adapter.connect(target);
+  try { return await fn(conn); } finally { target.adapter.close(conn); }
+}
+const wrap = (fn) => (req, res) =>
+  Promise.resolve(fn(req, res)).catch((e) => res.status(500).json({ detail: e.message }));
+
+const sqlPreview = (target, rec) => {
+  try { return executor.buildSqlForRecommendation(target, rec); }
+  catch (e) { return { error: e.message }; }
+};
+
+const targetSummary = (t) => ({
+  name: t.name, engine: t.dbEngine, host: t.host, port: t.port, database: t.database,
+  prerequisites: t.prerequisites,
+});
+
+/* ------------------------------- targets ------------------------------ */
+router.get("/targets", wrap(async (req, res) => res.json(targets.map(targetSummary))));
+
+/* ------------------------------- health ------------------------------- */
+router.get("/health", wrap(async (req, res) => {
+  const t = req.target;
+  try {
+    await withConn(t, (c) => t.adapter.ping(c));
+    const needed = (t.prerequisites.actions_needed || []).length;
+    res.json({ status: needed ? "degraded" : "ok", db_connection: "ok",
+               engine: t.dbEngine, database: t.name, prerequisites: t.prerequisites });
+  } catch (e) {
+    res.json({ status: "degraded", db_connection: "failed",
+               engine: t.dbEngine, database: t.name, error: e.message });
+  }
+}));
+
+/* ------------------------------- queries ------------------------------ */
+router.get("/queries/slow", wrap(async (req, res) => {
+  const rows = await withConn(req.target, (c) =>
+    req.target.adapter.getSlowQueries(c, parseInt(req.query.min_calls || "1", 10),
+                                         parseInt(req.query.limit || "30", 10)));
+  res.json(rows);
+}));
+
+/* ------------------------------- tables ------------------------------- */
+const fmtUnused = (raw) => raw.map((r) => ({
+  table: r.table_name, index: r.index_name, scans: r.idx_scan,
+  size_mb: Math.round((r.size_bytes / 1048576) * 100) / 100, indexdef: r.indexdef,
+}));
+
+router.get("/tables/unused-indexes", wrap(async (req, res) =>
+  res.json(fmtUnused(await withConn(req.target, (c) => req.target.adapter.getUnusedIndexes(c, req.target))))));
+router.get("/tables/growth", wrap(async (req, res) =>
+  res.json(await withConn(req.target, (c) => req.target.adapter.getTableGrowthStats(c)))));
+
+/* --------------------------- recommendations -------------------------- */
+router.get("/recommendations", wrap(async (req, res) => {
+  let recs = req.target.store.listRecommendations(req.query.status || null);
+  if (req.query.type) recs = recs.filter((r) => r.type === req.query.type);
+  res.json(recs.map((r) => ({ ...r, sql: sqlPreview(req.target, r) })));
+}));
+
+router.get("/recommendations/:id(\\d+)/sql", wrap(async (req, res) => {
+  const rec = req.target.store.getRecommendation(+req.params.id);
+  if (!rec) return res.status(404).json({ detail: "Recommendation not found" });
+  res.json(sqlPreview(req.target, rec));
+}));
+
+router.get("/recommendations/:id(\\d+)", wrap(async (req, res) => {
+  const rec = req.target.store.getRecommendation(+req.params.id);
+  if (!rec) return res.status(404).json({ detail: "Recommendation not found" });
+  res.json({ ...rec, sql: sqlPreview(req.target, rec) });
+}));
+
+router.post("/recommendations/:id(\\d+)/apply", wrap(async (req, res) => {
+  const rec = req.target.store.getRecommendation(+req.params.id);
+  if (!rec) return res.status(404).json({ detail: "Recommendation not found" });
+  if (!["pending", "failed", "skipped"].includes(rec.status)) {
+    return res.status(409).json({ detail: `Recommendation is already '${rec.status}'` });
+  }
+  const result = await executor.applyRecommendation(req.target, rec);
+  if (result.status === "failed") return res.status(500).json({ detail: `Apply failed: ${result.error}` });
+  res.json(result);
+}));
+
+router.post("/recommendations/:id(\\d+)/rollback", wrap(async (req, res) => {
+  const rec = req.target.store.getRecommendation(+req.params.id);
+  if (!rec) return res.status(404).json({ detail: "Recommendation not found" });
+  try { res.json(await executor.rollbackRecommendation(req.target, rec)); }
+  catch (e) { res.status(409).json({ detail: e.message }); }
+}));
+
+for (const [action, status] of [["acknowledge", "acknowledged"], ["reject", "rejected"], ["reopen", "pending"]]) {
+  router.post(`/recommendations/:id(\\d+)/${action}`, wrap(async (req, res) => {
+    const rec = req.target.store.updateStatus(+req.params.id, status);
+    if (!rec) return res.status(404).json({ detail: "Recommendation not found" });
+    res.json(rec);
+  }));
+}
+
+/* ------------------------------ analytics ----------------------------- */
+router.get("/analytics/config", wrap(async (req, res) => res.json({
+  engine: req.target.dbEngine, database: req.target.database, target: req.target.name,
+  databases: targets.map((t) => t.name),
+  scan_interval_minutes: settings.scanIntervalMinutes,
+  auto_create_indexes: settings.autoCreateIndexes,
+  auto_drop_unused_indexes: settings.autoDropUnusedIndexes,
+  min_improvement_pct: settings.minImprovementPct,
+  min_rows_for_flag: settings.minRowsForSeqScanFlag,
+  index_name_prefix: settings.indexNamePrefix,
+})));
+
+router.get("/analytics/actions", wrap(async (req, res) =>
+  res.json(req.target.store.listActions(parseInt(req.query.limit || "50", 10)))));
+
+router.get("/analytics/overview", wrap(async (req, res) => {
+  const t = req.target;
+  const all = t.store.listRecommendations();
+  const pending = all.filter((r) => r.status === "pending");
+  const byStatus = {};
+  for (const r of all) byStatus[r.status] = (byStatus[r.status] || 0) + 1;
+  const wasted = pending.filter((r) => r.type === "unused_index")
+    .reduce((s, r) => s + (r.detail.size_mb || 0), 0);
+  const [slow, growth] = await withConn(t, async (c) => [
+    await t.adapter.getSlowQueries(c, 1, 1), await t.adapter.getTableGrowthStats(c)]);
+  res.json({
+    pending_missing_index_count: pending.filter((r) => r.type === "missing_index").length,
+    pending_unused_index_count: pending.filter((r) => r.type === "unused_index").length,
+    total_recommendations_by_status: byStatus,
+    total_wasted_index_storage_mb: Math.round(wasted * 100) / 100,
+    slowest_query_right_now: slow[0] || null,
+    biggest_tables: growth.slice(0, 5),
+  });
+}));
+
+router.get("/analytics/tables/top-by-size", wrap(async (req, res) => {
+  const stats = await withConn(req.target, (c) => req.target.adapter.getTableGrowthStats(c));
+  res.json(stats.slice(0, parseInt(req.query.limit || "10", 10)));
+}));
+router.get("/analytics/queries/top-slow", wrap(async (req, res) =>
+  res.json(await withConn(req.target, (c) =>
+    req.target.adapter.getSlowQueries(c, 1, parseInt(req.query.limit || "10", 10))))));
+
+router.get("/analytics/wasted-storage", wrap(async (req, res) => {
+  const unused = req.target.store.listRecommendations("pending").filter((r) => r.type === "unused_index");
+  res.json({
+    unused_index_count: unused.length,
+    total_reclaimable_mb: Math.round(unused.reduce((s, r) => s + (r.detail.size_mb || 0), 0) * 100) / 100,
+    indexes: unused.map((r) => ({ table: r.table, index: r.detail.index_name, size_mb: r.detail.size_mb })),
+  });
+}));
+
+module.exports = { router, withConn, wrap };

package/src/scheduler.js

@@ -0,0 +1,132 @@
+"use strict";
+/** The autonomous analysis cycle: collect -> analyze -> verify -> execute.
+ *  Runs once per target database, sequentially, every scan interval. */
+const { settings } = require("./config");
+const { targets } = require("./targets");
+const executor = require("./executor");
+const { extractColumnsFromFilter, extractPartialPredicate } = require("./planParser");
+
+const log = (t, m) => console.log(`[scheduler] [${t.name}] ${m}`);
+
+async function columnsAlreadyIndexed(target, conn, table, columns) {
+  const existing = await target.adapter.getAllIndexesForTable(conn, table);
+  return existing.some((idx) => columns.every((col) => (idx.indexdef || "").includes(col)));
+}
+
+async function analyzeMissingIndexes(target, conn) {
+  const { adapter, store } = target;
+  const slow = await adapter.getSlowQueries(conn, 1, 50);
+  log(target, `fetched ${slow.length} candidate slow queries (${adapter.engine})`);
+
+  for (const q of slow) {
+    const text = String(q.query || "");
+    if (!text.trim().toUpperCase().startsWith("SELECT")) continue;
+
+    const { oldCost, findings, safeQuery } = await adapter.analyzeQuery(
+      conn, text, settings.minRowsForSeqScanFlag);
+    if (!findings.length) continue;
+
+    for (const f of findings) {
+      if (!f.table || !f.filter) continue;
+      let columns = extractColumnsFromFilter(f.filter);
+      let predicate = extractPartialPredicate(f.filter);
+      if (predicate && (!columns.length || (columns.length === 1 && columns[0] === predicate.column))) {
+        columns = [predicate.column];
+        if (!adapter.supportsPartialIndexes) predicate = null;
+      } else {
+        predicate = null;
+      }
+      if (!columns.length) continue;
+      if (await columnsAlreadyIndexed(target, conn, f.table, columns)) continue;
+
+      const desc = `${f.table}(${columns.join(", ")})` +
+        (predicate ? ` WHERE ${predicate.column} ${predicate.op}${predicate.value != null ? " " + predicate.value : ""}` : "");
+      try {
+        const baseCost = oldCost || f.cost || 1;
+        const { improvement, newCost } = await adapter.verifyIndexBenefit(
+          conn, safeQuery, f.table, columns, predicate, baseCost);
+        if (improvement >= settings.minImprovementPct) {
+          const detail = {
+            columns, reason: `Full scan on ${f.rows_scanned} rows`,
+            estimated_old_cost: baseCost, estimated_new_cost: newCost,
+          };
+          if (predicate) detail.predicate = predicate;
+          store.addRecommendation("missing_index", f.table, detail, improvement);
+          log(target, `recommendation: index on ${desc} -> ${improvement}% improvement`);
+        }
+      } catch (e) {
+        console.warn(`[scheduler] [${target.name}] verification failed for ${desc}: ${e.message}`);
+      }
+    }
+  }
+}
+
+async function analyzeUnusedIndexes(target, conn) {
+  const { adapter, store } = target;
+  const raw = await adapter.getUnusedIndexes(conn, target);
+  log(target, `found ${raw.length} unused indexes`);
+  for (const idx of raw) {
+    if (idx.index_name.startsWith(`${settings.indexNamePrefix}_`) &&
+        store.ownIndexCreatedWithin(idx.index_name, settings.unusedIndexGraceMinutes)) continue;
+    store.addRecommendation("unused_index", idx.table_name, {
+      index_name: idx.index_name,
+      size_mb: Math.round((idx.size_bytes / 1048576) * 100) / 100,
+      scans: idx.idx_scan,
+      original_index_definition: (idx.indexdef || "") + ";",
+    });
+  }
+}
+
+async function autoApplyPending(target) {
+  const { store } = target;
+  for (const rec of store.listRecommendations("pending")) {
+    if (rec.type === "missing_index" && !settings.autoCreateIndexes) continue;
+    if (rec.type === "unused_index" && !settings.autoDropUnusedIndexes) continue;
+    const fp = store.fingerprint(rec.type, rec.table, rec.detail);
+    if (store.recentlyActioned(fp, settings.reapplyCooldownMinutes)) {
+      store.updateStatus(rec.id, "skipped");
+      continue;
+    }
+    const result = await executor.applyRecommendation(target, rec);
+    log(target, `auto-applied rec #${rec.id} -> ${result.status}`);
+  }
+}
+
+async function runCycleFor(target) {
+  const { adapter, store } = target;
+  log(target, "starting analysis cycle...");
+  let conn;
+  try {
+    conn = await adapter.connect(target);
+    await analyzeMissingIndexes(target, conn);
+    await analyzeUnusedIndexes(target, conn);
+    try {
+      const tables = await adapter.getTableGrowthStats(conn);
+      store.recordSizeSnapshot(tables.reduce((s, t) => s + (t.total_size_bytes || 0), 0));
+    } catch (e) { console.warn(`[scheduler] [${target.name}] size snapshot failed:`, e.message); }
+    if (conn) { adapter.close(conn); conn = null; }
+    if (settings.autoCreateIndexes || settings.autoDropUnusedIndexes) await autoApplyPending(target);
+    log(target, "analysis cycle completed.");
+  } catch (e) {
+    console.error(`[scheduler] [${target.name}] cycle failed:`, e.message);
+  } finally {
+    if (conn) adapter.close(conn);
+  }
+}
+
+let running = false;
+async function runAnalysisCycle() {
+  if (running) return;
+  running = true;
+  try {
+    for (const target of targets) await runCycleFor(target);
+  } finally {
+    running = false;
+  }
+}
+
+function startScheduler() {
+  setInterval(runAnalysisCycle, settings.scanIntervalMinutes * 60e3);
+}
+
+module.exports = { runAnalysisCycle, startScheduler };