@amir83nasr/10router 0.1.0

package/src/lib/db/repos/proxyPoolsRepo.js

@@ -0,0 +1,109 @@
+import { v4 as uuidv4 } from "uuid";
+import { getAdapter } from "../driver.js";
+import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
+
+function rowToPool(row) {
+  if (!row) return null;
+  const extra = parseJson(row.data, {});
+  return {
+    ...extra,
+    id: row.id,
+    isActive: row.isActive === 1 || row.isActive === true,
+    testStatus: row.testStatus,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}
+
+function poolToRow(p) {
+  const { id, isActive, testStatus, createdAt, updatedAt, ...rest } = p;
+  return {
+    id,
+    isActive: isActive === false ? 0 : 1,
+    testStatus: testStatus ?? null,
+    data: stringifyJson(rest),
+    createdAt,
+    updatedAt,
+  };
+}
+
+function upsert(db, p) {
+  const r = poolToRow(p);
+  db.run(
+    `INSERT INTO proxyPools(id, isActive, testStatus, data, createdAt, updatedAt)
+     VALUES(?, ?, ?, ?, ?, ?)
+     ON CONFLICT(id) DO UPDATE SET
+       isActive=excluded.isActive, testStatus=excluded.testStatus,
+       data=excluded.data, updatedAt=excluded.updatedAt`,
+    [r.id, r.isActive, r.testStatus, r.data, r.createdAt, r.updatedAt],
+  );
+}
+
+export async function getProxyPools(filter = {}) {
+  const db = await getAdapter();
+  const where = [];
+  const params = [];
+  if (filter.isActive !== undefined) {
+    where.push("isActive = ?");
+    params.push(filter.isActive ? 1 : 0);
+  }
+  if (filter.testStatus) {
+    where.push("testStatus = ?");
+    params.push(filter.testStatus);
+  }
+  const sql = `SELECT * FROM proxyPools${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
+  const list = db.all(sql, params).map(rowToPool);
+  list.sort((a, b) => new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0));
+  return list;
+}
+
+export async function getProxyPoolById(id) {
+  const db = await getAdapter();
+  return rowToPool(db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]));
+}
+
+export async function createProxyPool(data) {
+  const db = await getAdapter();
+  const now = new Date().toISOString();
+  const pool = {
+    id: data.id || uuidv4(),
+    name: data.name,
+    proxyUrl: data.proxyUrl,
+    noProxy: data.noProxy || "",
+    type: data.type || "http",
+    isActive: data.isActive !== undefined ? data.isActive : true,
+    strictProxy: data.strictProxy === true,
+    testStatus: data.testStatus || "unknown",
+    lastTestedAt: data.lastTestedAt || null,
+    lastError: data.lastError || null,
+    createdAt: now,
+    updatedAt: now,
+  };
+  upsert(db, pool);
+  return pool;
+}
+
+export async function updateProxyPool(id, data) {
+  const db = await getAdapter();
+  let result = null;
+  db.transaction(() => {
+    const row = db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]);
+    if (!row) return;
+    const merged = { ...rowToPool(row), ...data, updatedAt: new Date().toISOString() };
+    upsert(db, merged);
+    result = merged;
+  });
+  return result;
+}
+
+export async function deleteProxyPool(id) {
+  const db = await getAdapter();
+  let removed = null;
+  db.transaction(() => {
+    const row = db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]);
+    if (!row) return;
+    removed = rowToPool(row);
+    db.run(`DELETE FROM proxyPools WHERE id = ?`, [id]);
+  });
+  return removed;
+}

package/src/lib/db/repos/requestDetailsRepo.js

@@ -0,0 +1,287 @@
+import { getAdapter } from "../driver.js";
+import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
+
+const DEFAULT_MAX_RECORDS = 200;
+const DEFAULT_BATCH_SIZE = 20;
+const DEFAULT_FLUSH_INTERVAL_MS = 5000;
+const DEFAULT_MAX_JSON_SIZE = 5 * 1024;
+const CONFIG_CACHE_TTL_MS = 5000;
+
+let cachedConfig = null;
+let cachedConfigTs = 0;
+
+async function getObservabilityConfig() {
+  if (cachedConfig && Date.now() - cachedConfigTs < CONFIG_CACHE_TTL_MS) return cachedConfig;
+  try {
+    const { getSettings } = await import("./settingsRepo.js");
+    const settings = await getSettings();
+    const envRequestLogs = process.env.ENABLE_REQUEST_LOGS;
+    if (envRequestLogs !== undefined) {
+      const enabled = envRequestLogs.toLowerCase() === "true";
+      cachedConfig = {
+        enabled,
+        maxRecords:
+          settings.observabilityMaxRecords ||
+          parseInt(process.env.OBSERVABILITY_MAX_RECORDS || String(DEFAULT_MAX_RECORDS), 10),
+        batchSize:
+          settings.observabilityBatchSize ||
+          parseInt(process.env.OBSERVABILITY_BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10),
+        flushIntervalMs:
+          settings.observabilityFlushIntervalMs ||
+          parseInt(
+            process.env.OBSERVABILITY_FLUSH_INTERVAL_MS || String(DEFAULT_FLUSH_INTERVAL_MS),
+            10,
+          ),
+        maxJsonSize:
+          (settings.observabilityMaxJsonSize ||
+            parseInt(process.env.OBSERVABILITY_MAX_JSON_SIZE || "5", 10)) * 1024,
+      };
+      cachedConfigTs = Date.now();
+      return cachedConfig;
+    }
+    const envFallback = process.env.OBSERVABILITY_ENABLED !== "false";
+    const uiFlag = typeof settings.enableObservability === "boolean";
+    const enabled = uiFlag ? settings.enableObservability : envFallback;
+
+    cachedConfig = {
+      enabled,
+      maxRecords:
+        settings.observabilityMaxRecords ||
+        parseInt(process.env.OBSERVABILITY_MAX_RECORDS || String(DEFAULT_MAX_RECORDS), 10),
+      batchSize:
+        settings.observabilityBatchSize ||
+        parseInt(process.env.OBSERVABILITY_BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10),
+      flushIntervalMs:
+        settings.observabilityFlushIntervalMs ||
+        parseInt(
+          process.env.OBSERVABILITY_FLUSH_INTERVAL_MS || String(DEFAULT_FLUSH_INTERVAL_MS),
+          10,
+        ),
+      maxJsonSize:
+        (settings.observabilityMaxJsonSize ||
+          parseInt(process.env.OBSERVABILITY_MAX_JSON_SIZE || "5", 10)) * 1024,
+    };
+  } catch {
+    cachedConfig = {
+      enabled: false,
+      maxRecords: DEFAULT_MAX_RECORDS,
+      batchSize: DEFAULT_BATCH_SIZE,
+      flushIntervalMs: DEFAULT_FLUSH_INTERVAL_MS,
+      maxJsonSize: DEFAULT_MAX_JSON_SIZE,
+    };
+  }
+  cachedConfigTs = Date.now();
+  return cachedConfig;
+}
+
+let writeBuffer = [];
+let flushTimer = null;
+let isFlushing = false;
+
+function sanitizeHeaders(headers) {
+  if (!headers || typeof headers !== "object") return {};
+  const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token", "api-key"];
+  const sanitized = { ...headers };
+  for (const key of Object.keys(sanitized)) {
+    if (sensitiveKeys.some((s) => key.toLowerCase().includes(s))) delete sanitized[key];
+  }
+  return sanitized;
+}
+
+export const __test__ = { sanitizeHeaders };
+
+function generateDetailId(model) {
+  const timestamp = new Date().toISOString();
+  const random = Math.random().toString(36).substring(2, 8);
+  const modelPart = model ? model.replace(/[^a-zA-Z0-9-]/g, "-") : "unknown";
+  return `${timestamp}-${random}-${modelPart}`;
+}
+
+function truncateField(obj, maxSize) {
+  const str = JSON.stringify(obj || {});
+  if (str.length > maxSize) {
+    return { _truncated: true, _originalSize: str.length, _preview: str.substring(0, 200) };
+  }
+  return obj || {};
+}
+
+async function flushToDatabase() {
+  if (isFlushing) return;
+  if (writeBuffer.length === 0) return;
+  isFlushing = true;
+  try {
+    // Drain entire buffer (loop in case more pushed during await)
+    while (writeBuffer.length > 0) {
+      const items = writeBuffer.splice(0, writeBuffer.length);
+      const db = await getAdapter();
+      const config = await getObservabilityConfig();
+
+      db.transaction(() => {
+        for (const item of items) {
+          if (!item.id) item.id = generateDetailId(item.model);
+          if (!item.timestamp) item.timestamp = new Date().toISOString();
+          if (item.request?.headers) item.request.headers = sanitizeHeaders(item.request.headers);
+
+          const record = {
+            id: item.id,
+            provider: item.provider || null,
+            model: item.model || null,
+            connectionId: item.connectionId || null,
+            timestamp: item.timestamp,
+            status: item.status || null,
+            latency: item.latency || {},
+            tokens: item.tokens || {},
+            request: truncateField(item.request, config.maxJsonSize),
+            providerRequest: truncateField(item.providerRequest, config.maxJsonSize),
+            providerResponse: truncateField(item.providerResponse, config.maxJsonSize),
+            response: truncateField(item.response, config.maxJsonSize),
+            pxpipe: item.pxpipe || undefined,
+          };
+
+          db.run(
+            `INSERT INTO requestDetails(id, timestamp, provider, model, connectionId, status, data) VALUES(?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET timestamp = excluded.timestamp, provider = excluded.provider, model = excluded.model, connectionId = excluded.connectionId, status = excluded.status, data = excluded.data`,
+            [
+              record.id,
+              record.timestamp,
+              record.provider,
+              record.model,
+              record.connectionId,
+              record.status,
+              stringifyJson(record),
+            ],
+          );
+        }
+
+        const cnt = db.get(`SELECT COUNT(*) as c FROM requestDetails`);
+        if (cnt && cnt.c > config.maxRecords) {
+          db.run(
+            `DELETE FROM requestDetails WHERE id IN (SELECT id FROM requestDetails ORDER BY timestamp ASC LIMIT ?)`,
+            [cnt.c - config.maxRecords],
+          );
+        }
+      });
+    }
+  } catch (e) {
+    console.error("[requestDetailsRepo] Batch write failed:", e);
+  } finally {
+    isFlushing = false;
+  }
+}
+
+export async function saveRequestDetail(detail) {
+  const config = await getObservabilityConfig();
+  if (!config.enabled) {
+    return;
+  }
+
+  writeBuffer.push(detail);
+
+  // Trigger immediate flush if batch threshold reached.
+  // flushToDatabase() drains entire buffer in a loop, so all pushes during await are persisted.
+  if (writeBuffer.length >= config.batchSize) {
+    if (flushTimer) {
+      clearTimeout(flushTimer);
+      flushTimer = null;
+    }
+    flushToDatabase().catch((e) => console.error("[requestDetailsRepo] flush err:", e));
+  } else if (!flushTimer) {
+    flushTimer = setTimeout(() => {
+      flushTimer = null;
+      flushToDatabase().catch(() => {});
+    }, config.flushIntervalMs);
+  }
+}
+
+export async function getRequestDetails(filter = {}) {
+  const db = await getAdapter();
+  const conds = [];
+  const params = [];
+
+  if (filter.provider) {
+    conds.push("provider = ?");
+    params.push(filter.provider);
+  }
+  if (filter.model) {
+    conds.push("model = ?");
+    params.push(filter.model);
+  }
+  if (filter.connectionId) {
+    conds.push("connectionId = ?");
+    params.push(filter.connectionId);
+  }
+  if (filter.status) {
+    conds.push("status = ?");
+    params.push(filter.status);
+  }
+  if (filter.startDate) {
+    conds.push("timestamp >= ?");
+    params.push(new Date(filter.startDate).toISOString());
+  }
+  if (filter.endDate) {
+    conds.push("timestamp <= ?");
+    params.push(new Date(filter.endDate).toISOString());
+  }
+
+  const where = conds.length ? `WHERE ${conds.join(" AND ")}` : "";
+  const cntRow = db.get(`SELECT COUNT(*) as c FROM requestDetails ${where}`, params);
+  const totalItems = cntRow ? cntRow.c : 0;
+
+  const page = filter.page || 1;
+  const pageSize = filter.pageSize || 50;
+  const totalPages = Math.ceil(totalItems / pageSize);
+  const offset = (page - 1) * pageSize;
+
+  const rows = db.all(
+    `SELECT data FROM requestDetails ${where} ORDER BY timestamp DESC LIMIT ? OFFSET ?`,
+    [...params, pageSize, offset],
+  );
+  const details = rows.map((r) => parseJson(r.data, {}));
+
+  return {
+    details,
+    pagination: {
+      page,
+      pageSize,
+      totalItems,
+      totalPages,
+      hasNext: page < totalPages,
+      hasPrev: page > 1,
+    },
+  };
+}
+
+export async function getDistinctProviders() {
+  const db = await getAdapter();
+  const rows = db.all(
+    `SELECT DISTINCT provider FROM requestDetails WHERE provider IS NOT NULL ORDER BY provider ASC`,
+  );
+  return rows.map((r) => r.provider);
+}
+
+export async function getRequestDetailById(id) {
+  const db = await getAdapter();
+  const row = db.get(`SELECT data FROM requestDetails WHERE id = ?`, [id]);
+  return row ? parseJson(row.data, null) : null;
+}
+
+const _shutdownHandler = async () => {
+  if (flushTimer) {
+    clearTimeout(flushTimer);
+    flushTimer = null;
+  }
+  if (writeBuffer.length > 0) await flushToDatabase();
+};
+
+function ensureShutdownHandler() {
+  process.off("beforeExit", _shutdownHandler);
+  process.off("SIGINT", _shutdownHandler);
+  process.off("SIGTERM", _shutdownHandler);
+  process.off("exit", _shutdownHandler);
+
+  process.on("beforeExit", _shutdownHandler);
+  process.on("SIGINT", _shutdownHandler);
+  process.on("SIGTERM", _shutdownHandler);
+  process.on("exit", _shutdownHandler);
+}
+
+ensureShutdownHandler();

package/src/lib/db/repos/settingsRepo.js

@@ -0,0 +1,123 @@
+import { getAdapter } from "../driver.js";
+import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
+
+const DEFAULT_HEADROOM_URL = process.env.HEADROOM_URL || "http://localhost:8787";
+
+const DEFAULT_SETTINGS = {
+  cloudEnabled: false,
+  tunnelEnabled: false,
+  tunnelUrl: "",
+  tunnelProvider: "cloudflare",
+  tailscaleEnabled: false,
+  tailscaleUrl: "",
+  stickyRoundRobinLimit: 3,
+  providerStrategies: {},
+  quotaVisibility: {},
+  comboStrategy: "fallback",
+  comboStickyRoundRobinLimit: 1,
+  comboStrategies: {},
+  capacityAdapter: {
+    vision: { enabled: true, roundRobin: false, models: [] },
+    pdf: { enabled: false, roundRobin: false, models: [] },
+    audioInput: { enabled: true, roundRobin: false, models: [] },
+    videoInput: { enabled: false, roundRobin: false, models: [] },
+  },
+  requireLogin: true,
+  requireApiKey: true,
+  tunnelDashboardAccess: true,
+  authMode: "password",
+  ssoType: "oidc",
+  oidcIssuerUrl: "",
+  oidcClientId: "",
+  oidcClientSecret: "",
+  oidcScopes: "openid profile email",
+  oidcLoginLabel: "Sign in with OIDC",
+  samlEntryPoint: "",
+  samlIssuer: "urn:10router:sp",
+  samlCert: "",
+  samlLoginLabel: "Sign in with SAML SSO",
+  samlAttributeEmail: "email",
+  samlAttributeName: "name",
+  enableObservability: false,
+  observabilityMaxRecords: 1000,
+  observabilityBatchSize: 20,
+  observabilityFlushIntervalMs: 5000,
+  observabilityMaxJsonSize: 5,
+  outboundProxyEnabled: false,
+  outboundProxyPoolId: "",
+  outboundProxyUrl: "",
+  outboundNoProxy: "",
+  rtkEnabled: true,
+  headroomEnabled: false,
+  headroomUrl: DEFAULT_HEADROOM_URL,
+  headroomCompressUserMessages: false,
+  headroomTimeoutMs: 3000,
+  cavemanEnabled: false,
+  cavemanLevel: "full",
+  ponytailEnabled: false,
+  ponytailLevel: "full",
+  pxpipeEnabled: false,
+  pxpipeAutoInstall: true,
+  pxpipeMinChars: 25000,
+  pxpipeTimeoutMs: 15000,
+};
+
+async function readRaw() {
+  const db = await getAdapter();
+  const row = db.get(`SELECT data FROM settings WHERE id = 1`);
+  return row ? parseJson(row.data, {}) : {};
+}
+
+// Merge raw settings with defaults; backward-compat for missing keys
+export function mergeWithDefaults(raw) {
+  const merged = { ...DEFAULT_SETTINGS, ...(raw || {}) };
+  for (const [key, defVal] of Object.entries(DEFAULT_SETTINGS)) {
+    if (merged[key] === undefined) {
+      if (
+        key === "outboundProxyEnabled" &&
+        typeof merged.outboundProxyUrl === "string" &&
+        merged.outboundProxyUrl.trim()
+      ) {
+        merged[key] = true;
+      } else {
+        merged[key] = defVal;
+      }
+    }
+  }
+  return merged;
+}
+
+export async function getSettings() {
+  const raw = await readRaw();
+  return mergeWithDefaults(raw);
+}
+
+// Atomic read-merge-write inside transaction (prevents losing concurrent updates)
+export async function updateSettings(updates) {
+  const db = await getAdapter();
+  let next;
+  db.transaction(function () {
+    const row = db.get(`SELECT data FROM settings WHERE id = 1`);
+    const current = row ? parseJson(row.data, {}) : {};
+    next = { ...current, ...updates };
+    db.run(
+      `INSERT INTO settings(id, data) VALUES(1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data`,
+      [stringifyJson(next)],
+    );
+  });
+  return mergeWithDefaults(next);
+}
+
+export async function isCloudEnabled() {
+  const settings = await getSettings();
+  return settings.cloudEnabled === true;
+}
+
+export async function getCloudUrl() {
+  const settings = await getSettings();
+  return settings.cloudUrl || process.env.CLOUD_URL || process.env.NEXT_PUBLIC_CLOUD_URL || "";
+}
+
+export async function exportSettings() {
+  return await readRaw();
+}