@outsidedata/dolex 0.1.0

package/src/mcp/tools/operation-log.ts

@@ -0,0 +1,110 @@
+/**
+ * Operation log — ring buffer of recent tool calls for bug reports.
+ * Captures sanitized metadata (patterns, data shapes, DSL structure)
+ * but never data values, connection strings, or file paths.
+ */
+
+export interface OperationMeta {
+  pattern?: string;
+  specId?: string;
+  alternativesCount?: number;
+  dataShape?: { rowCount: number; columnCount: number; columns: { name: string; type: string }[] };
+  dslStructure?: DslStructure;
+  sourceType?: string;
+  error?: string;
+  viewCount?: number;
+  filterCount?: number;
+  changes?: string[];
+}
+
+export interface DslStructure {
+  hasJoin: boolean;
+  hasFilter: boolean;
+  hasGroupBy: boolean;
+  hasHaving: boolean;
+  hasOrderBy: boolean;
+  hasLimit: boolean;
+  aggregates: string[];
+  windows: string[];
+}
+
+export interface OperationEntry {
+  toolName: string;
+  timestamp: number;
+  durationMs: number;
+  success: boolean;
+  meta: OperationMeta;
+}
+
+const MAX_ENTRIES = 10;
+
+const EMPTY_DSL_STRUCTURE: DslStructure = {
+  hasJoin: false, hasFilter: false, hasGroupBy: false,
+  hasHaving: false, hasOrderBy: false, hasLimit: false,
+  aggregates: [], windows: [],
+};
+
+export class OperationLog {
+  private entries: OperationEntry[] = [];
+
+  log(entry: OperationEntry): void {
+    this.entries.push(entry);
+    if (this.entries.length > MAX_ENTRIES) {
+      this.entries.shift();
+    }
+  }
+
+  getAll(): OperationEntry[] {
+    return [...this.entries].reverse();
+  }
+
+  getLast(): OperationEntry | undefined {
+    return this.entries.at(-1);
+  }
+
+  clear(): void {
+    this.entries = [];
+  }
+
+  get size(): number {
+    return this.entries.length;
+  }
+}
+
+export function extractDslStructure(query: unknown): DslStructure {
+  if (!query || typeof query !== 'object') {
+    return { ...EMPTY_DSL_STRUCTURE, aggregates: [], windows: [] };
+  }
+
+  const q = query as Record<string, unknown>;
+  const aggregates: string[] = [];
+  const windows: string[] = [];
+
+  if (Array.isArray(q.select)) {
+    for (const field of q.select) {
+      if (field && typeof field === 'object') {
+        const f = field as Record<string, unknown>;
+        if (f.aggregate && typeof f.aggregate === 'string') aggregates.push(f.aggregate);
+        if (f.window && typeof f.window === 'string') windows.push(f.window);
+      }
+    }
+  }
+
+  return {
+    hasJoin: Array.isArray(q.join) && q.join.length > 0,
+    hasFilter: Array.isArray(q.filter) && q.filter.length > 0,
+    hasGroupBy: Array.isArray(q.groupBy) && q.groupBy.length > 0,
+    hasHaving: Array.isArray(q.having) && q.having.length > 0,
+    hasOrderBy: Array.isArray(q.orderBy) && q.orderBy.length > 0,
+    hasLimit: q.limit != null,
+    aggregates: [...new Set(aggregates)],
+    windows: [...new Set(windows)],
+  };
+}
+
+export const operationLog = new OperationLog();
+
+/** Log an operation, silently ignoring errors so logging never breaks tool handlers. */
+export function logOperation(entry: OperationEntry): void {
+  try { operationLog.log(entry); } catch {}
+}

package/src/mcp/tools/query-source.ts

@@ -0,0 +1,69 @@
+/**
+ * MCP Tool: query_source
+ * Execute a DSL query against a source and return tabular results.
+ */
+
+import { z } from 'zod';
+import { dslQuerySchema } from './dsl-schemas.js';
+import { saveResult } from './result-cache.js';
+import { errorResponse, jsonResponse } from './shared.js';
+import { logOperation, extractDslStructure } from './operation-log.js';
+
+export const querySourceInputSchema = z.object({
+  sourceId: z.string().describe('Source ID from add_source'),
+  table: z.string().describe('Table name within the source'),
+  query: dslQuerySchema.describe('Declarative query (join, select, groupBy, filter, orderBy, limit). Use join to combine tables; use table.field dot notation for joined table fields. Example: { select: ["region", { field: "revenue", aggregate: "sum", as: "total" }], groupBy: ["region"], orderBy: [{ field: "total", direction: "desc" }], limit: 10 }'),
+  maxRows: z.number().optional().describe('Max rows to return (default: 20)'),
+});
+
+export function handleQuerySource(deps: { sourceManager: any }) {
+  return async (args: z.infer<typeof querySourceInputSchema>) => {
+    const start = Date.now();
+
+    const query = { ...args.query };
+    if (args.maxRows) {
+      query.limit = Math.min(args.maxRows, query.limit ?? Infinity);
+    }
+
+    const result = await deps.sourceManager.queryDsl(args.sourceId, args.table, query);
+    if (!result.ok) {
+      logOperation({
+        toolName: 'query_source',
+        timestamp: start,
+        durationMs: Date.now() - start,
+        success: false,
+        meta: {
+          dslStructure: extractDslStructure(args.query),
+          error: result.error,
+        },
+      });
+      return errorResponse(result.error);
+    }
+
+    const resultId = saveResult(result.rows, result.columns);
+
+    logOperation({
+      toolName: 'query_source',
+      timestamp: start,
+      durationMs: Date.now() - start,
+      success: true,
+      meta: {
+        dslStructure: extractDslStructure(args.query),
+        dataShape: {
+          rowCount: result.rows.length,
+          columnCount: result.columns.length,
+          columns: result.columns,
+        },
+      },
+    });
+
+    return jsonResponse({
+      resultId,
+      columns: result.columns,
+      rows: result.rows,
+      totalRows: result.totalRows,
+      truncated: result.truncated ?? false,
+      queryTimeMs: Date.now() - start,
+    });
+  };
+}

package/src/mcp/tools/refine.ts

@@ -0,0 +1,385 @@
+/**
+ * MCP Tool: refine_visualization
+ * Takes a current visualization spec and a refinement request,
+ * returns an updated spec.
+ *
+ * Supports both atomic VisualizationSpec and CompoundVisualizationSpec.
+ * Accepts specId from a previous visualize/refine call.
+ */
+
+import { z } from 'zod';
+import type {
+  VisualizationSpec,
+  CompoundVisualizationSpec,
+  RefineOutput,
+  ColorPaletteName,
+} from '../../types.js';
+import { isCompoundSpec } from '../../types.js';
+import { buildChartHtml, isHtmlPatternSupported } from '../../renderers/html/index.js';
+import { buildCompoundHtml } from '../../renderers/html/builders/compound.js';
+import { specStore } from '../spec-store.js';
+import { errorResponse, jsonResponse, htmlResponse } from './shared.js';
+import { logOperation } from './operation-log.js';
+
+export const refineInputSchema = z.object({
+  specId: z.string().describe('Spec ID from a previous visualize or refine call'),
+  refinement: z.string().describe('What to change — e.g., "sort by value descending", "use warm palette", "highlight Engineering and Science", "mute to 30% opacity"'),
+  selectAlternative: z.string().optional().describe('Switch to an alternative pattern by ID (e.g., "line"). Uses alternatives stored from the original visualize call.'),
+});
+
+// ─── PALETTE LOOKUP ─────────────────────────────────────────────────────────
+
+/** Map natural language color terms to valid ColorPaletteName values */
+const PALETTE_LOOKUP: [RegExp, ColorPaletteName, string][] = [
+  [/\bblue\s*red\b|\bblue[-_]?red\b/, 'blueRed', 'blueRed diverging palette'],
+  [/\bgreen\s*purple\b|\bgreen[-_]?purple\b/, 'greenPurple', 'greenPurple diverging palette'],
+  [/\bteal\s*orange\b|\bteal[-_]?orange\b/, 'tealOrange', 'tealOrange diverging palette'],
+  [/\bred\s*green\b|\bred[-_]?green\b/, 'redGreen', 'redGreen diverging palette'],
+  [/\btraffic\s*light\b|\bstoplight\b/, 'traffic-light', 'traffic-light palette'],
+  [/\bprofit[-\s]*loss\b/, 'profit-loss', 'profit-loss palette'],
+  [/\btemperature\b/, 'temperature', 'temperature palette'],
+  [/\bwarm\b|\borange\b/, 'warm', 'warm palette'],
+  [/\bpurple\b|\bviolet\b/, 'purple', 'purple palette'],
+  [/\bgreen\b/, 'green', 'green palette'],
+  [/\bblue\b|\bsequential\b|\bcool\b/, 'blue', 'blue palette'],
+  [/\bdiverging\b/, 'blueRed', 'blueRed diverging palette'],
+  [/\bcategorical\b|\bdistinct\b|\bqualitative\b/, 'categorical', 'categorical palette'],
+];
+
+function matchPalette(text: string): [ColorPaletteName, string] | null {
+  const lower = text.toLowerCase();
+  for (const [regex, palette, label] of PALETTE_LOOKUP) {
+    if (regex.test(lower)) return [palette, label];
+  }
+  return null;
+}
+
+// ─── HIGHLIGHT PARSING ──────────────────────────────────────────────────────
+
+/** Parse highlight values from a refinement string */
+function parseHighlight(refinement: string, lower: string): {
+  values: string[];
+  color?: string;
+  mutedOpacity?: number;
+} | null {
+  // "highlight X and Y" or "highlight X, Y, Z"
+  const highlightMatch = lower.match(
+    /highlight\s+(.+?)(?:\s+in\s+(#[0-9a-f]{3,8}|\w+))?$/
+  );
+  if (!highlightMatch) return null;
+
+  const rawValues = highlightMatch[1]
+    // Remove trailing "in <color>" that may have been greedily captured
+    .replace(/\s+in\s+#[0-9a-f]{3,8}$/i, '')
+    .replace(/\s+in\s+\w+$/i, '');
+
+  // Split on " and ", commas, or " & "
+  const values = rawValues
+    .split(/\s*(?:,\s*|\s+and\s+|\s*&\s*)\s*/)
+    .map(v => v.trim())
+    .filter(Boolean)
+    // Restore original casing from the refinement string
+    .map(v => {
+      const idx = refinement.toLowerCase().indexOf(v);
+      return idx >= 0 ? refinement.slice(idx, idx + v.length) : v;
+    });
+
+  if (values.length === 0) return null;
+
+  const result: { values: string[]; color?: string; mutedOpacity?: number } = { values };
+  if (highlightMatch[2]) result.color = highlightMatch[2];
+
+  return result;
+}
+
+/** Parse muted opacity from refinement */
+function parseMutedOpacity(lower: string): number | null {
+  const match = lower.match(/mute\w*\s+(?:to\s+)?(\d+)%\s*opacity/);
+  if (match) return parseInt(match[1], 10) / 100;
+  const match2 = lower.match(/opacity\s+(\d+)%/);
+  if (match2) return parseInt(match2[1], 10) / 100;
+  const match3 = lower.match(/mute\w*\s+(?:to\s+)?(?:0?\.\d+)/);
+  if (match3) {
+    const val = parseFloat(match3[0].match(/\d+\.\d+/)![0]);
+    return val > 0 && val <= 1 ? val : null;
+  }
+  return null;
+}
+
+// ─── APPLY REFINEMENT ───────────────────────────────────────────────────────
+
+/** Apply refinements to an atomic visualization spec */
+function applyRefinement(spec: VisualizationSpec, refinement: string): RefineOutput {
+  const changes: string[] = [];
+  const updated = JSON.parse(JSON.stringify(spec)) as VisualizationSpec;
+  const lower = refinement.toLowerCase();
+
+  // Sort refinements
+  if (lower.includes('sort') && lower.includes('desc')) {
+    if (updated.encoding.x) updated.encoding.x.sort = 'descending';
+    if (updated.encoding.y) updated.encoding.y.sort = 'descending';
+    changes.push('Applied descending sort');
+  } else if (lower.includes('sort') && lower.includes('asc')) {
+    if (updated.encoding.x) updated.encoding.x.sort = 'ascending';
+    if (updated.encoding.y) updated.encoding.y.sort = 'ascending';
+    changes.push('Applied ascending sort');
+  }
+
+  // Limit refinements
+  const limitMatch = lower.match(/(?:top|limit|first)\s*(\d+)/);
+  if (limitMatch) {
+    const limit = parseInt(limitMatch[1], 10);
+    updated.data = updated.data.slice(0, limit);
+    changes.push(`Limited data to top ${limit} rows`);
+  }
+
+  // Orientation refinements
+  if (lower.includes('horizontal') || lower.includes('flip')) {
+    const { x, y } = updated.encoding;
+    if (x && y) {
+      updated.encoding.x = y;
+      updated.encoding.y = x;
+      changes.push('Flipped axes for horizontal layout');
+    }
+  }
+
+  // Title refinements
+  const titleMatch = refinement.match(/title[:\s]+"?([^"]+)"?/i);
+  if (titleMatch) {
+    updated.title = titleMatch[1].trim();
+    changes.push(`Changed title to "${updated.title}"`);
+  }
+
+  // ── Palette refinements (replaces broken colorScheme block) ──
+  if (lower.includes('color') || lower.includes('palette') || lower.includes('scheme') ||
+      lower.includes('warm') || lower.includes('diverging') || lower.includes('sequential') ||
+      lower.includes('categorical')) {
+    const paletteResult = matchPalette(lower);
+    if (paletteResult) {
+      if (!updated.encoding.color) updated.encoding.color = {};
+      updated.encoding.color.palette = paletteResult[0];
+      changes.push(`Applied ${paletteResult[1]}`);
+    }
+  }
+
+  // ── Highlight refinements ──
+  if (lower.includes('highlight')) {
+    const highlight = parseHighlight(refinement, lower);
+    if (highlight && highlight.values.length > 0) {
+      // Validate highlight values against actual data values in the color field
+      const colorField = updated.encoding.color?.field;
+      if (colorField) {
+        const dataValues = new Set(updated.data.map(d => String(d[colorField]).toLowerCase()));
+        const validValues = highlight.values.filter(v => dataValues.has(v.toLowerCase()));
+        if (validValues.length > 0) {
+          if (!updated.encoding.color) updated.encoding.color = {};
+          updated.encoding.color.highlight = {
+            values: validValues,
+            ...(highlight.color ? { color: highlight.color } : {}),
+          };
+          changes.push(`Highlighted values: ${validValues.join(', ')}`);
+        }
+        // If no values match data, skip highlight (it's likely descriptive text like "highlight differences")
+      } else {
+        // No color field — apply highlight as-is (may be used for x/y field matching)
+        if (!updated.encoding.color) updated.encoding.color = {};
+        updated.encoding.color.highlight = {
+          values: highlight.values,
+          ...(highlight.color ? { color: highlight.color } : {}),
+        };
+        changes.push(`Highlighted values: ${highlight.values.join(', ')}`);
+      }
+    }
+  }
+
+  // ── Muted opacity refinements ──
+  if (lower.includes('mute') || lower.includes('opacity')) {
+    const opacity = parseMutedOpacity(lower);
+    if (opacity !== null) {
+      if (!updated.encoding.color) updated.encoding.color = {};
+      if (!updated.encoding.color.highlight) {
+        // Set opacity on existing highlight or create a placeholder
+        updated.encoding.color.highlight = { values: [], mutedOpacity: opacity };
+      } else {
+        updated.encoding.color.highlight.mutedOpacity = opacity;
+      }
+      changes.push(`Set muted opacity to ${opacity}`);
+    }
+  }
+
+  // ── colorBy refinements (for alluvial, sankey, chord) ──
+  const colorByMatch = lower.match(/color\s*by\s+(\w+)/);
+  if (colorByMatch) {
+    updated.config.colorBy = colorByMatch[1];
+    changes.push(`Set colorBy to "${colorByMatch[1]}"`);
+  }
+
+  // Percentage format
+  if (lower.includes('percent') || lower.includes('%')) {
+    if (updated.encoding.y) {
+      updated.encoding.y.format = '.1%';
+      changes.push('Applied percentage format to y-axis');
+    }
+  }
+
+  if (changes.length === 0) {
+    changes.push(`Refinement noted: "${refinement}" — manual adjustment may be needed`);
+    updated.config._pendingRefinement = refinement;
+  }
+
+  return { spec: updated, changes };
+}
+
+/** Apply refinements to a compound visualization spec */
+function applyCompoundRefinement(
+  spec: CompoundVisualizationSpec,
+  refinement: string
+): RefineOutput {
+  const changes: string[] = [];
+  const updated = JSON.parse(JSON.stringify(spec)) as CompoundVisualizationSpec;
+  const lower = refinement.toLowerCase();
+
+  // ── Remove table ──
+  if (lower.includes('remove') && lower.includes('table')) {
+    const chartView = updated.views.find(v => v.type === 'chart');
+    if (chartView?.chart) {
+      // Unwrap compound to atomic spec
+      const atomicSpec: VisualizationSpec = {
+        ...chartView.chart,
+        data: updated.data,
+      } as VisualizationSpec;
+      return { spec: atomicSpec, changes: ['Removed table, returned atomic chart spec'] };
+    }
+  }
+
+  // ── Layout changes ──
+  if (lower.includes('side by side') || lower.includes('columns') || lower.includes('right') || lower.includes('left')) {
+    updated.layout.type = 'columns';
+    changes.push('Changed layout to side-by-side columns');
+  } else if (lower.includes('stack') || lower.includes('rows') || lower.includes('below') || lower.includes('above')) {
+    updated.layout.type = 'rows';
+    changes.push('Changed layout to stacked rows');
+  }
+
+  // ── Change interaction field ──
+  const highlightFieldMatch = lower.match(/highlight\s+(?:by|on)\s+(\w+)/);
+  if (highlightFieldMatch) {
+    const field = highlightFieldMatch[1];
+    updated.interactions = [{ type: 'highlight', field }];
+    changes.push(`Changed highlight interaction field to "${field}"`);
+  }
+
+  // ── Table column visibility ──
+  const hideMatch = lower.match(/hide\s+(?:the\s+)?(\w+)\s+column/);
+  if (hideMatch) {
+    const colName = hideMatch[1].toLowerCase();
+    const tableView = updated.views.find(v => v.type === 'table');
+    if (tableView?.table?.columns) {
+      tableView.table.columns = tableView.table.columns.filter(
+        c => c.field.toLowerCase() !== colName && (c.title || '').toLowerCase() !== colName
+      );
+      changes.push(`Hidden "${colName}" column from table`);
+    }
+  }
+
+  // ── Title change ──
+  const titleMatch = refinement.match(/title[:\s]+"?([^"]+)"?/i);
+  if (titleMatch) {
+    updated.title = titleMatch[1].trim();
+    // Also update chart view title
+    const chartView = updated.views.find(v => v.type === 'chart');
+    if (chartView?.chart) chartView.chart.title = updated.title;
+    changes.push(`Changed title to "${updated.title}"`);
+  }
+
+  // ── Delegate chart-specific refinements to the chart view ──
+  if (changes.length === 0) {
+    const chartView = updated.views.find(v => v.type === 'chart');
+    if (chartView?.chart) {
+      const chartSpec: VisualizationSpec = {
+        ...chartView.chart,
+        data: updated.data,
+      } as VisualizationSpec;
+      const chartResult = applyRefinement(chartSpec, refinement);
+      // Write chart changes back to compound spec
+      const refinedChart = chartResult.spec as VisualizationSpec;
+      chartView.chart = {
+        pattern: refinedChart.pattern,
+        title: refinedChart.title,
+        encoding: refinedChart.encoding,
+        config: refinedChart.config,
+      };
+      updated.data = refinedChart.data;
+      changes.push(...chartResult.changes);
+    }
+  }
+
+  if (changes.length === 0) {
+    changes.push(`Refinement noted: "${refinement}" — manual adjustment may be needed`);
+  }
+
+  return { spec: updated, changes };
+}
+
+// ─── BUILD HTML FOR OUTPUT ──────────────────────────────────────────────────
+
+function buildOutputHtml(spec: VisualizationSpec | CompoundVisualizationSpec): string | undefined {
+  if (isCompoundSpec(spec)) {
+    return buildCompoundHtml(spec);
+  }
+  if (isHtmlPatternSupported(spec.pattern)) {
+    return buildChartHtml(spec);
+  }
+  return undefined;
+}
+
+// ─── HANDLER ────────────────────────────────────────────────────────────────
+
+export function handleRefine() {
+  return async (args: z.infer<typeof refineInputSchema>) => {
+    const start = Date.now();
+    let spec: VisualizationSpec | CompoundVisualizationSpec;
+
+    if (args.selectAlternative) {
+      const altSpec = specStore.getAlternative(args.specId, args.selectAlternative);
+      if (!altSpec) {
+        return errorResponse(`Alternative pattern "${args.selectAlternative}" not found for specId "${args.specId}". Check the alternatives list from the original visualize call.`);
+      }
+      spec = altSpec;
+    } else {
+      const stored = specStore.get(args.specId);
+      if (!stored) {
+        return errorResponse(`Spec "${args.specId}" not found. It may have expired. Re-run visualize to get a new specId.`);
+      }
+      spec = stored.spec;
+    }
+
+    const result = isCompoundSpec(spec)
+      ? applyCompoundRefinement(spec as CompoundVisualizationSpec, args.refinement)
+      : applyRefinement(spec as VisualizationSpec, args.refinement);
+
+    const outputHtml = buildOutputHtml(result.spec);
+    const newSpecId = specStore.updateSpec(args.specId, result.spec);
+    const body = { specId: newSpecId, changes: result.changes };
+
+    const pattern = isCompoundSpec(result.spec)
+      ? (result.spec as any).views?.find((v: any) => v.type === 'chart')?.chart?.pattern
+      : (result.spec as VisualizationSpec).pattern;
+    logOperation({
+      toolName: 'refine_visualization',
+      timestamp: start,
+      durationMs: Date.now() - start,
+      success: true,
+      meta: {
+        pattern,
+        specId: newSpecId,
+        changes: result.changes,
+      },
+    });
+
+    if (outputHtml) {
+      return htmlResponse(body, outputHtml);
+    }
+    return jsonResponse(body);
+  };
+}

package/src/mcp/tools/result-cache.ts

@@ -0,0 +1,61 @@
+/**
+ * In-memory result cache for query_source → visualize flow.
+ * Max 20 entries, 10-minute TTL.
+ */
+
+import { randomBytes } from 'crypto';
+
+interface CachedResult {
+  rows: Record<string, any>[];
+  columns: { name: string; type: string }[];
+  timestamp: number;
+}
+
+const MAX_ENTRIES = 20;
+const TTL_MS = 10 * 60 * 1000;
+
+const cache = new Map<string, CachedResult>();
+
+function evictExpired(): void {
+  const now = Date.now();
+  for (const [key, entry] of cache) {
+    if (now - entry.timestamp > TTL_MS) {
+      cache.delete(key);
+    }
+  }
+}
+
+export function saveResult(rows: Record<string, any>[], columns: { name: string; type: string }[]): string {
+  evictExpired();
+  while (cache.size >= MAX_ENTRIES) {
+    const oldest = cache.keys().next().value!;
+    cache.delete(oldest);
+  }
+  const resultId = 'qr-' + randomBytes(4).toString('hex');
+  cache.set(resultId, { rows, columns, timestamp: Date.now() });
+  return resultId;
+}
+
+export function getResult(resultId: string): { rows: Record<string, any>[]; columns: { name: string; type: string }[] } | null {
+  evictExpired();
+  const entry = cache.get(resultId);
+  if (!entry) return null;
+  return { rows: entry.rows, columns: entry.columns };
+}
+
+export function clearResultCache(): void {
+  cache.clear();
+}
+
+export function resultCacheSize(): number {
+  return cache.size;
+}
+
+export function resultCacheStats(): { entries: number; maxEntries: number; ttlMs: number; totalRows: number } {
+  evictExpired();
+  let totalRows = 0;
+  for (const entry of cache.values()) {
+    totalRows += entry.rows.length;
+  }
+  return { entries: cache.size, maxEntries: MAX_ENTRIES, ttlMs: TTL_MS, totalRows };
+}