@plmbr/notebook-intelligence 5.0.0

package/src/cell-output-bundle.ts

@@ -0,0 +1,176 @@
+// Copyright (c) Mehmet Bektas <mbektasgh@outlook.com>
+
+import { CodeCell } from '@jupyterlab/cells';
+import {
+  formatJupyterError,
+  joinMultilineString,
+  truncateToTokenCount
+} from './utils';
+import { IOutputContextItem } from './tokens';
+
+export const MAX_TOKENS_PER_OUTPUT = 4000;
+export const MAX_TOKENS_PER_TURN = 8000;
+// Above this raw base64 length (~150 KiB decoded), we drop the image and
+// emit a placeholder rather than blow up the context window.
+const MAX_IMAGE_BASE64_BYTES = 200_000;
+
+export type IMimeBundle = IOutputContextItem['mimeBundles'][number];
+export type IOutputContextBundle = IOutputContextItem;
+
+export interface IBundleOptions {
+  maxTokensPerOutput?: number;
+  maxTokensPerTurn?: number;
+  supportsVision?: boolean;
+}
+
+interface ICellLike {
+  outputArea: { model: { toJSON: () => any[] } };
+  model?: { sharedModel?: { getSource: () => string } };
+}
+
+export function cellOutputAsContextBundle(
+  cell: CodeCell | ICellLike,
+  opts: IBundleOptions = {}
+): IOutputContextBundle {
+  const maxOut = opts.maxTokensPerOutput ?? MAX_TOKENS_PER_OUTPUT;
+  const maxTurn = opts.maxTokensPerTurn ?? MAX_TOKENS_PER_TURN;
+  const supportsVision = opts.supportsVision ?? false;
+
+  const cellLike = cell as ICellLike;
+  const cellSource = cellLike.model?.sharedModel?.getSource() ?? '';
+  const outputs = cellLike.outputArea.model.toJSON();
+
+  const mimeBundles: IMimeBundle[] = [];
+  let truncated = false;
+  let isError = false;
+  let totalTokens = 0;
+
+  const remaining = (): number => Math.max(0, maxTurn - totalTokens);
+
+  const pushBundle = (mimeType: string, data: string): void => {
+    const perOutCap = Math.min(maxOut, remaining());
+    const {
+      text,
+      size,
+      truncated: didTruncate
+    } = truncateToTokenCount(data, perOutCap);
+    if (didTruncate) {
+      truncated = true;
+    }
+    if (size === 0 && text.length === 0) {
+      return;
+    }
+    mimeBundles.push({ mimeType, data: text, sizeTokens: size });
+    totalTokens += size;
+  };
+
+  for (const output of outputs) {
+    if (remaining() === 0) {
+      truncated = true;
+      break;
+    }
+
+    if (output.output_type === 'error') {
+      isError = true;
+      pushBundle('application/vnd.jupyter.error', formatJupyterError(output));
+      continue;
+    }
+
+    if (output.output_type === 'stream') {
+      pushBundle('text/plain', joinMultilineString(output.text));
+      continue;
+    }
+
+    // execute_result and display_data
+    const data = (output.data ?? {}) as Record<string, any>;
+
+    // DataFrames render as both text/html and text/plain; prefer the plain
+    // form — it's the formatted ASCII table, cheaper, and an LLM reads it
+    // just fine. text/html alone is stripped to plain text below.
+    // Use ``in`` rather than truthy check: an empty array ``[]`` is legal
+    // (if rare) nbformat for ``text/plain`` but falsy in JS, which would
+    // otherwise let the html branch fire over an explicit-but-empty plain
+    // entry.
+    if ('text/plain' in data) {
+      pushBundle('text/plain', joinMultilineString(data['text/plain']));
+    } else if ('text/html' in data) {
+      pushBundle(
+        'text/html',
+        stripHtml(joinMultilineString(data['text/html']))
+      );
+    }
+
+    if (data['application/vnd.plotly.v1+json']) {
+      pushBundle(
+        'application/vnd.plotly.v1+json',
+        summarizePlotly(data['application/vnd.plotly.v1+json'])
+      );
+    } else if (data['application/json']) {
+      pushBundle(
+        'application/json',
+        JSON.stringify(data['application/json'], null, 2)
+      );
+    }
+
+    for (const imgMime of ['image/png', 'image/jpeg']) {
+      if (!data[imgMime]) {
+        continue;
+      }
+      const raw = joinMultilineString(data[imgMime]);
+      if (!supportsVision) {
+        pushBundle(imgMime, '<image omitted: model lacks vision support>');
+      } else if (raw.length > MAX_IMAGE_BASE64_BYTES) {
+        pushBundle(imgMime, '<image omitted: too large for inline attachment>');
+        truncated = true;
+      } else if (!isValidBase64(raw)) {
+        pushBundle(imgMime, '<image omitted: invalid base64 payload>');
+      } else {
+        // Send raw base64; the server constructs the data URL after
+        // re-validating, so a forged POST can't inject markdown.
+        pushBundle(imgMime, raw);
+      }
+    }
+  }
+
+  return { cellSource, mimeBundles, isError, truncated };
+}
+
+function isValidBase64(s: string): boolean {
+  // Standard base64 alphabet only. No URL-safe chars; no characters that
+  // could break out of a markdown image URL on the server side.
+  const compact = s.replace(/\s+/g, '');
+  return /^[A-Za-z0-9+/]+=*$/.test(compact);
+}
+
+function stripHtml(html: string): string {
+  // NOTE: not a sanitizer. Output is intended only for LLM context; never
+  // route this back into the DOM without a real sanitizer.
+  // Replace block-level closers with a newline so structure survives, then
+  // strip the remaining tags. Cheap and good enough for LLM consumption.
+  return html
+    .replace(/<\/(p|div|tr|li|h[1-6]|br)\s*\/?>/gi, '\n')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<[^>]+>/g, '')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function summarizePlotly(payload: any): string {
+  const lines: string[] = [];
+  const traces = Array.isArray(payload?.data) ? payload.data : [];
+  const types = traces.map((t: any) => t?.type ?? 'trace').join(', ');
+  if (types) {
+    lines.push(`Plotly figure traces: ${types}`);
+  }
+  const layout = payload?.layout ?? {};
+  if (layout.title?.text) {
+    lines.push(`Title: ${layout.title.text}`);
+  }
+  if (layout.xaxis?.title?.text) {
+    lines.push(`X axis: ${layout.xaxis.title.text}`);
+  }
+  if (layout.yaxis?.title?.text) {
+    lines.push(`Y axis: ${layout.yaxis.title.text}`);
+  }
+  return lines.join('\n');
+}

package/src/cell-output-toolbar.ts

@@ -0,0 +1,232 @@
+// Copyright (c) Mehmet Bektas <mbektasgh@outlook.com>
+
+import { JupyterFrontEnd } from '@jupyterlab/application';
+import { CodeCell } from '@jupyterlab/cells';
+import { NotebookPanel } from '@jupyterlab/notebook';
+import { LabIcon } from '@jupyterlab/ui-components';
+import { CommandRegistry } from '@lumino/commands';
+import { IDisposable } from '@lumino/disposable';
+import { CellOutputActionFlag, NBIAPI } from './api';
+import { cellOutputHasError } from './utils';
+import sparkleSvgstr from '../style/icons/cell-toolbar-sparkle.svg';
+import chatSvgstr from '../style/icons/cell-toolbar-chat.svg';
+import bugSvgstr from '../style/icons/cell-toolbar-bug.svg';
+
+interface IToolbarAction {
+  id: string;
+  label: string;
+  title: string;
+  icon: LabIcon;
+  command: string;
+  /** Hide the button when this feature flag is disabled. */
+  featureFlag: CellOutputActionFlag;
+  /** Only show when the cell has at least one error output. */
+  requireError?: boolean;
+}
+
+const TOOLBAR_CLASS = 'nbi-cell-output-toolbar';
+const BUTTON_CLASS = 'nbi-cell-output-toolbar-button';
+
+// SVGs live under style/icons/ and are loaded via LabIcon to match the rest
+// of the project's icon plumbing. The source assets are Microsoft's
+// vscode-codicons (sparkle, comment-discussion, bug), CC BY 4.0 — the
+// license header lives inside each .svg file.
+const sparkleIcon = new LabIcon({
+  name: 'notebook-intelligence:cell-toolbar-sparkle',
+  svgstr: sparkleSvgstr
+});
+const chatIcon = new LabIcon({
+  name: 'notebook-intelligence:cell-toolbar-chat',
+  svgstr: chatSvgstr
+});
+const bugIcon = new LabIcon({
+  name: 'notebook-intelligence:cell-toolbar-bug',
+  svgstr: bugSvgstr
+});
+
+const ACTIONS: IToolbarAction[] = [
+  {
+    id: 'explain',
+    label: 'Explain',
+    title: "Explain this cell's output",
+    icon: sparkleIcon,
+    command: 'notebook-intelligence:editor-explain-this-output',
+    featureFlag: 'output_followup'
+  },
+  {
+    id: 'ask',
+    label: 'Ask',
+    title: 'Ask about this output',
+    icon: chatIcon,
+    command: 'notebook-intelligence:editor-ask-about-this-output',
+    featureFlag: 'output_followup'
+  },
+  {
+    id: 'troubleshoot',
+    label: 'Troubleshoot',
+    title: 'Troubleshoot the error in this cell',
+    icon: bugIcon,
+    command: 'notebook-intelligence:editor-troubleshoot-this-output',
+    featureFlag: 'explain_error',
+    requireError: true
+  }
+];
+
+/**
+ * Show a hover toolbar over Jupyter cell outputs that surfaces the existing
+ * Explain / Ask / Troubleshoot commands as one-click buttons.
+ *
+ * The toolbar respects the `output_toolbar` feature flag (whole-toolbar
+ * gate) and the per-action `explain_error` / `output_followup` flags so a
+ * locked-off feature stays locked off here too.
+ */
+export class CellOutputHoverToolbar implements IDisposable {
+  private _app: JupyterFrontEnd;
+  private _commands: CommandRegistry;
+  private _disposed = false;
+  private _activeArea: HTMLElement | null = null;
+  private _onMouseOver: (event: MouseEvent) => void;
+  private _onMouseLeave: () => void;
+
+  constructor(app: JupyterFrontEnd, commands: CommandRegistry) {
+    this._app = app;
+    // The Explain / Ask / Troubleshoot commands live on the context-menu's
+    // private CommandRegistry, not on `app.commands`, so callers must pass
+    // the same registry the menu uses.
+    this._commands = commands;
+    this._onMouseOver = this._handleMouseOver.bind(this);
+    // mouseleave only fires when the cursor exits the area entirely —
+    // descendants (including the toolbar itself) don't trigger it.
+    this._onMouseLeave = this._removeActiveToolbar.bind(this);
+    document.body.addEventListener('mouseover', this._onMouseOver);
+  }
+
+  get isDisposed(): boolean {
+    return this._disposed;
+  }
+
+  dispose(): void {
+    if (this._disposed) {
+      return;
+    }
+    this._disposed = true;
+    document.body.removeEventListener('mouseover', this._onMouseOver);
+    this._removeActiveToolbar();
+  }
+
+  private _handleMouseOver(event: MouseEvent): void {
+    if (!NBIAPI.config.cellOutputFeatures.output_toolbar.enabled) {
+      this._removeActiveToolbar();
+      return;
+    }
+    const target = event.target as HTMLElement | null;
+    if (!target) {
+      return;
+    }
+    const area = target.closest<HTMLElement>('.jp-Cell-outputArea');
+    if (!area) {
+      return;
+    }
+    if (area === this._activeArea) {
+      return;
+    }
+    this._removeActiveToolbar();
+    const cellEl = area.closest<HTMLElement>('.jp-Cell');
+    if (!cellEl) {
+      return;
+    }
+    const located = this._locateCell(cellEl);
+    if (!located) {
+      return;
+    }
+    const toolbar = this._buildToolbar(
+      located.panel,
+      located.cellIndex,
+      located.cell
+    );
+    if (!toolbar) {
+      return;
+    }
+    area.appendChild(toolbar);
+    this._activeArea = area;
+    area.addEventListener('mouseleave', this._onMouseLeave);
+  }
+
+  private _removeActiveToolbar(): void {
+    if (!this._activeArea) {
+      return;
+    }
+    this._activeArea.removeEventListener('mouseleave', this._onMouseLeave);
+    const existing = this._activeArea.querySelector(`.${TOOLBAR_CLASS}`);
+    if (existing) {
+      existing.remove();
+    }
+    this._activeArea = null;
+  }
+
+  private _locateCell(
+    cellEl: HTMLElement
+  ): { panel: NotebookPanel; cell: CodeCell; cellIndex: number } | null {
+    const widget = this._app.shell.currentWidget;
+    if (!(widget instanceof NotebookPanel)) {
+      return null;
+    }
+    const widgets = widget.content.widgets;
+    for (let i = 0; i < widgets.length; i++) {
+      const cell = widgets[i];
+      if (cell.node === cellEl && cell instanceof CodeCell) {
+        return { panel: widget, cell, cellIndex: i };
+      }
+    }
+    return null;
+  }
+
+  private _buildToolbar(
+    panel: NotebookPanel,
+    cellIndex: number,
+    cell: CodeCell
+  ): HTMLElement | null {
+    const features = NBIAPI.config.cellOutputFeatures;
+    const hasError = cellOutputHasError(cell);
+
+    const visible = ACTIONS.filter(a => {
+      if (!features[a.featureFlag].enabled) {
+        return false;
+      }
+      if (a.requireError && !hasError) {
+        return false;
+      }
+      return true;
+    });
+    if (visible.length === 0) {
+      return null;
+    }
+
+    const toolbar = document.createElement('div');
+    toolbar.className = TOOLBAR_CLASS;
+    toolbar.setAttribute('role', 'toolbar');
+    toolbar.setAttribute('aria-label', 'Notebook Intelligence cell actions');
+
+    for (const action of visible) {
+      const button = document.createElement('button');
+      button.type = 'button';
+      button.className = BUTTON_CLASS;
+      button.title = action.title;
+      button.setAttribute('aria-label', action.title);
+      action.icon.element({ container: button, tag: 'span' });
+      const label = document.createElement('span');
+      label.className = `${BUTTON_CLASS}-label`;
+      label.textContent = action.label;
+      button.appendChild(label);
+      button.addEventListener('click', event => {
+        event.stopPropagation();
+        // The editor commands act on the active cell, so activate the
+        // hovered one first.
+        panel.content.activeCellIndex = cellIndex;
+        void this._commands.execute(action.command);
+      });
+      toolbar.appendChild(button);
+    }
+    return toolbar;
+  }
+}

package/src/chat-progress-feedback.ts

@@ -0,0 +1,35 @@
+// Copyright (c) Mehmet Bektas <mbektasgh@outlook.com>
+
+// Small helpers for the chat sidebar's "Generating" progress indicator.
+// Pulled out so each piece (elapsed-time formatting, heartbeat staleness)
+// can be unit-tested without dragging JupyterLab into Jest's import graph.
+
+// 30 seconds without a heartbeat counts as "stalled" — long enough that a
+// healthy round-trip + retransmission delay can't account for it, short
+// enough that a real hang is signaled while there's still time to act.
+// The Claude heartbeat fires every 20s server-side, so 30s gives roughly
+// 1.5 expected intervals of slack before we change copy.
+export const HEARTBEAT_STALE_MS = 30_000;
+
+export function formatElapsedSeconds(totalSeconds: number): string {
+  const safe = Math.max(0, Math.floor(totalSeconds));
+  const hours = Math.floor(safe / 3600);
+  const minutes = Math.floor((safe % 3600) / 60);
+  const seconds = safe % 60;
+  const ss = String(seconds).padStart(2, '0');
+  if (hours > 0) {
+    return `${hours}:${String(minutes).padStart(2, '0')}:${ss}`;
+  }
+  return `${minutes}:${ss}`;
+}
+
+export function isHeartbeatStale(
+  lastHeartbeatAt: number | null,
+  now: number,
+  staleMs: number = HEARTBEAT_STALE_MS
+): boolean {
+  if (lastHeartbeatAt === null) {
+    return false;
+  }
+  return now - lastHeartbeatAt > staleMs;
+}