@xterm/addon-image 0.10.0-beta.3 → 0.10.0-beta.300

package/src/ImageAddon.ts

@@ -5,24 +5,72 @@
 
 import type { ITerminalAddon, IDisposable } from '@xterm/xterm';
 import type { ImageAddon as IImageApi } from '@xterm/addon-image';
+import { Emitter, type IEvent } from 'common/Event';
 import { IIPHandler } from './IIPHandler';
 import { ImageRenderer } from './ImageRenderer';
 import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage';
+import { KittyGraphicsHandler } from './kitty/KittyGraphicsHandler';
+import { KittyImageStorage } from './kitty/KittyImageStorage';
 import { SixelHandler } from './SixelHandler';
+import { SixelImageStorage } from './SixelImageStorage';
+import { IIPImageStorage } from './IIPImageStorage';
 import { ITerminalExt, IImageAddonOptions, IResetHandler } from './Types';
 
+
+/**
+ * Document VT features provided by this addon.
+ *
+ * @vt: #E[Supported via @xterm/addon-image.] DCS SIXEL "SIXEL Graphics"  "DCS Ps ; Ps ; Ps ; q Pt ST"  "Draw SIXEL image."
+ *
+ * Sixel support is provided by the addon @xterm/addon-image with these limitations:
+ * - immediate coloring (no shared palette, allows high color settings of `img2sixel`)
+ * - max. palette size of 4096 colors
+ * - max. pixel width of 16K
+ * - max. 25 MB per sixel sequence
+ * - VT340 cursor positioning (begin of last sixel data row)
+ *
+ * See [addon readme](https://github.com/xtermjs/xterm.js/tree/master/addons/addon-image) for more details.
+ *
+ *
+ * @vt: #E[Supported via @xterm/addon-image.] OSC 1337  "iTerm2 Commands"   "OSC 1337 ; Pt BEL"  "Custom iTerm2 commands."
+ *
+ * Only the inline image protocol (IIP) is supported by the addon @xterm/addon-image with
+ * the following limitations:
+ * - sequence:
+ *   - format: `OSC 1337 ; File=inline=1 ; size=<unencoded size> ; ... : <base64 payload> BEL`
+ *   - size param must be set and payload may not exceed CEIL(size * 4 / 3)
+ *   - strict base64 handling as of RFC4648 §4 (standard alphabet, optional padding,
+ *     no separator bytes allowed)
+ *   - supported params: size, name, width, height, preserveAspectRatio
+ * - image formats: PNG, JPEG and GIF
+ * - no animation support (renders first image of a GIF)
+ * - no multipart support
+ * - VT340 cursor positioning (begin of last sixel data row)
+ *
+ * See [addon readme](https://github.com/xtermjs/xterm.js/tree/master/addons/addon-image)
+ * and [iTerm2 IIP docs](https://iterm2.com/documentation-images.html) for more details.
+ *
+ *
+ * @vt: #E[Supported via @xterm/addon-image.] APC KITTY_GRAPHICS "Kitty Graphics"  "APC G Pt ST"  "Kitty Graphics Protocol."
+ *
+ * Kitty graphics support is provided by the addon @xterm/addon-image.
+ * Note that while basic image output already works, this is still work in progress.
+ */
+
 // default values of addon ctor options
 const DEFAULT_OPTIONS: IImageAddonOptions = {
   enableSizeReports: true,
   pixelLimit: 16777216, // limit to 4096 * 4096 pixels
   sixelSupport: true,
   sixelScrolling: true,
-  sixelPaletteLimit: 256,
-  sixelSizeLimit: 25000000,
+  sixelPaletteLimit: 4096,
+  sixelSizeLimit: 33554432,
   storageLimit: 128,
   showPlaceholder: true,
   iipSupport: true,
-  iipSizeLimit: 20000000
+  iipSizeLimit: 33554432,
+  kittySupport: true,
+  kittySizeLimit: 33554432
 };
 
 // max palette size supported by the sixel lib (compile time setting)
@@ -48,7 +96,7 @@ const enum GaStatus {
 }
 
 
-export class ImageAddon implements ITerminalAddon , IImageApi {
+export class ImageAddon implements ITerminalAddon, IImageApi {
   private _opts: IImageAddonOptions;
   private _defaultOpts: IImageAddonOptions;
   private _storage: ImageStorage | undefined;
@@ -56,6 +104,8 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
   private _disposables: IDisposable[] = [];
   private _terminal: ITerminalExt | undefined;
   private _handlers: Map<String, IResetHandler> = new Map();
+  private readonly _onImageAdded = new Emitter<void>();
+  public readonly onImageAdded: IEvent<void> = this._onImageAdded.event;
 
   constructor(opts?: Partial<IImageAddonOptions>) {
     this._opts = Object.assign({}, DEFAULT_OPTIONS, opts);
@@ -68,6 +118,7 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
     }
     this._disposables.length = 0;
     this._handlers.clear();
+    this._onImageAdded.dispose();
   }
 
   private _disposeLater(...args: IDisposable[]): void {
@@ -82,15 +133,11 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
     // internal data structures
     this._renderer = new ImageRenderer(terminal);
     this._storage = new ImageStorage(terminal, this._renderer, this._opts);
+    this._storage.onImageAdded = () => this._onImageAdded.fire();
 
     // enable size reports
     if (this._opts.enableSizeReports) {
-      // const windowOptions = terminal.getOption('windowOptions');
-      // windowOptions.getWinSizePixels = true;
-      // windowOptions.getCellSizePixels = true;
-      // windowOptions.getWinSizeChars = true;
-      // terminal.setOption('windowOptions', windowOptions);
-      const windowOps = terminal.options.windowOptions || {};
+      const windowOps = terminal.options.windowOptions ?? {};
       windowOps.getWinSizePixels = true;
       windowOps.getCellSizePixels = true;
       windowOps.getWinSizeChars = true;
@@ -129,7 +176,8 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
 
     // SIXEL handler
     if (this._opts.sixelSupport) {
-      const sixelHandler = new SixelHandler(this._opts, this._storage!, terminal);
+      const sixelStorage = new SixelImageStorage(this._storage!, this._opts, this._renderer!, terminal);
+      const sixelHandler = new SixelHandler(this._opts, sixelStorage, terminal);
       this._handlers.set('sixel', sixelHandler);
       this._disposeLater(
         terminal._core._inputHandler._parser.registerDcsHandler({ final: 'q' }, sixelHandler)
@@ -138,12 +186,25 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
 
     // iTerm IIP handler
     if (this._opts.iipSupport) {
-      const iipHandler = new IIPHandler(this._opts, this._renderer!, this._storage!, terminal);
+      const iipStorage = new IIPImageStorage(this._storage!);
+      const iipHandler = new IIPHandler(this._opts, this._renderer!, iipStorage, terminal);
       this._handlers.set('iip', iipHandler);
       this._disposeLater(
         terminal._core._inputHandler._parser.registerOscHandler(1337, iipHandler)
       );
     }
+
+    // Kitty graphics handler
+    if (this._opts.kittySupport) {
+      const kittyStorage = new KittyImageStorage(this._storage!);
+      const kittyHandler = new KittyGraphicsHandler(this._opts, this._renderer!, kittyStorage, terminal);
+      this._handlers.set('kitty', kittyHandler);
+      this._disposeLater(
+        kittyStorage,
+        kittyHandler,
+        terminal._core._inputHandler._parser.registerApcHandler({ final: 'G' }, kittyHandler)
+      );
+    }
   }
 
   // Note: storageLimit is skipped here to not intoduce a surprising side effect.
@@ -194,7 +255,7 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
   }
 
   private _report(s: string): void {
-    this._terminal?._core.coreService.triggerDataEvent(s);
+    this._terminal?._core.input(s, false);
   }
 
   private _decset(params: (number | number[])[]): boolean {

package/src/ImageRenderer.ts

@@ -5,11 +5,13 @@
 
 import { toRGBA8888 } from 'sixel/lib/Colors';
 import { IDisposable } from '@xterm/xterm';
-import { ICellSize, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types';
-import { Disposable, MutableDisposable, toDisposable } from 'vs/base/common/lifecycle';
+import { ICellSize, ImageLayer, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types';
+import { Disposable, MutableDisposable, toDisposable } from 'common/Lifecycle';
 
-const PLACEHOLDER_LENGTH = 4096;
-const PLACEHOLDER_HEIGHT = 24;
+const enum Constants {
+  PLACEHOLDER_LENGTH = 4096,
+  PLACEHOLDER_HEIGHT = 24
+}
 
 /**
  * ImageRenderer - terminal frontend extension:
@@ -18,8 +20,9 @@ const PLACEHOLDER_HEIGHT = 24;
  * - draw image tiles onRender
  */
 export class ImageRenderer extends Disposable implements IDisposable {
-  public canvas: HTMLCanvasElement | undefined;
-  private _ctx: CanvasRenderingContext2D | null | undefined;
+  /** @deprecated Kept for backward compat — points to top layer canvas. */
+  public get canvas(): HTMLCanvasElement | undefined { return this._layers.get('top')?.canvas; }
+  private _layers = new Map<ImageLayer, CanvasRenderingContext2D>();
   private _placeholder: HTMLCanvasElement | undefined;
   private _placeholderBitmap: ImageBitmap | undefined;
   private _optionsRefresh = this._register(new MutableDisposable());
@@ -38,7 +41,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
      * Only the DOM output canvas should be on the terminal's document,
      * which gets explicitly checked in `insertLayerToDom`.
      */
-    const canvas = (localDocument || document).createElement('canvas');
+    const canvas = (localDocument ?? document).createElement('canvas');
     canvas.width = width | 0;
     canvas.height = height | 0;
     return canvas;
@@ -86,6 +89,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
     });
     this._register(toDisposable(() => {
       this.removeLayerFromDom();
+      this.removeLayerFromDom('bottom');
       if (this._terminal._core && this._oldOpen) {
         this._terminal._core.open = this._oldOpen;
         this._oldOpen = undefined;
@@ -95,8 +99,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
         this._oldSetRenderer = undefined;
       }
       this._renderService = undefined;
-      this.canvas = undefined;
-      this._ctx = undefined;
+      this._layers.clear();
       this._placeholderBitmap?.close();
       this._placeholderBitmap = undefined;
       this._placeholder = undefined;
@@ -109,7 +112,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
   public showPlaceholder(value: boolean): void {
     if (value) {
       if (!this._placeholder && this.cellSize.height !== -1) {
-        this._createPlaceHolder(Math.max(this.cellSize.height + 1, PLACEHOLDER_HEIGHT));
+        this._createPlaceHolder(Math.max(this.cellSize.height + 1, Constants.PLACEHOLDER_HEIGHT));
       }
     } else {
       this._placeholderBitmap?.close();
@@ -124,7 +127,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
    * Forwarded from internal render service.
    */
   public get dimensions(): IRenderDimensions | undefined {
-    return this._renderService?.dimensions;
+    return this._terminal.dimensions;
   }
 
   /**
@@ -140,27 +143,38 @@ export class ImageRenderer extends Disposable implements IDisposable {
   /**
    * Clear a region of the image layer canvas.
    */
-  public clearLines(start: number, end: number): void {
-    this._ctx?.clearRect(
-      0,
-      start * (this.dimensions?.css.cell.height || 0),
-      this.dimensions?.css.canvas.width || 0,
-      (++end - start) * (this.dimensions?.css.cell.height || 0)
-    );
+  public clearLines(start: number, end: number, layer?: ImageLayer): void {
+    const y = start * (this.dimensions?.css.cell.height || 0);
+    const w = this.dimensions?.css.canvas.width || 0;
+    const h = (end + 1 - start) * (this.dimensions?.css.cell.height || 0);
+    if (!layer || layer === 'top') {
+      this._layers.get('top')?.clearRect(0, y, w, h);
+    }
+    if (!layer || layer === 'bottom') {
+      this._layers.get('bottom')?.clearRect(0, y, w, h);
+    }
   }
 
   /**
    * Clear whole image canvas.
    */
-  public clearAll(): void {
-    this._ctx?.clearRect(0, 0, this.canvas?.width || 0, this.canvas?.height || 0);
+  public clearAll(layer?: ImageLayer): void {
+    if (!layer || layer === 'top') {
+      const ctx = this._layers.get('top');
+      ctx?.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+    }
+    if (!layer || layer === 'bottom') {
+      const ctx = this._layers.get('bottom');
+      ctx?.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+    }
   }
 
   /**
    * Draw neighboring tiles on the image layer canvas.
    */
   public draw(imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void {
-    if (!this._ctx) {
+    const ctx = this._layers.get(imgSpec.layer);
+    if (!ctx) {
       return;
     }
     const { width, height } = this.cellSize;
@@ -187,7 +201,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
     // Note: For not pixel perfect aligned cells like in the DOM renderer
     // this will move a tile slightly to the top/left (subpixel range, thus ignore it).
     // FIX #34: avoid striping on displays with pixelDeviceRatio != 1 by ceiling height and width
-    this._ctx.drawImage(
+    ctx.drawImage(
       img,
       Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
       Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight)
@@ -227,7 +241,8 @@ export class ImageRenderer extends Disposable implements IDisposable {
    * Draw a line with placeholder on the image layer canvas.
    */
   public drawPlaceholder(col: number, row: number, count: number = 1): void {
-    if (this._ctx) {
+    const ctx = this._layers.get('top');
+    if (ctx) {
       const { width, height } = this.cellSize;
 
       // Don't try to draw anything, if we cannot get valid renderer metrics.
@@ -236,13 +251,13 @@ export class ImageRenderer extends Disposable implements IDisposable {
       }
 
       if (!this._placeholder) {
-        this._createPlaceHolder(Math.max(height + 1, PLACEHOLDER_HEIGHT));
+        this._createPlaceHolder(Math.max(height + 1, Constants.PLACEHOLDER_HEIGHT));
       } else if (height >= this._placeholder!.height) {
         this._createPlaceHolder(height + 1);
       }
       if (!this._placeholder) return;
-      this._ctx.drawImage(
-        this._placeholderBitmap || this._placeholder!,
+      ctx.drawImage(
+        this._placeholderBitmap ?? this._placeholder!,
         col * width,
         (row * height) % 2 ? 0 : 1,  // needs %2 offset correction
         width * count,
@@ -260,12 +275,13 @@ export class ImageRenderer extends Disposable implements IDisposable {
    * Checked once from `ImageStorage.render`.
    */
   public rescaleCanvas(): void {
-    if (!this.canvas) {
-      return;
-    }
-    if (this.canvas.width !== this.dimensions!.css.canvas.width || this.canvas.height !== this.dimensions!.css.canvas.height) {
-      this.canvas.width = this.dimensions!.css.canvas.width || 0;
-      this.canvas.height = this.dimensions!.css.canvas.height || 0;
+    const w = this.dimensions?.css.canvas.width || 0;
+    const h = this.dimensions?.css.canvas.height || 0;
+    for (const ctx of this._layers.values()) {
+      if (ctx.canvas.width !== w || ctx.canvas.height !== h) {
+        ctx.canvas.width = w;
+        ctx.canvas.height = h;
+      }
     }
   }
 
@@ -304,38 +320,68 @@ export class ImageRenderer extends Disposable implements IDisposable {
     this._renderService = this._terminal._core._renderService;
     this._oldSetRenderer = this._renderService.setRenderer.bind(this._renderService);
     this._renderService.setRenderer = (renderer: any) => {
-      this.removeLayerFromDom();
+      for (const key of [...this._layers.keys()]) {
+        this.removeLayerFromDom(key);
+      }
       this._oldSetRenderer?.call(this._renderService, renderer);
     };
   }
 
-  public insertLayerToDom(): void {
+  public insertLayerToDom(layer: ImageLayer = 'top'): void {
     // make sure that the terminal is attached to a document and to DOM
-    if (this.document && this._terminal._core.screenElement) {
-      if (!this.canvas) {
-        this.canvas = ImageRenderer.createCanvas(
-          this.document, this.dimensions?.css.canvas.width || 0,
-          this.dimensions?.css.canvas.height || 0
-        );
-        this.canvas.classList.add('xterm-image-layer');
-        this._terminal._core.screenElement.appendChild(this.canvas);
-        this._ctx = this.canvas.getContext('2d', { alpha: true, desynchronized: true });
-        this.clearAll();
-      }
-    } else {
+    if (!this.document || !this._terminal._core.screenElement) {
       console.warn('image addon: cannot insert output canvas to DOM, missing document or screenElement');
+      return;
+    }
+    if (this._layers.has(layer)) {
+      return;
     }
+    const canvas = ImageRenderer.createCanvas(
+      this.document, this.dimensions?.css.canvas.width || 0,
+      this.dimensions?.css.canvas.height || 0
+    );
+    canvas.classList.add(`xterm-image-layer-${layer}`);
+    const screenElement = this._terminal._core.screenElement;
+    // Use isolation to create a stacking context without overriding z-index,
+    // which would conflict with integrators (e.g. VS Code) that set their
+    // own z-index on the screen element.
+    screenElement.style.isolation = 'isolate';
+    if (layer === 'bottom') {
+      // Use z-index:-1 so it paints behind non-positioned text elements.
+      // The screen element needs to be a stacking context (via isolation)
+      // to contain the negative z-index, otherwise it would go behind the
+      // entire terminal.
+      canvas.style.zIndex = '-1';
+      screenElement.insertBefore(canvas, screenElement.firstChild);
+    } else {
+      // Explicit z-index ensures the image canvas reliably stacks above
+      // the text layer (DOM renderer rows). z-index: 0 is below the
+      // selection overlay (z-index: 1).
+      canvas.style.zIndex = '0';
+      screenElement.appendChild(canvas);
+    }
+    const ctx = canvas.getContext('2d', { alpha: true });
+    if (!ctx) {
+      canvas.remove();
+      return;
+    }
+    this._layers.set(layer, ctx);
+    this.clearAll(layer);
   }
 
-  public removeLayerFromDom(): void {
-    if (this.canvas) {
-      this._ctx = undefined;
-      this.canvas.remove();
-      this.canvas = undefined;
+  public removeLayerFromDom(layer: ImageLayer = 'top'): void {
+    const ctx = this._layers.get(layer);
+    if (ctx) {
+      ctx.canvas.remove();
+      this._layers.delete(layer);
     }
   }
 
-  private _createPlaceHolder(height: number = PLACEHOLDER_HEIGHT): void {
+  public hasLayer(layer: ImageLayer): boolean {
+    return this._layers.has(layer);
+  }
+
+  private _createPlaceHolder(height: number = Constants.PLACEHOLDER_HEIGHT): void {
     this._placeholderBitmap?.close();
     this._placeholderBitmap = undefined;
 
@@ -359,7 +405,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
     ctx.putImageData(imgData, 0, 0);
 
     // create placeholder line, width aligned to blueprint width
-    const width = (screen.width + bWidth - 1) & ~(bWidth - 1) || PLACEHOLDER_LENGTH;
+    const width = (screen.width + bWidth - 1) & ~(bWidth - 1) || Constants.PLACEHOLDER_LENGTH;
     this._placeholder = ImageRenderer.createCanvas(this.document, width, height);
     const ctx2 = this._placeholder.getContext('2d', { alpha: false });
     if (!ctx2) {