ntk 8.1.1 → 8.3.0

package/lib/glyphdirectory.js

@@ -0,0 +1,275 @@
+// The `_NTK_GLYPHD` glyph directory (docs/shared-glyphs.md): the naming half
+// of the display-wide shared glyph cache.
+//
+// The directory does not rasterize and knows nothing about fonts. It owns
+// three things and nothing else: the ICCCM manager selection that names it,
+// the map from page tokens to server-side glyphsets, and the compact local
+// id space inside each set — it is the single writer of lids, which is what
+// keeps them dense enough for CompositeGlyphs8. Presence is a bit it tracks
+// per (token, member key): clients report uploads (`_NTK_GLYPHD_ADDED`), and
+// later askers are told "already there".
+//
+// Lifetime contract, load-bearing for the client's XID-reuse fence: while
+// the directory owns the selection it frees nothing. Eviction opens a new
+// *generation* — fresh sets advertised, old ones no longer handed out; the
+// retired sets stay pinned by this connection (their ids are the cheap part;
+// Xorg dedupes the pixel storage by content anyway) and die with it, or
+// when the last client alias drops, whichever is later.
+//
+// Every ntk app embeds one of these behind the same code path: the first app
+// on a display self-elects (lib/sharedglyphs.js calls `claim()` when nobody
+// owns the selection), a standalone directory process would claim it the
+// same way and simply outlive the apps.
+
+import x11 from 'x11';
+
+import GlyphSet from './glyphset.js';
+import {
+  GLYPHD,
+  MAX_LIDS,
+  encodeGlyphdReply,
+  parseGlyphdRequest
+} from './glyphdwire.js';
+
+/**
+ * Directory-side policy: the byte budget across the current generation's
+ * pages. Uploads reported by clients accumulate against it; exceeding it
+ * opens a new generation. It bounds *growth*, not total server memory —
+ * retired generations froze at whatever size they reached.
+ */
+export const DEFAULT_SHARED_GLYPHS_POLICY = {
+  budgetBytes: 32 << 20
+};
+
+const TIME_TIMEOUT = 2000;
+
+export default class GlyphDirectory {
+  /**
+   * @param {App} app the connection the directory rides
+   * @param {object} [options] `{ budgetBytes }` over DEFAULT_SHARED_GLYPHS_POLICY
+   */
+  constructor(app, options) {
+    this.app = app;
+    this.X = app.X;
+    this.policy = { ...DEFAULT_SHARED_GLYPHS_POLICY, ...options };
+    this.window = null;
+    this.generation = 1;
+    this.pages = new Map(); // token -> { set, members: Map(key -> {lid, present}), bytes }
+    this.active = false;
+    this._atoms = null;
+    this._bytes = 0; // uploads reported into the current generation
+    this._retired = []; // GlyphSets of past generations: pinned, never freed
+    this._onXEvent = this._onXEvent.bind(this);
+  }
+
+  /** the selection endpoint's window id, 0 before claim() */
+  get wid() {
+    return this.window ? this.window.id : 0;
+  }
+
+  /**
+   * Claim the manager selection (ICCCM 2.8): an unmapped window, a real
+   * timestamp, `SetSelectionOwner`, and a `MANAGER` announcement on the root
+   * so waiting clients adopt the new directory.
+   *
+   * @returns {Promise<number>} the window id of whoever owns the selection
+   *   afterwards — ours when the claim won, the winner's when it raced and
+   *   lost, 0 when the server refused the whole dance
+   */
+  async claim() {
+    const X = this.X;
+    try {
+      const names = [GLYPHD.selection, GLYPHD.ensure, GLYPHD.added, GLYPHD.done, GLYPHD.time];
+      const [selection, ensure, added, done, time] = await Promise.all(
+        names.map((name) => this._atom(name))
+      );
+      this._atoms = { selection, ensure, added, done, time };
+      this.window = this.app.createWindow({
+        width: 1,
+        height: 1,
+        eventMask: x11.eventMask.PropertyChange
+      });
+      // ICCCM 2.1: never CurrentTime. The property-append trick gives a real
+      // server timestamp; 0 (CurrentTime) is the documented fallback when
+      // the server will not answer, which every server ntk targets accepts.
+      const stamp = await this._serverTime();
+      X.SetSelectionOwner(this.window.id, selection, stamp);
+      // SetSelectionOwner is void, and a race is settled by whoever the
+      // server says owns the selection now
+      const owner = await new Promise((resolve, reject) =>
+        X.GetSelectionOwner(selection, (err, wid) => (err ? reject(err) : resolve(wid)))
+      );
+      if (owner !== this.window.id) {
+        this.window.destroy();
+        this.window = null;
+        return owner;
+      }
+      this.active = true;
+      this.window.on('message', (ev) => this._onMessage(ev));
+      // SelectionClear (a successor took over) carries the owner in
+      // ev.owner, not ev.wid, so the Window wrapper never sees it — raw
+      // listener, the same reason the clipboard has one
+      X.on('event', this._onXEvent);
+      // announce (ICCCM 2.8): MANAGER on the root with the acquisition
+      // timestamp, StructureNotify mask — what surviving clients of a dead
+      // directory are listening for
+      await this.app
+        .rootWindow()
+        .sendClientMessage(GLYPHD.manager, [stamp, selection, this.window.id], {
+          mask: x11.eventMask.StructureNotify
+        });
+      return this.window.id;
+    } catch {
+      if (this.window) {
+        this.window.destroy();
+        this.window = null;
+      }
+      this.active = false;
+      return 0;
+    }
+  }
+
+  /**
+   * Stop answering. The sets stay: clients still hold aliases into them, and
+   * the fence contract ("frees nothing while it owns the selection") has
+   * already ended with the ownership — nothing here may free what a reply
+   * has advertised, so everything is simply kept until the connection ends.
+   */
+  deactivate() {
+    if (!this.active) return;
+    this.active = false;
+    this.X.removeListener('event', this._onXEvent);
+  }
+
+  _onXEvent(ev) {
+    // SelectionClear: another manager took the selection with a later
+    // timestamp — the ICCCM handover. It answers from here on.
+    if (ev.type === 29 && this.window && ev.owner === this.window.id) this.deactivate();
+  }
+
+  _onMessage(ev) {
+    if (!this.active || !this._atoms) return;
+    if (ev.message_type === this._atoms.ensure) {
+      this._serve(ev, true).catch(() => {});
+    } else if (ev.message_type === this._atoms.added) {
+      this._serve(ev, false).catch(() => {});
+    }
+  }
+
+  /**
+   * One RPC: read the request property off the client's mailbox (the read
+   * deletes it), answer, confirm with DONE. Failures on the client's side —
+   * a mailbox that vanished mid-request — are its own; every cross-client
+   * request here swallows its error rather than warning about a dead peer.
+   */
+  async _serve(ev, isEnsure) {
+    const X = this.X;
+    const [mailbox, serial, property] = ev.data;
+    const done = (status) => {
+      X.SendClientMessage(mailbox, mailbox, this._atoms.done, 32, [serial, status], 0, () => true);
+    };
+    const prop = await new Promise((resolve) =>
+      X.GetProperty(1, mailbox, property, 0, 0, 0x1fffffff, (err, res) => resolve(err ? null : res))
+    );
+    const request = prop && prop.data ? parseGlyphdRequest(prop.data) : null;
+    if (!request || !request.token) return done(0);
+    if (isEnsure) {
+      const entries = this._ensure(request);
+      if (!entries) return done(0);
+      const reply = encodeGlyphdReply({
+        serial,
+        generation: this.generation,
+        gsid: this.pages.get(request.token).set.id,
+        entries
+      });
+      X.ChangeProperty(0, mailbox, property, property, 8, reply, () => true);
+      return done(1);
+    }
+    this._added(request);
+    return done(1);
+  }
+
+  /**
+   * Allocate (or look up) a lid per member key, in request order. The single
+   * writer of the lid space is what keeps ids compact — dense from 0 in
+   * first-asked order, whichever client asked.
+   * @returns {Array<{lid, present}>|null} null refuses the request
+   */
+  _ensure({ token, keys }) {
+    let page = this.pages.get(token);
+    if (!page) {
+      page = { set: new GlyphSet(this.app), members: new Map(), bytes: 0 };
+      this.pages.set(token, page);
+    }
+    const entries = new Array(keys.length);
+    for (let i = 0; i < keys.length; i++) {
+      let member = page.members.get(keys[i]);
+      if (!member) {
+        if (page.members.size >= MAX_LIDS) return null;
+        member = { lid: page.members.size, present: false };
+        page.members.set(keys[i], member);
+      }
+      entries[i] = member;
+    }
+    return entries;
+  }
+
+  /**
+   * A client uploaded: flip the presence bits so later askers skip the
+   * rasterization, and count the bytes against the generation budget. A
+   * token this generation does not advertise (retired since the client
+   * bound) is ignored — the uploader still draws from its own alias, and
+   * the next generation simply re-earns those glyphs.
+   */
+  _added({ token, keys, bytes }) {
+    const page = this.pages.get(token);
+    if (!page) return;
+    for (const key of keys) {
+      const member = page.members.get(key);
+      if (member) member.present = true;
+    }
+    page.bytes += bytes;
+    this._bytes += bytes;
+    if (this._bytes > this.policy.budgetBytes) this._openGeneration();
+  }
+
+  /**
+   * Eviction, directory style: no set is freed — the current pages retire
+   * (pinned here so not even a GC finalizer can free what was advertised)
+   * and the next ensure of each token starts a fresh set in a fresh
+   * generation. Clients bound to a retired (gsid, generation) notice the
+   * mismatch in their next reply and freeze that page.
+   */
+  _openGeneration() {
+    for (const page of this.pages.values()) this._retired.push(page.set);
+    this.pages.clear();
+    this._bytes = 0;
+    this.generation++;
+  }
+
+  // a current server timestamp, via the zero-length property append trick
+  // (the same one the clipboard uses): the PropertyNotify it provokes
+  // carries the server's clock
+  _serverTime() {
+    return new Promise((resolve) => {
+      const finish = (time) => {
+        clearTimeout(timer);
+        this.window.removeListener('property', onProperty);
+        resolve(time);
+      };
+      const onProperty = (ev) => {
+        if (ev.atom === this._atoms.time && ev.state === 0) finish(ev.time >>> 0);
+      };
+      const timer = setTimeout(() => finish(0), TIME_TIMEOUT);
+      timer.unref?.();
+      this.window.on('property', onProperty);
+      this.X.ChangeProperty(2, this.window.id, this._atoms.time, this.X.atoms.STRING, 8, Buffer.alloc(0));
+    });
+  }
+
+  _atom(name) {
+    return new Promise((resolve, reject) =>
+      this.X.InternAtom(false, name, (err, atom) => (err ? reject(err) : resolve(atom)))
+    );
+  }
+}

package/lib/glyphdwire.js

@@ -0,0 +1,142 @@
+// The `_NTK_GLYPHD` wire vocabulary (docs/shared-glyphs.md): the atoms and
+// the property payload codecs shared by the directory (lib/glyphdirectory.js)
+// and the client (lib/sharedglyphs.js). Pure functions, no X in sight —
+// which is what makes the protocol unit-testable without a server.
+
+/** selection + message vocabulary */
+export const GLYPHD = {
+  selection: '_NTK_GLYPHD',
+  ensure: '_NTK_GLYPHD_ENSURE',
+  added: '_NTK_GLYPHD_ADDED',
+  done: '_NTK_GLYPHD_DONE',
+  property: '_NTK_GLYPHD_RPC',
+  time: '_NTK_GLYPHD_TIME',
+  manager: 'MANAGER'
+};
+
+/** wire format version of the RPC properties */
+export const RPC_VERSION = 1;
+
+/** lids are compact so CompositeGlyphs can use narrow encodings; a page
+ *  cannot outgrow 16-bit ids (fonts cap at 65535 glyphs; corner populations
+ *  are far smaller). The directory refuses rather than overflows. */
+export const MAX_LIDS = 65536;
+
+const FLAG_INDICES = 1;
+
+/**
+ * The ensure/added request property (format 8, little-endian):
+ *
+ *     u8   version (1)
+ *     u8   flags (bit 0: members are u32 font glyph indices, not strings)
+ *     u16  token byte length T
+ *     u32  member count N
+ *     u32  uploaded bytes (added only; 0 in an ensure)
+ *     T    token bytes
+ *     N    members — u32 each, or (u16 length + bytes) each
+ */
+export function encodeGlyphdRequest({ token, indices, keys, bytes = 0 }) {
+  const tokenBuf = Buffer.from(token, 'utf8');
+  let size = 12 + tokenBuf.length;
+  const keyBufs = indices ? null : keys.map((k) => Buffer.from(String(k), 'utf8'));
+  size += indices ? keys.length * 4 : keyBufs.reduce((n, b) => n + 2 + b.length, 0);
+  const buf = Buffer.alloc(size);
+  buf.writeUInt8(RPC_VERSION, 0);
+  buf.writeUInt8(indices ? FLAG_INDICES : 0, 1);
+  buf.writeUInt16LE(tokenBuf.length, 2);
+  buf.writeUInt32LE(keys.length, 4);
+  buf.writeUInt32LE(bytes, 8);
+  tokenBuf.copy(buf, 12);
+  let o = 12 + tokenBuf.length;
+  for (let i = 0; i < keys.length; i++) {
+    if (indices) {
+      buf.writeUInt32LE(keys[i] >>> 0, o);
+      o += 4;
+    } else {
+      const kb = keyBufs[i];
+      buf.writeUInt16LE(kb.length, o);
+      kb.copy(buf, o + 2);
+      o += 2 + kb.length;
+    }
+  }
+  return buf;
+}
+
+/** @returns {{token, indices, keys, bytes}|null} null when malformed */
+export function parseGlyphdRequest(buf) {
+  try {
+    if (buf.length < 12 || buf.readUInt8(0) !== RPC_VERSION) return null;
+    const indices = (buf.readUInt8(1) & FLAG_INDICES) !== 0;
+    const tokenLen = buf.readUInt16LE(2);
+    const count = buf.readUInt32LE(4);
+    const bytes = buf.readUInt32LE(8);
+    let o = 12;
+    if (o + tokenLen > buf.length) return null;
+    const token = buf.toString('utf8', o, o + tokenLen);
+    o += tokenLen;
+    const keys = new Array(count);
+    for (let i = 0; i < count; i++) {
+      if (indices) {
+        if (o + 4 > buf.length) return null;
+        keys[i] = buf.readUInt32LE(o);
+        o += 4;
+      } else {
+        if (o + 2 > buf.length) return null;
+        const len = buf.readUInt16LE(o);
+        if (o + 2 + len > buf.length) return null;
+        keys[i] = buf.toString('utf8', o + 2, o + 2 + len);
+        o += 2 + len;
+      }
+    }
+    return { token, indices, keys, bytes };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The ensure reply property (format 8), member entries in request order:
+ *
+ *     u8   version (1)
+ *     u8   status (1 — a refusal sends no reply property at all)
+ *     u16  0
+ *     u32  serial (echo of the request's)
+ *     u32  generation
+ *     u32  gsid
+ *     u32  member count N
+ *     N    u32 each: bit 31 = present, bits 0..30 = lid
+ */
+export function encodeGlyphdReply({ serial, generation, gsid, entries }) {
+  const buf = Buffer.alloc(20 + entries.length * 4);
+  buf.writeUInt8(RPC_VERSION, 0);
+  buf.writeUInt8(1, 1);
+  buf.writeUInt32LE(serial >>> 0, 4);
+  buf.writeUInt32LE(generation >>> 0, 8);
+  buf.writeUInt32LE(gsid >>> 0, 12);
+  buf.writeUInt32LE(entries.length, 16);
+  entries.forEach((e, i) => {
+    buf.writeUInt32LE(((e.present ? 0x80000000 : 0) | e.lid) >>> 0, 20 + i * 4);
+  });
+  return buf;
+}
+
+/** @returns {{serial, generation, gsid, entries: [{lid, present}]}|null} */
+export function parseGlyphdReply(buf) {
+  try {
+    if (buf.length < 20 || buf.readUInt8(0) !== RPC_VERSION) return null;
+    if (buf.readUInt8(1) !== 1) return null;
+    const serial = buf.readUInt32LE(4);
+    const generation = buf.readUInt32LE(8);
+    const gsid = buf.readUInt32LE(12);
+    const count = buf.readUInt32LE(16);
+    if (buf.length < 20 + count * 4) return null;
+    const entries = new Array(count);
+    for (let i = 0; i < count; i++) {
+      const word = buf.readUInt32LE(20 + i * 4);
+      entries[i] = { lid: word & 0x7fffffff, present: (word & 0x80000000) !== 0 };
+    }
+    return { serial, generation, gsid, entries };
+  } catch {
+    return null;
+  }
+}

package/lib/glyphset.js

@@ -9,14 +9,41 @@ const registry = new FinalizationRegistry(({ Render, X, id }) => {
 
 export default class GlyphSet {
   constructor(app, format) {
-    const X = app.X;
-    this.X = X;
+    this._bind(app);
+    this.Render.CreateGlyphSet(this.id, format || this.Render.a8);
+    this._register();
+  }
+
+  /**
+   * A new id for an existing glyphset — RENDER `ReferenceGlyphSet`. XIDs are
+   * display-global, so `gsid` may name a set another connection created; the
+   * server refcounts, and the underlying set stays alive as long as any
+   * reference to it does — including across its creator's disconnect. That
+   * is what a cross-process glyph cache stands on (docs/shared-glyphs.md).
+   * `destroy()` drops only this alias.
+   *
+   * @param {App} app
+   * @param {number} gsid existing glyphset XID, ours or another client's
+   * @returns {GlyphSet}
+   */
+  static referenceTo(app, gsid) {
+    const gs = Object.create(GlyphSet.prototype);
+    gs._bind(app);
+    gs.Render.ReferenceGlyphSet(gs.id, gsid);
+    gs._register();
+    return gs;
+  }
+
+  _bind(app) {
+    this.X = app.X;
     this.display = app.display;
     this.Render = app.display.Render;
-    this.id = X.AllocID();
-    this.Render.CreateGlyphSet(this.id, format || this.Render.a8);
+    this.id = this.X.AllocID();
+  }
+
+  _register() {
     this._owned = true;
-    registry.register(this, { Render: this.Render, X, id: this.id }, this);
+    registry.register(this, { Render: this.Render, X: this.X, id: this.id }, this);
   }
 
   addGlyphs(glyphs) {

package/lib/index.js

@@ -19,6 +19,7 @@ import {
 import { decodeKey, groupForState } from './keyboard.js';
 import Pixmap from './pixmap.js';
 import Picture from './picture.js';
+import Region from './region.js';
 import { Image, decodeImage, loadImage } from './image.js';
 import {
   ImageData,
@@ -48,6 +49,8 @@ import {
 } from './rasterize.js';
 import { DEFAULT_MASK_POLICY } from './maskcluster.js';
 import { DEFAULT_SHAPE_POLICY } from './shapeglyphs.js';
+import { DEFAULT_SHARED_GLYPHS_POLICY } from './glyphdirectory.js';
+import { warmSharedGlyphs } from './text/glyphs.js';
 import { TextLayout } from './text/layout.js';
 import SvgView from './widgets/svgview.js';
 import { cssColor, cssColorStraight, premultiply } from './color.js';
@@ -95,7 +98,11 @@ const DEFAULT_BUFFER_REQUESTS = { maxSize: 64 * 1024 };
  *   `'direct'` or `'off'`; an object sets the direct backend's knobs too (see
  *   docs/context-gles.md) — and
  *   { onXError } — called with X protocol errors no request callback
- *   claimed (default: console.warn).
+ *   claimed (default: console.warn) —
+ *   { sharedGlyphs } — the cross-process shared glyph cache
+ *   (docs/shared-glyphs.md): on by default, `false` turns it off (as does
+ *   the `NTK_NO_SHARED_GLYPHS` environment variable), an object sets the
+ *   directory-side budget (`{ budgetBytes }`).
  *   x11's { bufferRequests } defaults to a 64 KB output buffer here, which
  *   makes a frame one socket write; pass false to write per request.
  * @param {function} [callback] optional node-style callback
@@ -150,6 +157,15 @@ export function createClient(options, callback) {
           if (process.env.NTK_STRICT_COLORS) Render.strictColors = true;
           display.GLX = glxError ? null : GLX;
 
+          const app = new App(display, appOptions);
+          // Build the visual -> picture format table (issue #295) now: a
+          // picture is bound synchronously, so the table has to be here
+          // before anything draws rather than fetched at the first paint.
+          // It costs the connection nothing and no wait — node-x11 asked
+          // `QueryPictFormats` itself while requiring RENDER and kept the
+          // reply, so this only reads it. A server that cannot
+          // answer leaves every format picked from the depth, as before.
+          app.pictFormats().catch(() => null);
           // RENDER requires the client to declare its version before using
           // requests beyond 0.0, and the reply is what gates them afterwards
           // — CreateSolidFill needs 0.10 (see App#solidPicture). 0.11 is the
@@ -176,7 +192,6 @@ export function createClient(options, callback) {
             });
             updateKeyboardMapping(display.min_keycode, display.max_keycode);
 
-            const app = new App(display, appOptions);
             // Under a policy that could pick the direct backend, find out
             // during the handshake whether it can: two extension queries,
             // running alongside the round trips already in flight here, so
@@ -225,6 +240,9 @@ export {
   resolveCursorShape,
   Pixmap,
   Picture,
+  // server-side XFIXES regions (docs/context-2d.md#region-clips):
+  // app.createRegion(rects) makes one, ctx.clipRegion(region) clips to it
+  Region,
   Surface,
   // fill/stroke styles the 2d context hands back (docs/context-2d.md) —
   // exported for `instanceof`, not to be constructed directly
@@ -255,6 +273,11 @@ export {
   DEFAULT_RASTER_POLICY,
   DEFAULT_MASK_POLICY,
   DEFAULT_SHAPE_POLICY,
+  // cross-process shared glyphs (docs/shared-glyphs.md): the directory-side
+  // budget, and the prewarm that makes a first paint of already-shared text
+  // rasterize and upload nothing
+  DEFAULT_SHARED_GLYPHS_POLICY,
+  warmSharedGlyphs,
   TextLayout,
   SvgView,
   cssColor,