@matdata/yasqe 5.20.2 → 6.0.0

package/grammar/tokenizer.ts

@@ -1,6 +1,6 @@
-import CodeMirror from "codemirror";
+import type { StringStream, StreamParser, IndentContext } from "@codemirror/language";
 export interface State {
-  tokenize: (stream: CodeMirror.StringStream, state: State) => string;
+  tokenize: (stream: StringStream, state: State) => string;
   inLiteral: "SINGLE" | "DOUBLE" | undefined;
   errorStartPos: number | undefined;
   errorEndPos: number | undefined;
@@ -52,7 +52,54 @@ export interface Token {
   start: number;
 }
 import * as grammar from "./_tokenizer-table.js";
-export default function (config: CodeMirror.EditorConfiguration): CodeMirror.Mode<State> {
+
+export interface TokenizerConfig {
+  indentUnit?: number;
+}
+
+/**
+ * All token style names that the tokenizer can emit (the space-separated
+ * parts of the `style` strings). Exported so that the CodeMirror 6 language
+ * definition can build a token table and a highlight style for them.
+ */
+export const TOKEN_STYLES = [
+  "ws",
+  "comment",
+  "variable-3",
+  "atom",
+  "meta",
+  "number",
+  "string",
+  "string-2",
+  "punc",
+  "keyword",
+  "error",
+  "bracket",
+  "bracket-level-0",
+  "bracket-level-1",
+  "bracket-level-2",
+  "bracket-level-3",
+  "bracket-paren",
+  "bracket-mismatch",
+] as const;
+export type TokenStyle = (typeof TOKEN_STYLES)[number];
+
+export function copyState(s: State): State {
+  return {
+    ...s,
+    possibleCurrent: s.possibleCurrent.slice(),
+    possibleNext: s.possibleNext.slice(),
+    stack: s.stack.slice(),
+    variables: { ...s.variables },
+    prefixes: { ...s.prefixes },
+    constructVariables: { ...s.constructVariables },
+    whereVariables: { ...s.whereVariables },
+    pendingToken: s.pendingToken ? { ...s.pendingToken } : undefined,
+    bracketStack: s.bracketStack.map((b) => ({ ...b })),
+  };
+}
+
+export default function (config: TokenizerConfig = {}): StreamParser<State> {
   const ll1_table = grammar.table;
 
   const IRI_REF = '<[^<>"`|{}^\\\x00-\x20]*>';
@@ -324,7 +371,7 @@ export default function (config: CodeMirror.EditorConfiguration): CodeMirror.Mod
     return possibles;
   }
 
-  function tokenBase(stream: CodeMirror.StringStream, state: State) {
+  function tokenBase(stream: StringStream, state: State): string {
     // If we have a pending token from a previous split, consume it from stream and return style
     if (state.pendingToken) {
       const token = state.pendingToken;
@@ -764,7 +811,7 @@ export default function (config: CodeMirror.EditorConfiguration): CodeMirror.Mod
     //		"*[;,?[or([verbPath,verbSimple]),objectList]]": 1,
   };
 
-  function indent(state: State, textAfter: string) {
+  function indent(state: State, textAfter: string, indentUnit: number) {
     //just avoid we don't indent multi-line  literals
     if (state.inLiteral) return 0;
     if (
@@ -800,12 +847,21 @@ export default function (config: CodeMirror.EditorConfiguration): CodeMirror.Mod
           n += dn;
         }
       }
-      return n * (config.indentUnit ?? 2);
+      return n * indentUnit;
     }
   }
 
   return {
+    name: "sparql11",
     token: tokenBase,
+    copyState,
+    indent: (state: State, textAfter: string, context: IndentContext) =>
+      indent(state, textAfter, context.unit ?? config.indentUnit ?? 2),
+    languageData: {
+      commentTokens: { line: "#" },
+      indentOnInput: /^\s*[\}\]\)]$/,
+      closeBrackets: { brackets: ["(", "[", "{", "'", '"'] },
+    },
     startState: function (): State {
       return {
         tokenize: tokenBase,
@@ -839,7 +895,5 @@ export default function (config: CodeMirror.EditorConfiguration): CodeMirror.Mod
         bracketLevel: 0,
       };
     },
-    indent: indent,
-    electricChars: "}])",
   };
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@matdata/yasqe",
-  "description": "Yet Another SPARQL Query Editor",
-  "version": "5.20.2",
+  "description": "MatGUI query editor: a modern SPARQL query editor (formerly Yasgui/YASQE).",
+  "version": "6.0.0",
   "main": "build/yasqe.min.js",
   "types": "build/ts/src/index.d.ts",
   "license": "MIT",
@@ -17,22 +17,27 @@
     "Semantic Web",
     "Linked Data"
   ],
-  "bugs": "https://github.com/Matdata-eu/Yasgui/issues/",
+  "bugs": "https://github.com/Matdata-eu/Matgui/issues/",
   "repository": {
     "type": "git",
-    "url": "https://github.com/Matdata-eu/Yasgui.git",
+    "url": "https://github.com/Matdata-eu/Matgui.git",
     "directory": "packages/yasqe"
   },
   "dependencies": {
+    "@codemirror/autocomplete": "^6.20.3",
+    "@codemirror/commands": "^6.11.0",
+    "@codemirror/language": "^6.12.4",
+    "@codemirror/search": "^6.7.2",
+    "@codemirror/state": "^6.7.2",
+    "@codemirror/view": "^6.43.10",
+    "@lezer/highlight": "^1.2.3",
     "@matdata/yasgui-utils": "^4.6.1",
-    "codemirror": "^5.51.0",
     "leaflet": "^1.9.4",
     "lodash-es": "^4.17.15",
     "query-string": "^6.10.1",
     "sparql-formatter": "^1.0.2"
   },
   "devDependencies": {
-    "@types/codemirror": "0.0.100",
     "@types/leaflet": "^1.9.21",
     "@types/lodash-es": "^4.17.3",
     "@types/node": "^22.5.4"

package/src/autocompleters/index.ts

@@ -1,8 +1,7 @@
-import { default as Yasqe, Token, Hint, Position, Config, HintFn, HintConfig } from "../";
+import { default as Yasqe, Token, Hint, Position, Config, HintFn, HintList } from "../";
 import Trie from "../trie";
 import { EventEmitter } from "events";
 import { take } from "lodash-es";
-import CodeMirror from "codemirror";
 import "./show-hint.scss";
 export interface CompleterConfig {
   onInitialize?: (this: CompleterConfig, yasqe: Yasqe) => void; //allows for e.g. registering event listeners in yasqe, like the prefix autocompleter does
@@ -173,39 +172,39 @@ export class Completer extends EventEmitter {
         });
     return Promise.resolve([]);
   }
-  public autocomplete(fromAutoShow: boolean) {
+  /**
+   * Returns whether the cursor is currently at a position this completer can complete.
+   * Also takes care of showing/hiding the "press ctrl-space" notification.
+   */
+  public isValidCompletionPosition(): boolean {
+    return this.isValidPosition();
+  }
+
+  /**
+   * Returns a hint function for the current cursor position, or `false` when this completer should not
+   * (or cannot) show completions right now. The caller (Yasqe) is responsible for opening the completion popup.
+   */
+  public autocomplete(fromAutoShow: boolean): HintFn | false {
     //this part goes before the autoshow check, as we _would_ like notification showing to indicate a user can press ctrl-space
     if (!this.isValidPosition()) return false;
-    const previousCompletionItem = this.yasqe.state.completionActive;
-
-    // Showhint by defaults takes the autocomplete start position (the location of the cursor at the time of starting the autocompletion).
-    const cursor = this.yasqe.getDoc().getCursor();
-    if (
-      // When the cursor goes before current completionItem (e.g. using arrow keys), it would close the autocompletions.
-      // We want the autocompletion to be active at whatever point we are in the token, so let's modify this start pos with the start pos of the token
-      previousCompletionItem &&
-      cursor.sticky && // Is undefined at the end of the token, otherwise it is set as either "before" or "after" (The movement of the cursor)
-      cursor.ch !== previousCompletionItem.startPos.ch
-    ) {
-      this.yasqe.state.completionActive.startPos = cursor;
-    } else if (previousCompletionItem && !cursor.sticky && cursor.ch < previousCompletionItem.startPos.ch) {
-      // A similar thing happens when pressing backspace, CodeMirror will close this autocomplete when 'startLen' changes downward
-      cursor.sticky = previousCompletionItem.startPos.sticky;
-      this.yasqe.state.completionActive.startPos.ch = cursor.ch;
-      this.yasqe.state.completionActive.startLen--;
-    }
     if (
       fromAutoShow && // from autoShow, i.e. this gets called each time the editor content changes
-      (!this.config.autoShow || this.yasqe.state.completionActive) // Don't show  and don't create a new instance when its already active
+      (!this.config.autoShow || this.yasqe.isAutocompletionActive()) // Don't show  and don't create a new instance when its already active
     ) {
       return false;
     }
+    return this.getHintFn();
+  }
 
+  /**
+   * Hint function that computes the hints for the token at the current cursor position
+   */
+  public getHintFn(): HintFn {
     const getHints: HintFn = () => {
-      return this.getHints(this.yasqe.getCompleteToken()).then((list) => {
+      return this.getHints(this.yasqe.getCompleteToken()).then((list): HintList => {
         const cur = this.yasqe.getDoc().getCursor();
         const token: AutocompletionToken = this.yasqe.getCompleteToken();
-        const hintResult = {
+        return {
           list: list,
           from: <Position>{
             line: cur.line,
@@ -216,37 +215,9 @@ export class Completer extends EventEmitter {
             ch: token.end,
           },
         };
-        CodeMirror.on(hintResult, "shown", () => {
-          this.yasqe.emit("autocompletionShown", (this.yasqe as any).state.completionActive.widget);
-        });
-        CodeMirror.on(hintResult, "close", () => {
-          this.yasqe.emit("autocompletionClose");
-        });
-        return hintResult;
       });
     };
-
-    getHints.async = false; //in their code, async means using a callback
-    //we always return a promise, which should be properly handled regardless of this val
-    var hintConfig: HintConfig = {
-      closeCharacters: /[\s>"]/,
-      completeSingle: false,
-      hint: getHints,
-      container: this.yasqe.rootEl,
-      // Override these actions back to use their default function
-      // Otherwise these would navigate to the start/end of the suggestion list, while this can also be accomplished with PgUp and PgDn
-      extraKeys: {
-        Home: (yasqe, event) => {
-          yasqe.getDoc().setCursor({ ch: 0, line: event.data.from.line });
-        },
-        End: (yasqe, event) => {
-          yasqe.getDoc().setCursor({ ch: yasqe.getLine(event.data.to.line).length, line: event.data.to.line });
-        },
-      },
-      ...this.yasqe.config.hintConfig,
-    };
-    this.yasqe.showHint(hintConfig);
-    return true;
+    return getHints;
   }
 }
 

package/src/autocompleters/show-hint.scss

@@ -1,13 +1,11 @@
-//This is a copy of the codemirror show-hint css
-//Including this ourselves, as we don't want this css to be auto-scoped to yasqe
+// Styling for the autocompletion tooltip (CodeMirror 6 `@codemirror/autocomplete`).
+// The tooltip gets the legacy `CodeMirror-hints` class (see editor/autocompletion.ts) so that
+// existing consumers / tests keep working.
+// Including this ourselves, as we don't want this css to be auto-scoped to yasqe
+.cm-tooltip.CodeMirror-hints,
 .CodeMirror-hints {
-  position: absolute;
   z-index: 10;
   overflow: hidden;
-  list-style: none;
-
-  margin: 0;
-  padding: 2px;
 
   -webkit-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2);
   -moz-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2);
@@ -18,10 +16,18 @@
   background: white;
   font-size: 90%;
   font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, Courier, monospace;
-  max-height: 20em;
-  overflow-y: auto;
+
+  & > ul {
+    list-style: none;
+    margin: 0;
+    padding: 2px;
+    max-height: 20em;
+    overflow-y: auto;
+    font-family: inherit;
+  }
 }
 
+.CodeMirror-hints > ul > li,
 .CodeMirror-hint {
   margin: 0;
   padding: 0 4px;
@@ -29,8 +35,10 @@
   white-space: pre;
   color: black;
   cursor: pointer;
+  line-height: 1.4;
 }
 
+.CodeMirror-hints > ul > li[aria-selected],
 li.CodeMirror-hint-active {
   background: #08f;
   color: white;

package/src/autocompleters/variables.ts

@@ -1,4 +1,5 @@
 import * as autocompleter from "./";
+import { runMode } from "../editor/tokenizerRunner";
 
 var conf: autocompleter.CompleterConfig = {
   name: "variables",
@@ -16,33 +17,28 @@ var conf: autocompleter.CompleterConfig = {
     if (!token || token.string.length == 0) return []; //nothing to autocomplete
     const distinctVars: { [varname: string]: any } = {};
     const vars: string[] = [];
-    //do this outside of codemirror. I expect jquery to be faster here (just finding dom elements with classnames)
-    //and: this'll still work when the query is incorrect (i.e., when simply typing '?')
-    const atoms = yasqe.getWrapperElement().querySelectorAll(".cm-atom");
-    for (let i = 0; i < atoms.length; i++) {
-      const atom = atoms[i];
-      var variable = atom.innerHTML;
-      if (variable[0] === "?" || variable[0] === "$") {
-        //ok, lets check if the next element in the div is an atom as well. In that case, they belong together (may happen sometimes when query is not syntactically valid)
-        // var nextElClass = nextEl.attr("class");
-        const nextEl: HTMLElement = <any>atom.nextSibling;
-        if (nextEl && nextEl.className && nextEl.className.indexOf("cm-atom") >= 0) {
-          variable += nextEl.innerText;
-        }
-        if (distinctVars[variable]) continue; //already in list
-        //skip single questionmarks
-        if (variable.length <= 1) continue;
+    // Tokenize the whole query (independent of what is rendered in the viewport), collecting all
+    // variable tokens. This still works when the query is syntactically incorrect (e.g. when simply typing '?').
+    const atoms: string[] = [];
+    runMode(yasqe.getValue(), (stringVal: string, style: string | null) => {
+      if (style === "atom" && (stringVal[0] === "?" || stringVal[0] === "$")) {
+        atoms.push(stringVal);
+      }
+    });
+    for (const variable of atoms) {
+      if (distinctVars[variable]) continue; //already in list
+      //skip single questionmarks
+      if (variable.length <= 1) continue;
 
-        //it should match our token ofcourse
-        if (variable.indexOf(token.string) !== 0) continue;
+      //it should match our token ofcourse
+      if (variable.indexOf(token.string) !== 0) continue;
 
-        //skip exact matches
-        if (variable === token.string) continue;
+      //skip exact matches
+      if (variable === token.string) continue;
 
-        //store in map so we have a unique list
-        distinctVars[variable] = true;
-        vars.push(variable);
-      }
+      //store in map so we have a unique list
+      distinctVars[variable] = true;
+      vars.push(variable);
     }
     return vars.sort();
   },

package/src/defaults.ts

@@ -6,8 +6,6 @@
  */
 import { default as Yasqe, Config, PlainRequestConfig } from "./";
 import * as queryString from "query-string";
-import CodeMirror from "codemirror";
-//need to pass Yasqe object as argument, as the imported version might not have inherited all (e.g. `fold`) props of Codemirror yet
 export default function get() {
   const prefixCcApi = null; // null = use bundled prefix data (packages/yasqe/src/prefixes.json, refreshed each build)
   const config: Omit<Config, "requestConfig"> = {
@@ -23,11 +21,9 @@ SELECT * WHERE {
     tabMode: "indent",
     lineNumbers: true,
     lineWrapping: true,
-    foldGutter: {
-      rangeFinder: new (<any>CodeMirror).fold.combine((<any>CodeMirror).fold.brace, (<any>CodeMirror).fold.prefix),
-    },
+    foldGutter: true, // Folding of braces and prefix blocks is handled by the built-in SPARQL fold service
     collapsePrefixesOnLoad: false,
-    gutters: ["gutterErrorBar", "gutterConstructWarning", "CodeMirror-linenumbers", "CodeMirror-foldgutter"],
+    gutters: ["gutterErrorBar", "gutterConstructWarning"],
     matchBrackets: true,
     fixedGutter: true,
     syntaxErrorCheck: true,

package/src/editor/autocompletion.ts

@@ -0,0 +1,147 @@
+/**
+ * CodeMirror 6 autocompletion bridge.
+ *
+ * YASQE's autocompleters (see ../autocompleters) were written against the CodeMirror 5
+ * `show-hint` addon: a "hint function" returns a list of `Hint`s together with the range
+ * that should be replaced. This module adapts that model onto `@codemirror/autocomplete`:
+ *
+ * - A single `CompletionSource` is registered. Whenever it is queried it asks the host
+ *   (the `Yasqe` facade) for the currently active hint function and converts its result
+ *   into a `CompletionResult`.
+ * - `Hint.from` / `Hint.to` are honoured per completion by using a custom `apply`.
+ * - Open/close transitions are observed through `completionStatus` and reported to the host
+ *   so that the legacy `autocompletionShown` / `autocompletionClose` events keep working.
+ */
+import { EditorState, Extension, Text } from "@codemirror/state";
+import { EditorView, ViewUpdate } from "@codemirror/view";
+import {
+  autocompletion,
+  Completion,
+  CompletionContext,
+  CompletionResult,
+  completionStatus,
+  completionKeymap,
+  startCompletion,
+  closeCompletion,
+} from "@codemirror/autocomplete";
+import { keymap } from "@codemirror/view";
+import { Prec } from "@codemirror/state";
+import type { Position } from "./tokenizerRunner";
+import { posToOffset, offsetToPos } from "./doc";
+
+export interface Hint {
+  text: string;
+  displayText?: string;
+  className?: string;
+  /** Kept for API compatibility; custom rendering is not supported by the CM6 bridge. */
+  render?: (el: HTMLElement, self: Hint, data: any) => void;
+  from?: Position;
+  to?: Position;
+}
+export interface HintList {
+  list: Hint[];
+  from: Position;
+  to: Position;
+}
+export type HintFn = { async?: boolean } & (() => Promise<HintList | undefined> | HintList | undefined);
+
+export interface AutocompletionHost {
+  /** Returns the hint function that should be used for the current completion session (if any) */
+  getHintFn: () => HintFn | undefined;
+  onShown?: (tooltipDom: HTMLElement | null) => void;
+  onClose?: () => void;
+}
+
+export { posToOffset, offsetToPos };
+
+function hintToCompletion(hint: Hint): Completion {
+  return {
+    label: hint.displayText || hint.text,
+    type: hint.className,
+    apply: (view: EditorView, _completion: Completion, from: number, to: number) => {
+      const doc = view.state.doc;
+      const replaceFrom = hint.from ? posToOffset(doc, hint.from) : from;
+      const replaceTo = hint.to ? posToOffset(doc, hint.to) : to;
+      const start = Math.min(replaceFrom, replaceTo);
+      const end = Math.max(replaceFrom, replaceTo);
+      view.dispatch({
+        changes: { from: start, to: end, insert: hint.text },
+        selection: { anchor: start + hint.text.length },
+        userEvent: "input.complete",
+      });
+    },
+  };
+}
+
+export function hintListToCompletionResult(hintList: HintList, doc: Text): CompletionResult | null {
+  if (!hintList || !hintList.list || hintList.list.length === 0) return null;
+  const from = posToOffset(doc, hintList.from);
+  const to = posToOffset(doc, hintList.to);
+  return {
+    from: Math.min(from, to),
+    to: Math.max(from, to),
+    options: hintList.list.map(hintToCompletion),
+    // Completers already filtered and ordered their suggestions
+    filter: false,
+  };
+}
+
+function createCompletionSource(host: AutocompletionHost) {
+  return async (context: CompletionContext): Promise<CompletionResult | null> => {
+    const hintFn = host.getHintFn();
+    if (!hintFn) return null;
+    const result = await hintFn();
+    if (!result || context.aborted) return null;
+    return hintListToCompletionResult(result, context.state.doc);
+  };
+}
+
+/**
+ * Autocompletion extension. `host.getHintFn()` is consulted every time CodeMirror queries for completions.
+ */
+export function sparqlAutocompletion(host: AutocompletionHost): Extension {
+  const statusListener = EditorView.updateListener.of((update: ViewUpdate) => {
+    const before = completionStatus(update.startState);
+    const after = completionStatus(update.state);
+    if (before === null && after !== null) {
+      host.onShown?.(update.view.dom.querySelector<HTMLElement>(".cm-tooltip-autocomplete"));
+    } else if (before !== null && after === null) {
+      host.onClose?.();
+    }
+  });
+  return [
+    autocompletion({
+      override: [createCompletionSource(host)],
+      // YASQE decides itself when to open the popup (see Yasqe.autocomplete)
+      activateOnTyping: false,
+      closeOnBlur: true,
+      // We register the navigation keys ourselves (below) so that `Ctrl-Space` is exclusively handled by
+      // YASQE's `extraKeys` (which builds the hint function before opening the popup).
+      defaultKeymap: false,
+      icons: false,
+      // CM5 rendered the complete hint list; CM6 pages by 100 items by default. Render everything so that
+      // consumers that inspect the popup (and the existing tests) see the full list.
+      maxRenderedOptions: 100000,
+      // CM5 accepted a completion immediately after the popup opened; CM6 ignores Enter/arrows during the first
+      // 75ms (and lets the key fall through, inserting a newline). Keep the CM5 behaviour.
+      interactionDelay: 0,
+      // Keep the CM5 class names around so existing styling / tests keep working
+      tooltipClass: () => "CodeMirror-hints",
+      optionClass: (completion) => (completion.type ? completion.type : ""),
+    }),
+    Prec.highest(keymap.of(completionKeymap.filter((binding) => binding.key !== "Ctrl-Space"))),
+    statusListener,
+  ];
+}
+
+export function isCompletionActive(state: EditorState): boolean {
+  return completionStatus(state) !== null;
+}
+
+export function openCompletion(view: EditorView): boolean {
+  return startCompletion(view);
+}
+
+export function hideCompletion(view: EditorView): boolean {
+  return closeCompletion(view);
+}