@eriknaslund/skiss 0.5.0 → 0.6.0

package/README.md

@@ -69,7 +69,9 @@ Or without installing: `npx @eriknaslund/skiss diagram model.skiss`.
 
 `skiss import schema.linkml.yaml` goes the other way: it reads a LinkML schema, YAML or JSON, and writes Skiss. What the projection could not carry — patterns, `required`, `mixins`, the names it had to convert — is one line on standard error before the diagnostics, because a sketch is a smaller language than LinkML and that line says by how much. `-o`, `-` and the diagnostics work as they do for `compile`; `--strict` exits 1 when the projected sketch has a diagnostic, and also when anything was dropped. An input that is not a schema at all — not YAML, or YAML no schema can be read out of — produces no sketch, says why on standard error, and exits 2, the code every command uses for unreadable input.
 
-`npm install @eriknaslund/skiss` also gives you the library: `compile`, `formatDiagnostic`, `parse`, `resolve`, `toLinkML`, `serialize`, `toMermaid`, `toSkiss`, `fromLinkML`, `formatDropped`, `importLinkML` and `VERSION`, browser-safe.
+`npm install @eriknaslund/skiss` also gives you the library: `compile`, `formatDiagnostic`, `parse`, `resolve`, `toLinkML`, `serialize`, `toMermaid`, `toSkiss`, `fromLinkML`, `formatDropped`, `importLinkML`, `tokenizeLine` and `VERSION`, browser-safe.
+
+`tokenizeLine` is there for editors: it says where the words of one line are — `{ kind, from, to }` for each run of characters, so a host can colour them. It is line-based and stateless, as the language is: one line in, its tokens out, nothing carried from the line before, and nothing said about a run it does not recognise, so a half-typed line keeps the colours it has earned. It is not a parser and reports nothing; diagnostics stay `parse` and `resolve`'s ([ADR 0010](docs/adr/0010-tokeniser-in-the-package.md)).
 
 Editors that render Skiss live are separate projects built on this package. This repository is the language, the library and the command line.
 
@@ -87,7 +89,7 @@ Editors that render Skiss live are separate projects built on this package. This
 
 ## Status
 
-0.5.0, published on npm as [`@eriknaslund/skiss`](https://www.npmjs.com/package/@eriknaslund/skiss). The language parses, resolves and diagnoses; `skiss diagram` draws it, `skiss compile` writes LinkML that the real LinkML toolchain validates in CI, and `skiss import` reads a LinkML schema back into a sketch. The library exports the same in both directions and is browser-safe.
+0.6.0, published on npm as [`@eriknaslund/skiss`](https://www.npmjs.com/package/@eriknaslund/skiss). The language parses, resolves and diagnoses; `skiss diagram` draws it, `skiss compile` writes LinkML that the real LinkML toolchain validates in CI, and `skiss import` reads a LinkML schema back into a sketch. The library exports the same in both directions and is browser-safe.
 
 The [specification](docs/SPEC.md) is at 0.3 and moves when the language does, which is not with every release. Not built: editing a projected schema so that what Skiss cannot express survives the edit (SPEC §8.1). [CHANGELOG.md](CHANGELOG.md) is what changed when.
 

package/dist/cli.js

@@ -197,6 +197,10 @@ function splitTrailer(text) {
   return trailer;
 }
 var WORD = /[\p{L}\p{N}_-]+/uy;
+function wordAt(text, i) {
+  WORD.lastIndex = i;
+  return WORD.exec(text)?.[0];
+}
 var OPERATORS = ["*", ":", "@", "~", "=", ".", "|", "<", ",", "["];
 function tokenize(head) {
   const tokens = [];
@@ -207,11 +211,10 @@ function tokenize(head) {
       i++;
       continue;
     }
-    WORD.lastIndex = i;
-    const word = WORD.exec(head);
-    if (word !== null) {
-      tokens.push({ kind: "word", text: word[0], col: i, end: i + word[0].length });
-      i += word[0].length;
+    const word = wordAt(head, i);
+    if (word !== void 0) {
+      tokens.push({ kind: "word", text: word, col: i, end: i + word.length });
+      i += word.length;
       continue;
     }
     if (c === "[" && head[i + 1] === "]") {
@@ -1667,7 +1670,7 @@ function notYAML(error) {
 }
 
 // src/index.ts
-var VERSION = "0.5.0";
+var VERSION = "0.6.0";
 
 // src/cli.ts
 var HELP = `Usage: skiss <command> [options]

package/dist/index.d.ts

@@ -205,6 +205,51 @@ declare function formatDropped(dropped: Dropped[]): string;
 
 declare function toSkiss(doc: Document): string;
 
+/**
+ * Where the words of a Skiss line are, so an editor can colour them.
+ *
+ * This is not a parser and holds no language logic (AGENTS.md §2): nothing
+ * here decides whether a line is valid, what it means or what it compiles to.
+ * It reads the shapes of SPEC §3 and §4 far enough to say "this run of
+ * characters is a class name, that one is an operator", and says nothing at
+ * all about a run it does not recognise, which is what leaves a half-typed
+ * line looking calm instead of wrong. Diagnostics stay the compiler's, and
+ * reach the editor through the gutter.
+ *
+ * Every line is read on its own, as the language is (SPEC §4): no state is
+ * carried from one line to the next, so an edit is never more than one line
+ * of work.
+ *
+ * The productions come from `parse.ts` — the name shapes, the primitive
+ * table, what a word is and where a trailer starts — so that the colours and
+ * the compiler can never disagree about them (ADR 0010).
+ */
+/** What a run of characters is, as far as colour is concerned. */
+type TokenKind = 'class' | 'field' | 'system' | 'primitive'
+/** A lowercase word after `:` that is no primitive. The compiler warns; the colour does not. */
+ | 'type' | 'enum'
+/** The operators of SPEC §4, `<` included, and the `.` of a `= Class.field` join. */
+ | 'operator'
+/** The `*` that marks the identifier. */
+ | 'marker'
+/** A `#` trailer, the `#` included. */
+ | 'description'
+/** A `?` trailer, the `?` included. */
+ | 'doubt'
+/** A whole `#` line at column 0. */
+ | 'comment';
+/** A run of one line, from `from` up to but not including `to`. */
+interface Token {
+    kind: TokenKind;
+    from: number;
+    to: number;
+}
+/**
+ * The tokens of one line, in the order they appear, never overlapping. A blank
+ * line, and a line of nothing the tokenizer recognises, has none.
+ */
+declare function tokenizeLine(line: string): Token[];
+
 interface ImportResult {
     /** The projected sketch as canonical Skiss text, `\n`-terminated. */
     output: string;
@@ -223,6 +268,6 @@ interface ResolvedDocument extends Document {
 }
 declare function resolve(doc: Document): ResolvedDocument;
 
-declare const VERSION = "0.5.0";
+declare const VERSION = "0.6.0";
 
-export { type ClassNode, type CompileOptions, type CompileResult, type Diagnostic, type DiagnosticCode, type Document, type Dropped, type FieldNode, type FromLinkMLResult, type ImportResult, type JoinRef, type LinkMLAnnotations, type LinkMLAttribute, type LinkMLClass, type LinkMLCompileOptions, type LinkMLEnum, type LinkMLOptions, type LinkMLSchema, type MermaidCompileOptions, type MermaidOptions, type Name, type Primitive, type ResolvedDocument, type SerializeFormat, type Severity, type TypeRef, VERSION, compile, formatDiagnostic, formatDropped, fromLinkML, importLinkML, parse, resolve, serialize, toLinkML, toMermaid, toSkiss };
+export { type ClassNode, type CompileOptions, type CompileResult, type Diagnostic, type DiagnosticCode, type Document, type Dropped, type FieldNode, type FromLinkMLResult, type ImportResult, type JoinRef, type LinkMLAnnotations, type LinkMLAttribute, type LinkMLClass, type LinkMLCompileOptions, type LinkMLEnum, type LinkMLOptions, type LinkMLSchema, type MermaidCompileOptions, type MermaidOptions, type Name, type Primitive, type ResolvedDocument, type SerializeFormat, type Severity, type Token, type TokenKind, type TypeRef, VERSION, compile, formatDiagnostic, formatDropped, fromLinkML, importLinkML, parse, resolve, serialize, toLinkML, toMermaid, toSkiss, tokenizeLine };

package/dist/index.js

@@ -187,6 +187,10 @@ function splitTrailer(text) {
   return trailer;
 }
 var WORD = /[\p{L}\p{N}_-]+/uy;
+function wordAt(text, i) {
+  WORD.lastIndex = i;
+  return WORD.exec(text)?.[0];
+}
 var OPERATORS = ["*", ":", "@", "~", "=", ".", "|", "<", ",", "["];
 function tokenize(head) {
   const tokens = [];
@@ -197,11 +201,10 @@ function tokenize(head) {
       i++;
       continue;
     }
-    WORD.lastIndex = i;
-    const word = WORD.exec(head);
-    if (word !== null) {
-      tokens.push({ kind: "word", text: word[0], col: i, end: i + word[0].length });
-      i += word[0].length;
+    const word = wordAt(head, i);
+    if (word !== void 0) {
+      tokens.push({ kind: "word", text: word, col: i, end: i + word.length });
+      i += word.length;
       continue;
     }
     if (c === "[" && head[i + 1] === "]") {
@@ -518,36 +521,36 @@ function parseType(c, colon) {
     first
   );
 }
-function unexpected(token, line, kind, order) {
+function unexpected(token2, line, kind, order) {
   const modifiers = kind === "field" ? ["*", ":", "@", "="] : ["<", "@", "~"];
-  if (modifiers.includes(token.kind)) {
+  if (modifiers.includes(token2.kind)) {
     return fail(
       "E_UNPARSABLE",
-      `\`${token.text}\` is out of order; a ${kind} line reads: ${order}`,
+      `\`${token2.text}\` is out of order; a ${kind} line reads: ${order}`,
       line,
-      token
+      token2
     );
   }
-  if (token.kind === "<") {
+  if (token2.kind === "<") {
     return fail(
       "E_UNPARSABLE",
       "`<` marks inheritance and belongs on the class line: `Child < Parent`",
       line,
-      token
+      token2
     );
   }
-  if (token.text === "?") {
+  if (token2.text === "?") {
     return fail(
       "E_UNPARSABLE",
       "`?` attached to a word is not a marker; a doubt needs whitespace before `?` and after it",
       line,
-      token
+      token2
     );
   }
-  if (token.text === "#") {
-    return fail("E_UNPARSABLE", "A description needs whitespace before `#`", line, token);
+  if (token2.text === "#") {
+    return fail("E_UNPARSABLE", "A description needs whitespace before `#`", line, token2);
   }
-  return fail("E_UNPARSABLE", `Unexpected \`${token.text}\` on a ${kind} line`, line, token);
+  return fail("E_UNPARSABLE", `Unexpected \`${token2.text}\` on a ${kind} line`, line, token2);
 }
 
 // src/resolve.ts
@@ -1633,6 +1636,204 @@ var nothing = (element, detail) => ({
   dropped: [{ kind: UNREADABLE, element, detail }]
 });
 
+// src/highlight.ts
+var TRAILING_WS = /[ \t]+$/;
+function tokenizeLine(line) {
+  const text = line.replace(TRAILING_WS, "");
+  const first = text[0];
+  if (first === void 0) {
+    return [];
+  }
+  if (first === "#") {
+    return [{ kind: "comment", from: 0, to: text.length }];
+  }
+  const { head, trailers } = splitTrailer2(text);
+  return [...headTokens(head, isWs(first)), ...trailers];
+}
+function splitTrailer2(text) {
+  let start = -1;
+  for (let i = 1; i < text.length; i++) {
+    if (!isWs(text[i - 1])) {
+      continue;
+    }
+    if (text[i] === "#" || isDoubtMarker(text, i)) {
+      start = i;
+      break;
+    }
+  }
+  if (start < 0) {
+    return { head: text, trailers: [] };
+  }
+  const head = text.slice(0, start);
+  if (text[start] === "?") {
+    return { head, trailers: [{ kind: "doubt", from: start, to: text.length }] };
+  }
+  let doubt = -1;
+  for (let i = start + 1; i < text.length; i++) {
+    if (isDoubtMarker(text, i)) {
+      doubt = i;
+      break;
+    }
+  }
+  if (doubt < 0) {
+    return { head, trailers: [{ kind: "description", from: start, to: text.length }] };
+  }
+  return {
+    head,
+    trailers: [
+      // The whitespace between the two belongs to neither.
+      {
+        kind: "description",
+        from: start,
+        to: text.slice(0, doubt).replace(TRAILING_WS, "").length
+      },
+      { kind: "doubt", from: doubt, to: text.length }
+    ]
+  };
+}
+function scan(head) {
+  const raws = [];
+  let i = 0;
+  while (i < head.length) {
+    const c = head[i] ?? "";
+    if (isWs(c)) {
+      i += 1;
+      continue;
+    }
+    const word = wordAt(head, i);
+    if (word !== void 0) {
+      raws.push({ text: word, from: i, to: i + word.length, word: true });
+      i += word.length;
+      continue;
+    }
+    if (c === "[" && head[i + 1] === "]") {
+      raws.push({ text: "[]", from: i, to: i + 2, word: false });
+      i += 2;
+      continue;
+    }
+    const ch = String.fromCodePoint(head.codePointAt(i) ?? 0);
+    raws.push({ text: ch, from: i, to: i + ch.length, word: false });
+    i += ch.length;
+  }
+  return raws;
+}
+var token = (kind, raw) => ({ kind, from: raw.from, to: raw.to });
+function headTokens(head, field) {
+  const raws = scan(head);
+  const tokens = [];
+  let i = 0;
+  const name2 = raws[0];
+  if (name2?.word === true && (field ? FIELD_NAME : CLASS_NAME).test(name2.text)) {
+    tokens.push(token(field ? "field" : "class", name2));
+    i = 1;
+  }
+  while (i < raws.length) {
+    const raw = raws[i];
+    i += 1;
+    if (raw === void 0 || raw.word) {
+      continue;
+    }
+    switch (raw.text) {
+      case "*":
+        tokens.push(token("marker", raw));
+        break;
+      case ":":
+        tokens.push(token("operator", raw));
+        i = typeTokens(raws, i, tokens);
+        break;
+      case "@":
+        tokens.push(token("operator", raw));
+        i = nameToken(raws, i, tokens, "system", SYSTEM_NAME);
+        break;
+      case "~":
+      case "<":
+        tokens.push(token("operator", raw));
+        i = nameToken(raws, i, tokens, "class", CLASS_NAME);
+        break;
+      case "=":
+        tokens.push(token("operator", raw));
+        i = joinTokens(raws, i, tokens);
+        break;
+      case "|":
+      case "[]":
+        tokens.push(token("operator", raw));
+        break;
+      default:
+        break;
+    }
+  }
+  return tokens;
+}
+function nameToken(raws, i, tokens, kind, shape) {
+  const raw = raws[i];
+  if (raw === void 0 || !raw.word) {
+    return i;
+  }
+  if (shape.test(raw.text)) {
+    tokens.push(token(kind, raw));
+  }
+  return i + 1;
+}
+function joinTokens(raws, start, tokens) {
+  let i = nameToken(raws, start, tokens, "class", CLASS_NAME);
+  const dot = raws[i];
+  if (dot === void 0 || dot.text !== "." || dot.word) {
+    return i;
+  }
+  tokens.push(token("operator", dot));
+  i += 1;
+  return nameToken(raws, i, tokens, "field", FIELD_NAME);
+}
+function typeTokens(raws, start, tokens) {
+  const first = raws[start];
+  if (first === void 0 || !first.word) {
+    return start;
+  }
+  const words = [first];
+  const operators = [];
+  let pipes = 0;
+  let i = start + 1;
+  while (raws[i]?.text === "|") {
+    const pipe = raws[i];
+    if (pipe === void 0) {
+      break;
+    }
+    operators.push(pipe);
+    pipes += 1;
+    i += 1;
+    const value = raws[i];
+    if (value === void 0 || !value.word) {
+      break;
+    }
+    words.push(value);
+    i += 1;
+  }
+  const many = raws[i];
+  if (many !== void 0 && many.text === "[]" && !many.word) {
+    operators.push(many);
+    i += 1;
+  }
+  const typed = pipes > 0 ? words.filter((word) => ENUM_VALUE.test(word.text)).map((word) => token("enum", word)) : nameTypeTokens(first);
+  tokens.push(
+    ...[...typed, ...operators.map((raw) => token("operator", raw))].sort(
+      (a, b) => a.from - b.from
+    )
+  );
+  return i;
+}
+function nameTypeTokens(word) {
+  if (primitiveFor(word.text) !== void 0) {
+    return [token("primitive", word)];
+  }
+  if (CLASS_NAME.test(word.text)) {
+    return [token("class", word)];
+  }
+  if (/^[a-z]/.test(word.text)) {
+    return [token("type", word)];
+  }
+  return [];
+}
+
 // src/import.ts
 import { parse as parseYAML } from "yaml";
 function importLinkML(text) {
@@ -1657,7 +1858,7 @@ function notYAML(error) {
 }
 
 // src/index.ts
-var VERSION = "0.5.0";
+var VERSION = "0.6.0";
 export {
   VERSION,
   compile,
@@ -1670,6 +1871,7 @@ export {
   serialize,
   toLinkML,
   toMermaid,
-  toSkiss
+  toSkiss,
+  tokenizeLine
 };
 //# sourceMappingURL=index.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eriknaslund/skiss",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "A text notation for sketching data models, compiled to LinkML",
   "type": "module",
   "license": "MIT",