@citisen/litearea 0.1.0

package/src/core/format.ts

@@ -0,0 +1,71 @@
+// ─── format: the small string helpers the messages share ────────────────────
+//
+// Kept apart from the vocabulary so that a grammar can raise its own diagnostics
+// with the same phrasing the built-in ones use, and so that a translated host can
+// reuse the substitution instead of writing a second one.
+
+/** The most members a message will name before it stops counting. */
+const DEFAULT_LIST_LIMIT = 12
+
+/**
+ * Substitute `{name}` placeholders in a template.
+ *
+ * A placeholder with no matching key is left exactly as it was rather than
+ * blanked: a visible `{allowed}` in a message is a bug report, and an empty gap
+ * is a mystery.
+ * @param template - the template, with `{name}` placeholders.
+ * @param values - the substitutions.
+ * @returns the filled text.
+ */
+export function fillTemplate(template: string, values: Readonly<Record<string, string>>): string {
+  return String(template).replace(/\{(\w+)\}/g, (match, key: string) =>
+    Object.hasOwn(values, key) ? String(values[key]) : match,
+  )
+}
+
+/**
+ * A readable list: `a`, `a or b`, `a, b, or c`.
+ *
+ * The conjunction is a parameter because the same list reads differently as
+ * "expected one of X, Y, or Z" and as "X, Y, and Z are installed".
+ * @param items - the members.
+ * @param options - `conjunction` (`'or'` by default) and `limit`, the most
+ *   members to name before summarising the rest as `… and N more`.
+ * @returns the phrase.
+ */
+export function listPhrase(
+  items: readonly string[],
+  options?: { conjunction?: string; limit?: number },
+): string {
+  const conjunction = options?.conjunction ?? 'or'
+  const limit = options?.limit ?? DEFAULT_LIST_LIMIT
+  const shown = items.slice(0, limit)
+  const rest = items.length - shown.length
+  if (shown.length === 0) return ''
+  if (shown.length === 1) {
+    return rest > 0 ? `${String(shown[0])} and ${String(rest)} more` : String(shown[0])
+  }
+  // Two members take no comma before the conjunction. "circle, or square" is not how
+  // anyone writes it, and every diagnostic that lists a two-member vocabulary would
+  // have said it that way.
+  if (shown.length === 2) {
+    const pair = `${String(shown[0])} ${conjunction} ${String(shown[1])}`
+    return rest > 0 ? `${pair}, and ${String(rest)} more` : pair
+  }
+  const head = shown.slice(0, -1).join(', ')
+  const tail = shown[shown.length - 1]
+  const phrase = `${head}, ${conjunction} ${String(tail)}`
+  return rest > 0 ? `${phrase}, and ${String(rest)} more` : phrase
+}
+
+/**
+ * Collapse the whitespace in a snippet of user text so it can be quoted inside a
+ * one-line message.
+ * @param text - the text.
+ * @param limit - the longest result, ellipsised beyond it.
+ * @returns the excerpt.
+ */
+export function excerpt(text: string, limit = 24): string {
+  const flat = text.replace(/\s+/g, ' ').trim()
+  return flat.length <= limit ? flat : `${flat.slice(0, limit - 1)}…`
+}

package/src/core/grammar.ts

@@ -0,0 +1,40 @@
+// ─── grammar: the two identity functions authors actually want ──────────────
+//
+// Neither does anything at runtime, and that is not a joke at the reader's
+// expense: `defineGrammar` exists so `State` is inferred from `analyze` and then
+// enforced everywhere else. Written as a bare object literal, a grammar whose
+// `analyze` returns `{ entries: Entry[] }` would have every hook's `context.state`
+// typed as `unknown`, and the author would spend the afternoon casting.
+//
+//     const grammar = defineGrammar({
+//       id: 'my-language',
+//       rules: [...],
+//       analyze: (text) => ({ lines: text.split('\n') }),   // State is inferred
+//       compose: [{
+//         id: 'values',
+//         range: (context) => ({ from: 0, to: context.caret }),
+//         items: (context) => context.state.lines.map(...),  // typed, not cast
+//       }],
+//     })
+
+import type { CompletionSource, Grammar } from './types.js'
+
+/**
+ * Declare a grammar, inferring its state type from `analyze`.
+ * @param grammar - the grammar.
+ * @returns the same grammar, typed.
+ */
+export function defineGrammar<State = unknown>(grammar: Grammar<State>): Grammar<State> {
+  return grammar
+}
+
+/**
+ * Declare a completion source, inferring the state type from its context.
+ * @param source - the source.
+ * @returns the same source, typed.
+ */
+export function defineCompletion<State = unknown>(
+  source: CompletionSource<State>,
+): CompletionSource<State> {
+  return source
+}

package/src/core/hover.ts

@@ -0,0 +1,129 @@
+// ─── hover: what the pointer is resting on ─────────────────────────────────
+//
+// A tooltip is only worth showing when it says something the text does not, and
+// there are exactly three things that qualify: a problem, a semantic mark the
+// grammar made, and the grammar's own explanation of a token. They are consulted
+// in that order and the first one that has something to say wins.
+//
+// A diagnostic leads deliberately. The other two describe what the text IS, and
+// the user resting the pointer on a red squiggle is asking what is WRONG with it
+// — answering "circle: a full disc" there would be technically true and useless,
+// and it is exactly the behaviour that makes people stop hovering.
+
+import type { Decoration, Diagnostic, Grammar, HoverContext, HoverInfo, Severity } from './types.js'
+import type { Inspection } from './inspect.js'
+import { containsOffset, lineAt, lineStarts, tokenAt, wordInfoAt } from './text.js'
+import { isResolvedGrammar, resolveGrammar, type ResolvedGrammar } from './scan.js'
+
+/** The word a severity is shown under, as VSCode titles its hovered diagnostics. */
+const SEVERITY_TITLE: Record<Severity, string> = {
+  error: 'Error',
+  warning: 'Warning',
+  info: 'Info',
+  hint: 'Hint',
+}
+
+/**
+ * Turn one diagnostic into a tooltip.
+ *
+ * Exported because a host that shows its own marker list wants the same wording,
+ * and because a test should be able to assert the tooltip without going through
+ * the ordering rules below.
+ * @param diagnostic - the diagnostic.
+ * @returns the tooltip.
+ */
+export function diagnosticHover(diagnostic: Diagnostic): HoverInfo {
+  return {
+    kind: 'diagnostic',
+    title: SEVERITY_TITLE[diagnostic.severity],
+    detail: diagnostic.source,
+    body: diagnostic.detail === undefined ? diagnostic.message : `${diagnostic.message}\n\n${diagnostic.detail}`,
+    range: { from: diagnostic.from, to: diagnostic.to },
+  }
+}
+
+/**
+ * Resolve what a hover should show at an offset.
+ *
+ * @param inspection - the current inspection.
+ * @param grammar - the language.
+ * @param offset - the character offset the pointer resolved to.
+ * @returns the tooltip, or undefined when there is nothing to say.
+ */
+export function resolveHover<State>(
+  inspection: Inspection<State>,
+  grammar: Grammar<State> | ResolvedGrammar<State>,
+  offset: number,
+): HoverInfo | undefined {
+  const resolved = isResolvedGrammar(grammar) ? grammar : resolveGrammar(grammar)
+  const { text, tokens, diagnostics, decorations, state } = inspection
+
+  // ── a problem outranks a description ─────────────────────────────────────
+  // The narrowest diagnostic wins when several overlap, so a squiggle inside a
+  // wider warning still explains itself rather than its container.
+  let covering: Diagnostic | undefined
+  for (const diagnostic of diagnostics) {
+    if (!containsOffset(diagnostic, offset)) continue
+    if (covering === undefined || diagnostic.to - diagnostic.from < covering.to - covering.from) {
+      covering = diagnostic
+    }
+  }
+  if (covering !== undefined) return diagnosticHover(covering)
+
+  // ── a mark the grammar made, combined with its own account of the token ──
+  // Both, rather than one or the other. A decoration's title is short by design ("in
+  // effect: Geist Mono") and the grammar's description is rich but says nothing about
+  // the mark, so a tooltip that showed only one of them would always be missing the
+  // half the reader wanted.
+  const decoration: Decoration | undefined = decorations.find((entry) => containsOffset(entry, offset))
+  const described = describeAt(inspection, resolved, offset)
+  if (decoration?.title !== undefined) {
+    return {
+      kind: 'decoration',
+      title: decoration.title,
+      detail: described?.detail ?? described?.title,
+      body: described?.body,
+      range: { from: decoration.from, to: decoration.to },
+    }
+  }
+
+  return described
+}
+
+/**
+ * Ask the grammar what a token is.
+ *
+ * Split out because the decoration branch above wants the same answer and must not
+ * build a second context for it — one parse, one description, however many tooltips
+ * are made from it.
+ * @param inspection - the current inspection.
+ * @param resolved - the resolved grammar.
+ * @param offset - the offset being described.
+ * @returns the description, or undefined when the grammar has none.
+ */
+function describeAt<State>(
+  inspection: Inspection<State>,
+  resolved: ResolvedGrammar<State>,
+  offset: number,
+): HoverInfo | undefined {
+  if (resolved.grammar.describe === undefined) return undefined
+  const { text, tokens, diagnostics, state } = inspection
+  const found = tokenAt(tokens, offset)
+  // Whitespace is painted like anything else, so a token IS found in blank space — and a
+  // grammar handed one will describe it, which is how resting the pointer on a gap produced a
+  // tooltip naming the fallback scope. There is no thing under the pointer to explain, so the
+  // grammar is told there is no token; a grammar that wants to say something about the
+  // POSITION can still do so, because `describe` is called either way.
+  const token = found === undefined || found.text.trim() === '' ? undefined : found
+  const context: HoverContext<State> = {
+    text,
+    offset,
+    token,
+    word: wordInfoAt(text, offset, resolved.wordChars),
+    line: lineAt(text, offset, lineStarts(text)),
+    tokens,
+    diagnostics,
+    state,
+  }
+  return resolved.grammar.describe(context) ?? undefined
+}

package/src/core/index.ts

@@ -0,0 +1,98 @@
+// ─── the engine's public surface ────────────────────────────────────────────
+//
+// Everything here is pure: text and a grammar go in, values come out. Nothing in
+// this module touches the DOM, which is why `scripts/browser-check.mjs` is needed
+// for only one claim and everything else is asserted in plain Node.
+
+export type {
+  CheckRule,
+  Completion,
+  CompletionContext,
+  CompletionRow,
+  CompletionSource,
+  CompletionTrigger,
+  Decoration,
+  DiagnoseContext,
+  Diagnostic,
+  Grammar,
+  HoverContext,
+  HoverInfo,
+  LineInfo,
+  MatchRule,
+  Range,
+  RegionRule,
+  ResolvedVocabulary,
+  Rule,
+  RuleContext,
+  RuleMatch,
+  Scope,
+  ScopeSpec,
+  Severity,
+  SuggestionItem,
+  Token,
+  VocabularyContext,
+  VocabularyEntry,
+  WordInfo,
+  WordsRule,
+  WordsSource,
+} from './types.js'
+
+export { excerpt, fillTemplate, listPhrase } from './format.js'
+
+export {
+  clamp,
+  containsOffset,
+  isEmptyRange,
+  isOffset,
+  isWordChar,
+  lineAt,
+  lineIndexAt,
+  lineStarts,
+  scopeAt,
+  tokenAfter,
+  tokenAt,
+  tokenBefore,
+  tokensOnLine,
+  wordInfoAt,
+} from './text.js'
+
+export {
+  asResolvedVocabulary,
+  defineVocabulary,
+  resolveWordsSource,
+  vocabularyWords,
+  type VocabularySpec,
+} from './vocabulary.js'
+
+export {
+  isResolvedGrammar,
+  resolveGrammar,
+  scan,
+  type ResolvedGrammar,
+  type ScanResult,
+} from './scan.js'
+
+export { buildSegments, segmentClasses, type PaintSegment, type SegmentInput } from './segments.js'
+
+export { inspect, normalizeDiagnostics, type Inspection } from './inspect.js'
+
+export {
+  fuzzyMatch,
+  highlightSegments,
+  isWordStart,
+  rank,
+  type FuzzyMatch,
+  type Ranked,
+  type RankOptions,
+} from './rank.js'
+
+export {
+  applyCompletion,
+  complete,
+  type AppliedCompletion,
+  type CompletionRequest,
+} from './complete.js'
+
+export { diagnosticHover, resolveHover } from './hover.js'
+
+export { defineCompletion, defineGrammar } from './grammar.js'

package/src/core/inspect.ts

@@ -0,0 +1,198 @@
+// ─── inspect: the one call the editor makes per keystroke ───────────────────
+//
+// Everything the editor needs to know about a document is computed here, once,
+// from one grammar. The paint, the squiggles, the semantic decorations, and the
+// structural analysis come out of a single call and are stored together, so the
+// completion list and the coloured text cannot be looking at different documents.
+//
+// That is a deliberate inversion of how the editors this library replaces were
+// built. Each of them re-derived the tokens, the parse, and the suggestions
+// separately, from the same string, in three places — and the three drifted, so
+// a word could be painted as a valid value while the completer thought it was
+// unknown and the diagnostic pointed at a range that had already moved.
+
+import type {
+  CheckRule,
+  Decoration,
+  DiagnoseContext,
+  Diagnostic,
+  Grammar,
+  Token,
+} from './types.js'
+import { listPhrase } from './format.js'
+import { isResolvedGrammar, resolveGrammar, scan, type ResolvedGrammar } from './scan.js'
+import { resolveWordsSource } from './vocabulary.js'
+
+/** Everything one pass over a document produced. */
+export interface Inspection<State = unknown> {
+  /** The text this was computed from, kept so nothing has to guess its version. */
+  text: string
+  tokens: Token[]
+  /** Lexical problems, the grammar's `checks`, and its `validate`, together. */
+  diagnostics: Diagnostic[]
+  /** Semantic ranges that are not tokens. */
+  decorations: Decoration[]
+  /** Whatever `Grammar.analyze` returned. */
+  state: State
+}
+
+/**
+ * Run a grammar's declarative checks over the tokens.
+ *
+ * A check is the shape most vocabulary mistakes actually take — "this token is
+ * one of these words or it is wrong" — and expressing it declaratively means the
+ * message, the severity, and the code are declared once instead of being
+ * reassembled by hand at every site.
+ * @param source - the document.
+ * @param grammar - the resolved grammar.
+ * @param tokens - the scanned tokens.
+ * @param state - the analysis, so a check may consult a vocabulary the machine supplied.
+ * @returns the diagnostics the checks raised.
+ */
+function runChecks<State>(
+  source: string,
+  grammar: ResolvedGrammar<State>,
+  tokens: readonly Token[],
+  state: State,
+): Diagnostic[] {
+  const checks = grammar.grammar.checks
+  if (checks === undefined || checks.length === 0) return []
+  const diagnostics: Diagnostic[] = []
+  // The ANALYSIS, not `initialState`. A `WordsSource` function is documented as receiving
+  // the grammar's analysis, and the point of a dynamic vocabulary is that it comes from
+  // outside the document — so a check whose allowed set is resolved from `analyze` must
+  // see it, or the two halves of one declaration would disagree about what is legal.
+  const context = { text: source, state }
+
+  for (const check of checks as readonly CheckRule<State>[]) {
+    const members = check.allow === undefined ? undefined : resolveWordsSource(check.allow, context)
+    const vocabulary = check.allow === undefined ? undefined : check.allow
+    const caseSensitive =
+      typeof vocabulary === 'object' && vocabulary !== null && !Array.isArray(vocabulary)
+        ? (vocabulary as { caseSensitive?: boolean }).caseSensitive === true
+        : false
+    const fold = (word: string): string => (caseSensitive ? word : word.toLowerCase())
+    const set =
+      members === undefined ? undefined : new Set(members.map((member) => fold(member)))
+    /** Lines already reported, for a check that only wants one complaint each. */
+    const reportedLines = new Set<number>()
+
+    for (const token of tokens) {
+      // Whitespace is a token like any other, which means a check written against `'*'` would
+      // otherwise underline every space and report it as a misspelling. No one has ever
+      // wanted a diagnostic about a space.
+      if (token.text.trim() === '') continue
+      if (!check.scopes.includes('*') && !check.scopes.includes(token.scope)) continue
+      if (check.except !== undefined && check.except.test(token.text)) continue
+      if (set !== undefined && set.has(fold(token.text))) continue
+      if (check.perLine === true) {
+        if (reportedLines.has(token.line)) continue
+        reportedLines.add(token.line)
+      }
+      diagnostics.push({
+        from: token.from,
+        to: token.to,
+        severity: check.severity ?? 'error',
+        message: check.message
+          .replace(/\{word\}/g, token.text)
+          .replace(/\{allowed\}/g, listPhrase(members ?? [])),
+        code: check.code,
+        detail: check.detail,
+        source: grammar.grammar.id,
+      })
+    }
+  }
+  return diagnostics
+}
+
+/**
+ * Inspect a document: paint, problems, decorations, and structure in one pass.
+ * @param source - the document.
+ * @param grammar - the language, as written or already resolved.
+ * @returns everything the editor needs for one text.
+ */
+export function inspect<State>(
+  source: string,
+  grammar: Grammar<State> | ResolvedGrammar<State>,
+): Inspection<State> {
+  const resolved = isResolvedGrammar(grammar) ? grammar : resolveGrammar(grammar)
+  const scanned = scan(source, resolved)
+  const declared = resolved.grammar
+  const diagnostics: Diagnostic[] = [...scanned.diagnostics]
+
+  diagnostics.push(...runChecks(source, resolved, scanned.tokens, scanned.state))
+
+  if (declared.validate !== undefined) {
+    const context: DiagnoseContext<State> = {
+      text: source,
+      tokens: scanned.tokens,
+      state: scanned.state,
+      report: (problem) => {
+        diagnostics.push({
+          from: problem.from,
+          to: problem.to,
+          severity: problem.severity ?? 'error',
+          message: problem.message,
+          code: problem.code ?? 'validate',
+          detail: problem.detail,
+          source: declared.id,
+        })
+      },
+    }
+    declared.validate(context)
+  }
+
+  const decorations = declared.decorate === undefined ? [] : [...declared.decorate(source, scanned.state)]
+
+  return {
+    text: source,
+    tokens: scanned.tokens,
+    diagnostics: normalizeDiagnostics(diagnostics),
+    decorations: normalizeDecorations(decorations, source.length),
+    state: scanned.state,
+  }
+}
+
+/**
+ * Sort diagnostics by position and drop duplicates.
+ *
+ * Two rules can legitimately notice the same mistake — a vocabulary's own
+ * rejection and a grammar's validator both know that `nope` is not a colour —
+ * and drawing the underline twice makes it darker rather than more informative.
+ * Duplicates are compared on position, code, and message, so two genuinely
+ * different complaints about one word both survive.
+ * @param diagnostics - the collected diagnostics.
+ * @returns the deduplicated, ordered list.
+ */
+export function normalizeDiagnostics(diagnostics: readonly Diagnostic[]): Diagnostic[] {
+  const seen = new Set<string>()
+  const out: Diagnostic[] = []
+  for (const diagnostic of diagnostics) {
+    const key = `${diagnostic.from}:${diagnostic.to}:${diagnostic.code ?? ''}:${diagnostic.message}`
+    if (seen.has(key)) continue
+    seen.add(key)
+    out.push(diagnostic)
+  }
+  return out.sort((left, right) => left.from - right.from || left.to - right.to)
+}
+
+/**
+ * Clamp decorations into the document and drop the ones that came out empty.
+ *
+ * A grammar computing decorations from a stale parse can hand over a range that
+ * no longer exists, and a decoration outside the text would only produce a span
+ * nobody can see.
+ * @param decorations - the declared decorations.
+ * @param length - the document's length.
+ * @returns the usable decorations, ordered.
+ */
+function normalizeDecorations(decorations: readonly Decoration[], length: number): Decoration[] {
+  const out: Decoration[] = []
+  for (const decoration of decorations) {
+    const from = Math.max(0, Math.min(decoration.from, length))
+    const to = Math.max(from, Math.min(decoration.to, length))
+    if (to <= from) continue
+    out.push({ ...decoration, from, to })
+  }
+  return out.sort((left, right) => left.from - right.from || left.to - right.to)
+}