@hydranium/core 1.0.0-next.385 → 1.0.0-next.387

package/src/langium/residency/cst-residency-service.ts

@@ -23,6 +23,7 @@ import {
    URI
 } from '@hydranium/langium';
 import { type HydraniumTextDocuments } from '../../documents/hydranium-text-documents.js';
+import { type ModelLedger } from '../../documents/model-ledger.js';
 import { type LogNameOptions } from '../diagnostics/logger.js';
 import { type ServerSharedServicesMinimal } from '../shared-services.js';
 
@@ -90,19 +91,17 @@ export interface CstResidencyOptions extends LogNameOptions {
 export const CST_REHYDRATION_RESET_STATE: DocumentState = DocumentState.Changed;
 
 /**
- * Whether `document`'s CST has been shed by a residency policy: the AST root is
- * resident but its `$cstNode` was nulled. The built-state and present-root guards
- * exclude the `fromModel` INVALID placeholder and not-yet-parsed documents, which
- * legitimately have no CST. The single definition of "shed", shared by
- * {@link CstResidencyService} and the document builder's rehydrate-on-re-entry
- * seam so the two cannot drift.
+ * Whether `document` has a resident AST without a CST, and `ledger` holds the
+ * text to graft one from. That text is the one a residency policy recorded when
+ * it shed the CST, or the serialized text of a `fromModel` document. A
+ * not-yet-parsed document has no CST to graft, and nor has a `fromModel`
+ * document without a serializer. The single definition of "shed", shared by the
+ * document builder's rehydrate-on-re-entry seam and the server-state snapshot
+ * so the two cannot drift.
  */
-export function isCstShed(document: LangiumDocument): boolean {
-   return (
-      document.state >= DocumentState.Parsed &&
-      document.parseResult?.value !== undefined &&
-      document.parseResult.value.$cstNode === undefined
-   );
+export function isCstShed(document: LangiumDocument, ledger: ModelLedger): boolean {
+   const root = document.parseResult?.value;
+   return document.state >= DocumentState.Parsed && root !== undefined && root.$cstNode === undefined && ledger.textOf(root) !== undefined;
 }
 
 /**
@@ -485,8 +484,9 @@ export class DefaultCstResidencyService implements CstResidencyService {
     * references. Only a `Validated` document qualifies: the idle timer can fire
     * while a build waits between documents, and a build that indexes a
     * document's references without their CST drops them from the reference
-    * index. Its root needs a recorded version, because `shed` then records no
-    * text and the graft refuses it. A document whose graft diverged on its
+    * index. Its root needs a recorded version, since without one the ledger
+    * keeps no text to graft the CST back from, and `shed` refuses it for a
+    * caller that skips this check. A document whose graft diverged on its
     * current text would cost a parse on every re-entry. Checked before
     * {@link isSheddable}, so an override of that policy keeps these guards.
     */
@@ -515,22 +515,21 @@ export class DefaultCstResidencyService implements CstResidencyService {
    /**
     * Shed one document's CST: null every AST node's `$cstNode` and every
     * reference's `$refNode`. Returns the number of `$cstNode`s cleared (0 if the
-    * document is unparsed or already shed).
+    * document is unparsed, already shed, or has no recorded version).
     */
    protected shed(document: LangiumDocument): number {
       const root = document.parseResult?.value;
-      if (root === undefined || root.$cstNode === undefined) {
+      const ledger = this.services.workspace.ModelLedger;
+      // Without a version the ledger keeps no text, and without the text no
+      // build can graft the CST back or tell the document is shed.
+      if (root === undefined || root.$cstNode === undefined || ledger.versionOf(root) === UNRECORDED_VERSION) {
          return 0;
       }
       // Readers of the parsed text, the `'text-diagnostics'` fingerprint among
       // them, otherwise see none once the CST is gone. Recorded in the ledger
       // directly: no root is produced, and reconciling it with the store would
       // step a closed document's sequence for a root behind its text.
-      const ledger = this.services.workspace.ModelLedger;
-      const version = ledger.versionOf(root);
-      if (version !== UNRECORDED_VERSION) {
-         ledger.record(root, version, ledger.textOf(root));
-      }
+      ledger.record(root, ledger.versionOf(root), ledger.textOf(root));
       let cleared = 0;
       for (const node of AstUtils.streamAst(root)) {
          if (node.$cstNode) {

package/src/langium/workspace/hydranium-langium-document-factory.ts

@@ -9,11 +9,19 @@
 
 import {
    type AstNode,
+   type AstReflection,
    AstUtils,
    DefaultLangiumDocumentFactory,
+   DocumentState,
+   isAstNode,
+   isMultiReference,
    type LangiumDocument,
+   type Linker,
+   type MultiReference,
    type Mutable,
    type ParseResult,
+   type Reference,
+   type ReferenceInfo,
    type URI
 } from '@hydranium/langium';
 import { type CancellationToken } from 'vscode-languageserver-protocol';
@@ -44,10 +52,18 @@ interface WithSerializer {
  * scope descriptions pointing at them, stay valid. A `fromString` document
  * already carries its source, so only `fromModel` needs help.
  *
+ * The first build grafts the CST of that text onto the model, as for a shed
+ * document, which gives every reference the `$refNode` the reference index
+ * needs. A model that does not pair node for node with its serialized text is
+ * re-parsed instead, and its nodes are replaced: from its file, or from the
+ * text it holds when no file is at its URI, since reading the URI would throw.
+ * While the document is open, the store's text wins.
+ *
  * Degrades to Langium's text-less `fromModel` when no serializer is bound for
- * the URI's language (the serializer throws or is absent) — the document still
- * works; only a re-read of it would yield empty text, which never happens for a
- * build-once virtual document.
+ * the URI's language (the serializer throws or is absent). Its first build
+ * re-reads it from the store or its file, replacing its nodes, as Langium
+ * does. With neither, it keeps its nodes but gets no CST, so its references
+ * stay out of the reference index and it is not relinked.
  *
  * Also links the model's container properties (`$container` /
  * `$containerProperty` / `$containerIndex`), which Langium's parser sets via
@@ -71,9 +87,13 @@ export class HydraniumLangiumDocumentFactory extends DefaultLangiumDocumentFacto
    protected readonly modelLedger: () => ModelLedger;
    /** A thunk: the residency service resolves `LangiumDocuments`, which is built with this factory. */
    protected readonly cstResidency: () => CstResidencyService;
+   /** The code-built documents, which are re-read from the text they hold when no file is at their URI. */
+   protected readonly codeBuiltDocuments = new WeakSet<LangiumDocument>();
+   protected readonly astReflection: AstReflection;
 
    constructor(services: ServerSharedServicesMinimal) {
       super(services);
+      this.astReflection = services.AstReflection;
       this.versionSyncService = () => services.workspace.VersionSyncService;
       this.modelLedger = () => services.workspace.ModelLedger;
       this.cstResidency = () => services.workspace.CstResidencyService;
@@ -109,12 +129,67 @@ export class HydraniumLangiumDocumentFactory extends DefaultLangiumDocumentFacto
       // Make the code-built AST as structurally complete as a parsed one — the
       // parser links containers, `fromModel` does not. In place (identity kept).
       AstUtils.linkContentToContainer(model, { deep: true });
+      // The parser fills in the defaults a model built in code may leave out.
+      // Filled before serializing, since a serializer may read them, and
+      // without them the model does not pair with the parse of its text.
+      for (const node of AstUtils.streamAst(model)) {
+         AstUtils.assignMandatoryProperties(this.astReflection, node);
+      }
       const text = this.serializeModel(model, uri);
       if (text === undefined) {
          return super.fromModel(model, uri);
       }
       const parseResult: ParseResult<T> = { value: model, parserErrors: [], lexerErrors: [] };
-      return this.createLangiumDocument<T>(parseResult, uri, undefined, text);
+      const document = this.createLangiumDocument<T>(parseResult, uri, undefined, text);
+      this.codeBuiltDocuments.add(document);
+      const linker = this.serviceRegistry.getServices(uri).references.Linker;
+      document.references = AstUtils.streamAst(model)
+         .flatMap(node => AstUtils.streamReferences(node))
+         .toArray()
+         .flatMap(info => this.rebuildReference(linker, info) ?? []);
+      return document;
+   }
+
+   /**
+    * Replace a code-set reference with one the linker built, holding the same
+    * target. A relink resets and the reference index reads only such
+    * references, and only those listed in `document.references`. The first
+    * build unlinks them before its link phase, which resolves and indexes them.
+    * The target is kept in the field Langium's references hold it in, so a
+    * linker whose references have no such field resolves it on first read.
+    * `undefined` when the reference has no slot to go back in.
+    */
+   protected rebuildReference(
+      linker: Linker,
+      { reference, container, property, index }: ReferenceInfo
+   ): Reference | MultiReference | undefined {
+      const owner: unknown = Reflect.get(container, property);
+      if (index !== undefined && !Array.isArray(owner)) {
+         return undefined;
+      }
+      let rebuilt: Reference | MultiReference;
+      if (isMultiReference(reference)) {
+         // Read through the field, as `items` would resolve a linker-built reference.
+         const items = '_items' in reference ? reference._items : reference.items;
+         const multi = linker.buildMultiReference(container, property, reference.$refNode, reference.$refText);
+         if ('_items' in multi) {
+            multi._items = Array.isArray(items) ? [...items] : undefined;
+         }
+         rebuilt = multi;
+      } else {
+         const target = '_ref' in reference ? reference._ref : reference.ref;
+         const single = linker.buildReference(container, property, reference.$refNode, reference.$refText);
+         if ('_ref' in single) {
+            single._ref = isAstNode(target) ? target : undefined;
+         }
+         rebuilt = single;
+      }
+      if (Array.isArray(owner) && index !== undefined) {
+         owner[index] = rebuilt;
+      } else {
+         Reflect.set(container, property, rebuilt);
+      }
+      return rebuilt;
    }
 
    protected override createLangiumDocument<T extends AstNode = AstNode>(
@@ -127,7 +202,10 @@ export class HydraniumLangiumDocumentFactory extends DefaultLangiumDocumentFacto
       // Without `textDocument` Langium creates one at version 0 on first read,
       // which needs a language for `uri` that a folder URI does not have.
       const parsed = parseResult.value.$cstNode?.root.fullText ?? text ?? textDocument?.getText();
-      this.versionSyncService().modelProduced(document, { version: this.versionOfParsedText(uri, parsed, textDocument?.version ?? 0) });
+      const version = this.versionOfParsedText(uri, parsed, textDocument?.version ?? 0);
+      // A `fromModel` root has no CST to hold its serialized text. Recorded, it
+      // is the text a build grafts a CST from, keeping the model's nodes.
+      this.versionSyncService().modelProduced(document, { version, text: parseResult.value.$cstNode ? undefined : text });
       return document;
    }
 
@@ -171,8 +249,12 @@ export class HydraniumLangiumDocumentFactory extends DefaultLangiumDocumentFacto
       // documents, and a reset now would drop their build state.
       const root = document.parseResult.value;
       const parsedFrom = root.$cstNode === undefined ? this.modelLedger().textOf(root) : undefined;
+      const codeBuilt = await this.codeBuiltSource(document);
       if (parsedFrom !== undefined) {
-         const text = this.textDocuments?.get(document.uri.toString())?.getText() ?? (await this.fileSystemProvider.readFile(document.uri));
+         const text =
+            codeBuilt?.text ??
+            this.textDocuments?.get(document.uri.toString())?.getText() ??
+            (await this.fileSystemProvider.readFile(document.uri));
          if (text === parsedFrom) {
             this.cstResidency().rehydrate(document);
             if (root.$cstNode === undefined) {
@@ -180,8 +262,11 @@ export class HydraniumLangiumDocumentFactory extends DefaultLangiumDocumentFacto
             }
          }
       }
-      const stored = this.textDocuments?.get(document.uri.toString())?.version;
-      const updated = await super.update(document, cancellationToken);
+      const stored = codeBuilt === undefined ? this.textDocuments?.get(document.uri.toString())?.version : codeBuilt.version;
+      const updated =
+         codeBuilt === undefined
+            ? await super.update(document, cancellationToken)
+            : await this.reparse(document, codeBuilt.text, cancellationToken);
       const version =
          stored ??
          this.versionOfParsedText(
@@ -193,6 +278,49 @@ export class HydraniumLangiumDocumentFactory extends DefaultLangiumDocumentFacto
       return updated;
    }
 
+   /**
+    * The text a code-built document is read from when no file is at its URI:
+    * the store's while it is open, else the text it holds, serialized at
+    * creation or last read, as `VirtualFileSystemProvider` serves it.
+    * `undefined` for any other document, which is read as Langium reads it.
+    * The store's version is read with its text, since an edit can land
+    * before the caller resumes. Held text has no version.
+    */
+   protected async codeBuiltSource(document: LangiumDocument): Promise<{ text: string; version: number | undefined } | undefined> {
+      if (!this.codeBuiltDocuments.has(document) || (await this.fileSystemProvider.exists(document.uri))) {
+         return undefined;
+      }
+      const open = this.textDocuments?.get(document.uri.toString());
+      return open === undefined
+         ? { text: document.textDocument.getText(), version: undefined }
+         : { text: open.getText(), version: open.version };
+   }
+
+   /**
+    * Langium's `update` with `text` in place of the text it reads from the
+    * store or the URI. Mirrors `DefaultLangiumDocumentFactory.update` of
+    * Langium 4.3.1, so a Langium upgrade has to compare the two. From
+    * Langium 4.4, an override of `getUpdatedText` replaces it.
+    */
+   protected async reparse<T extends AstNode = AstNode>(
+      document: Mutable<LangiumDocument<T>>,
+      text: string,
+      cancellationToken: CancellationToken
+   ): Promise<LangiumDocument<T>> {
+      const textDocument = this.textDocuments?.get(document.uri.toString());
+      Object.defineProperty(
+         document,
+         'textDocument',
+         textDocument ? { value: textDocument } : { get: this.createTextDocumentGetter(document.uri, text) }
+      );
+      if (document.parseResult.value.$cstNode?.root.fullText !== text) {
+         document.parseResult = await this.parseAsync<T>(document.uri, text, cancellationToken);
+         (document.parseResult.value as Mutable<AstNode>).$document = document;
+      }
+      document.state = DocumentState.Parsed;
+      return document;
+   }
+
    /**
     * `version`, or {@link STALE_VERSION} when the store holds `uri` open with
     * other text than `parsed`: text read from the file while an open landed.

package/src/node/event-loop-monitor.ts

@@ -52,9 +52,7 @@ export function startEventLoopMonitor(shared: ServerSharedServicesMinimal, optio
    const onStallSnapshot =
       options.onStallSnapshot ??
       ((maxMs: number) =>
-         shared.Logger.for('Process').info(
-            formatServerState(shared.workspace.LangiumDocuments, `auto: event-loop stall ${Math.round(maxMs)}ms`)
-         ));
+         shared.Logger.for('Process').info(formatServerState(shared.workspace, `auto: event-loop stall ${Math.round(maxMs)}ms`)));
    const histogram = monitorEventLoopDelay({ resolution: 10 });
    histogram.enable();
    const interval = setInterval(() => {

package/src/node/server-state-snapshot.ts

@@ -9,6 +9,8 @@
 
 import { Format, type Stopwatch, SystemClock } from '@hydranium/protocol';
 import { DocumentState, type LangiumDocuments } from '@hydranium/langium';
+import { type ModelLedger } from '../documents/model-ledger.js';
+import { type ServerSharedServicesMinimal } from '../langium/shared-services.js';
 import { isCstShed } from '../langium/residency/cst-residency-service.js';
 import { performance } from 'node:perf_hooks';
 import * as v8 from 'node:v8';
@@ -58,9 +60,15 @@ export interface ServerStateSnapshotOptions {
  * logs the result, e.g. `tracer.info(formatServerState(...))`), matching
  * `formatProcessMemory` / `formatPodMemory`. Tracks deltas across calls so
  * subsequent snapshots show "since last snapshot" rates. Optional sections are
- * requested via `options` ({@link ServerStateSnapshotOptions}).
+ * requested via `options` ({@link ServerStateSnapshotOptions}). `workspace`
+ * adds the document counts. Its ledger tells a shed document from one that
+ * never had a CST.
  */
-export function formatServerState(documents?: LangiumDocuments, label?: string, options?: ServerStateSnapshotOptions): string {
+export function formatServerState(
+   workspace?: Pick<ServerSharedServicesMinimal['workspace'], 'LangiumDocuments' | 'ModelLedger'>,
+   label?: string,
+   options?: ServerStateSnapshotOptions
+): string {
    const sinceSample = new SystemClock().stopwatch();
    const mem = process.memoryUsage();
    const heap = v8.getHeapStatistics();
@@ -110,7 +118,8 @@ export function formatServerState(documents?: LangiumDocuments, label?: string,
       const eluPct = eluTotal > 0 ? Math.round((elu.active / eluTotal) * 100) : 0;
       lines.push(`  loop      utilization ${eluPct}% (cumulative since process start)`);
    }
-   if (documents) {
+   if (workspace) {
+      const { LangiumDocuments: documents, ModelLedger: ledger } = workspace;
       const counts = countDocumentsByState(documents);
       const total = Object.values(counts).reduce((sum, n) => sum + n, 0);
       const breakdown = Object.entries(counts)
@@ -118,9 +127,9 @@ export function formatServerState(documents?: LangiumDocuments, label?: string,
          .map(([state, n]) => `${state}: ${n}`)
          .join(', ');
       lines.push(`  documents ${total} tracked${breakdown ? ` (${breakdown})` : ''}`);
-      // CST-residency breakdown — only meaningful (and only emitted) once a residency
-      // policy has shed at least one document; silent under the always-keep default.
-      const residency = countCstResidency(documents);
+      // CST-residency breakdown, emitted once a document waits for a CST graft:
+      // shed by a residency policy, or built from code and not built yet.
+      const residency = countCstResidency(documents, ledger);
       if (residency.shed > 0) {
          lines.push(`  cst       ${residency.resident} resident, ${residency.shed} shed`);
       }
@@ -143,17 +152,17 @@ export function formatServerState(documents?: LangiumDocuments, label?: string,
 
 /**
  * Count documents whose CST is resident vs. shed by a residency policy
- * ({@link isCstShed}). A document can also be in neither bucket — not-yet-parsed
- * or a synthetic placeholder with no `$cstNode` at all (nothing to be "resident"
- * or "shed") — which is intentionally counted as neither; `resident + shed` is
- * therefore ≤ total tracked, with the remainder explained by the state breakdown
- * on the preceding `documents` line.
+ * ({@link isCstShed}). A document can also be in neither bucket: a
+ * not-yet-parsed document, a synthetic placeholder, or a `fromModel` document
+ * without a serializer has no CST and no text to graft one from. So
+ * `resident + shed` is at most the total tracked, with the remainder explained
+ * by the state breakdown on the preceding `documents` line.
  */
-function countCstResidency(documents: LangiumDocuments): { resident: number; shed: number } {
+function countCstResidency(documents: LangiumDocuments, ledger: ModelLedger): { resident: number; shed: number } {
    let resident = 0;
    let shed = 0;
    for (const doc of documents.all) {
-      if (isCstShed(doc)) {
+      if (isCstShed(doc, ledger)) {
          shed++;
       } else if (doc.parseResult?.value?.$cstNode !== undefined) {
          resident++;