@mpeggroup/mpeg-sdl-parser 4.0.0

package/src/completion/completion-rules.ts

@@ -0,0 +1,447 @@
+import { HierarchicalSearch } from "./hierarchical-map.ts";
+import { alignedModifierRules } from "./rules/aligned-modifier-rules.ts";
+import { parameterValueListRules } from "./rules/parameter-value-list-rules.ts";
+import { stringDefinitionRules } from "./rules/string-definition-rules.ts";
+import { utfStringLiteralRules } from "./rules/utf-string-literal-rules.ts";
+import { whileStatementRules } from "./rules/while-statement-rules.ts";
+import { computedElementaryTypeRules } from "./rules/computed-elementary-type-rules.ts";
+import { elementaryTypeRules } from "./rules/elementary-type-rules.ts";
+import { compoundStatementRules } from "./rules/compound-statement-rules.ts";
+import { ifStatementRules } from "./rules/if-statement-rules.ts";
+import { forStatementRules } from "./rules/for-statement-rules.ts";
+import { switchStatementRules } from "./rules/switch-statement-rules.ts";
+import { caseClauseRules } from "./rules/case-clause-rules.ts";
+import { defaultClauseRules } from "./rules/default-clause-rules.ts";
+import { classDeclarationRules } from "./rules/class-declaration-rules.ts";
+import { expandableModifierRules } from "./rules/expandable-modifier-rules.ts";
+import { classDefinitionRules } from "./rules/class-definition-rules.ts";
+import { assignmentExpressionRules } from "./rules/assignment-expression-rules.ts";
+import { doStatementRules } from "./rules/do-statement-rules.ts";
+import { elementaryTypeDefinitionRules } from "./rules/elementary-type-definition-rules.ts";
+import { lengthAttributeRules } from "./rules/length-attribute-rules.ts";
+import { extendsModifierRules } from "./rules/extends-modifier-rules.ts";
+import { parameterListRules } from "./rules/parameter-list-rules.ts";
+import { parameterRules } from "./rules/parameter-rules.ts";
+import { binaryExpressionRules } from "./rules/binary-expression-rules.ts";
+import { unaryExpressionRules } from "./rules/unary-expression-rules.ts";
+import { arrayElementAccessRules } from "./rules/array-element-access-rules.ts";
+import { classMemberAccessRules } from "./rules/class-member-access-rules.ts";
+import { lengthofExpressionRules } from "./rules/lengthof-expression-rules.ts";
+import { bitModifierRules } from "./rules/bit-modifier-rules.ts";
+import { extendedClassIdRangeRules } from "./rules/extended-class-id-range-rules.ts";
+import { classIdRules } from "./rules/class-id-rules.ts";
+import { classIdRangeRules } from "./rules/class-id-range-rules.ts";
+import { expressionStatementRules } from "./rules/expression-statement-rules.ts";
+import { mapDefinitionRules } from "./rules/map-definition-rules.ts";
+import { arrayDefinitionRules } from "./rules/array-definition-rules.ts";
+import { implicitArrayDimensionRules } from "./rules/implicit-array-dimension-rules.ts";
+import { explicitArrayDimensionRules } from "./rules/explicit-array-dimension-rules.ts";
+import { partialArrayDimensionRules } from "./rules/partial-array-dimension-rules.ts";
+import { computedArrayDefinitionRules } from "./rules/computed-array-definition-rules.ts";
+import { mapDeclarationRules } from "./rules/map-declaration-rules.ts";
+import { mapEntryRules } from "./rules/map-entry-rules.ts";
+import { aggregateOutputValueRules } from "./rules/aggregate-output-value-rules.ts";
+import { elementaryTypeOutputValueRules } from "./rules/elementary-type-output-value-rules.ts";
+import { InternalScannerError } from "../scanner-error.ts";
+import { specificationRules } from "./rules/specification-rules.ts";
+import { tokenIdToSyntaxStringMapping } from "../lezer/props/syntactic-token-node-prop-source.ts";
+
+import getLogger from "../util/logger.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../lezer/parser.terms.ts";
+import { parser } from "../lezer/parser.ts";
+
+const nodeTypes = parser.nodeSet.types;
+
+const logger = getLogger("completionRules");
+
+/**
+ * A map from parent token type ID to a HierarchicalSearch of previous sibling token type IDs to expected token type IDs.
+ */
+export const completionRulesByParentMap: Map<
+  number,
+  HierarchicalSearch<number[]>
+> = new Map();
+
+/**
+ * A map from token type ID to potential starting syntactic tokens.
+ */
+export const potentialSyntacticTokensByTokenTypeId: Map<number, string[]> =
+  new Map();
+
+type CompletionRule = {
+  previous: number | number[];
+  expected: number | number[];
+};
+
+function addRulesForParent(
+  parentTokenId: number,
+  rules: CompletionRule | CompletionRule[],
+) {
+  if (completionRulesByParentMap.has(parentTokenId)) {
+    throw new InternalScannerError(
+      `Completion rules for parent token ${
+        nodeTypes[parentTokenId].name
+      } already exist`,
+    );
+  }
+
+  if (!Array.isArray(rules)) {
+    rules = [rules];
+  }
+
+  if (rules.length === 0) {
+    logger.warn(
+      `No completion rules provided for parent token ${
+        nodeTypes[parentTokenId].name
+      }`,
+    );
+    return;
+  }
+
+  const hierarchicalSearch = new HierarchicalSearch<number[]>();
+
+  for (const rule of rules) {
+    const previousTokens = Array.isArray(rule.previous)
+      ? rule.previous
+      : [rule.previous];
+    const expectedTokens = Array.isArray(rule.expected)
+      ? rule.expected
+      : [rule.expected];
+
+    hierarchicalSearch.set(previousTokens, expectedTokens);
+  }
+
+  completionRulesByParentMap.set(parentTokenId, hierarchicalSearch);
+
+  logger.debug(
+    `Added completion rules for parent token ${nodeTypes[parentTokenId].name}`,
+  );
+}
+
+/**
+ * Validate that all TokenTypeIds have completion rules defined, except for those that are intentionally excluded.
+ */
+Object.values(TokenTypeId).forEach((tokenTypeId) => {
+  switch (tokenTypeId) {
+    case TokenTypeId.AlignedModifier:
+      addRulesForParent(TokenTypeId.AlignedModifier, alignedModifierRules);
+      break;
+    case TokenTypeId.AggregateOutputValue:
+      addRulesForParent(
+        TokenTypeId.AggregateOutputValue,
+        aggregateOutputValueRules,
+      );
+      break;
+    case TokenTypeId.ArrayDefinition:
+      addRulesForParent(TokenTypeId.ArrayDefinition, arrayDefinitionRules);
+      break;
+    case TokenTypeId.AssignmentExpression:
+      addRulesForParent(
+        TokenTypeId.AssignmentExpression,
+        assignmentExpressionRules,
+      );
+      break;
+    case TokenTypeId.ArrayElementAccess:
+      addRulesForParent(
+        TokenTypeId.ArrayElementAccess,
+        arrayElementAccessRules,
+      );
+      break;
+    case TokenTypeId.BinaryExpression:
+      addRulesForParent(TokenTypeId.BinaryExpression, binaryExpressionRules);
+      break;
+    case TokenTypeId.BitModifier:
+      addRulesForParent(TokenTypeId.BitModifier, bitModifierRules);
+      break;
+    case TokenTypeId.CaseClause:
+      addRulesForParent(TokenTypeId.CaseClause, caseClauseRules);
+      break;
+    case TokenTypeId.ClassDefinition:
+      addRulesForParent(TokenTypeId.ClassDefinition, classDefinitionRules);
+      break;
+    case TokenTypeId.ClassDeclaration:
+      addRulesForParent(TokenTypeId.ClassDeclaration, classDeclarationRules);
+      break;
+    case TokenTypeId.ClassId:
+      addRulesForParent(TokenTypeId.ClassId, classIdRules);
+      break;
+    case TokenTypeId.ClassIdRange:
+      addRulesForParent(TokenTypeId.ClassIdRange, classIdRangeRules);
+      break;
+    case TokenTypeId.ClassMemberAccess:
+      addRulesForParent(TokenTypeId.ClassMemberAccess, classMemberAccessRules);
+      break;
+    case TokenTypeId.CompoundStatement:
+      addRulesForParent(TokenTypeId.CompoundStatement, compoundStatementRules);
+      break;
+    case TokenTypeId.ComputedArrayDefinition:
+      addRulesForParent(
+        TokenTypeId.ComputedArrayDefinition,
+        computedArrayDefinitionRules,
+      );
+      break;
+    case TokenTypeId.ComputedElementaryTypeDefinition:
+      addRulesForParent(
+        TokenTypeId.ComputedElementaryTypeDefinition,
+        computedElementaryTypeRules,
+      );
+      break;
+    case TokenTypeId.DefaultClause:
+      addRulesForParent(TokenTypeId.DefaultClause, defaultClauseRules);
+      break;
+    case TokenTypeId.DoStatement:
+      addRulesForParent(TokenTypeId.DoStatement, doStatementRules);
+      break;
+    case TokenTypeId.ElementaryType:
+      addRulesForParent(TokenTypeId.ElementaryType, elementaryTypeRules);
+      break;
+    case TokenTypeId.ElementaryTypeDefinition:
+      addRulesForParent(
+        TokenTypeId.ElementaryTypeDefinition,
+        elementaryTypeDefinitionRules,
+      );
+      break;
+    case TokenTypeId.ElementaryTypeOutputValue:
+      addRulesForParent(
+        TokenTypeId.ElementaryTypeOutputValue,
+        elementaryTypeOutputValueRules,
+      );
+      break;
+    case TokenTypeId.ExpandableModifier:
+      addRulesForParent(
+        TokenTypeId.ExpandableModifier,
+        expandableModifierRules,
+      );
+      break;
+    case TokenTypeId.ExplicitArrayDimension:
+      addRulesForParent(
+        TokenTypeId.ExplicitArrayDimension,
+        explicitArrayDimensionRules,
+      );
+      break;
+    case TokenTypeId.ExpressionStatement:
+      addRulesForParent(
+        TokenTypeId.ExpressionStatement,
+        expressionStatementRules,
+      );
+      break;
+    case TokenTypeId.ExtendedClassIdRange:
+      addRulesForParent(
+        TokenTypeId.ExtendedClassIdRange,
+        extendedClassIdRangeRules,
+      );
+      break;
+    case TokenTypeId.ExtendsModifier:
+      addRulesForParent(TokenTypeId.ExtendsModifier, extendsModifierRules);
+      break;
+    case TokenTypeId.ForStatement:
+      addRulesForParent(TokenTypeId.ForStatement, forStatementRules);
+      break;
+    case TokenTypeId.IfStatement:
+      addRulesForParent(TokenTypeId.IfStatement, ifStatementRules);
+      break;
+    case TokenTypeId.ImplicitArrayDimension:
+      addRulesForParent(
+        TokenTypeId.ImplicitArrayDimension,
+        implicitArrayDimensionRules,
+      );
+      break;
+    case TokenTypeId.LengthAttribute:
+      addRulesForParent(TokenTypeId.LengthAttribute, lengthAttributeRules);
+      break;
+    case TokenTypeId.LengthofExpression:
+      addRulesForParent(
+        TokenTypeId.LengthofExpression,
+        lengthofExpressionRules,
+      );
+      break;
+    case TokenTypeId.MapDeclaration:
+      addRulesForParent(TokenTypeId.MapDeclaration, mapDeclarationRules);
+      break;
+    case TokenTypeId.MapDefinition:
+      addRulesForParent(TokenTypeId.MapDefinition, mapDefinitionRules);
+      break;
+    case TokenTypeId.MapEntry:
+      addRulesForParent(TokenTypeId.MapEntry, mapEntryRules);
+      break;
+    case TokenTypeId.Parameter:
+      addRulesForParent(TokenTypeId.Parameter, parameterRules);
+      break;
+    case TokenTypeId.ParameterList:
+      addRulesForParent(TokenTypeId.ParameterList, parameterListRules);
+      break;
+    case TokenTypeId.ParameterValueList:
+      addRulesForParent(
+        TokenTypeId.ParameterValueList,
+        parameterValueListRules,
+      );
+      break;
+    case TokenTypeId.PartialArrayDimension:
+      addRulesForParent(
+        TokenTypeId.PartialArrayDimension,
+        partialArrayDimensionRules,
+      );
+      break;
+    case TokenTypeId.StringDefinition:
+      addRulesForParent(TokenTypeId.StringDefinition, stringDefinitionRules);
+      break;
+    case TokenTypeId.Specification:
+      addRulesForParent(TokenTypeId.Specification, specificationRules);
+      break;
+    case TokenTypeId.SwitchStatement:
+      addRulesForParent(TokenTypeId.SwitchStatement, switchStatementRules);
+      break;
+    case TokenTypeId.UnaryExpression:
+      addRulesForParent(TokenTypeId.UnaryExpression, unaryExpressionRules);
+      break;
+    case TokenTypeId.UtfStringLiteral:
+      addRulesForParent(TokenTypeId.UtfStringLiteral, utfStringLiteralRules);
+      break;
+    case TokenTypeId.WhileStatement:
+      addRulesForParent(TokenTypeId.WhileStatement, whileStatementRules);
+      break;
+    // Tokens that do not require completion rules
+    case TokenTypeId._break:
+    case TokenTypeId._case:
+    case TokenTypeId._class:
+    case TokenTypeId._const:
+    case TokenTypeId._default:
+    case TokenTypeId._do:
+    case TokenTypeId._else:
+    case TokenTypeId._extends:
+    case TokenTypeId._for:
+    case TokenTypeId._if:
+    case TokenTypeId._switch:
+    case TokenTypeId._while:
+    case TokenTypeId.abstract:
+    case TokenTypeId.Addition:
+    case TokenTypeId.AlignmentBitCount8:
+    case TokenTypeId.AlignmentBitCount16:
+    case TokenTypeId.AlignmentBitCount32:
+    case TokenTypeId.AlignmentBitCount64:
+    case TokenTypeId.AlignmentBitCount128:
+    case TokenTypeId.aligned:
+    case TokenTypeId.Assignment:
+    case TokenTypeId.base64string:
+    case TokenTypeId.Base64StringLiteral:
+    case TokenTypeId.Base64StringLiteralCharacters:
+    case TokenTypeId.BinaryLiteral:
+    case TokenTypeId.bit:
+    case TokenTypeId.BitwiseAnd:
+    case TokenTypeId.BitwiseOr:
+    case TokenTypeId.BitwiseShiftLeft:
+    case TokenTypeId.BitwiseShiftRight:
+    case TokenTypeId.CloseBrace:
+    case TokenTypeId.CloseBracket:
+    case TokenTypeId.CloseParenthesis:
+    case TokenTypeId.Colon:
+    case TokenTypeId.Comma:
+    case TokenTypeId.Comment:
+    case TokenTypeId.computed:
+    case TokenTypeId.DecimalLiteral:
+    case TokenTypeId.Division:
+    case TokenTypeId.DoubleQuote:
+    case TokenTypeId.expandable:
+    case TokenTypeId.float:
+    case TokenTypeId.FloatingPointLiteral:
+    case TokenTypeId.HexadecimalLiteral:
+    case TokenTypeId.Identifier:
+    case TokenTypeId.int:
+    case TokenTypeId.IntegerLiteral:
+    case TokenTypeId.legacy:
+    case TokenTypeId.lengthof:
+    case TokenTypeId.LogicalAnd:
+    case TokenTypeId.LogicalOr:
+    case TokenTypeId.LookAhead:
+    case TokenTypeId.map:
+    case TokenTypeId.Modulus:
+    case TokenTypeId.MultipleCharacterLiteral:
+    case TokenTypeId.MultipleCharacterLiteralCharacters:
+    case TokenTypeId.Multiplication:
+    case TokenTypeId.OpenBrace:
+    case TokenTypeId.OpenBracket:
+    case TokenTypeId.OpenParenthesis:
+    case TokenTypeId.Period:
+    case TokenTypeId.PostfixDecrement:
+    case TokenTypeId.PostfixIncrement:
+    case TokenTypeId.RangeOperator:
+    case TokenTypeId.reserved:
+    case TokenTypeId.RelationalEqual:
+    case TokenTypeId.RelationalGreaterThan:
+    case TokenTypeId.RelationalGreaterThanOrEqual:
+    case TokenTypeId.RelationalLessThan:
+    case TokenTypeId.RelationalLessThanOrEqual:
+    case TokenTypeId.RelationalNotEqual:
+    case TokenTypeId.Semicolon:
+    case TokenTypeId.SingleQuote:
+    case TokenTypeId.Subtraction:
+    case TokenTypeId.UnaryNegation:
+    case TokenTypeId.UnaryPlus:
+    case TokenTypeId.unsigned:
+    case TokenTypeId.utf16string:
+    case TokenTypeId.utf8list:
+    case TokenTypeId.utf8string:
+    case TokenTypeId.utfstring:
+    case TokenTypeId.UtfPrefix:
+    case TokenTypeId.UtfStringLiteralCharacters:
+    case TokenTypeId.Whitespace:
+      break;
+    default: {
+      const exhaustiveCheck: never = tokenTypeId;
+      throw new InternalScannerError(
+        "Unreachable code reached, tokenTypeId == " + exhaustiveCheck,
+      );
+    }
+  }
+});
+
+// now build the potentialSyntacticTokensByTokenTypeId map
+// get the potential starting syntactic tokens for each token type ID
+Object.values(TokenTypeId).forEach((tokenTypeId) => {
+  const resolvedTokenTypeIds: Set<number> = new Set();
+
+  const resolveTokenTypeId = (tokenTypeId: number) => {
+    if (completionRulesByParentMap.has(tokenTypeId)) {
+      // recurse
+      const childHierarchicalSearch = completionRulesByParentMap.get(
+        tokenTypeId,
+      )!;
+      const childPotentialTokenTypeIds = childHierarchicalSearch.get([-1]);
+
+      if (childPotentialTokenTypeIds) {
+        for (const childTokenTypeId of childPotentialTokenTypeIds) {
+          resolveTokenTypeId(childTokenTypeId);
+        }
+      }
+    } else {
+      // this should be a syntactic token type ID
+      resolvedTokenTypeIds.add(tokenTypeId);
+    }
+  };
+
+  resolveTokenTypeId(tokenTypeId);
+
+  if (resolvedTokenTypeIds.size > 0) {
+    // now convert the resolvedTokenTypeIds to syntactic token strings and order them
+    const syntacticTokens: string[] = [];
+
+    for (const tokenTypeId of resolvedTokenTypeIds) {
+      const syntaxString = tokenIdToSyntaxStringMapping.get(tokenTypeId);
+      if (syntaxString) {
+        syntacticTokens.push(syntaxString);
+      }
+    }
+
+    syntacticTokens.sort();
+
+    potentialSyntacticTokensByTokenTypeId.set(tokenTypeId, syntacticTokens);
+
+    logger.debug(
+      `Added potential syntactic tokens for parent token ${
+        nodeTypes[tokenTypeId].name
+      }: ${syntacticTokens.join(" ")}`,
+    );
+  }
+});

package/src/completion/get-potential-syntactic-tokens.ts

@@ -0,0 +1,54 @@
+import type { TreeCursor } from "@lezer/common";
+import { getPotentialTokenTypeIds } from "./get-potential-token-type-ids.ts";
+import { potentialSyntacticTokensByTokenTypeId } from "./completion-rules.ts";
+import getLogger from "../util/logger.ts";
+
+const logger = getLogger("getPotentialSyntacticTokens");
+
+/**
+ * Given a TreeCursor positioned at a node, returns the potential syntactic tokens at that position.
+ *
+ * @param cursor A TreeCursor positioned at a node.
+ *
+ * @returns An array of potential syntactic tokens, or undefined if none found.
+ */
+export function getPotentialSyntacticTokens(
+  cursor: TreeCursor,
+): string[] | undefined {
+  const potentialTokenTypeIds = getPotentialTokenTypeIds(cursor);
+
+  if (!potentialTokenTypeIds || (potentialTokenTypeIds.length === 0)) {
+    logger.debug(
+      "No potentialTokenTypeIds so no potential syntactic tokens",
+    );
+    return undefined;
+  }
+
+  const potentialSyntacticTokens: string[] = [];
+
+  potentialTokenTypeIds.forEach((potentialTokenTypeId) => {
+    const syntacticTokens = potentialSyntacticTokensByTokenTypeId.get(
+      potentialTokenTypeId,
+    );
+
+    if (syntacticTokens) {
+      potentialSyntacticTokens.push(...syntacticTokens);
+    }
+  });
+
+  if (potentialSyntacticTokens.length === 0) {
+    logger.debug(
+      "No potential syntactic tokens",
+    );
+    return undefined;
+  }
+
+  // sort and remove duplicates
+  const uniqueSortedTokens = Array.from(new Set(potentialSyntacticTokens))
+    .sort();
+
+  logger.debug(
+    `Potential syntactic tokens: ${uniqueSortedTokens.join(" ")}`,
+  );
+  return uniqueSortedTokens;
+}

package/src/completion/get-potential-token-type-ids.ts

@@ -0,0 +1,116 @@
+import type { NodeType, TreeCursor } from "@lezer/common";
+import { completionRulesByParentMap } from "./completion-rules.ts";
+import getLogger from "../util/logger.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../lezer/parser.terms.ts";
+import { parser } from "../lezer/parser.ts";
+
+const nodeTypes = parser.nodeSet.types;
+
+const logger = getLogger("getPotentialTokenTypeIds");
+
+/**
+ * Given a TreeCursor positioned at a node, returns the potential token types at that position.
+ *
+ * @param cursor The TreeCursor positioned at a node.
+ *
+ * @returns An array of potential NodeType IDs, or undefined if none found.
+ */
+export function getPotentialTokenTypeIds(
+  cursor: TreeCursor,
+): number[] | undefined {
+  // No potential tokens if currently on a comment
+  if (cursor.type.id === TokenTypeId.Comment) {
+    logger.debug("No potential tokens in comments");
+    return undefined;
+  }
+
+  // firstly clone the cursor to avoid breaking the traversal of the original cursor
+  const parentCursorClone = cursor.node.cursor();
+  const siblingCursorClone = cursor.node.cursor();
+
+  // look for parent token type, ignoring whitespace and errors
+  const parentTokenTypesIds: number[] = [];
+
+  while (parentCursorClone.parent()) {
+    if (
+      (parentCursorClone.type.id !== TokenTypeId.Whitespace) &&
+      !parentCursorClone.type.isError
+    ) {
+      parentTokenTypesIds.unshift(parentCursorClone.type.id);
+      break;
+    }
+  }
+
+  if (parentTokenTypesIds.length === 0) {
+    logger.debug("No parent token type found, assuming global scope");
+    parentTokenTypesIds.push(TokenTypeId.Specification);
+  }
+
+  // look for previous sibling token types, ignoring whitespace, errors and comments
+  const previousSiblingTokenTypes: NodeType[] = [];
+
+  while (siblingCursorClone.prevSibling()) {
+    if (
+      (siblingCursorClone.type.id !== TokenTypeId.Whitespace) &&
+      !siblingCursorClone.type.isError &&
+      (siblingCursorClone.type.id !== TokenTypeId.Comment)
+    ) {
+      previousSiblingTokenTypes.unshift(siblingCursorClone.type);
+    }
+  }
+
+  logger.debug(
+    "parentTokenTypes: " +
+      parentTokenTypesIds.map((id) => nodeTypes[id].name).join(", "),
+  );
+
+  if (previousSiblingTokenTypes.length > 0) {
+    logger.debug(
+      "previousSiblingTokenTypes: " +
+        previousSiblingTokenTypes.map((type) => type.name).join(", "),
+    );
+  } else {
+    logger.debug("No previous sibling token type found");
+  }
+
+  const potentialCompletionRules = completionRulesByParentMap
+    .get(parentTokenTypesIds[parentTokenTypesIds.length - 1]);
+
+  if (!potentialCompletionRules) {
+    logger.debug(
+      "No expected tokens map found for parentTokenType: " +
+        nodeTypes[parentTokenTypesIds[parentTokenTypesIds.length - 1]].name,
+    );
+
+    return undefined;
+  }
+
+  const potentialTokenTypes = potentialCompletionRules.get(
+    previousSiblingTokenTypes.length > 0
+      ? [-1, ...previousSiblingTokenTypes.map((type) => type.id)]
+      : [-1],
+  );
+  if (!potentialTokenTypes || potentialTokenTypes.length === 0) {
+    logger.debug(
+      "No expected tokens found for parentTokenTypes: " +
+        parentTokenTypesIds.map((id) => nodeTypes[id].name).join(", ") +
+        (previousSiblingTokenTypes.length > 0
+          ? " and previousSiblingTokenTypes: " +
+            previousSiblingTokenTypes.map((type) => type.name).join(", ")
+          : ""),
+    );
+    return undefined;
+  }
+
+  // sort and remove duplicates
+  const uniqueSortedTokenTypes = Array.from(new Set(potentialTokenTypes)).sort((
+    a,
+    b,
+  ) => a - b);
+  logger.debug(
+    `Potential token types: ${uniqueSortedTokenTypes.join(" ")}`,
+  );
+  return uniqueSortedTokenTypes;
+}

package/src/completion/hierarchical-map.ts

@@ -0,0 +1,43 @@
+/**
+ * A hierarchical search structure that allows for searching with progressively shorter suffixes of ID arrays.
+ */
+export class HierarchicalSearch<T> {
+  private store = new Map<string, T>();
+
+  constructor(initialData?: [number[], T][]) {
+    if (initialData) {
+      for (const [ids, value] of initialData) {
+        this.set(ids, value);
+      }
+    }
+  }
+
+  private serializeKey(ids: number[]): string {
+    return ids.join(",");
+  }
+
+  set(ids: number[], value: T): void {
+    this.store.set(this.serializeKey(ids), value);
+  }
+
+  get(ids: number[]): T | undefined {
+    // Try exact match first
+    const exactKey = this.serializeKey(ids);
+
+    if (this.store.has(exactKey)) {
+      return this.store.get(exactKey);
+    }
+
+    // Try progressively shorter suffixes
+    for (let i = 1; i < ids.length; i++) {
+      const suffix = ids.slice(i);
+      const key = this.serializeKey(suffix);
+
+      if (this.store.has(key)) {
+        return this.store.get(key);
+      }
+    }
+
+    return undefined;
+  }
+}