@astryxdesign/core 0.7.4 → 0.7.5-canary.0418c47

package/src/CodeBlock/languages/dockerfile.ts

@@ -0,0 +1,43 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/dockerfile.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `dockerfile` language value for CodeBlock: Dockerfile: `dockerfile`, `docker`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/dockerfile
+ */
+
+import {
+  DOUBLE_QUOTED,
+  HASH_COMMENT,
+  NUMBER,
+  SINGLE_QUOTED,
+  defineCodeLanguage,
+} from '../grammar';
+
+/** Dockerfile: `dockerfile`, `docker`. */
+export const dockerfile = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'dockerfile',
+    identifiers: ['dockerfile', 'docker'],
+    fileExtensions: ['dockerfile'],
+    fileNames: ['Dockerfile', 'Containerfile'],
+  },
+  () => ({
+    defaultType: 'variable',
+    patterns: [
+      {type: 'comment', regex: HASH_COMMENT},
+      {
+        type: 'keyword',
+        regex:
+          /^[ \t]*(?:FROM|RUN|CMD|LABEL|EXPOSE|ENV|ADD|COPY|ENTRYPOINT|VOLUME|USER|WORKDIR|ARG|ONBUILD|STOPSIGNAL|HEALTHCHECK|SHELL)\b/im,
+      },
+      {type: 'keyword', regex: /\bAS\b/},
+      {type: 'string', regex: DOUBLE_QUOTED},
+      {type: 'string', regex: SINGLE_QUOTED},
+      {type: 'variable', regex: /\$\{?\w+\}?/},
+      {type: 'number', regex: NUMBER},
+      {type: 'operator', regex: /[&|\\=]+/},
+    ],
+  }),
+);

package/src/CodeBlock/languages/graphql.ts

@@ -0,0 +1,30 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/graphql.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `graphql` language value for CodeBlock: GraphQL: `graphql`, `gql`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/graphql
+ */
+
+import {HASH_COMMENT, cLike, defineCodeLanguage} from '../grammar';
+
+/** GraphQL: `graphql`, `gql`. */
+export const graphql = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'graphql',
+    identifiers: ['graphql', 'gql'],
+    fileExtensions: ['graphql', 'gql'],
+    fileNames: [],
+  },
+  () =>
+    cLike(
+      'query mutation subscription fragment on type input enum interface union scalar schema extend directive implements repeatable true false null',
+      [
+        {type: 'variable', regex: /\$\w+/},
+        {type: 'constant', regex: /@\w+/},
+        {type: 'string', regex: /"""[\s\S]*?"""/},
+      ],
+      [HASH_COMMENT],
+    ),
+);

package/src/CodeBlock/languages/java.ts

@@ -0,0 +1,28 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/java.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `java` language value for CodeBlock: Java: `java`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/java
+ */
+
+import {cLike, defineCodeLanguage} from '../grammar';
+
+/** Java: `java`. */
+export const java = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'java',
+    identifiers: ['java'],
+    fileExtensions: ['java'],
+    fileNames: [],
+  },
+  () =>
+    cLike(
+      'abstract assert boolean break byte case catch char class const continue default do double else enum extends final finally float for if implements import instanceof int interface long native new package private protected public return short static super switch synchronized this throw throws transient try var void volatile while record sealed permits yield true false null',
+      [
+        {type: 'constant', regex: /@\w+/},
+        {type: 'string', regex: /"""[\s\S]*?"""/},
+      ],
+    ),
+);

package/src/CodeBlock/languages/lua.ts

@@ -0,0 +1,21 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/lua.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `lua` language value for CodeBlock: Lua: `lua`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/lua
+ */
+
+import {cLike, defineCodeLanguage} from '../grammar';
+
+/** Lua: `lua`. */
+export const lua = /* @__PURE__ */ defineCodeLanguage(
+  {name: 'lua', identifiers: ['lua'], fileExtensions: ['lua'], fileNames: []},
+  () =>
+    cLike(
+      'and break do else elseif end false for function goto if in local nil not or repeat return then true until while',
+      [{type: 'string', regex: /\[(=*)\[[\s\S]*?\]\1\]/}],
+      [/--\[(=*)\[[\s\S]*?\]\1\]/, /--[^\n]*/],
+    ),
+);

package/src/CodeBlock/languages/objective-c.ts

@@ -0,0 +1,26 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/objective-c.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `objectiveC` language value for CodeBlock: Objective-C: `objective-c`, `objc`, `m`, `mm`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/objective-c
+ */
+
+import {CPP_WORDS, DIRECTIVE, cLike, defineCodeLanguage} from '../grammar';
+
+/** Objective-C: `objective-c`, `objc`, `m`, `mm`. */
+export const objectiveC = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'objective-c',
+    identifiers: ['objective-c', 'objc', 'm', 'mm'],
+    fileExtensions: ['m', 'mm'],
+    fileNames: [],
+  },
+  () =>
+    cLike(`${CPP_WORDS} id self super nil Nil YES NO SEL BOOL instancetype`, [
+      {type: 'keyword', regex: DIRECTIVE},
+      {type: 'string', regex: /@"(?:[^"\\\n]|\\.)*"/},
+      {type: 'keyword', regex: /@\w+/},
+    ]),
+);

package/src/CodeBlock/languages/powershell.ts

@@ -0,0 +1,31 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/powershell.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `powershell` language value for CodeBlock: PowerShell
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/powershell
+ */
+
+import {HASH_COMMENT, cLike, defineCodeLanguage} from '../grammar';
+
+/** PowerShell: `powershell`, `ps1`, `pwsh`. */
+export const powershell = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'powershell',
+    identifiers: ['powershell', 'ps1', 'pwsh'],
+    fileExtensions: ['ps1', 'psm1', 'psd1'],
+    fileNames: [],
+  },
+  () =>
+    cLike(
+      'begin break catch class continue data do dynamicparam else elseif end exit filter finally for foreach function if in param process return switch throw trap try until using while',
+      [
+        {type: 'comment', regex: /<#[\s\S]*?#>/},
+        {type: 'variable', regex: /\$[\w:]+/},
+        {type: 'function', regex: /\b[A-Z][a-z]+-[A-Z]\w*/},
+        {type: 'operator', regex: /-[a-z]+\b/},
+      ],
+      [HASH_COMMENT],
+    ),
+);

package/src/CodeBlock/languages/ruby.ts

@@ -0,0 +1,29 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/ruby.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `ruby` language value for CodeBlock: Ruby: `ruby`, `rb`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/ruby
+ */
+
+import {HASH_COMMENT, cLike, defineCodeLanguage} from '../grammar';
+
+/** Ruby: `ruby`, `rb`. */
+export const ruby = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'ruby',
+    identifiers: ['ruby', 'rb'],
+    fileExtensions: ['rb', 'rake', 'gemspec'],
+    fileNames: ['Gemfile', 'Rakefile'],
+  },
+  () =>
+    cLike(
+      'alias and begin break case class def defined do else elsif end ensure false for if in module next nil not or redo rescue retry return self super then true undef unless until when while yield require attr_accessor',
+      [
+        {type: 'constant', regex: /:\w+/},
+        {type: 'variable', regex: /@{1,2}\w+|\$\w+/},
+      ],
+      [HASH_COMMENT],
+    ),
+);

package/src/CodeBlock/languages/thrift.ts

@@ -0,0 +1,32 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/thrift.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `thrift` language value for CodeBlock: Thrift IDL: `thrift`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/thrift
+ */
+
+import {
+  BLOCK_COMMENT,
+  HASH_COMMENT,
+  LINE_COMMENT,
+  cLike,
+  defineCodeLanguage,
+} from '../grammar';
+
+/** Thrift IDL: `thrift`. */
+export const thrift = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'thrift',
+    identifiers: ['thrift'],
+    fileExtensions: ['thrift'],
+    fileNames: [],
+  },
+  () =>
+    cLike(
+      'namespace include cpp_include typedef enum struct union exception service extends oneway void throws const required optional bool byte i8 i16 i32 i64 double string binary list set map true false',
+      [],
+      [BLOCK_COMMENT, LINE_COMMENT, HASH_COMMENT],
+    ),
+);

package/src/CodeBlock/languages/toml.ts

@@ -0,0 +1,36 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages/toml.ts
+ * @input Shared grammar builders from ../grammar
+ * @output The `toml` language value for CodeBlock: TOML: `toml`
+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/toml
+ */
+
+import {DOUBLE_QUOTED, HASH_COMMENT, defineCodeLanguage} from '../grammar';
+
+/** TOML: `toml`. */
+export const toml = /* @__PURE__ */ defineCodeLanguage(
+  {
+    name: 'toml',
+    identifiers: ['toml'],
+    fileExtensions: ['toml'],
+    fileNames: [],
+  },
+  () => ({
+    defaultType: 'variable',
+    patterns: [
+      {type: 'comment', regex: HASH_COMMENT},
+      {type: 'tag', regex: /^[ \t]*\[\[?[^\]\n]+\]\]?/m},
+      {type: 'property', regex: /^[ \t]*[\w."'-]+(?=\s*=)/m},
+      {type: 'string', regex: /"""[\s\S]*?"""|'''[\s\S]*?'''/},
+      {type: 'string', regex: DOUBLE_QUOTED},
+      {type: 'string', regex: /'[^'\n]*'/},
+      {type: 'constant', regex: /\b(?:true|false)\b/},
+      {type: 'number', regex: /[+-]?\b\d[\d_:.TZ+-]*\b/},
+      {type: 'operator', regex: /=/},
+      // eslint-disable-next-line no-useless-escape
+      {type: 'punctuation', regex: /[\[\]{},.]/},
+    ],
+  }),
+);

package/src/CodeBlock/languages.test.ts

@@ -0,0 +1,402 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file languages.test.ts
+ * @input tokenizer.ts and the language modules
+ * @output CodeBlock FR12 (built-in identifiers) and FR13 (language modules)
+ *   tokenizer evidence
+ * @position CodeBlock FR12, FR13 evidence
+ */
+
+import {describe, expect, it} from 'vitest';
+import {tokenize} from './tokenizer';
+import {dockerfile} from './languages/dockerfile';
+import {graphql} from './languages/graphql';
+import {java} from './languages/java';
+import {lua} from './languages/lua';
+import {objectiveC} from './languages/objective-c';
+import {powershell} from './languages/powershell';
+import {ruby} from './languages/ruby';
+import {thrift} from './languages/thrift';
+import {toml} from './languages/toml';
+import {
+  createCodeLanguage,
+  isCodeLanguage,
+  languageKey,
+  listedLanguage,
+  listedLanguageForFile,
+} from './grammar';
+import type {CodeLanguage} from './grammar';
+
+/** Token types painted for `code`, by the text they cover. */
+function painted(
+  code: string,
+  language: string | CodeLanguage,
+): Map<string, Set<string>> {
+  const lines = code.split('\n');
+  const out = new Map<string, Set<string>>();
+  tokenize(code, language).forEach((tokens, index) => {
+    for (const token of tokens) {
+      const text = lines[index].slice(token.start, token.end);
+      const types = out.get(token.type) ?? new Set<string>();
+      types.add(text);
+      out.set(token.type, types);
+    }
+  });
+  return out;
+}
+
+type Case = {
+  ids: string[];
+  code: string;
+  comment?: string;
+  string?: string;
+  number?: string;
+  keyword: string;
+};
+
+const NATIVE: Case[] = [
+  {
+    ids: ['typescript', 'ts', 'javascript', 'js', 'tsx', 'jsx'],
+    code: '// c\nconst n = 3; const s = "x";',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'const',
+  },
+  {
+    ids: ['json'],
+    code: '{"s": "x", "n": 3}',
+    string: '"x"',
+    number: '3',
+    keyword: '',
+  },
+  {
+    ids: ['html', 'xml', 'svg'],
+    code: '<!-- c -->\n<a href="x">t</a>',
+    comment: '<!-- c -->',
+    string: '"x"',
+    keyword: '',
+  },
+  {
+    ids: ['css', 'scss', 'less'],
+    code: '/* c */\na { width: 3px; content: "x"; }',
+    comment: '/* c */',
+    string: '"x"',
+    number: '3px',
+    keyword: '',
+  },
+  {
+    ids: ['python', 'py'],
+    code: '# c\ndef f(): return "x", 3',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: 'def',
+  },
+  {
+    ids: ['bash', 'sh', 'zsh', 'shell'],
+    code: '# c\nif true; then echo "x" 3; fi',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: 'if',
+  },
+  {
+    ids: ['php'],
+    code: '<?php\n// c\nfunction f() { return "x" . 3; }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'function',
+  },
+  {
+    ids: ['hack'],
+    code: '// c\nfunction f(): vec<int> { return vec["x", 3]; }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'function',
+  },
+  {
+    ids: ['yaml', 'yml'],
+    code: '# c\nname: "x"\nn: 3',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: '',
+  },
+  {
+    ids: ['markdown', 'md'],
+    code: '# Title\n\nSome `code` here.',
+    keyword: '# Title',
+  },
+  {
+    ids: ['c', 'h'],
+    code: '#include <stdio.h>\n// c\nint n = 42; char *s = "x";',
+    comment: '// c',
+    string: '"x"',
+    number: '42',
+    keyword: 'int',
+  },
+  {
+    ids: ['cpp', 'c++', 'cc', 'cxx', 'hpp', 'hh'],
+    code: '// c\nauto v = std::vector<int>{1};\nconst char* s = "x";',
+    comment: '// c',
+    string: '"x"',
+    number: '1',
+    keyword: 'auto',
+  },
+  {
+    ids: ['sql'],
+    code: "-- c\nSELECT id FROM t WHERE n > 7 AND s = 'x';",
+    comment: '-- c',
+    string: "'x'",
+    number: '7',
+    keyword: 'SELECT',
+  },
+  {
+    ids: ['rust', 'rs'],
+    code: '// c\nfn main() { let n = 3; let s = "x"; }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'fn',
+  },
+  {
+    ids: ['kotlin', 'kt', 'kts'],
+    code: '// c\nfun main() { val n = 3; val s = "x" }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'fun',
+  },
+  {
+    ids: ['swift'],
+    code: '// c\nfunc f() { let n = 3; let s = "x" }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'func',
+  },
+  {
+    ids: ['go', 'golang'],
+    code: '// c\nfunc main() { n := 3; s := "x" }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'func',
+  },
+];
+
+const MODULES: (Case & {module: CodeLanguage})[] = [
+  {
+    module: graphql,
+    ids: ['graphql', 'gql'],
+    code: '# c\nquery Q { user(id: "x", n: 3) { name } }',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: 'query',
+  },
+  {
+    module: java,
+    ids: ['java'],
+    code: '// c\npublic class A { int n = 3; String s = "x"; }',
+    comment: '// c',
+    string: '"x"',
+    number: '3',
+    keyword: 'public',
+  },
+  {
+    module: thrift,
+    ids: ['thrift'],
+    code: '# c\nstruct U { 1: required string s = "x"; }',
+    comment: '# c',
+    string: '"x"',
+    number: '1',
+    keyword: 'struct',
+  },
+  {
+    module: objectiveC,
+    ids: ['objective-c', 'objc', 'm', 'mm'],
+    code: '// c\n@interface A : NSObject\nint n = 3; NSString *s = @"x";',
+    comment: '// c',
+    string: '@"x"',
+    number: '3',
+    keyword: '@interface',
+  },
+  {
+    module: toml,
+    ids: ['toml'],
+    code: '# c\n[pkg]\nname = "x"\nn = 3',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: '',
+  },
+  {
+    module: powershell,
+    ids: ['powershell', 'ps1', 'pwsh'],
+    code: '# c\nfunction F { $n = 3; $s = "x" }',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: 'function',
+  },
+  {
+    module: ruby,
+    ids: ['ruby', 'rb'],
+    code: '# c\ndef f\n  n = 3; s = "x"\nend',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: 'def',
+  },
+  {
+    module: lua,
+    ids: ['lua'],
+    code: '-- c\nlocal function f() local n = 3; local s = "x" end',
+    comment: '-- c',
+    string: '"x"',
+    number: '3',
+    keyword: 'local',
+  },
+  {
+    module: dockerfile,
+    ids: ['dockerfile', 'docker'],
+    code: '# c\nFROM node:20 AS build\nRUN echo "x" 3',
+    comment: '# c',
+    string: '"x"',
+    number: '3',
+    keyword: 'FROM',
+  },
+];
+
+function expectPaints(map: Map<string, Set<string>>, c: Case) {
+  for (const type of ['comment', 'string', 'number', 'keyword'] as const) {
+    const text = c[type];
+    if (text) {
+      expect(map.get(type)).toContain(text);
+    }
+  }
+}
+
+describe('built-in languages (FR12)', () => {
+  for (const c of NATIVE) {
+    for (const id of c.ids) {
+      it(`paints ${id} with no setup`, () => {
+        expectPaints(painted(c.code, id), c);
+      });
+    }
+  }
+
+  it('paints a call, not the operand before `!=`, and Rust macros', () => {
+    const go = painted('if err != nil { f(1) }', 'go');
+    expect(go.get('function')).toEqual(new Set(['f']));
+    expect(go.get('operator')).toContain('!=');
+    expect(painted('println!("x"); a != b', 'rust').get('function')).toEqual(
+      new Set(['println!']),
+    );
+  });
+
+  it('reads a raw string with inner quotes as one string', () => {
+    const rust = 'let j = r#"{"id": 1, "type": "user"}"#; let u = user;';
+    expect(painted(rust, 'rust').get('string')).toEqual(
+      new Set(['r#"{"id": 1, "type": "user"}"#']),
+    );
+    const cpp = 'auto j = R"json({"id": ")"})json"; int n;';
+    expect(painted(cpp, 'cpp').get('string')).toEqual(
+      new Set(['R"json({"id": ")"})json"']),
+    );
+  });
+
+  it('reads prefixed C++ and extended-delimiter Swift raw strings as one string', () => {
+    expect(painted('auto s = u8R"(a "b" c)";', 'cpp').get('string')).toEqual(
+      new Set(['u8R"(a "b" c)"']),
+    );
+    expect(painted('auto s = LR"x(say "y")x";', 'cpp').get('string')).toEqual(
+      new Set(['LR"x(say "y")x"']),
+    );
+    expect(painted('let s = #"He said "hi""#', 'swift').get('string')).toEqual(
+      new Set(['#"He said "hi""#']),
+    );
+  });
+
+  it('renders an identifier with no grammar as plain code', () => {
+    expect(tokenize('const x = 1;', 'zig').flat()).toEqual([]);
+    expect(tokenize('', 'rust')).toEqual([[]]);
+  });
+});
+
+describe('language values (FR13)', () => {
+  for (const c of MODULES) {
+    it(`paints ${c.module.name} only when its value is passed`, () => {
+      for (const id of c.ids) {
+        // A string never resolves to a module.
+        expect(tokenize(c.code, id).flat()).toEqual([]);
+      }
+      expectPaints(painted(c.code, c.module), c);
+    });
+  }
+
+  it('carries its name, identifiers, file extensions, and file names', () => {
+    expect(thrift.name).toBe('thrift');
+    expect(objectiveC.identifiers).toContain('objc');
+    expect(graphql.fileExtensions).toEqual(['graphql', 'gql']);
+    expect(dockerfile.fileNames).toContain('Dockerfile');
+    expect(java.fileNames).toEqual([]);
+    expect(languageKey(objectiveC)).toBe('objective-c');
+  });
+
+  it('makes a custom language from a tokenize function', () => {
+    const seen: string[] = [];
+    const zig = createCodeLanguage({
+      name: 'zig',
+      identifiers: ['zig'],
+      tokenize: (code, language) => {
+        seen.push(language);
+        return [{type: 'keyword', start: 0, end: 3}];
+      },
+    });
+    expect(zig.fileExtensions).toEqual([]);
+    expect(zig.fileNames).toEqual([]);
+    expect(painted('pub fn\nx', zig).get('keyword')).toEqual(new Set(['pub']));
+    expect(seen).toEqual(['zig']);
+    expect(isCodeLanguage(zig)).toBe(true);
+  });
+
+  it('finds a listed value by identifier, later entries first', () => {
+    const gqlToo = createCodeLanguage({
+      name: 'gql2',
+      identifiers: ['gql'],
+      tokenize: () => [],
+    });
+    expect(listedLanguage([graphql], 'gql')).toBe(graphql);
+    expect(listedLanguage([graphql, gqlToo], 'gql')).toBe(gqlToo);
+    expect(listedLanguage([graphql], 'thrift')).toBeUndefined();
+    expect(listedLanguage(undefined, 'gql')).toBeUndefined();
+  });
+
+  it('finds a listed value by file name or extension', () => {
+    const list = [graphql, dockerfile, ruby];
+    expect(listedLanguageForFile(list, 'schema.graphql', 'graphql')).toBe(
+      graphql,
+    );
+    expect(listedLanguageForFile(list, 'Dockerfile', null)).toBe(dockerfile);
+    expect(listedLanguageForFile(list, 'Gemfile', null)).toBe(ruby);
+    expect(listedLanguageForFile(list, 'a.rs', 'rs')).toBeUndefined();
+  });
+
+  it('ignores values that are not language values', () => {
+    const fake = {
+      name: 'thrift',
+      identifiers: ['thrift'],
+    } as unknown as CodeLanguage;
+    expect(isCodeLanguage(fake)).toBe(false);
+    expect(tokenize('struct U {}', fake).flat()).toEqual([]);
+    expect(powershell.name).toBe('powershell');
+    expect(lua.name).toBe('lua');
+    expect(toml.name).toBe('toml');
+  });
+});