@xincli/opentui-core 0.4.6

package/src/lib/tree-sitter/parsers-config.ts

@@ -0,0 +1,81 @@
+/**
+ * This file contains the configuration for the defaulttree-sitter parsers.
+ * It is used by ./assets/update.ts to generate the default-parsers.ts file.
+ * For changes here to be reflected in the default-parsers.ts file, you need to run `bun run ./assets/update.ts`
+ */
+export default {
+  parsers: [
+    {
+      filetype: "javascript",
+      aliases: ["javascriptreact"],
+      wasm: "https://github.com/tree-sitter/tree-sitter-javascript/releases/download/v0.25.0/tree-sitter-javascript.wasm",
+      queries: {
+        highlights: [
+          "https://raw.githubusercontent.com/tree-sitter/tree-sitter-javascript/refs/heads/master/queries/highlights.scm",
+        ],
+      },
+    },
+    {
+      filetype: "typescript",
+      aliases: ["typescriptreact"],
+      wasm: "https://github.com/tree-sitter/tree-sitter-typescript/releases/download/v0.23.2/tree-sitter-typescript.wasm",
+      queries: {
+        highlights: [
+          "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/ecma/highlights.scm",
+          "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/typescript/highlights.scm",
+        ],
+      },
+    },
+    {
+      filetype: "markdown",
+      wasm: "https://github.com/tree-sitter-grammars/tree-sitter-markdown/releases/download/v0.5.1/tree-sitter-markdown.wasm",
+      queries: {
+        highlights: [
+          // Using local file to preserve custom modifications
+          "./assets/markdown/highlights.scm",
+        ],
+        injections: [
+          "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/markdown/injections.scm",
+        ],
+      },
+      injectionMapping: {
+        nodeTypes: {
+          inline: "markdown_inline",
+          pipe_table_cell: "markdown_inline",
+        },
+        infoStringMap: {
+          javascript: "javascript",
+          js: "javascript",
+          jsx: "javascriptreact",
+          javascriptreact: "javascriptreact",
+          typescript: "typescript",
+          ts: "typescript",
+          tsx: "typescriptreact",
+          typescriptreact: "typescriptreact",
+          markdown: "markdown",
+          md: "markdown",
+        },
+      },
+    },
+    {
+      filetype: "markdown_inline",
+      wasm: "https://github.com/tree-sitter-grammars/tree-sitter-markdown/releases/download/v0.5.1/tree-sitter-markdown_inline.wasm",
+      queries: {
+        highlights: [
+          // NOTE: Based on the last working version of the query, newer versions are adapted to neovim breaking changes
+          // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/99ddf573531c4dbe53f743ecbc1595af5eb1d32f/queries/markdown_inline/highlights.scm",
+          "./assets/markdown_inline/highlights.scm",
+        ],
+      },
+    },
+    {
+      filetype: "zig",
+      wasm: "https://github.com/tree-sitter-grammars/tree-sitter-zig/releases/download/v1.1.2/tree-sitter-zig.wasm",
+      queries: {
+        highlights: [
+          "https://github.com/nvim-treesitter/nvim-treesitter/raw/refs/heads/master/queries/zig/highlights.scm",
+        ],
+      },
+    },
+  ],
+}

package/src/lib/tree-sitter/resolve-ft.test.ts

@@ -0,0 +1,55 @@
+import { test, expect } from "bun:test"
+import { extensionToFiletype, infoStringToFiletype, pathToFiletype } from "./resolve-ft.js"
+
+test("pathToFiletype only resolves actual paths", () => {
+  expect(pathToFiletype("tsx")).toBeUndefined()
+  expect(pathToFiletype("components/Button.tsx")).toBe("typescriptreact")
+})
+
+test("pathToFiletype resolves common extension aliases to parser ids", () => {
+  expect(pathToFiletype("src/index.mjs")).toBe("javascript")
+  expect(pathToFiletype("src/index.cts")).toBe("typescript")
+  expect(pathToFiletype("src/index.mtsx")).toBe("typescriptreact")
+  expect(pathToFiletype("src/module.cc")).toBe("cpp")
+  expect(pathToFiletype("src/module.hxx")).toBe("cpp")
+  expect(pathToFiletype("src/config.hrl")).toBe("erlang")
+  expect(pathToFiletype("src/main.hs")).toBe("haskell")
+  expect(pathToFiletype("src/main.ml")).toBe("ocaml")
+  expect(pathToFiletype("src/main.scala")).toBe("scala")
+  expect(pathToFiletype("src/config.zon")).toBe("zig")
+  expect(pathToFiletype("src/script.sh")).toBe("bash")
+})
+
+test("pathToFiletype resolves common basenames", () => {
+  expect(pathToFiletype("Dockerfile")).toBe("dockerfile")
+  expect(pathToFiletype("Containerfile")).toBe("dockerfile")
+  expect(pathToFiletype("Makefile")).toBe("make")
+  expect(pathToFiletype("Rakefile")).toBe("ruby")
+  expect(pathToFiletype(".bashrc")).toBe("bash")
+  expect(pathToFiletype(".vimrc")).toBe("vim")
+})
+
+test("infoStringToFiletype normalizes markdown fence labels", () => {
+  expect(infoStringToFiletype("tsx")).toBe("typescriptreact")
+  expect(infoStringToFiletype("TSX title=Button.tsx")).toBe("typescriptreact")
+  expect(infoStringToFiletype(".jsx")).toBe("javascriptreact")
+  expect(infoStringToFiletype("Button.tsx")).toBe("typescriptreact")
+  expect(infoStringToFiletype("Dockerfile")).toBe("dockerfile")
+  expect(infoStringToFiletype("bash")).toBe("bash")
+})
+
+test("extensionToFiletype can be extended by consumers", () => {
+  const previous = extensionToFiletype.get("foo")
+
+  try {
+    extensionToFiletype.set("foo", "custom")
+    expect(infoStringToFiletype("foo")).toBe("custom")
+    expect(pathToFiletype("example.foo")).toBe("custom")
+  } finally {
+    if (previous === undefined) {
+      extensionToFiletype.delete("foo")
+    } else {
+      extensionToFiletype.set("foo", previous)
+    }
+  }
+})

package/src/lib/tree-sitter/resolve-ft.ts

@@ -0,0 +1,189 @@
+import path from "node:path"
+
+export const extensionToFiletype: Map<string, string> = new Map([
+  ["astro", "astro"],
+  ["bash", "bash"],
+  ["c", "c"],
+  ["cc", "cpp"],
+  ["cjs", "javascript"],
+  ["clj", "clojure"],
+  ["cljs", "clojure"],
+  ["cljc", "clojure"],
+  ["cpp", "cpp"],
+  ["cxx", "cpp"],
+  ["cs", "csharp"],
+  ["cts", "typescript"],
+  ["ctsx", "typescriptreact"],
+  ["dart", "dart"],
+  ["diff", "diff"],
+  ["edn", "clojure"],
+  ["go", "go"],
+  ["gemspec", "ruby"],
+  ["groovy", "groovy"],
+  ["h", "c"],
+  ["handlebars", "handlebars"],
+  ["hbs", "handlebars"],
+  ["hpp", "cpp"],
+  ["hxx", "cpp"],
+  ["h++", "cpp"],
+  ["hh", "cpp"],
+  ["hrl", "erlang"],
+  ["hs", "haskell"],
+  ["htm", "html"],
+  ["html", "html"],
+  ["ini", "ini"],
+  ["js", "javascript"],
+  ["jsx", "javascriptreact"],
+  ["jl", "julia"],
+  ["json", "json"],
+  ["ksh", "bash"],
+  ["kt", "kotlin"],
+  ["kts", "kotlin"],
+  ["latex", "latex"],
+  ["less", "less"],
+  ["lua", "lua"],
+  ["markdown", "markdown"],
+  ["md", "markdown"],
+  ["mdown", "markdown"],
+  ["mkd", "markdown"],
+  ["mjs", "javascript"],
+  ["ml", "ocaml"],
+  ["mli", "ocaml"],
+  ["mts", "typescript"],
+  ["mtsx", "typescriptreact"],
+  ["patch", "diff"],
+  ["php", "php"],
+  ["pl", "perl"],
+  ["pm", "perl"],
+  ["ps1", "powershell"],
+  ["psm1", "powershell"],
+  ["py", "python"],
+  ["pyi", "python"],
+  ["r", "r"],
+  ["rb", "ruby"],
+  ["rake", "ruby"],
+  ["rs", "rust"],
+  ["ru", "ruby"],
+  ["sass", "sass"],
+  ["sc", "scala"],
+  ["scala", "scala"],
+  ["scss", "scss"],
+  ["sh", "bash"],
+  ["sql", "sql"],
+  ["svelte", "svelte"],
+  ["swift", "swift"],
+  ["ts", "typescript"],
+  ["tsx", "typescriptreact"],
+  ["tex", "latex"],
+  ["toml", "toml"],
+  ["vue", "vue"],
+  ["vim", "vim"],
+  ["xml", "xml"],
+  ["xsl", "xsl"],
+  ["yaml", "yaml"],
+  ["yml", "yaml"],
+  ["zig", "zig"],
+  ["zon", "zig"],
+  ["zsh", "bash"],
+  ["c++", "cpp"],
+  ["erl", "erlang"],
+  ["exs", "elixir"],
+  ["ex", "elixir"],
+  ["elm", "elm"],
+  ["fsharp", "fsharp"],
+  ["fs", "fsharp"],
+  ["fsx", "fsharp"],
+  ["fsscript", "fsharp"],
+  ["fsi", "fsharp"],
+  ["java", "java"],
+  ["css", "css"],
+])
+
+export const basenameToFiletype: Map<string, string> = new Map([
+  [".bash_aliases", "bash"],
+  [".bash_logout", "bash"],
+  [".bash_profile", "bash"],
+  [".bashrc", "bash"],
+  [".kshrc", "bash"],
+  [".profile", "bash"],
+  [".vimrc", "vim"],
+  [".zlogin", "bash"],
+  [".zlogout", "bash"],
+  [".zprofile", "bash"],
+  [".zshenv", "bash"],
+  [".zshrc", "bash"],
+  ["appfile", "ruby"],
+  ["berksfile", "ruby"],
+  ["brewfile", "ruby"],
+  ["cheffile", "ruby"],
+  ["containerfile", "dockerfile"],
+  ["dockerfile", "dockerfile"],
+  ["fastfile", "ruby"],
+  ["gemfile", "ruby"],
+  ["gnumakefile", "make"],
+  ["gvimrc", "vim"],
+  ["guardfile", "ruby"],
+  ["makefile", "make"],
+  ["podfile", "ruby"],
+  ["rakefile", "ruby"],
+  ["thorfile", "ruby"],
+  ["vagrantfile", "ruby"],
+])
+
+function normalizeFiletypeToken(value: string): string | undefined {
+  const normalizedValue = value.trim().replace(/^\./, "").toLowerCase()
+  return normalizedValue || undefined
+}
+
+function getBasename(value: string): string | undefined {
+  const normalizedValue = value.trim().replaceAll("\\", "/")
+  if (!normalizedValue) return undefined
+
+  const basename = path.posix.basename(normalizedValue).toLowerCase()
+  return basename || undefined
+}
+
+export function extToFiletype(extension: string): string | undefined {
+  const normalizedExtension = normalizeFiletypeToken(extension)
+  if (!normalizedExtension) return undefined
+
+  return extensionToFiletype.get(normalizedExtension)
+}
+
+export function pathToFiletype(path: string): string | undefined {
+  if (typeof path !== "string") return undefined
+
+  const basename = getBasename(path)
+  if (!basename) return undefined
+
+  const basenameFiletype = basenameToFiletype.get(basename)
+  if (basenameFiletype) {
+    return basenameFiletype
+  }
+
+  const lastDot = basename.lastIndexOf(".")
+  if (lastDot === -1 || lastDot === basename.length - 1) {
+    return undefined
+  }
+
+  const extension = basename.substring(lastDot + 1)
+  return extToFiletype(extension)
+}
+
+export function infoStringToFiletype(infoString: string): string | undefined {
+  if (typeof infoString !== "string") return undefined
+
+  const token = infoString.trim().split(/\s+/, 1)[0]
+  const directBasenameMatch = basenameToFiletype.get(token.toLowerCase())
+  if (directBasenameMatch) return directBasenameMatch
+
+  const normalizedToken = normalizeFiletypeToken(token)
+  if (!normalizedToken) return undefined
+
+  return (
+    basenameToFiletype.get(normalizedToken) ??
+    pathToFiletype(normalizedToken) ??
+    extToFiletype(normalizedToken) ??
+    normalizedToken
+  )
+}

package/src/lib/tree-sitter/types.ts

@@ -0,0 +1,132 @@
+export interface HighlightRange {
+  startCol: number
+  endCol: number
+  group: string
+}
+
+export interface HighlightResponse {
+  line: number
+  highlights: HighlightRange[]
+  droppedHighlights: HighlightRange[]
+}
+
+export interface HighlightMeta {
+  isInjection?: boolean
+  injectionLang?: string
+  containsInjection?: boolean
+  conceal?: string | null // Value from (#set! conceal "...") predicate
+  concealLines?: string | null // Value from (#set! conceal_lines "...") predicate - indicates the whole line should be concealed
+}
+
+export type SimpleHighlight = [number, number, string, HighlightMeta?]
+
+export interface InjectionMapping {
+  // Maps tree-sitter node types to target filetypes
+  nodeTypes?: { [nodeType: string]: string }
+  // Maps info string content (e.g., from code blocks) to target filetypes
+  infoStringMap?: { [infoString: string]: string }
+}
+
+export interface FiletypeParserOptions {
+  filetype: string
+  aliases?: string[]
+  queries: {
+    highlights: string[] // Array of URLs or local file paths to fetch highlight queries from
+    injections?: string[] // Array of URLs or local file paths to fetch injection queries from
+  }
+  wasm: string // URL or local file path to the language parser WASM file
+  injectionMapping?: InjectionMapping // Optional mapping for injection handling
+}
+
+export interface BufferState {
+  id: number
+  version: number
+  content: string
+  filetype: string
+  hasParser: boolean
+}
+
+export interface ParsedBuffer extends BufferState {
+  hasParser: true
+}
+
+export type TreeSitterWorkerLogType = "log" | "error" | "warn"
+
+export type TreeSitterWorkerRequest =
+  | { type: "INIT"; dataPath: string }
+  | { type: "ADD_FILETYPE_PARSER"; filetypeParser: FiletypeParserOptions }
+  | { type: "PRELOAD_PARSER"; filetype: string; messageId: string }
+  | {
+      type: "INITIALIZE_PARSER"
+      bufferId: number
+      version: number
+      content: string
+      filetype: string
+      messageId: string
+    }
+  | { type: "HANDLE_EDITS"; bufferId: number; version: number; content: string; edits: Edit[] }
+  | { type: "GET_PERFORMANCE"; messageId: string }
+  | { type: "RESET_BUFFER"; bufferId: number; version: number; content: string; edits: Edit[] }
+  | { type: "DISPOSE_BUFFER"; bufferId: number }
+  | { type: "ONESHOT_HIGHLIGHT"; content: string; filetype: string; messageId: string }
+  | { type: "UPDATE_DATA_PATH"; dataPath: string; messageId: string }
+  | { type: "CLEAR_CACHE"; messageId: string }
+
+export type TreeSitterWorkerResponse =
+  | { type: "INIT_RESPONSE"; error?: string }
+  | {
+      type: "PARSER_INIT_RESPONSE"
+      bufferId: number
+      messageId: string
+      hasParser: boolean
+      warning?: string
+      error?: string
+    }
+  | { type: "HIGHLIGHT_RESPONSE"; bufferId: number; version: number; highlights: HighlightResponse[] }
+  | { type: "PRELOAD_PARSER_RESPONSE"; messageId: string; hasParser: boolean }
+  | { type: "BUFFER_DISPOSED"; bufferId: number }
+  | { type: "PERFORMANCE_RESPONSE"; performance: PerformanceStats; messageId: string }
+  | {
+      type: "ONESHOT_HIGHLIGHT_RESPONSE"
+      messageId: string
+      hasParser: boolean
+      highlights?: SimpleHighlight[]
+      warning?: string
+      error?: string
+    }
+  | { type: "UPDATE_DATA_PATH_RESPONSE"; messageId: string; error?: string }
+  | { type: "CLEAR_CACHE_RESPONSE"; messageId: string; error?: string }
+  | { type: "WARNING"; bufferId?: number; warning: string }
+  | { type: "ERROR"; bufferId?: number; error: string }
+  | { type: "WORKER_LOG"; logType: TreeSitterWorkerLogType; data: unknown[] }
+
+export interface TreeSitterClientEvents {
+  "highlights:response": [bufferId: number, version: number, highlights: HighlightResponse[]]
+  "buffer:initialized": [bufferId: number, hasParser: boolean]
+  "buffer:disposed": [bufferId: number]
+  "worker:log": [logType: TreeSitterWorkerLogType, message: string]
+  error: [error: string, bufferId?: number]
+  warning: [warning: string, bufferId?: number]
+}
+
+export interface TreeSitterClientOptions {
+  dataPath: string // Directory for storing downloaded parsers and queries
+  workerPath?: string | URL
+  initTimeout?: number // Timeout in milliseconds for worker initialization, defaults to 10000
+}
+
+export interface Edit {
+  startIndex: number
+  oldEndIndex: number
+  newEndIndex: number
+  startPosition: { row: number; column: number }
+  oldEndPosition: { row: number; column: number }
+  newEndPosition: { row: number; column: number }
+}
+
+export interface PerformanceStats {
+  averageParseTime: number
+  parseTimes: number[]
+  averageQueryTime: number
+  queryTimes: number[]
+}

package/src/lib/tree-sitter/update-assets.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env bun
+
+import { runUpdateAssetsCli } from "./assets/update.js"
+
+export { runUpdateAssetsCli, updateAssets } from "./assets/update.js"
+export type { UpdateOptions } from "./assets/update.js"
+
+if (import.meta.main) {
+  await runUpdateAssetsCli()
+}