@xincli/opentui-core 0.4.6 → 0.4.7

package/src/lib/tree-sitter/assets/update.ts

@@ -1,360 +0,0 @@
-#!/usr/bin/env bun
-
-import { readFile, writeFile, mkdir } from "fs/promises"
-import * as path from "path"
-import { DownloadUtils } from "../download-utils.js"
-import { parseArgs } from "util"
-import type { FiletypeParserOptions } from "../types.js"
-import { readdir } from "fs/promises"
-import { fileURLToPath } from "node:url"
-
-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-
-interface ParsersConfig {
-  parsers: FiletypeParserOptions[]
-}
-
-interface GeneratedParser {
-  filetype: string
-  aliases?: string[]
-  languagePath: string
-  highlightsPath: string
-  injectionsPath?: string
-  injectionMapping?: any
-}
-
-export interface UpdateOptions {
-  /** Path to parsers-config.json */
-  configPath: string
-  /** Directory where .wasm and .scm files will be downloaded */
-  assetsDir: string
-  /** Path where the generated TypeScript file will be written */
-  outputPath: string
-}
-
-function getDefaultOptions(): UpdateOptions {
-  return {
-    configPath: path.resolve(__dirname, "../parsers-config"),
-    assetsDir: path.resolve(__dirname),
-    outputPath: path.resolve(__dirname, "../default-parsers.ts"),
-  }
-}
-
-async function loadConfig(configPath: string): Promise<ParsersConfig> {
-  let ext = path.extname(configPath)
-  let resolvedConfigPath = configPath
-
-  if (ext === "") {
-    const files = await readdir(path.dirname(configPath))
-    const file = files.find(
-      (file) =>
-        file.startsWith(path.basename(configPath)) &&
-        (file.endsWith(".json") || file.endsWith(".ts") || file.endsWith(".js")),
-    )
-    if (!file) {
-      throw new Error(`No config file found for ${configPath}`)
-    }
-    resolvedConfigPath = path.join(path.dirname(configPath), file)
-    ext = path.extname(resolvedConfigPath)
-  }
-
-  if (ext === ".json") {
-    const configContent = await readFile(resolvedConfigPath, "utf-8")
-    return JSON.parse(configContent)
-  } else if (ext === ".ts" || ext === ".js") {
-    const { default: configContent } = await import(resolvedConfigPath)
-    return configContent
-  }
-  throw new Error(`Unsupported config file extension: ${ext}`)
-}
-
-async function downloadLanguage(
-  filetype: string,
-  languageUrl: string,
-  assetsDir: string,
-  outputPath: string,
-): Promise<string> {
-  const languageDir = path.join(assetsDir, filetype)
-  const languageFilename = path.basename(languageUrl)
-  const languagePath = path.join(languageDir, languageFilename)
-
-  const result = await DownloadUtils.downloadToPath(languageUrl, languagePath)
-
-  if (result.error) {
-    throw new Error(`Failed to download language for ${filetype}: ${result.error}`)
-  }
-
-  return "./" + path.relative(path.dirname(outputPath), languagePath)
-}
-
-async function downloadAndCombineQueries(
-  filetype: string,
-  queryUrls: string[],
-  assetsDir: string,
-  outputPath: string,
-  queryType: "highlights" | "injections",
-  configPath: string,
-): Promise<string> {
-  const queriesDir = path.join(assetsDir, filetype)
-  const queryPath = path.join(queriesDir, `${queryType}.scm`)
-
-  const queryContents: string[] = []
-
-  for (let i = 0; i < queryUrls.length; i++) {
-    const queryUrl = queryUrls[i]
-
-    if (queryUrl.startsWith("./")) {
-      console.log(`    Using local query ${i + 1}/${queryUrls.length}: ${queryUrl}`)
-
-      try {
-        const localPath = path.resolve(path.dirname(configPath), queryUrl)
-        const content = await readFile(localPath, "utf-8")
-
-        if (content.trim()) {
-          queryContents.push(content)
-          console.log(`    ✓ Loaded ${content.split("\n").length} lines from local file`)
-        }
-      } catch (error) {
-        console.warn(`Failed to read local query from ${queryUrl}: ${error}`)
-        continue
-      }
-    } else {
-      console.log(`    Downloading query ${i + 1}/${queryUrls.length}: ${queryUrl}`)
-
-      try {
-        const response = await fetch(queryUrl)
-        if (!response.ok) {
-          console.warn(`Failed to download query from ${queryUrl}: ${response.statusText}`)
-          continue
-        }
-
-        const content = await response.text()
-        if (content.trim()) {
-          queryContents.push(`; Query from: ${queryUrl}\n${content}`)
-          console.log(`    ✓ Downloaded ${content.split("\n").length} lines`)
-        }
-      } catch (error) {
-        console.warn(`Failed to download query from ${queryUrl}: ${error}`)
-        continue
-      }
-    }
-  }
-
-  const combinedContent = queryContents.join("\n\n")
-  await writeFile(queryPath, combinedContent, "utf-8")
-
-  console.log(`  Combined ${queryContents.length} queries into ${queryPath}`)
-
-  return "./" + path.relative(path.dirname(outputPath), queryPath)
-}
-
-async function generateDefaultParsersFile(parsers: GeneratedParser[], outputPath: string): Promise<void> {
-  const assetPaths = parsers
-    .map((parser) => {
-      const safeFiletype = parser.filetype.replace(/[^a-zA-Z0-9]/g, "_")
-      const lines = [
-        `const ${safeFiletype}_highlights = await resolveBundledFilePath(`,
-        `  () => import("${parser.highlightsPath}" as string, { with: { type: "file" } }),`,
-        `  "${parser.highlightsPath}",`,
-        `  import.meta.url,`,
-        `)`,
-        `const ${safeFiletype}_language = await resolveBundledFilePath(`,
-        `  () => import("${parser.languagePath}" as string, { with: { type: "file" } }),`,
-        `  "${parser.languagePath}",`,
-        `  import.meta.url,`,
-        `)`,
-      ]
-
-      if (parser.injectionsPath) {
-        lines.push(
-          `const ${safeFiletype}_injections = await resolveBundledFilePath(`,
-          `  () => import("${parser.injectionsPath}" as string, { with: { type: "file" } }),`,
-          `  "${parser.injectionsPath}",`,
-          `  import.meta.url,`,
-          `)`,
-        )
-      }
-
-      return lines.join("\n")
-    })
-    .join("\n")
-
-  const parserDefinitions = parsers
-    .map((parser) => {
-      const safeFiletype = parser.filetype.replace(/[^a-zA-Z0-9]/g, "_")
-      const queriesLines = [`          highlights: [${safeFiletype}_highlights],`]
-
-      if (parser.injectionsPath) {
-        queriesLines.push(`          injections: [${safeFiletype}_injections],`)
-      }
-
-      const injectionMappingLine = parser.injectionMapping
-        ? `        injectionMapping: ${JSON.stringify(parser.injectionMapping, null, 10)},`
-        : ""
-      const aliasesLine = parser.aliases?.length ? `        aliases: ${JSON.stringify(parser.aliases)},` : ""
-
-      return `      {
-        filetype: "${parser.filetype}",
-${aliasesLine ? aliasesLine + "\n" : ""}        queries: {
-${queriesLines.join("\n")}
-        },
-        wasm: ${safeFiletype}_language,${injectionMappingLine ? "\n" + injectionMappingLine : ""}
-      }`
-    })
-    .join(",\n")
-
-  const fileContent = `// This file is generated by assets/update.ts - DO NOT EDIT MANUALLY
-// Run 'bun assets/update.ts' to regenerate this file
-// Last generated: ${new Date().toISOString()}
-
-import type { FiletypeParserOptions } from "./types.js"
-import { resolveBundledFilePath } from "../../platform/runtime.js"
-
-// Cached parsers to avoid re-resolving paths on every call
-let _cachedParsers: Promise<FiletypeParserOptions[]> | undefined
-
-export function getParsers(): Promise<FiletypeParserOptions[]> {
-  if (!_cachedParsers) {
-    _cachedParsers = loadParsers()
-  }
-  return _cachedParsers
-}
-
-async function loadParsers(): Promise<FiletypeParserOptions[]> {
-${assetPaths}
-
-  return [
-${parserDefinitions},
-  ]
-}
-`
-
-  await mkdir(path.dirname(outputPath), { recursive: true })
-  await writeFile(outputPath, fileContent, "utf-8")
-  console.log(`Generated ${path.basename(outputPath)} with ${parsers.length} parsers`)
-}
-
-async function main(options?: Partial<UpdateOptions>): Promise<void> {
-  const opts = { ...getDefaultOptions(), ...options }
-
-  try {
-    console.log("Loading parsers configuration...")
-    console.log(`  Config: ${opts.configPath}`)
-    console.log(`  Assets Dir: ${opts.assetsDir}`)
-    console.log(`  Output: ${opts.outputPath}`)
-
-    const config = await loadConfig(opts.configPath)
-
-    console.log(`Found ${config.parsers.length} parsers to process`)
-
-    const generatedParsers: GeneratedParser[] = []
-
-    for (const parser of config.parsers) {
-      console.log(`Processing ${parser.filetype}...`)
-
-      console.log(`  Downloading language...`)
-      const languagePath = await downloadLanguage(parser.filetype, parser.wasm, opts.assetsDir, opts.outputPath)
-
-      console.log(`  Downloading ${parser.queries.highlights.length} highlight queries...`)
-      const highlightsPath = await downloadAndCombineQueries(
-        parser.filetype,
-        parser.queries.highlights,
-        opts.assetsDir,
-        opts.outputPath,
-        "highlights",
-        opts.configPath,
-      )
-
-      let injectionsPath: string | undefined
-      if (parser.queries.injections && parser.queries.injections.length > 0) {
-        console.log(`  Downloading ${parser.queries.injections.length} injection queries...`)
-        injectionsPath = await downloadAndCombineQueries(
-          parser.filetype,
-          parser.queries.injections,
-          opts.assetsDir,
-          opts.outputPath,
-          "injections",
-          opts.configPath,
-        )
-      }
-
-      generatedParsers.push({
-        filetype: parser.filetype,
-        aliases: parser.aliases,
-        languagePath,
-        highlightsPath,
-        injectionsPath,
-        injectionMapping: parser.injectionMapping,
-      })
-
-      console.log(`  ✓ Completed ${parser.filetype}`)
-    }
-
-    console.log("Generating output file...")
-    await generateDefaultParsersFile(generatedParsers, opts.outputPath)
-
-    console.log("✅ Update completed successfully!")
-  } catch (error) {
-    console.error("❌ Update failed:", error)
-    process.exit(1)
-  }
-}
-
-function parseCLIArgs(): Partial<UpdateOptions> | null {
-  try {
-    const { values } = parseArgs({
-      args: process.argv.slice(2),
-      options: {
-        config: { type: "string" },
-        assets: { type: "string" },
-        output: { type: "string" },
-        help: { type: "boolean" },
-      },
-      strict: true,
-    })
-
-    if (values.help) {
-      const command = path.basename(Bun.argv[1] ?? "update-assets.js")
-
-      console.log(`Usage: bun ${command} [options]
-
-Options:
-  --config <path>  Path to parsers-config.json
-  --assets <path>  Directory where .wasm and .scm files will be downloaded
-  --output <path>  Path where the generated TypeScript file will be written
-  --help           Show this help message
-
-Examples:
-  # Use default paths (for OpenTUI core development)
-  bun ${command}
-
-  # Use custom paths (for application integration)
-  bun ${command} --config ./my-parsers.json --assets ./src/parsers --output ./src/parsers.ts
-`)
-      process.exit(0)
-    }
-
-    const options: Partial<UpdateOptions> = {}
-    if (values.config) options.configPath = path.resolve(values.config)
-    if (values.assets) options.assetsDir = path.resolve(values.assets)
-    if (values.output) options.outputPath = path.resolve(values.output)
-
-    return Object.keys(options).length > 0 ? options : null
-  } catch (error) {
-    console.error(`Error parsing arguments: ${error}`)
-    console.log("Run with --help for usage information")
-    process.exit(1)
-  }
-}
-
-export function runUpdateAssetsCli(): Promise<void> {
-  const cliOptions = parseCLIArgs()
-  return main(cliOptions || undefined)
-}
-
-if (import.meta.main) {
-  await runUpdateAssetsCli()
-}
-
-export { main as updateAssets }

package/src/lib/tree-sitter/assets.d.ts

@@ -1,9 +0,0 @@
-declare module "*.scm" {
-  const value: string
-  export default value
-}
-
-declare module "*.wasm" {
-  const value: string
-  export default value
-}

package/src/lib/tree-sitter/cache.test.ts

@@ -1,291 +0,0 @@
-import { test, expect, beforeEach, beforeAll, afterAll, describe } from "bun:test"
-import { TreeSitterClient, addDefaultParsers } from "./client.js"
-import { createServer, type Server } from "node:http"
-import { tmpdir } from "node:os"
-import { dirname, join, resolve } from "node:path"
-import { mkdir, readdir, stat, writeFile } from "node:fs/promises"
-import { readFileSync } from "node:fs"
-import { fileURLToPath } from "node:url"
-import type { FiletypeParserOptions } from "./types.js"
-
-describe("TreeSitterClient Caching", () => {
-  let dataPath: string
-  let testServer: Server | undefined
-  const TEST_PORT = 55231
-  const TEST_HOST = "127.0.0.1"
-  const BASE_URL = `http://${TEST_HOST}:${TEST_PORT}`
-  const DATA_ROOT = join(tmpdir(), "tree-sitter-cache-test")
-
-  beforeAll(async () => {
-    const assetsDir = resolve(dirname(fileURLToPath(import.meta.url)), "assets")
-
-    testServer = createServer((req, res) => {
-      const url = new URL(req.url ?? "/", BASE_URL)
-      const filePath = join(assetsDir, url.pathname)
-      res.end(readFileSync(filePath))
-    })
-
-    await new Promise<void>((resolve, reject) => {
-      testServer!.once("error", reject)
-      testServer!.listen(TEST_PORT, TEST_HOST, () => {
-        testServer!.off("error", reject)
-        resolve()
-      })
-    })
-
-    await mkdir(DATA_ROOT, { recursive: true })
-  })
-
-  afterAll(async () => {
-    if (testServer) {
-      await new Promise<void>((resolve, reject) => {
-        testServer!.close((error) => {
-          if (error) reject(error)
-          else resolve()
-        })
-      })
-      testServer = undefined
-    }
-  })
-
-  beforeEach(async () => {
-    dataPath = join(DATA_ROOT, Math.random().toString(36).slice(2))
-    await mkdir(dataPath, { recursive: true })
-  })
-
-  test("should create storage directories on initialization", async () => {
-    const client = new TreeSitterClient({ dataPath })
-    await client.initialize()
-
-    const languagesDir = join(dataPath, "tree-sitter", "languages")
-    const queriesDir = join(dataPath, "tree-sitter", "queries")
-
-    const languagesStat = await stat(languagesDir)
-    const queriesStat = await stat(queriesDir)
-
-    expect(languagesStat.isDirectory()).toBe(true)
-    expect(queriesStat.isDirectory()).toBe(true)
-
-    await client.destroy()
-  })
-
-  test("should cache downloaded language files", async () => {
-    const client = new TreeSitterClient({ dataPath })
-    await client.initialize()
-
-    // Add URL-based parser for this test
-    client.addFiletypeParser({
-      filetype: "javascript",
-      queries: {
-        highlights: [`${BASE_URL}/javascript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
-    })
-
-    const hasParser = await client.preloadParser("javascript")
-    expect(hasParser).toBe(true)
-
-    const languagesDir = join(dataPath, "tree-sitter", "languages")
-    const cachedFiles = await readdir(languagesDir)
-
-    expect(cachedFiles).toContain("tree-sitter-javascript.wasm")
-
-    await client.destroy()
-  })
-
-  test("should cache downloaded highlight queries", async () => {
-    const client = new TreeSitterClient({ dataPath })
-    await client.initialize()
-
-    // Add URL-based parser for this test
-    client.addFiletypeParser({
-      filetype: "javascript",
-      queries: {
-        highlights: [`${BASE_URL}/javascript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
-    })
-
-    const hasParser = await client.preloadParser("javascript")
-    expect(hasParser).toBe(true)
-
-    const queriesDir = join(dataPath, "tree-sitter", "queries")
-    const cachedQueries = await readdir(queriesDir)
-
-    const scmFiles = cachedQueries.filter((file) => file.endsWith(".scm"))
-    expect(scmFiles.length).toBeGreaterThan(0)
-
-    await client.destroy()
-  })
-
-  // TODO: This is flaky, there must be a more reliable way to test this
-  test.skip("should reuse cached files across client instances", async () => {
-    const jsParser: FiletypeParserOptions = {
-      filetype: "javascript",
-      queries: {
-        highlights: [`${BASE_URL}/javascript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
-    }
-
-    let client1 = new TreeSitterClient({ dataPath })
-    await client1.initialize()
-    client1.addFiletypeParser(jsParser)
-
-    console.log("=== First client (should download) ===")
-    const start1 = Date.now()
-    const hasParser1 = await client1.preloadParser("javascript")
-    const duration1 = Date.now() - start1
-    expect(hasParser1).toBe(true)
-
-    await client1.destroy()
-
-    let client2 = new TreeSitterClient({ dataPath })
-    await client2.initialize()
-    client2.addFiletypeParser(jsParser)
-
-    console.log("=== Second client (should use cache) ===")
-    const start2 = Date.now()
-    const hasParser2 = await client2.preloadParser("javascript")
-    const duration2 = Date.now() - start2
-    expect(hasParser2).toBe(true)
-
-    console.log(`First client: ${duration1}ms, Second client: ${duration2}ms`)
-
-    expect(duration2).toBeLessThanOrEqual(duration1)
-    expect(duration2).toBeLessThan(100) // Should be very fast with cache
-
-    await client2.destroy()
-  })
-
-  test("should handle multiple parsers with independent caching", async () => {
-    const client = new TreeSitterClient({ dataPath })
-    await client.initialize()
-
-    // Add URL-based parsers for this test
-    client.addFiletypeParser({
-      filetype: "javascript",
-      queries: {
-        highlights: [`${BASE_URL}/javascript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
-    })
-    client.addFiletypeParser({
-      filetype: "typescript",
-      queries: {
-        highlights: [`${BASE_URL}/typescript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/typescript/tree-sitter-typescript.wasm`,
-    })
-
-    const hasJS = await client.preloadParser("javascript")
-    const hasTS = await client.preloadParser("typescript")
-
-    expect(hasJS).toBe(true)
-    expect(hasTS).toBe(true)
-
-    const languagesDir = join(dataPath, "tree-sitter", "languages")
-    const cachedFiles = await readdir(languagesDir)
-
-    expect(cachedFiles).toContain("tree-sitter-javascript.wasm")
-    expect(cachedFiles).toContain("tree-sitter-typescript.wasm")
-
-    const queriesDir = join(dataPath, "tree-sitter", "queries")
-    const cachedQueries = await readdir(queriesDir)
-    const scmFiles = cachedQueries.filter((file) => file.endsWith(".scm"))
-
-    expect(scmFiles.length).toBe(2)
-
-    await client.destroy()
-  })
-
-  test("should store files in dataPath subdirectories", async () => {
-    const client = new TreeSitterClient({ dataPath })
-    await client.initialize()
-
-    // Add URL-based parser for this test
-    client.addFiletypeParser({
-      filetype: "javascript",
-      queries: {
-        highlights: [`${BASE_URL}/javascript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
-    })
-
-    const hasParser = await client.preloadParser("javascript")
-    expect(hasParser).toBe(true)
-
-    const languagesDir = join(dataPath, "tree-sitter", "languages")
-    const queriesDir = join(dataPath, "tree-sitter", "queries")
-
-    const languagesStat = await stat(languagesDir)
-    const queriesStat = await stat(queriesDir)
-
-    expect(languagesStat.isDirectory()).toBe(true)
-    expect(queriesStat.isDirectory()).toBe(true)
-
-    const cachedFiles = await readdir(languagesDir)
-    expect(cachedFiles).toContain("tree-sitter-javascript.wasm")
-
-    await client.destroy()
-  })
-
-  test("should reject when tree-sitter cache directories cannot be created", async () => {
-    const blockedDataPath = join(DATA_ROOT, "blocked-" + Math.random().toString(36).slice(2))
-    await mkdir(blockedDataPath, { recursive: true })
-    await writeFile(join(blockedDataPath, "tree-sitter"), "blocked")
-
-    const client = new TreeSitterClient({ dataPath: blockedDataPath })
-
-    await expect(client.initialize()).rejects.toThrow()
-
-    await client.destroy()
-  })
-
-  test("should handle data path changes", async () => {
-    const initialDataPath = join(DATA_ROOT, "initial-" + Math.random().toString(36).slice(2))
-    const newDataPath = join(DATA_ROOT, "new-" + Math.random().toString(36).slice(2))
-
-    await mkdir(initialDataPath, { recursive: true })
-    await mkdir(newDataPath, { recursive: true })
-
-    const client = new TreeSitterClient({ dataPath: initialDataPath })
-    await client.initialize()
-
-    // Add URL-based parsers for this test
-    client.addFiletypeParser({
-      filetype: "javascript",
-      queries: {
-        highlights: [`${BASE_URL}/javascript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
-    })
-
-    const hasParser1 = await client.preloadParser("javascript")
-    expect(hasParser1).toBe(true)
-
-    const initialLanguagesDir = join(initialDataPath, "tree-sitter", "languages")
-    const initialFiles = await readdir(initialLanguagesDir)
-    expect(initialFiles).toContain("tree-sitter-javascript.wasm")
-
-    await client.setDataPath(newDataPath)
-
-    // Add typescript parser for the new data path
-    client.addFiletypeParser({
-      filetype: "typescript",
-      queries: {
-        highlights: [`${BASE_URL}/typescript/highlights.scm`],
-      },
-      wasm: `${BASE_URL}/typescript/tree-sitter-typescript.wasm`,
-    })
-
-    const hasParser2 = await client.preloadParser("typescript")
-    expect(hasParser2).toBe(true)
-
-    const newLanguagesDir = join(newDataPath, "tree-sitter", "languages")
-    const newFiles = await readdir(newLanguagesDir)
-    expect(newFiles).toContain("tree-sitter-typescript.wasm")
-
-    await client.destroy()
-  })
-})