@fairyhunter13/opentui-core 0.1.88 → 0.1.90

package/src/lib/tree-sitter/assets/update.ts

@@ -0,0 +1,331 @@
+#!/usr/bin/env bun
+
+import { readFile, writeFile, mkdir } from "fs/promises"
+import * as path from "path"
+import { DownloadUtils } from "../download-utils.js"
+import { parseArgs } from "util"
+import type { FiletypeParserOptions } from "../types.js"
+import { readdir } from "fs/promises"
+
+interface ParsersConfig {
+  parsers: FiletypeParserOptions[]
+}
+
+interface GeneratedParser {
+  filetype: string
+  languagePath: string
+  highlightsPath: string
+  injectionsPath?: string
+  injectionMapping?: any
+}
+
+export interface UpdateOptions {
+  /** Path to parsers-config.json */
+  configPath: string
+  /** Directory where .wasm and .scm files will be downloaded */
+  assetsDir: string
+  /** Path where the generated TypeScript file will be written */
+  outputPath: string
+}
+
+function getDefaultOptions(): UpdateOptions {
+  return {
+    configPath: path.resolve(__dirname, "../parsers-config"),
+    assetsDir: path.resolve(__dirname),
+    outputPath: path.resolve(__dirname, "../default-parsers.ts"),
+  }
+}
+
+async function loadConfig(configPath: string): Promise<ParsersConfig> {
+  let ext = path.extname(configPath)
+  let resolvedConfigPath = configPath
+
+  if (ext === "") {
+    const files = await readdir(path.dirname(configPath))
+    const file = files.find(
+      (file) =>
+        file.startsWith(path.basename(configPath)) &&
+        (file.endsWith(".json") || file.endsWith(".ts") || file.endsWith(".js")),
+    )
+    if (!file) {
+      throw new Error(`No config file found for ${configPath}`)
+    }
+    resolvedConfigPath = path.join(path.dirname(configPath), file)
+    ext = path.extname(resolvedConfigPath)
+  }
+
+  if (ext === ".json") {
+    const configContent = await readFile(resolvedConfigPath, "utf-8")
+    return JSON.parse(configContent)
+  } else if (ext === ".ts" || ext === ".js") {
+    const { default: configContent } = await import(resolvedConfigPath)
+    return configContent
+  }
+  throw new Error(`Unsupported config file extension: ${ext}`)
+}
+
+async function downloadLanguage(
+  filetype: string,
+  languageUrl: string,
+  assetsDir: string,
+  outputPath: string,
+): Promise<string> {
+  const languageDir = path.join(assetsDir, filetype)
+  const languageFilename = path.basename(languageUrl)
+  const languagePath = path.join(languageDir, languageFilename)
+
+  const result = await DownloadUtils.downloadToPath(languageUrl, languagePath)
+
+  if (result.error) {
+    throw new Error(`Failed to download language for ${filetype}: ${result.error}`)
+  }
+
+  return "./" + path.relative(path.dirname(outputPath), languagePath)
+}
+
+async function downloadAndCombineQueries(
+  filetype: string,
+  queryUrls: string[],
+  assetsDir: string,
+  outputPath: string,
+  queryType: "highlights" | "injections",
+  configPath: string,
+): Promise<string> {
+  const queriesDir = path.join(assetsDir, filetype)
+  const queryPath = path.join(queriesDir, `${queryType}.scm`)
+
+  const queryContents: string[] = []
+
+  for (let i = 0; i < queryUrls.length; i++) {
+    const queryUrl = queryUrls[i]
+
+    if (queryUrl.startsWith("./")) {
+      console.log(`    Using local query ${i + 1}/${queryUrls.length}: ${queryUrl}`)
+
+      try {
+        const localPath = path.resolve(path.dirname(configPath), queryUrl)
+        const content = await readFile(localPath, "utf-8")
+
+        if (content.trim()) {
+          queryContents.push(content)
+          console.log(`    ✓ Loaded ${content.split("\n").length} lines from local file`)
+        }
+      } catch (error) {
+        console.warn(`Failed to read local query from ${queryUrl}: ${error}`)
+        continue
+      }
+    } else {
+      console.log(`    Downloading query ${i + 1}/${queryUrls.length}: ${queryUrl}`)
+
+      try {
+        const response = await fetch(queryUrl)
+        if (!response.ok) {
+          console.warn(`Failed to download query from ${queryUrl}: ${response.statusText}`)
+          continue
+        }
+
+        const content = await response.text()
+        if (content.trim()) {
+          queryContents.push(`; Query from: ${queryUrl}\n${content}`)
+          console.log(`    ✓ Downloaded ${content.split("\n").length} lines`)
+        }
+      } catch (error) {
+        console.warn(`Failed to download query from ${queryUrl}: ${error}`)
+        continue
+      }
+    }
+  }
+
+  const combinedContent = queryContents.join("\n\n")
+  await writeFile(queryPath, combinedContent, "utf-8")
+
+  console.log(`  Combined ${queryContents.length} queries into ${queryPath}`)
+
+  return "./" + path.relative(path.dirname(outputPath), queryPath)
+}
+
+async function generateDefaultParsersFile(parsers: GeneratedParser[], outputPath: string): Promise<void> {
+  const imports = parsers
+    .map((parser) => {
+      const safeFiletype = parser.filetype.replace(/[^a-zA-Z0-9]/g, "_")
+      const lines = [
+        `import ${safeFiletype}_highlights from "${parser.highlightsPath}" with { type: "file" }`,
+        `import ${safeFiletype}_language from "${parser.languagePath}" with { type: "file" }`,
+      ]
+      if (parser.injectionsPath) {
+        lines.push(`import ${safeFiletype}_injections from "${parser.injectionsPath}" with { type: "file" }`)
+      }
+      return lines.join("\n")
+    })
+    .join("\n")
+
+  const parserDefinitions = parsers
+    .map((parser) => {
+      const safeFiletype = parser.filetype.replace(/[^a-zA-Z0-9]/g, "_")
+      const queriesLines = [
+        `          highlights: [resolve(dirname(fileURLToPath(import.meta.url)), ${safeFiletype}_highlights)],`,
+      ]
+      if (parser.injectionsPath) {
+        queriesLines.push(
+          `          injections: [resolve(dirname(fileURLToPath(import.meta.url)), ${safeFiletype}_injections)],`,
+        )
+      }
+
+      const injectionMappingLine = parser.injectionMapping
+        ? `        injectionMapping: ${JSON.stringify(parser.injectionMapping, null, 10)},`
+        : ""
+
+      return `      {
+        filetype: "${parser.filetype}",
+        queries: {
+${queriesLines.join("\n")}
+        },
+        wasm: resolve(dirname(fileURLToPath(import.meta.url)), ${safeFiletype}_language),${injectionMappingLine ? "\n" + injectionMappingLine : ""}
+      }`
+    })
+    .join(",\n")
+
+  const fileContent = `// This file is generated by assets/update.ts - DO NOT EDIT MANUALLY
+// Run 'bun assets/update.ts' to regenerate this file
+// Last generated: ${new Date().toISOString()}
+
+import type { FiletypeParserOptions } from "./types"
+import { resolve, dirname } from "path"
+import { fileURLToPath } from "url"
+
+${imports}
+
+// Cached parsers to avoid re-resolving paths on every call
+let _cachedParsers: FiletypeParserOptions[] | undefined
+
+export function getParsers(): FiletypeParserOptions[] {
+  if (!_cachedParsers) {
+    _cachedParsers = [
+${parserDefinitions},
+    ]
+  }
+  return _cachedParsers
+}
+`
+
+  await mkdir(path.dirname(outputPath), { recursive: true })
+  await writeFile(outputPath, fileContent, "utf-8")
+  console.log(`Generated ${path.basename(outputPath)} with ${parsers.length} parsers`)
+}
+
+async function main(options?: Partial<UpdateOptions>): Promise<void> {
+  const opts = { ...getDefaultOptions(), ...options }
+
+  try {
+    console.log("Loading parsers configuration...")
+    console.log(`  Config: ${opts.configPath}`)
+    console.log(`  Assets Dir: ${opts.assetsDir}`)
+    console.log(`  Output: ${opts.outputPath}`)
+
+    const config = await loadConfig(opts.configPath)
+
+    console.log(`Found ${config.parsers.length} parsers to process`)
+
+    const generatedParsers: GeneratedParser[] = []
+
+    for (const parser of config.parsers) {
+      console.log(`Processing ${parser.filetype}...`)
+
+      console.log(`  Downloading language...`)
+      const languagePath = await downloadLanguage(parser.filetype, parser.wasm, opts.assetsDir, opts.outputPath)
+
+      console.log(`  Downloading ${parser.queries.highlights.length} highlight queries...`)
+      const highlightsPath = await downloadAndCombineQueries(
+        parser.filetype,
+        parser.queries.highlights,
+        opts.assetsDir,
+        opts.outputPath,
+        "highlights",
+        opts.configPath,
+      )
+
+      let injectionsPath: string | undefined
+      if (parser.queries.injections && parser.queries.injections.length > 0) {
+        console.log(`  Downloading ${parser.queries.injections.length} injection queries...`)
+        injectionsPath = await downloadAndCombineQueries(
+          parser.filetype,
+          parser.queries.injections,
+          opts.assetsDir,
+          opts.outputPath,
+          "injections",
+          opts.configPath,
+        )
+      }
+
+      generatedParsers.push({
+        filetype: parser.filetype,
+        languagePath,
+        highlightsPath,
+        injectionsPath,
+        injectionMapping: parser.injectionMapping,
+      })
+
+      console.log(`  ✓ Completed ${parser.filetype}`)
+    }
+
+    console.log("Generating output file...")
+    await generateDefaultParsersFile(generatedParsers, opts.outputPath)
+
+    console.log("✅ Update completed successfully!")
+  } catch (error) {
+    console.error("❌ Update failed:", error)
+    process.exit(1)
+  }
+}
+
+function parseCLIArgs(): Partial<UpdateOptions> | null {
+  try {
+    const { values } = parseArgs({
+      args: Bun.argv.slice(2),
+      options: {
+        config: { type: "string" },
+        assets: { type: "string" },
+        output: { type: "string" },
+        help: { type: "boolean" },
+      },
+      strict: true,
+    })
+
+    if (values.help) {
+      console.log(`Usage: bun update.ts [options]
+
+Options:
+  --config <path>  Path to parsers-config.json
+  --assets <path>  Directory where .wasm and .scm files will be downloaded
+  --output <path>  Path where the generated TypeScript file will be written
+  --help           Show this help message
+
+Examples:
+  # Use default paths (for OpenTUI core development)
+  bun update.ts
+
+  # Use custom paths (for application integration)
+  bun update.ts --config ./my-parsers.json --assets ./src/parsers --output ./src/parsers.ts
+`)
+      process.exit(0)
+    }
+
+    const options: Partial<UpdateOptions> = {}
+    if (values.config) options.configPath = path.resolve(values.config)
+    if (values.assets) options.assetsDir = path.resolve(values.assets)
+    if (values.output) options.outputPath = path.resolve(values.output)
+
+    return Object.keys(options).length > 0 ? options : null
+  } catch (error) {
+    console.error(`Error parsing arguments: ${error}`)
+    console.log("Run with --help for usage information")
+    process.exit(1)
+  }
+}
+
+if (import.meta.main) {
+  const cliOptions = parseCLIArgs()
+  main(cliOptions || undefined)
+}
+
+export { main as updateAssets }

package/src/lib/tree-sitter/assets.d.ts

@@ -0,0 +1,9 @@
+declare module "*.scm" {
+  const value: string
+  export default value
+}
+
+declare module "*.wasm" {
+  const value: string
+  export default value
+}

package/src/lib/tree-sitter/cache.test.ts

@@ -0,0 +1,270 @@
+import { test, expect, beforeEach, beforeAll, afterAll, describe } from "bun:test"
+import { TreeSitterClient, addDefaultParsers } from "./client.js"
+import { tmpdir } from "node:os"
+import { join, resolve } from "node:path"
+import { mkdir, readdir, stat } from "node:fs/promises"
+import { readFileSync } from "node:fs"
+import type { FiletypeParserOptions } from "./types.js"
+
+describe("TreeSitterClient Caching", () => {
+  let dataPath: string
+  let testServer: any
+  const TEST_PORT = 55231
+  const BASE_URL = `http://localhost:${TEST_PORT}`
+
+  beforeAll(async () => {
+    const assetsDir = resolve(__dirname, "assets")
+    testServer = Bun.serve({
+      port: TEST_PORT,
+      fetch(req) {
+        const url = new URL(req.url)
+        const filePath = join(assetsDir, url.pathname)
+        return new Response(readFileSync(filePath))
+      },
+    })
+  })
+
+  afterAll(async () => {
+    if (testServer) {
+      testServer.stop()
+    }
+  })
+
+  beforeEach(async () => {
+    dataPath = join(tmpdir(), "tree-sitter-cache-test-" + Math.random().toString(36).slice(2))
+    await mkdir(dataPath, { recursive: true })
+  })
+
+  test("should create storage directories on initialization", async () => {
+    const client = new TreeSitterClient({ dataPath })
+    await client.initialize()
+
+    const languagesDir = join(dataPath, "tree-sitter", "languages")
+    const queriesDir = join(dataPath, "tree-sitter", "queries")
+
+    const languagesStat = await stat(languagesDir)
+    const queriesStat = await stat(queriesDir)
+
+    expect(languagesStat.isDirectory()).toBe(true)
+    expect(queriesStat.isDirectory()).toBe(true)
+
+    await client.destroy()
+  })
+
+  test("should cache downloaded language files", async () => {
+    const client = new TreeSitterClient({ dataPath })
+    await client.initialize()
+
+    // Add URL-based parser for this test
+    client.addFiletypeParser({
+      filetype: "javascript",
+      queries: {
+        highlights: [`${BASE_URL}/javascript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
+    })
+
+    const hasParser = await client.preloadParser("javascript")
+    expect(hasParser).toBe(true)
+
+    const languagesDir = join(dataPath, "tree-sitter", "languages")
+    const cachedFiles = await readdir(languagesDir)
+
+    expect(cachedFiles).toContain("tree-sitter-javascript.wasm")
+
+    await client.destroy()
+  })
+
+  test("should cache downloaded highlight queries", async () => {
+    const client = new TreeSitterClient({ dataPath })
+    await client.initialize()
+
+    // Add URL-based parser for this test
+    client.addFiletypeParser({
+      filetype: "javascript",
+      queries: {
+        highlights: [`${BASE_URL}/javascript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
+    })
+
+    const hasParser = await client.preloadParser("javascript")
+    expect(hasParser).toBe(true)
+
+    const queriesDir = join(dataPath, "tree-sitter", "queries")
+    const cachedQueries = await readdir(queriesDir)
+
+    const scmFiles = cachedQueries.filter((file) => file.endsWith(".scm"))
+    expect(scmFiles.length).toBeGreaterThan(0)
+
+    await client.destroy()
+  })
+
+  // TODO: This is flaky, there must be a more reliable way to test this
+  test.skip("should reuse cached files across client instances", async () => {
+    const jsParser: FiletypeParserOptions = {
+      filetype: "javascript",
+      queries: {
+        highlights: [`${BASE_URL}/javascript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
+    }
+
+    let client1 = new TreeSitterClient({ dataPath })
+    await client1.initialize()
+    client1.addFiletypeParser(jsParser)
+
+    console.log("=== First client (should download) ===")
+    const start1 = Date.now()
+    const hasParser1 = await client1.preloadParser("javascript")
+    const duration1 = Date.now() - start1
+    expect(hasParser1).toBe(true)
+
+    await client1.destroy()
+
+    let client2 = new TreeSitterClient({ dataPath })
+    await client2.initialize()
+    client2.addFiletypeParser(jsParser)
+
+    console.log("=== Second client (should use cache) ===")
+    const start2 = Date.now()
+    const hasParser2 = await client2.preloadParser("javascript")
+    const duration2 = Date.now() - start2
+    expect(hasParser2).toBe(true)
+
+    console.log(`First client: ${duration1}ms, Second client: ${duration2}ms`)
+
+    expect(duration2).toBeLessThanOrEqual(duration1)
+    expect(duration2).toBeLessThan(100) // Should be very fast with cache
+
+    await client2.destroy()
+  })
+
+  test("should handle multiple parsers with independent caching", async () => {
+    const client = new TreeSitterClient({ dataPath })
+    await client.initialize()
+
+    // Add URL-based parsers for this test
+    client.addFiletypeParser({
+      filetype: "javascript",
+      queries: {
+        highlights: [`${BASE_URL}/javascript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
+    })
+    client.addFiletypeParser({
+      filetype: "typescript",
+      queries: {
+        highlights: [`${BASE_URL}/typescript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/typescript/tree-sitter-typescript.wasm`,
+    })
+
+    const hasJS = await client.preloadParser("javascript")
+    const hasTS = await client.preloadParser("typescript")
+
+    expect(hasJS).toBe(true)
+    expect(hasTS).toBe(true)
+
+    const languagesDir = join(dataPath, "tree-sitter", "languages")
+    const cachedFiles = await readdir(languagesDir)
+
+    expect(cachedFiles).toContain("tree-sitter-javascript.wasm")
+    expect(cachedFiles).toContain("tree-sitter-typescript.wasm")
+
+    const queriesDir = join(dataPath, "tree-sitter", "queries")
+    const cachedQueries = await readdir(queriesDir)
+    const scmFiles = cachedQueries.filter((file) => file.endsWith(".scm"))
+
+    expect(scmFiles.length).toBe(2)
+
+    await client.destroy()
+  })
+
+  test("should store files in dataPath subdirectories", async () => {
+    const client = new TreeSitterClient({ dataPath })
+    await client.initialize()
+
+    // Add URL-based parser for this test
+    client.addFiletypeParser({
+      filetype: "javascript",
+      queries: {
+        highlights: [`${BASE_URL}/javascript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
+    })
+
+    const hasParser = await client.preloadParser("javascript")
+    expect(hasParser).toBe(true)
+
+    const languagesDir = join(dataPath, "tree-sitter", "languages")
+    const queriesDir = join(dataPath, "tree-sitter", "queries")
+
+    const languagesStat = await stat(languagesDir)
+    const queriesStat = await stat(queriesDir)
+
+    expect(languagesStat.isDirectory()).toBe(true)
+    expect(queriesStat.isDirectory()).toBe(true)
+
+    const cachedFiles = await readdir(languagesDir)
+    expect(cachedFiles).toContain("tree-sitter-javascript.wasm")
+
+    await client.destroy()
+  })
+
+  test("should handle directory creation errors gracefully", async () => {
+    const invalidDataPath = "/invalid/path/that/cannot/be/created"
+    const client = new TreeSitterClient({ dataPath: invalidDataPath })
+
+    await expect(client.initialize()).rejects.toThrow()
+
+    await client.destroy()
+  })
+
+  test("should handle data path changes", async () => {
+    const initialDataPath = join(tmpdir(), "tree-sitter-initial-" + Math.random().toString(36).slice(2))
+    const newDataPath = join(tmpdir(), "tree-sitter-new-" + Math.random().toString(36).slice(2))
+
+    await mkdir(initialDataPath, { recursive: true })
+    await mkdir(newDataPath, { recursive: true })
+
+    const client = new TreeSitterClient({ dataPath: initialDataPath })
+    await client.initialize()
+
+    // Add URL-based parsers for this test
+    client.addFiletypeParser({
+      filetype: "javascript",
+      queries: {
+        highlights: [`${BASE_URL}/javascript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/javascript/tree-sitter-javascript.wasm`,
+    })
+
+    const hasParser1 = await client.preloadParser("javascript")
+    expect(hasParser1).toBe(true)
+
+    const initialLanguagesDir = join(initialDataPath, "tree-sitter", "languages")
+    const initialFiles = await readdir(initialLanguagesDir)
+    expect(initialFiles).toContain("tree-sitter-javascript.wasm")
+
+    await client.setDataPath(newDataPath)
+
+    // Add typescript parser for the new data path
+    client.addFiletypeParser({
+      filetype: "typescript",
+      queries: {
+        highlights: [`${BASE_URL}/typescript/highlights.scm`],
+      },
+      wasm: `${BASE_URL}/typescript/tree-sitter-typescript.wasm`,
+    })
+
+    const hasParser2 = await client.preloadParser("typescript")
+    expect(hasParser2).toBe(true)
+
+    const newLanguagesDir = join(newDataPath, "tree-sitter", "languages")
+    const newFiles = await readdir(newLanguagesDir)
+    expect(newFiles).toContain("tree-sitter-typescript.wasm")
+
+    await client.destroy()
+  })
+})