textbringer-tree-sitter 1.0.0

data/exe/textbringer-tree-sitter

@@ -0,0 +1,513 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "fileutils"
+require "open-uri"
+require "rbconfig"
+require "tmpdir"
+require "open3"
+
+module TextbringerTreeSitterCLI
+  FAVEOD_VERSION = "v4.11"
+
+  # Faveod tarball に含まれる parser
+  FAVEOD_PARSERS = %w[
+    bash c c-sharp cobol embedded-template groovy haml html
+    java javascript json pascal php python ruby rust
+  ].freeze
+
+  # 要ビルド（Faveod に含まれない）
+  BUILD_PARSERS = {
+    hcl: {
+      repo: "mitchellh/tree-sitter-hcl",
+      branch: "main",
+      build_cmd: ->(src_dir, out_file) {
+        "c++ -shared -fPIC -O2 -std=c++14 -I#{src_dir}/src #{src_dir}/src/parser.c #{src_dir}/src/scanner.cc -o #{out_file}"
+      }
+    },
+    yaml: {
+      repo: "tree-sitter-grammars/tree-sitter-yaml",
+      branch: "master",
+      build_cmd: ->(src_dir, out_file) {
+        "cc -shared -fPIC -O2 -I#{src_dir}/src #{src_dir}/src/parser.c #{src_dir}/src/scanner.c -o #{out_file}"
+      }
+    },
+    go: {
+      repo: "tree-sitter/tree-sitter-go",
+      branch: "master",
+      build_cmd: ->(src_dir, out_file) {
+        "cc -shared -fPIC -O2 -I#{src_dir}/src #{src_dir}/src/parser.c -o #{out_file}"
+      }
+    },
+    typescript: {
+      repo: "tree-sitter/tree-sitter-typescript",
+      branch: "master",
+      subdir: "typescript",
+      build_cmd: ->(src_dir, out_file) {
+        "cc -shared -fPIC -O2 -I#{src_dir}/src #{src_dir}/src/parser.c #{src_dir}/src/scanner.c -o #{out_file}"
+      }
+    },
+    tsx: {
+      repo: "tree-sitter/tree-sitter-typescript",
+      branch: "master",
+      subdir: "tsx",
+      build_cmd: ->(src_dir, out_file) {
+        "cc -shared -fPIC -O2 -I#{src_dir}/src #{src_dir}/src/parser.c #{src_dir}/src/scanner.c -o #{out_file}"
+      }
+    },
+    sql: {
+      repo: "DerekStride/tree-sitter-sql",
+      branch: "main",
+      build_cmd: ->(src_dir, out_file) {
+        "cc -shared -fPIC -O2 -I#{src_dir}/src #{src_dir}/src/parser.c #{src_dir}/src/scanner.c -o #{out_file}"
+      }
+    },
+    markdown: {
+      repo: "tree-sitter-grammars/tree-sitter-markdown",
+      branch: "split_parser",
+      commit: "9a23c1a",  # LANGUAGE_VERSION 14 (ruby_tree_sitter 互換)
+      subdir: "tree-sitter-markdown",
+      build_cmd: ->(src_dir, out_file) {
+        "cc -shared -fPIC -O2 -I#{src_dir}/src #{src_dir}/src/parser.c #{src_dir}/src/scanner.c -o #{out_file}"
+      }
+    }
+  }.freeze
+
+  class << self
+    def platform
+      os = case RbConfig::CONFIG["host_os"]
+           when /darwin/i then "darwin"
+           when /linux/i then "linux"
+           else "unknown"
+           end
+
+      arch = case RbConfig::CONFIG["host_cpu"]
+             when /arm64|aarch64/i then "arm64"
+             when /x86_64|amd64/i then "x64"
+             else "unknown"
+             end
+
+      "#{os}-#{arch}"
+    end
+
+    def faveod_platform
+      case platform
+      when "darwin-arm64" then "macos-arm64"
+      when "darwin-x64" then "macos-x64"
+      when "linux-x64" then "linux-x64"
+      when "linux-arm64" then "linux-arm64"
+      else platform
+      end
+    end
+
+    def dylib_ext
+      case RbConfig::CONFIG["host_os"]
+      when /darwin/i then ".dylib"
+      else ".so"
+      end
+    end
+
+    def parser_dir
+      File.expand_path("~/.textbringer/parsers/#{platform}")
+    end
+
+    def parser_installed?(language)
+      # c-sharp -> csharp の変換
+      lang_name = language.to_s.gsub("-", "")
+      filename = "libtree-sitter-#{language}#{dylib_ext}"
+      File.exist?(File.join(parser_dir, filename))
+    end
+
+    def download_faveod_parser(language)
+      # Faveod tarball から特定の parser をインストール
+      filename = "libtree-sitter-#{language}#{dylib_ext}"
+      dest_path = File.join(parser_dir, filename)
+
+      if File.exist?(dest_path)
+        puts "#{language}: already installed"
+        return true
+      end
+
+      url = "https://github.com/Faveod/tree-sitter-parsers/releases/download/#{FAVEOD_VERSION}/tree-sitter-parsers-#{FAVEOD_VERSION.delete('v')}-#{faveod_platform}.tar.gz"
+
+      puts "Downloading #{language} from Faveod..."
+
+      Dir.mktmpdir do |tmpdir|
+        tarball = File.join(tmpdir, "parsers.tar.gz")
+
+        begin
+          URI.open(url, "rb") do |remote|
+            File.open(tarball, "wb") { |f| f.write(remote.read) }
+          end
+        rescue OpenURI::HTTPError => e
+          puts "  Error: Failed to download: #{e.message}"
+          return false
+        end
+
+        extract_dir = File.join(tmpdir, "extracted")
+        FileUtils.mkdir_p(extract_dir)
+        system("tar", "-xzf", tarball, "-C", extract_dir, out: File::NULL, err: File::NULL)
+
+        src = Dir.glob("#{extract_dir}/**/#{filename}").first
+        if src
+          FileUtils.mkdir_p(parser_dir)
+          FileUtils.cp(src, dest_path)
+          FileUtils.chmod(0o755, dest_path)
+          puts "  -> #{dest_path}"
+          true
+        else
+          puts "  Error: #{language} not found in tarball"
+          false
+        end
+      end
+    end
+
+    def build_parser(language)
+      lang_sym = language.to_s.gsub("-", "_").to_sym
+      info = BUILD_PARSERS[lang_sym]
+
+      unless info
+        puts "Error: Unknown build-required language: #{language}"
+        puts "Available: #{BUILD_PARSERS.keys.join(', ')}"
+        return false
+      end
+
+      filename = "libtree-sitter-#{language}#{dylib_ext}"
+      dest_path = File.join(parser_dir, filename)
+
+      if File.exist?(dest_path)
+        puts "#{language}: already installed"
+        return true
+      end
+
+      puts "Building #{language} parser..."
+      puts "  Repository: #{info[:repo]}"
+
+      Dir.mktmpdir do |tmpdir|
+        repo_dir = File.join(tmpdir, "repo")
+
+        puts "  Cloning..."
+        if info[:commit]
+          # 特定コミットが必要な場合は shallow clone できない
+          _, status = Open3.capture2e("git", "clone", "-b", info[:branch],
+                                      "https://github.com/#{info[:repo]}.git", repo_dir)
+          unless status.success?
+            puts "  Error: Failed to clone repository"
+            return false
+          end
+
+          puts "  Checking out #{info[:commit]}..."
+          _, status = Open3.capture2e("git", "-C", repo_dir, "checkout", info[:commit])
+          unless status.success?
+            puts "  Error: Failed to checkout commit #{info[:commit]}"
+            return false
+          end
+        else
+          _, status = Open3.capture2e("git", "clone", "--depth", "1", "-b", info[:branch],
+                                      "https://github.com/#{info[:repo]}.git", repo_dir)
+          unless status.success?
+            puts "  Error: Failed to clone repository"
+            return false
+          end
+        end
+
+        src_dir = info[:subdir] ? File.join(repo_dir, info[:subdir]) : repo_dir
+        build_cmd = info[:build_cmd].call(src_dir, dest_path)
+
+        puts "  Building..."
+        FileUtils.mkdir_p(parser_dir)
+        output, status = Open3.capture2e(build_cmd)
+
+        unless status.success?
+          puts "  Error: Build failed"
+          puts output
+          return false
+        end
+
+        FileUtils.chmod(0o755, dest_path)
+        puts "  -> #{dest_path}"
+        true
+      end
+    end
+
+    def get_parser(language)
+      lang = language.to_s
+
+      # Faveod に含まれるか
+      if FAVEOD_PARSERS.include?(lang)
+        download_faveod_parser(lang)
+      # ビルドが必要か
+      elsif BUILD_PARSERS.key?(lang.gsub("-", "_").to_sym)
+        build_parser(lang)
+      else
+        puts "Error: Unknown language: #{language}"
+        puts ""
+        puts "Faveod prebuilt: #{FAVEOD_PARSERS.join(', ')}"
+        puts "Build required: #{BUILD_PARSERS.keys.join(', ')}"
+        false
+      end
+    end
+
+    def list_parsers
+      puts "=== Faveod Prebuilt Parsers ==="
+      FAVEOD_PARSERS.each do |lang|
+        status = parser_installed?(lang) ? "✓" : " "
+        puts "  [#{status}] #{lang}"
+      end
+
+      puts ""
+      puts "=== Build-required Parsers ==="
+      BUILD_PARSERS.keys.sort.each do |lang|
+        status = parser_installed?(lang) ? "✓" : " "
+        puts "  [#{status}] #{lang}"
+      end
+
+      puts ""
+      puts "Use 'textbringer-tree-sitter get <lang>' to install"
+    end
+
+    def get_all
+      puts "Installing all Faveod prebuilt parsers..."
+      puts ""
+
+      url = "https://github.com/Faveod/tree-sitter-parsers/releases/download/#{FAVEOD_VERSION}/tree-sitter-parsers-#{FAVEOD_VERSION.delete('v')}-#{faveod_platform}.tar.gz"
+
+      puts "Downloading from Faveod..."
+      puts "  URL: #{url}"
+
+      Dir.mktmpdir do |tmpdir|
+        tarball = File.join(tmpdir, "parsers.tar.gz")
+
+        begin
+          URI.open(url, "rb") do |remote|
+            File.open(tarball, "wb") { |f| f.write(remote.read) }
+          end
+        rescue OpenURI::HTTPError => e
+          puts "  Error: Failed to download: #{e.message}"
+          return
+        end
+
+        extract_dir = File.join(tmpdir, "extracted")
+        FileUtils.mkdir_p(extract_dir)
+        system("tar", "-xzf", tarball, "-C", extract_dir, out: File::NULL, err: File::NULL)
+
+        FileUtils.mkdir_p(parser_dir)
+
+        Dir.glob("#{extract_dir}/**/libtree-sitter-*#{dylib_ext}").each do |src|
+          filename = File.basename(src)
+          dest = File.join(parser_dir, filename)
+
+          if File.exist?(dest)
+            puts "  #{filename} -> already installed"
+          else
+            FileUtils.cp(src, dest)
+            FileUtils.chmod(0o755, dest)
+            puts "  #{filename} -> OK"
+          end
+        end
+      end
+
+      puts ""
+      puts "Done!"
+    end
+
+    def node_map_dir
+      File.expand_path("~/.textbringer/tree_sitter/node_maps")
+    end
+
+    def guess_face(node_name)
+      name = node_name.to_s.downcase
+      case name
+      when /comment/
+        :comment
+      when /string|heredoc|char_literal|template_string/
+        :string
+      when /number|integer|float|decimal|numeric/
+        :number
+      when /\b(if|else|elsif|unless|case|when|while|until|for|do|end|begin|rescue|ensure|return|yield|break|next|redo|retry|raise|class|module|def|alias|defined|super|self|nil|true|false|and|or|not|in|fn|func|function|let|const|var|import|export|from|as|try|catch|finally|throw|async|await|match|loop|struct|enum|impl|trait|pub|mut|ref|use|mod|crate|where|type|interface|package|extends|implements|static|final|abstract|native|synchronized|volatile|transient|new|this|instanceof|goto|switch|default|continue|assert|with|pass|lambda|nonlocal|global|del|except|exec|print|elif|is)\b/
+        :keyword
+      when /keyword|reserved/
+        :keyword
+      when /constant|boolean/
+        :constant
+      when /function_name|method_name|call|invocation/
+        :function_name
+      when /type|class_name|struct_name|interface_name/
+        :type
+      when /variable|identifier|name/
+        :variable
+      when /operator|binary_op|unary_op/
+        :operator
+      when /punctuation|delimiter|bracket|paren|brace/
+        :punctuation
+      when /marker|heading|header/
+        :keyword
+      else
+        nil
+      end
+    end
+
+    def generate_node_map(language)
+      lang = language.to_s.gsub("-", "_")
+      filename = "libtree-sitter-#{language}#{dylib_ext}"
+      parser_path = File.join(parser_dir, filename)
+
+      unless File.exist?(parser_path)
+        puts "Error: Parser not installed. Run 'textbringer-tree-sitter get #{language}' first."
+        return false
+      end
+
+      require "tree_sitter"
+
+      begin
+        ts_lang = TreeSitter::Language.load(lang, parser_path)
+      rescue => e
+        puts "Error: Failed to load parser: #{e.message}"
+        return false
+      end
+
+      # ノードタイプを収集してヒューリスティックでマッピング
+      mappings = {}
+      unmapped = []
+
+      ts_lang.symbol_count.times do |i|
+        name = ts_lang.symbol_name(i)
+        next if name.nil? || name.empty?
+        next if name.start_with?("_")  # internal nodes
+        next if name.length == 1       # single char tokens
+        next if name =~ /^[^a-zA-Z]/   # non-alphabetic
+
+        face = guess_face(name)
+        if face
+          mappings[face] ||= []
+          mappings[face] << name unless mappings[face].include?(name)
+        else
+          unmapped << name unless unmapped.include?(name)
+        end
+      end
+
+      # テンプレート生成
+      const_name = lang.upcase
+      output = <<~RUBY
+        # frozen_string_literal: true
+
+        # Auto-generated node map for #{language}
+        # Generated by: textbringer-tree-sitter get #{language} --generate-map
+        #
+        # Review and customize as needed.
+        # Uncommented mappings are heuristic guesses.
+        # Commented lines are unmapped nodes.
+
+        module Textbringer
+          module TreeSitter
+            module NodeMaps
+              #{const_name}_FEATURES = {
+      RUBY
+
+      # マッピング済みを出力
+      mappings.each do |face, nodes|
+        nodes_str = nodes.join(" ")
+        output << "          #{face}: %i[#{nodes_str}],\n"
+      end
+
+      output << "        }.freeze\n\n"
+
+      # 未マッピングをコメントで出力
+      if unmapped.any?
+        output << "        # Unmapped nodes (add to appropriate face if needed):\n"
+        unmapped.each do |name|
+          output << "        #   #{name}\n"
+        end
+        output << "\n"
+      end
+
+      output << <<~RUBY
+              #{const_name} = #{const_name}_FEATURES.flat_map { |face, nodes|
+                nodes.map { |node| [node, face] }
+              }.to_h.freeze
+
+              # 自動登録
+              register(:#{lang}, #{const_name})
+            end
+          end
+        end
+      RUBY
+
+      # ファイルに書き出し
+      FileUtils.mkdir_p(node_map_dir)
+      dest_path = File.join(node_map_dir, "#{lang}.rb")
+      File.write(dest_path, output)
+
+      puts "Generated node map: #{dest_path}"
+      puts "  Mapped: #{mappings.values.flatten.size} nodes"
+      puts "  Unmapped: #{unmapped.size} nodes (commented)"
+      puts ""
+      puts "Edit the file to customize mappings, then add to ~/.textbringer.rb:"
+      puts "  require '#{dest_path}'"
+      true
+    end
+
+    def show_help
+      puts <<~HELP
+        textbringer-tree-sitter - Parser manager for textbringer-tree-sitter
+
+        Usage:
+          textbringer-tree-sitter list                List available parsers
+          textbringer-tree-sitter get <lang>          Download/build parser + generate node_map
+          textbringer-tree-sitter get <lang> --no-map Download/build parser only (skip node_map)
+          textbringer-tree-sitter get-all             Install all Faveod prebuilt parsers
+          textbringer-tree-sitter generate-map <lang> (Re)generate node_map for installed parser
+          textbringer-tree-sitter path                Show parser directory
+
+        Examples:
+          textbringer-tree-sitter get markdown        Build parser + generate node_map
+          textbringer-tree-sitter get ruby            Download Ruby parser + generate node_map
+          textbringer-tree-sitter get-all             Install all prebuilt parsers (no node_maps)
+      HELP
+    end
+
+    def run(args)
+      command = args[0]
+
+      case command
+      when "list"
+        list_parsers
+      when "get"
+        lang = args[1]
+        if lang.nil?
+          puts "Error: Please specify a language"
+          puts "Usage: textbringer-tree-sitter get <lang>"
+          exit 1
+        end
+        skip_map = args.include?("--no-map")
+        success = get_parser(lang)
+        if success && !skip_map
+          puts ""
+          generate_node_map(lang)
+        end
+        exit(success ? 0 : 1)
+      when "get-all"
+        get_all
+      when "generate-map"
+        lang = args[1]
+        if lang.nil?
+          puts "Error: Please specify a language"
+          puts "Usage: textbringer-tree-sitter generate-map <lang>"
+          exit 1
+        end
+        success = generate_node_map(lang)
+        exit(success ? 0 : 1)
+      when "path"
+        puts parser_dir
+      when "help", "--help", "-h", nil
+        show_help
+      else
+        puts "Unknown command: #{command}"
+        show_help
+        exit 1
+      end
+    end
+  end
+end
+
+TextbringerTreeSitterCLI.run(ARGV)

data/ext/textbringer_tree_sitter/extconf.rb

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# gem install 時にプリビルド済み parser を自動ダウンロード
+# Faveod/tree-sitter-parsers から tarball を取得して展開
+
+require "fileutils"
+require "open-uri"
+require "rbconfig"
+require "tmpdir"
+
+def platform
+  os = case RbConfig::CONFIG["host_os"]
+       when /darwin/i then "darwin"
+       when /linux/i then "linux"
+       else "unknown"
+       end
+
+  arch = case RbConfig::CONFIG["host_cpu"]
+         when /arm64|aarch64/i then "arm64"
+         when /x86_64|amd64/i then "x64"
+         else "unknown"
+         end
+
+  "#{os}-#{arch}"
+end
+
+def faveod_platform
+  case platform
+  when "darwin-arm64" then "macos-arm64"
+  when "darwin-x64" then "macos-x64"
+  when "linux-x64" then "linux-x64"
+  when "linux-arm64" then "linux-arm64"
+  else platform
+  end
+end
+
+def dylib_ext
+  case RbConfig::CONFIG["host_os"]
+  when /darwin/i then ".dylib"
+  else ".so"
+  end
+end
+
+PARSER_DIR = File.expand_path("~/.textbringer/parsers/#{platform}")
+FAVEOD_VERSION = "v4.11"
+
+# 自動インストールする言語
+DEFAULT_PARSERS = %w[ruby python javascript json bash]
+
+def download_and_extract_parsers
+  url = "https://github.com/Faveod/tree-sitter-parsers/releases/download/#{FAVEOD_VERSION}/tree-sitter-parsers-#{FAVEOD_VERSION.delete('v')}-#{faveod_platform}.tar.gz"
+
+  puts "  Downloading parsers from Faveod..."
+  puts "  URL: #{url}"
+
+  Dir.mktmpdir do |tmpdir|
+    tarball = File.join(tmpdir, "parsers.tar.gz")
+
+    begin
+      URI.open(url, "rb") do |remote|
+        File.open(tarball, "wb") do |local|
+          local.write(remote.read)
+        end
+      end
+    rescue OpenURI::HTTPError => e
+      puts "  Error: Failed to download: #{e.message}"
+      return false
+    end
+
+    # 展開
+    extract_dir = File.join(tmpdir, "extracted")
+    FileUtils.mkdir_p(extract_dir)
+
+    system("tar", "-xzf", tarball, "-C", extract_dir)
+
+    # parser ファイルを探してコピー
+    Dir.glob("#{extract_dir}/**/libtree-sitter-*#{dylib_ext}").each do |src|
+      filename = File.basename(src)
+      # libtree-sitter-{lang}.dylib の形式から lang を抽出
+      lang = filename.sub(/^libtree-sitter-/, "").sub(/#{Regexp.escape(dylib_ext)}$/, "")
+
+      if DEFAULT_PARSERS.include?(lang)
+        dest = File.join(PARSER_DIR, filename)
+        unless File.exist?(dest)
+          FileUtils.cp(src, dest)
+          FileUtils.chmod(0o755, dest)
+          puts "    #{lang} -> OK"
+        else
+          puts "    #{lang} -> already installed"
+        end
+      end
+    end
+  end
+
+  true
+end
+
+puts ""
+puts "=" * 60
+puts "textbringer-tree-sitter: Installing default parsers"
+puts "=" * 60
+puts "Platform: #{platform}"
+puts "Directory: #{PARSER_DIR}"
+puts ""
+
+FileUtils.mkdir_p(PARSER_DIR)
+download_and_extract_parsers
+
+puts ""
+puts "For additional parsers (HCL, YAML, Go, etc.):"
+puts "  $ textbringer-tree-sitter get hcl"
+puts "  $ textbringer-tree-sitter list"
+puts "=" * 60
+puts ""
+
+# extconf.rb は Makefile を生成する必要がある
+File.write("Makefile", <<~MAKEFILE)
+  all:
+  \t@:
+  install:
+  \t@:
+  clean:
+  \t@:
+MAKEFILE

data/lib/textbringer/tree_sitter/node_maps/bash.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Textbringer
+  module TreeSitter
+    module NodeMaps
+      BASH_FEATURES = {
+        comment: %i[comment],
+        string: %i[
+          string
+          raw_string
+          heredoc_body
+          heredoc_start
+          ansi_c_string
+        ],
+        keyword: %i[
+          if
+          then
+          else
+          elif
+          fi
+          for
+          while
+          until
+          do
+          done
+          case
+          esac
+          in
+          function
+          select
+          time
+          coproc
+        ],
+        number: %i[],
+        constant: %i[],
+        function_name: %i[function_definition],
+        variable: %i[
+          variable_name
+          special_variable_name
+        ],
+        type: %i[],
+        operator: %i[
+          file_redirect
+          heredoc_redirect
+          herestring_redirect
+        ],
+        punctuation: %i[],
+        builtin: %i[],
+        property: %i[]
+      }.freeze
+
+      BASH = BASH_FEATURES.flat_map { |face, nodes|
+        nodes.map { |node| [node, face] }
+      }.to_h.freeze
+    end
+  end
+end