kompo 0.2.0 → 0.3.0

data/lib/kompo/tasks/make_fs_c.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'fileutils'
+require 'find'
+
+module Kompo
+  # Struct to hold file data for embedding
+  KompoFile = Struct.new(:path, :bytes)
+
+  # Generate fs.c containing embedded file data
+  # Required context:
+  #   - gems: Array of gem paths to embed
+  #   - ruby_std_libs: Array of Ruby standard library paths to embed
+  #   - bundler_config: Path to .bundle/config (if using Gemfile)
+  # Dependencies provide:
+  #   - WorkDir.path
+  #   - CopyProjectFiles.entrypoint_path
+  class MakeFsC < Taski::Task
+    exports :path
+
+    # File extensions to skip when embedding
+    SKIP_EXTENSIONS = %w[
+      .so .bundle .c .h .o .java .jar .gz .dat .sqlite3 .exe
+      .gem .out .png .jpg .jpeg .gif .bmp .ico .svg .webp .ttf .data
+    ].freeze
+
+    # Directory names to prune when traversing
+    PRUNE_DIRS = %w[.git ports logs spec .github docs exe _ruby].freeze
+
+    def run
+      @work_dir = WorkDir.path
+      @path = File.join(@work_dir, 'fs.c')
+
+      # Get original paths for Ruby standard library cache support
+      # When Ruby is restored from cache, standard library paths are from the original build
+      # We need to embed Ruby stdlib files with those original paths so the VFS can find them
+      # Project files and gems use current work_dir paths (no replacement needed)
+      @original_ruby_install_dir = InstallRuby.original_ruby_install_dir
+      @current_ruby_install_dir = InstallRuby.ruby_install_dir
+
+      # Initialize .kompoignore handler
+      project_dir = Taski.args.fetch(:project_dir, Taski.env.working_directory) || Taski.env.working_directory
+      @kompo_ignore = KompoIgnore.new(project_dir)
+      puts "Using .kompoignore from #{project_dir}" if @kompo_ignore.enabled?
+
+      @file_bytes = []
+      @paths = []
+      @file_sizes = [0]
+
+      group('Collecting files') do
+        embed_paths = collect_embed_paths
+
+        embed_paths.each do |embed_path|
+          expand_path = File.expand_path(embed_path)
+          unless File.exist?(expand_path)
+            warn "warn: #{expand_path} does not exist. Skipping."
+            next
+          end
+
+          if File.directory?(expand_path)
+            process_directory(expand_path)
+          else
+            add_file(expand_path)
+          end
+        end
+        puts "Collected #{@file_sizes.size - 1} files"
+      end
+
+      group('Generating fs.c') do
+        context = build_template_context
+
+        template_path = File.join(__dir__, '..', '..', 'fs.c.erb')
+        template = ERB.new(File.read(template_path))
+        File.write(@path, template.result(binding))
+        puts "Generated: fs.c (#{@file_bytes.size} bytes)"
+      end
+    end
+
+    def clean
+      return unless @path && File.exist?(@path)
+
+      FileUtils.rm_f(@path)
+      puts 'Cleaned up fs.c'
+    end
+
+    private
+
+    def collect_embed_paths
+      paths = []
+
+      # 1. Project files (entrypoint + additional files/directories specified by user)
+      #    Created by: CopyProjectFiles
+      paths << CopyProjectFiles.entrypoint_path
+      paths += CopyProjectFiles.additional_paths
+
+      # 2. Gemfile and Gemfile.lock (if exists)
+      #    Created by: CopyGemfile
+      if CopyGemfile.gemfile_exists
+        paths << File.join(@work_dir, 'Gemfile')
+        paths << File.join(@work_dir, 'Gemfile.lock')
+
+        # 3. Bundle directory (.bundle/config and bundle/ruby/X.Y.Z/gems/...)
+        #    Created by: BundleInstall
+        paths << BundleInstall.bundler_config_path
+        paths << BundleInstall.bundle_ruby_dir
+      end
+
+      # 4. Ruby standard library
+      #    Retrieved by: CheckStdlibs
+      paths += CheckStdlibs.paths
+
+      paths.compact
+    end
+
+    def process_directory(dir_path)
+      # Resolve base directory to ensure symlink safety
+      real_base = File.realpath(dir_path)
+
+      Find.find(dir_path) do |path|
+        # Prune certain directories
+        if File.directory?(path)
+          base = File.basename(path)
+          Find.prune if PRUNE_DIRS.any? { |d| base == d || path.end_with?("/#{d}") }
+          next
+        end
+
+        # Skip symlinks that escape the base directory (symlink traversal prevention)
+        if File.symlink?(path)
+          real_path = File.realpath(path)
+          unless real_path.start_with?(real_base)
+            warn "warn: Skipping symlink escaping base directory: #{path} -> #{real_path}"
+            next
+          end
+        end
+
+        # Skip certain file extensions
+        next if SKIP_EXTENSIONS.any? { |ext| path.end_with?(ext) }
+        next if path.end_with?('selenium-manager')
+
+        # Skip files matching .kompoignore patterns
+        next if should_ignore?(path)
+
+        add_file(path)
+      end
+    end
+
+    # Check if a file should be ignored based on .kompoignore patterns
+    # Only applies to files under work_dir (Ruby standard library is excluded)
+    def should_ignore?(absolute_path)
+      return false unless @kompo_ignore&.enabled?
+
+      # Only apply .kompoignore to work_dir files (project files and gems)
+      # Ruby standard library paths are outside work_dir and should not be filtered
+      return false unless absolute_path.start_with?(@work_dir)
+
+      relative_path = absolute_path.sub("#{@work_dir}/", '')
+      if @kompo_ignore.ignore?(relative_path)
+        puts "Ignoring (via .kompoignore): #{relative_path}"
+        true
+      else
+        false
+      end
+    end
+
+    def add_file(path)
+      # Keep original paths for VFS - the caching system already ensures
+      # the same work_dir path is reused across builds via metadata.json
+      # Ruby's $LOAD_PATH uses work_dir paths, so embedded files must match.
+      embedded_path = if @current_ruby_install_dir != @original_ruby_install_dir && path.start_with?(@current_ruby_install_dir)
+                        # Ruby install dir path replacement for cache compatibility (when paths differ)
+                        path.sub(@current_ruby_install_dir, @original_ruby_install_dir)
+                      else
+                        path
+                      end
+
+      puts "#{path} -> #{embedded_path}" if path != embedded_path
+
+      # Use binread for binary-safe reading (preserves exact bytes without encoding conversion)
+      content = File.binread(path)
+      bytes = content.bytes
+      byte_size = content.bytesize
+      path_bytes = embedded_path.bytes << 0 # null-terminated
+
+      file = KompoFile.new(path_bytes, bytes)
+
+      @file_bytes.concat(file.bytes)
+      @paths.concat(file.path)
+      prev_size = @file_sizes.last
+      @file_sizes << (prev_size + byte_size)
+    end
+
+    def build_template_context
+      FsCTemplateContext.new(
+        work_dir: @work_dir
+      )
+    end
+
+    # Struct for fs.c template variables
+    FsCTemplateContext = Struct.new(:work_dir, keyword_init: true)
+  end
+end

data/lib/kompo/tasks/make_main_c.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'fileutils'
+
+module Kompo
+  # Generate main.c from ERB template
+  # Required context:
+  #   - exts: Array of [so_path, init_func] pairs from native gem builds
+  # Dependencies provide:
+  #   - WorkDir.path
+  #   - CopyProjectFiles.entrypoint_path
+  class MakeMainC < Taski::Task
+    exports :path
+
+    def run
+      work_dir = WorkDir.path
+      @path = File.join(work_dir, 'main.c')
+
+      return if File.exist?(@path)
+
+      template_path = File.join(__dir__, '..', '..', 'main.c.erb')
+      template = ERB.new(File.read(template_path))
+
+      # Build context object for ERB template
+      context = build_template_context
+
+      File.write(@path, template.result(binding))
+      puts 'Generated: main.c'
+    end
+
+    def clean
+      return unless @path && File.exist?(@path)
+
+      FileUtils.rm_f(@path)
+      puts 'Cleaned up main.c'
+    end
+
+    private
+
+    def build_template_context
+      project_dir = Taski.args.fetch(:project_dir, Taski.env.working_directory) || Taski.env.working_directory
+      work_dir = WorkDir.path
+      entrypoint = CopyProjectFiles.entrypoint_path
+
+      TemplateContext.new(
+        exts: BuildNativeGem.exts || [],
+        work_dir: work_dir,
+        work_dir_entrypoint: entrypoint,
+        project_dir: project_dir,
+        has_gemfile: CopyGemfile.gemfile_exists
+      )
+    end
+
+    # Simple struct to hold template variables
+    TemplateContext = Struct.new(
+      :exts,
+      :work_dir,
+      :work_dir_entrypoint,
+      :project_dir,
+      :has_gemfile,
+      keyword_init: true
+    )
+  end
+end

data/lib/kompo/tasks/packing.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'shellwords'
+
+module Kompo
+  # Section to compile the final binary.
+  # Switches implementation based on the current platform.
+  # Uses CollectDependencies's exported values for dependencies.
+  class Packing < Taski::Section
+    interfaces :output_path
+
+    def impl
+      macos? ? ForMacOS : ForLinux
+    end
+
+    # Common helper methods shared between macOS and Linux implementations
+    module CommonHelpers
+      private
+
+      def get_ruby_cflags(ruby_install_dir)
+        ruby_pc = File.join(ruby_install_dir, 'lib', 'pkgconfig', 'ruby.pc')
+        output, = Open3.capture2('pkg-config', '--cflags', ruby_pc, err: File::NULL)
+        Shellwords.split(output.chomp)
+      end
+
+      def get_ruby_mainlibs(ruby_install_dir)
+        ruby_pc = File.join(ruby_install_dir, 'lib', 'pkgconfig', 'ruby.pc')
+        output, = Open3.capture2('pkg-config', '--variable=MAINLIBS', ruby_pc, err: File::NULL)
+        output.chomp
+      end
+
+      def get_ldflags(work_dir, ruby_major_minor)
+        makefiles = Dir.glob("#{work_dir}/bundle/ruby/#{ruby_major_minor}.0/gems/*/ext/*/Makefile")
+        flags = makefiles.flat_map do |makefile|
+          content = File.read(makefile)
+          ldflags = content.scan(/^ldflags\s+= (.*)/).flatten
+          ldflags += content.scan(/^LDFLAGS\s+= (.*)/).flatten
+          ldflags.flat_map { |f| f.split(' ') }
+        end
+        flags.uniq.select { |f| f.start_with?('-L') }
+      end
+
+      def get_libpath(work_dir, ruby_major_minor)
+        makefiles = Dir.glob("#{work_dir}/bundle/ruby/#{ruby_major_minor}.0/gems/*/ext/*/Makefile")
+        makefiles.flat_map do |makefile|
+          content = File.read(makefile)
+          content.scan(/^LIBPATH = (.*)/).flatten
+        end.compact.flat_map { |p| p.split(' ') }.uniq
+                 .reject { |p| p.start_with?('-Wl,-rpath,') || !p.start_with?('-L/') }
+      end
+
+      def get_extlibs(ruby_build_path, ruby_version)
+        exts_mk_files = Dir.glob(File.join(ruby_build_path, "ruby-#{ruby_version}", 'ext', '**', 'exts.mk'))
+        exts_mk_files.flat_map do |file|
+          File.read(file).scan(/^EXTLIBS\s+= (.*)/).flatten
+        end.compact.flat_map { |l| l.split(' ') }.uniq
+      end
+
+      def get_gem_libs(work_dir, ruby_major_minor)
+        makefiles = Dir.glob("#{work_dir}/bundle/ruby/#{ruby_major_minor}.0/gems/*/ext/*/Makefile")
+        makefiles.flat_map do |makefile|
+          File.read(makefile).scan(/^LIBS = (.*)/).flatten
+        end.compact.flat_map { |l| l.split(' ') }.uniq
+                 .map { |l| l.start_with?('-l') ? l : "-l#{File.basename(l, '.a').delete_prefix('lib')}" }
+      end
+    end
+
+    # macOS implementation - compiles with clang and Homebrew paths
+    class ForMacOS < Taski::Task
+      include CommonHelpers
+
+      # macOS system libraries
+      SYSTEM_LIBS = %w[pthread m c].freeze
+      # macOS frameworks
+      FRAMEWORKS = %w[Foundation CoreFoundation Security].freeze
+
+      def run
+        work_dir = CollectDependencies.work_dir
+        deps = CollectDependencies.deps
+        ext_paths = CollectDependencies.ext_paths
+        enc_files = CollectDependencies.enc_files
+        @output_path = CollectDependencies.output_path
+
+        command = build_command(work_dir, deps, ext_paths, enc_files)
+
+        group('Compiling binary (macOS)') do
+          system(*command) or raise 'Failed to compile final binary'
+          puts "Binary size: #{File.size(@output_path) / 1024 / 1024} MB"
+        end
+
+        puts "Successfully created: #{@output_path}"
+      end
+
+      private
+
+      def build_command(work_dir, deps, ext_paths, enc_files)
+        ruby_static_lib = "-lruby.#{deps.ruby_major_minor}-static"
+
+        [
+          'clang',
+          '-O3',
+          get_ruby_cflags(deps.ruby_install_dir),
+          # IMPORTANT: kompo_lib must come FIRST to override Homebrew-installed versions
+          "-L#{deps.kompo_lib}",
+          get_ldflags(work_dir, deps.ruby_major_minor),
+          "-L#{deps.ruby_lib}",
+          # Also add build path for static library lookup
+          "-L#{File.join(deps.ruby_build_path, "ruby-#{deps.ruby_version}")}",
+          # Add library paths for dependencies (Homebrew on macOS)
+          Shellwords.split(deps.deps_lib_paths),
+          get_libpath(work_dir, deps.ruby_major_minor),
+          '-fstack-protector-strong',
+          '-Wl,-dead_strip', # Remove unused code/data
+          '-Wl,-no_deduplicate',  # Allow duplicate symbols from Ruby YJIT and kompo-vfs
+          '-Wl,-export_dynamic',  # Export symbols to dynamic symbol table
+          deps.main_c,
+          deps.fs_c,
+          # Link kompo_wrap FIRST (before Ruby) to override libc symbols
+          '-lkompo_wrap',
+          ext_paths,
+          enc_files,
+          ruby_static_lib,
+          get_libs(deps.ruby_install_dir, work_dir, deps.ruby_build_path, deps.ruby_version, deps.ruby_major_minor),
+          '-o', @output_path
+        ].flatten
+      end
+
+      def get_libs(ruby_install_dir, work_dir, ruby_build_path, ruby_version, ruby_major_minor)
+        main_libs = get_ruby_mainlibs(ruby_install_dir)
+        ruby_std_gem_libs = get_extlibs(ruby_build_path, ruby_version)
+        gem_libs = get_gem_libs(work_dir, ruby_major_minor)
+
+        all_libs = [main_libs.split(' '), gem_libs, ruby_std_gem_libs].flatten
+                                                                      .select { |l| l.match?(/-l\w/) }.uniq
+                                                                      .reject { |l| l == '-ldl' } # macOS doesn't have libdl
+
+        # Separate system libs from other libs
+        other_libs = all_libs.reject { |l| SYSTEM_LIBS.any? { |sys| l == "-l#{sys}" } }
+
+        [
+          other_libs,
+          '-lkompo_fs',
+          # System libraries
+          SYSTEM_LIBS.map { |l| "-l#{l}" },
+          # Frameworks
+          FRAMEWORKS.flat_map { |f| ['-framework', f] }
+        ].flatten
+      end
+    end
+
+    # Linux implementation - compiles with gcc and pkg-config paths
+    class ForLinux < Taski::Task
+      include CommonHelpers
+
+      # Libraries that must be dynamically linked
+      DYN_LINK_LIBS = %w[pthread dl m c].freeze
+
+      def run
+        work_dir = CollectDependencies.work_dir
+        deps = CollectDependencies.deps
+        ext_paths = CollectDependencies.ext_paths
+        enc_files = CollectDependencies.enc_files
+        @output_path = CollectDependencies.output_path
+
+        command = build_command(work_dir, deps, ext_paths, enc_files)
+
+        group('Compiling binary (Linux)') do
+          system(*command) or raise 'Failed to compile final binary'
+          puts "Binary size: #{File.size(@output_path) / 1024 / 1024} MB"
+        end
+
+        puts "Successfully created: #{@output_path}"
+      end
+
+      private
+
+      def build_command(work_dir, deps, ext_paths, enc_files)
+        # Linux uses libruby-static.a (not libruby.X.Y-static.a like macOS)
+        ruby_static_lib = '-lruby-static'
+
+        [
+          'gcc',
+          '-O3',
+          '-no-pie', # Required: Rust std lib is not built with PIC
+          get_ruby_cflags(deps.ruby_install_dir),
+          # IMPORTANT: kompo_lib must come FIRST to override system-installed versions
+          "-L#{deps.kompo_lib}",
+          get_ldflags(work_dir, deps.ruby_major_minor),
+          "-L#{deps.ruby_lib}",
+          # Also add build path for static library lookup
+          "-L#{File.join(deps.ruby_build_path, "ruby-#{deps.ruby_version}")}",
+          # Add library paths for dependencies (from pkg-config)
+          Shellwords.split(deps.deps_lib_paths),
+          get_libpath(work_dir, deps.ruby_major_minor),
+          '-fstack-protector-strong',
+          '-rdynamic', '-Wl,-export-dynamic',
+          deps.main_c,
+          deps.fs_c,
+          '-Wl,-Bstatic',
+          '-Wl,--start-group',
+          ext_paths,
+          enc_files,
+          ruby_static_lib,
+          get_libs(deps.ruby_install_dir, work_dir, deps.ruby_build_path, deps.ruby_version, deps.ruby_major_minor),
+          '-o', @output_path
+        ].flatten
+      end
+
+      def get_libs(ruby_install_dir, work_dir, ruby_build_path, ruby_version, ruby_major_minor)
+        main_libs = get_ruby_mainlibs(ruby_install_dir)
+        ruby_std_gem_libs = get_extlibs(ruby_build_path, ruby_version)
+        gem_libs = get_gem_libs(work_dir, ruby_major_minor)
+
+        dyn_link_libs = DYN_LINK_LIBS.map { |l| "-l#{l}" }
+
+        all_libs = [main_libs.split(' '), gem_libs, ruby_std_gem_libs].flatten
+                                                                      .select { |l| l.match?(/-l\w/) }.uniq
+
+        static_libs, dyn_libs = all_libs.partition { |l| !dyn_link_libs.include?(l) }
+
+        dyn_libs << '-lc'
+        dyn_libs.unshift('-Wl,-Bdynamic')
+
+        [static_libs, '-Wl,--end-group', '-lkompo_fs', '-lkompo_wrap', dyn_libs].flatten
+      end
+    end
+
+    private
+
+    def macos?
+      RUBY_PLATFORM.include?('darwin')
+    end
+  end
+end

data/lib/kompo/tasks/ruby_build_path.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Kompo
+  # Section to get the ruby-build path.
+  # Switches implementation based on whether ruby-build is already installed.
+  class RubyBuildPath < Taski::Section
+    interfaces :path
+
+    def impl
+      ruby_build_installed? ? Installed : Install
+    end
+
+    # Use existing ruby-build installation
+    class Installed < Taski::Task
+      def run
+        path_output, = Open3.capture2('which', 'ruby-build', err: File::NULL)
+        @path = path_output.chomp
+        puts "ruby-build path: #{@path}"
+        version_output, = Open3.capture2(@path, '--version', err: File::NULL)
+        puts "ruby-build version: #{version_output.chomp}"
+      end
+    end
+
+    # Install ruby-build via git clone and return the path
+    class Install < Taski::Task
+      def run
+        puts 'ruby-build not found. Installing via git...'
+        install_dir = File.expand_path('~/.ruby-build')
+
+        if Dir.exist?(install_dir)
+          system('git', '-C', install_dir, 'pull', '--quiet')
+        else
+          system('git', 'clone', 'https://github.com/rbenv/ruby-build.git', install_dir)
+        end
+
+        @path = File.join(install_dir, 'bin', 'ruby-build')
+        raise 'Failed to install ruby-build' unless File.executable?(@path)
+
+        puts "ruby-build installed at: #{@path}"
+        version_output, = Open3.capture2(@path, '--version', err: File::NULL)
+        puts "ruby-build version: #{version_output.chomp}"
+      end
+    end
+
+    private
+
+    def ruby_build_installed?
+      _, status = Open3.capture2('which', 'ruby-build', err: File::NULL)
+      status.success?
+    end
+  end
+end

data/lib/kompo/tasks/work_dir.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "securerandom"
+require "tmpdir"
+
+module Kompo
+  # Create a temporary working directory and change into it
+  class WorkDir < Taski::Task
+    exports :path, :original_dir
+
+    # Marker file to identify Kompo-created work directories
+    MARKER_FILE = ".kompo_work_dir_marker"
+
+    def run
+      @original_dir = Dir.pwd
+
+      # Check if Ruby cache exists and use its work_dir path for $LOAD_PATH compatibility
+      ruby_version = Taski.args.fetch(:ruby_version, RUBY_VERSION)
+      kompo_cache = Taski.args.fetch(:kompo_cache, File.expand_path("~/.kompo/cache"))
+      cache_metadata_path = File.join(kompo_cache, ruby_version, "metadata.json")
+
+      if File.exist?(cache_metadata_path)
+        begin
+          metadata = JSON.parse(File.read(cache_metadata_path))
+          cached_work_dir = metadata["work_dir"]
+
+          if cached_work_dir
+            # Check if the directory exists and belongs to us (has marker) or doesn't exist at all
+            # In CI environments, the temp directory is cleaned between runs, so we recreate it
+            if Dir.exist?(cached_work_dir)
+              marker_path = File.join(cached_work_dir, MARKER_FILE)
+              if File.exist?(marker_path)
+                # Directory exists and has our marker - reuse it
+                @path = cached_work_dir
+                puts "Using cached work directory: #{@path}"
+                return
+              else
+                # Directory exists but wasn't created by Kompo - don't use it
+                warn "warn: #{cached_work_dir} exists but is not a Kompo work directory, creating new one"
+              end
+            else
+              # Directory doesn't exist - recreate it (common in CI after cache restore)
+              FileUtils.mkdir_p(cached_work_dir)
+              File.write(File.join(cached_work_dir, MARKER_FILE), "kompo-work-dir")
+              @path = cached_work_dir
+              puts "Recreated cached work directory: #{@path}"
+              return
+            end
+          end
+        rescue JSON::ParserError
+          # Fall through to create new work_dir
+        end
+      end
+
+      # No valid cache, create new work_dir
+      tmpdir = Dir.mktmpdir(SecureRandom.uuid)
+      # Resolve symlinks to get the real path
+      # On macOS, /var/folders is a symlink to /private/var/folders
+      # If we don't resolve this, paths won't match at runtime
+      @path = File.realpath(tmpdir)
+
+      # Create marker file to identify this as a Kompo work directory
+      File.write(File.join(@path, MARKER_FILE), "kompo-work-dir")
+
+      puts "Working directory: #{@path}"
+    end
+
+    def clean
+      return unless @path && Dir.exist?(@path)
+
+      # Only remove if marker file exists (confirms this is a Kompo work directory)
+      marker_path = File.join(@path, MARKER_FILE)
+      unless File.exist?(marker_path)
+        puts "Skipping cleanup: #{@path} is not a Kompo work directory"
+        return
+      end
+
+      FileUtils.rm_rf(@path)
+      puts "Cleaned up working directory: #{@path}"
+    end
+  end
+end

data/lib/kompo/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Kompo
-  VERSION = "0.2.0"
+  VERSION = '0.3.0'
+  KOMPO_VFS_MIN_VERSION = '0.5.0'
 end