canopus 0.1.0

data/lib/canopus/project/search.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+require_relative "search_worker"
+
+class Canopus::Project::Search
+  SearchWorker = Canopus::Project.const_get(:SearchWorker, false)
+  private_constant :SearchWorker
+
+  Match = Struct.new(:path, :line, :column, :byte_offset, :text, :match, keyword_init: true)
+  DEFAULT_MAX_SIZE = 10 * 1024 * 1024
+  Child = Struct.new(:pid, :input, :output, :queue, :writer, :reader, :waiter)
+  private_constant :Child
+
+  def initialize(project)
+    @project = project
+  end
+
+  # Results are ordered by relative path and byte offset, even with workers.
+  def call(pattern, workers: 4, max_size: DEFAULT_MAX_SIZE, extensions: nil, limit: nil,
+    case_sensitive: true, cancelled: nil, paths: nil, &block)
+    raise ArgumentError, "workers must be between 1 and 32" unless workers.is_a?(Integer) && workers.between?(1, 32)
+    raise ArgumentError, "limit must be positive" if limit && (!limit.is_a?(Integer) || !limit.positive?)
+    raise ArgumentError, "max_size must be nonnegative" if max_size && (!max_size.is_a?(Integer) || max_size.negative?)
+    max_size = [max_size || SearchWorker::MAX_FILE_BYTES, SearchWorker::MAX_FILE_BYTES].min
+    expression = pattern.is_a?(Regexp) ? pattern : Regexp.new(Regexp.escape(pattern), case_sensitive ? 0 : Regexp::IGNORECASE)
+    expression = Regexp.new(expression.source, expression.options, timeout: expression.timeout || 0.25)
+    files = []
+    (paths || @project.files(extensions: extensions, max_size: max_size)).each do |relative|
+      SearchWorker.check_cancelled(cancelled)
+      raise ArgumentError, "invalid search path" unless relative.is_a?(String) && relative.valid_encoding? && !relative.include?("\0")
+      @project.path(relative)
+      files << relative
+    end
+    files.sort!
+    return [] if files.empty?
+    results = []
+    receive = receiver(results)
+    if workers > 1 && files.length > 1
+      parallel(files, expression, [workers, files.length].min, max_size, limit, cancelled, &receive)
+    else
+      SearchWorker.scan(@project.root, files, expression, max_size, limit, cancelled: cancelled, &receive)
+    end
+    SearchWorker.check_cancelled(cancelled)
+    results.each(&block) if block
+    results
+  rescue SearchWorker::Cancelled
+    []
+  end
+
+  private
+
+  def receiver(results)
+    path = number = offset = raw = text = nil
+    lambda do |message|
+      case message[0]
+      when :line
+        _, path, number, offset, raw = message
+        text = raw.chomp.freeze
+      when :matches
+        message[1].each do |column, first, last|
+          results << Match.new(path: path, line: number, column: column,
+            byte_offset: offset + first, text: text, match: raw.byteslice(first, last - first))
+        end
+      when :error then raise IOError, "search worker: #{message[1]}"
+      when :done then nil
+      else raise IOError, "invalid search worker response"
+      end
+    end
+  end
+
+  def parallel(files, expression, count, max_size, limit, cancelled)
+    children, first = [], 0
+    count.times do |index|
+      SearchWorker.check_cancelled(cancelled)
+      length = files.length / count + (index < files.length % count ? 1 : 0)
+      config = [@project.root, files.slice(first, length), expression, max_size, limit, expression.timeout]
+      children << start_child(config)
+      first += length
+    end
+    remaining = limit
+    children.each do |child|
+      loop do
+        SearchWorker.check_cancelled(cancelled)
+        begin
+          message = child.queue.pop(true)
+        rescue ThreadError
+          sleep 0.005
+          next
+        end
+        raise message if message.is_a?(Exception)
+        break if message[0] == :done
+        if message[0] == :matches && remaining
+          message[1] = message[1].first(remaining)
+          remaining -= message[1].length
+        end
+        yield message
+        return if remaining == 0
+      end
+    end
+  ensure
+    children&.each { |child| stop_child(child) }
+  end
+
+  def start_child(config)
+    child_input, input = IO.pipe
+    output, child_output = IO.pipe
+    child = Child.new(nil, input, output, SizedQueue.new(2))
+        child.pid = Process.spawn({"RUBYOPT" => nil, "RUBYLIB" => nil}, RbConfig.ruby, "--disable-gems",
+          File.expand_path("search_worker/runner.rb", __dir__), in: child_input, out: child_output, err: File::NULL)
+    child.waiter = Process.detach(child.pid)
+    child_input.close
+    child_output.close
+    child.writer = Thread.new do
+      SearchWorker.write_frame(input, config)
+    rescue StandardError => error
+      enqueue(child, error)
+    ensure
+      input.close unless input.closed?
+    end
+    child.reader = Thread.new do
+      loop do
+        message = SearchWorker.read_frame(output)
+        break unless enqueue(child, message)
+        break if message[0] == :done
+      end
+    rescue StandardError => error
+      enqueue(child, error)
+    end
+    child
+  rescue Exception
+    stop_child(child) if child
+    [child_input, child_output, input, output].compact.each { |io| io.close unless io.closed? }
+    raise
+  end
+
+  def stop_child(child)
+    child.queue.close
+    if child.waiter && !child.waiter.join(0)
+      signal(child, "TERM")
+      unless child.waiter.join(0.2)
+        signal(child, "KILL")
+        child.waiter.join(0.5)
+      end
+    end
+    [child.input, child.output].compact.each { |io| io.close unless io.closed? }
+    [child.writer, child.reader].compact.each do |thread|
+      thread.kill unless thread.join(0.2)
+      thread.join(0.1)
+    end
+  end
+
+  def enqueue(child, message)
+    child.queue.push(message)
+  rescue ClosedQueueError
+    false
+  end
+
+  def signal(child, name)
+    Process.kill(name, child.pid)
+  rescue Errno::ESRCH, Errno::EINVAL
+    nil
+  end
+end

data/lib/canopus/project/search_worker/cancelled.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+class Canopus::Project::SearchWorker::Cancelled < StandardError; end

data/lib/canopus/project/search_worker/runner.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Canopus
+  Project = Class.new
+end
+
+require_relative "../search_worker"
+
+Canopus::Project.const_get(:SearchWorker, false).run

data/lib/canopus/project/search_worker.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "../regexp_compat"
+require_relative "../match_data_compat"
+
+    # This protocol only connects this file to its own Ruby parent. Marshal is
+    # never accepted from a file, network peer, plugin, or external executable.
+module Canopus::Project::SearchWorker
+  MAX_FILE_BYTES = 64 << 20
+  MAX_FRAME_BYTES = MAX_FILE_BYTES + (64 << 10)
+  READ_BYTES = 64 << 10
+  MATCH_BATCH = 128
+
+  def self.check_cancelled(callback)
+    raise Cancelled if callback&.call
+  end
+
+  def self.write_frame(io, value)
+    bytes = Marshal.dump(value)
+    raise IOError, "search frame exceeds safety limit" if bytes.bytesize > MAX_FRAME_BYTES
+    io.write([bytes.bytesize].pack("N"))
+    io.write(bytes)
+    io.flush
+  end
+
+  def self.read_frame(io)
+    header = io.read(4)
+    raise EOFError, "search worker ended before completion" unless header && header.bytesize == 4
+    size = header.unpack1("N")
+    raise IOError, "invalid search frame size" unless size.between?(1, MAX_FRAME_BYTES)
+    bytes = io.read(size)
+    raise EOFError, "truncated search frame" unless bytes && bytes.bytesize == size
+    Marshal.load(bytes)
+  rescue TypeError, ArgumentError => error
+    raise IOError, "invalid search frame: #{error.message}"
+  end
+
+  def self.scan(root, paths, expression, max_size, limit, cancelled: nil)
+    total = 0
+    paths.each do |relative|
+      check_cancelled(cancelled)
+      absolute = File.expand_path(relative, root)
+      raise ArgumentError, "path outside project" unless absolute.start_with?(root + File::SEPARATOR)
+      source = read_source(absolute, max_size, cancelled)
+      next unless source
+      offset = 0
+      source.each_line.with_index(1) do |line, number|
+        check_cancelled(cancelled)
+        matches, sent_line = [], false
+        Canopus.with_regexp_timeout(expression) do
+          line.to_enum(:scan, expression).each do
+            match = Regexp.last_match
+            check_cancelled(cancelled)
+            unless sent_line
+              yield [:line, relative, number, offset, line]
+              sent_line = true
+            end
+            first = match.respond_to?(:bytebegin) ? match.bytebegin(0) : match.pre_match.bytesize
+            matches << [match.begin(0) + 1, first, first + match[0].bytesize]
+            total += 1
+            if matches.length == MATCH_BATCH || (limit && total >= limit)
+              yield [:matches, matches]
+              matches = []
+            end
+            return if limit && total >= limit
+          end
+        end
+        yield [:matches, matches] unless matches.empty?
+        offset += line.bytesize
+      end
+    end
+  end
+
+  def self.read_source(path, max_size, cancelled)
+    flags = File::RDONLY
+    # Windows defines NONBLOCK as 1, which aliases WRONLY in File.open.
+    flags |= File::NONBLOCK unless RUBY_PLATFORM.match?(/mswin|mingw/)
+    File.open(path, flags) do |file|
+      file.binmode
+      return unless file.stat.file? && file.size <= max_size
+      source = +"".b
+      while (chunk = file.read(READ_BYTES))
+        check_cancelled(cancelled)
+        source << chunk
+        return if source.bytesize > max_size || chunk.include?("\0")
+      end
+      source.force_encoding(Encoding::UTF_8)
+      source if source.valid_encoding?
+    end
+  rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR, Errno::ELOOP
+    nil
+  end
+
+  def self.run
+    STDIN.binmode
+    STDOUT.binmode
+    root, paths, expression, max_size, limit, timeout = read_frame(STDIN)
+    expression = Regexp.new(expression.source, expression.options, timeout: timeout)
+    scan(root, paths, expression, max_size, limit) { |message| write_frame(STDOUT, message) }
+    write_frame(STDOUT, [:done])
+  rescue StandardError => error
+    write_frame(STDOUT, [:error, "#{error.class}: #{error.message}".byteslice(0, 4096)])
+    write_frame(STDOUT, [:done])
+  end
+end
+
+require_relative "search_worker/cancelled"
+Canopus::Project.send(:private_constant, :SearchWorker)

data/lib/canopus/project/tree.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Canopus
+  class Project::Tree
+    Entry = Data.define(:path, :name, :depth, :directory, :expanded)
+    attr_reader :expanded
+    def initialize(files, expanded: Set.new)
+      @expanded, @children = expanded, Hash.new { |hash, key| hash[key] = {} }
+      files.each do |path|
+        parent = ""
+        pieces = path.split("/")
+        pieces.each_with_index do |piece, index|
+          current = parent.empty? ? piece : "#{parent}/#{piece}"
+          @children[parent][piece] = [current, index < pieces.length - 1 || path.end_with?("/")]
+          parent = current
+        end
+      end
+      @children.transform_values! { |children| children.sort_by { |name, (_, directory)| [directory ? 0 : 1, name.downcase, name] } }
+    end
+    def toggle(path)
+      @expanded.include?(path) ? @expanded.delete(path) : @expanded.add(path)
+      @visible = nil
+    end
+    def reveal(path)
+      pieces = path.split("/")[0...-1]
+      pieces.length.times { |index| @expanded.add(pieces.first(index + 1).join("/")) }
+      @visible = nil
+    end
+    def visible
+      @visible ||= begin
+        output, stack = [], [["", 0]]
+        until stack.empty?
+          path, depth = stack.pop
+          @children.fetch(path, []).reverse_each do |name, (child, directory)|
+            stack << [Entry.new(child, name, depth, directory, @expanded.include?(child)), nil]
+          end if path.is_a?(String)
+          if path.is_a?(Entry)
+            output << path
+            stack << [path.path, path.depth + 1] if path.directory && path.expanded
+          end
+        end
+        output.freeze
+      end
+    end
+  end
+end

data/lib/canopus/project/watcher.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class Canopus::Project::Watcher
+  Event = Struct.new(:type, :path, keyword_init: true)
+  attr_reader :backend, :snapshot
+
+  # Native adapters implement #poll(timeout:) and #close. Polling is always
+  # available and also reconciles coalesced native directory notifications.
+  def initialize(project, interval: 0.5, native: nil)
+    @project = project
+    @interval = interval
+    @native = native
+    @backend = native ? :native : :polling
+    @snapshot = capture
+    @running = false
+  end
+
+  def poll(timeout: 0)
+    if @native
+      return [] if @native.poll(timeout: timeout).empty?
+    elsif timeout.positive?
+      sleep(timeout)
+    end
+    current = capture
+    events = (@snapshot.keys - current.keys).map { |path| Event.new(type: :deleted, path: path) }
+    current.each do |path, stamp|
+      type = !@snapshot.key?(path) ? :created : @snapshot[path] != stamp ? :modified : nil
+      events << Event.new(type: type, path: path) if type
+    end
+    @snapshot = current
+    events.sort_by(&:path)
+  end
+
+  def start(&block)
+    raise ArgumentError, "watch callback required" unless block
+    return self if @running
+    @running = true
+    @thread = Thread.new { poll(timeout: @interval).each(&block) while @running }
+    self
+  end
+
+  def close
+    @running = false
+    @thread&.join unless @thread == Thread.current
+    @native&.close
+    self
+  end
+
+  private
+
+  def capture
+    @project.files.to_h do |relative|
+      stat = File.stat(@project.path(relative))
+      [relative, [stat.ino, stat.size, stat.mtime, stat.ctime]]
+    rescue Errno::ENOENT
+      [relative, nil]
+    end.reject { |_, stamp| stamp.nil? }
+  end
+end

data/lib/canopus/project.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "set"
+require "fileutils"
+require "tempfile"
+
+module Canopus
+  class Project
+    attr_reader :root
+
+    def initialize(root)
+      @root = File.realpath(root)
+      raise ArgumentError, "project root must be a directory" unless File.directory?(@root)
+    end
+
+    def path(relative)
+      absolute = File.expand_path(relative, root)
+      raise ArgumentError, "path outside project" unless absolute == root || absolute.start_with?(root + File::SEPARATOR)
+      absolute
+    end
+
+    # Paths are relative to root, sorted, and never include .git directories.
+    # Directory symlinks are opt-in and visited at most once per inode.
+    def files(include_hidden: true, follow_symlinks: false, include_symlinks: false, include_directories: false, extensions: nil, max_size: nil)
+      return enum_for(__method__, include_hidden: include_hidden, follow_symlinks: follow_symlinks,
+        include_symlinks: include_symlinks, include_directories: include_directories, extensions: extensions, max_size: max_size) unless block_given?
+      extensions = extensions&.map { |extension| extension.start_with?(".") ? extension : ".#{extension}" }&.to_set
+      visited = Set.new
+      base_rules = IgnoreMatcher.new
+      exclude = File.join(root, ".git", "info", "exclude")
+      base_rules = base_rules.add(File.read(exclude), base: "") if File.file?(exclude)
+      walk = lambda do |directory, rules|
+        stat = File.stat(path(directory))
+        return unless visited.add?([stat.dev, stat.ino])
+        %w[.gitignore .ignore].each do |name|
+          source = path(directory.empty? ? name : File.join(directory, name))
+          rules = rules.add(File.read(source, encoding: "UTF-8"), base: directory) if File.file?(source)
+        end
+        Dir.children(path(directory)).sort.each do |name|
+          next if name == ".git" || (!include_hidden && name.start_with?("."))
+          relative = directory.empty? ? name : File.join(directory, name)
+          next if relative == ".canopus/trash"
+          absolute = path(relative)
+          begin
+            entry = File.lstat(absolute)
+            symbolic = entry.symlink?
+            if symbolic && !follow_symlinks
+              next unless include_symlinks
+              next if rules.ignored?(relative)
+              next if extensions && !extensions.include?(File.extname(name))
+              yield relative
+              next
+            end
+            entry = File.stat(absolute) if symbolic
+            next if rules.ignored?(relative, directory: entry.directory?)
+            if entry.directory?
+              yield relative + "/" if include_directories
+              walk.call(relative, rules)
+            elsif entry.file?
+              next if max_size && entry.size > max_size
+              next if extensions && !extensions.include?(File.extname(name))
+              yield relative
+            end
+          rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+            next # An editor walk tolerates concurrent deletion and inaccessible files.
+          end
+        end
+      end
+      walk.call("", base_rules)
+      self
+    end
+
+    def ignored?(relative, directory: File.directory?(path(relative)))
+      pieces = relative.sub(%r{\A\./}, "").split("/")
+      rules = IgnoreMatcher.new
+      base = ""
+      exclude = File.join(root, ".git", "info", "exclude")
+      rules = rules.add(File.read(exclude), base: "") if File.file?(exclude)
+      pieces.each_with_index do |piece, index|
+        return true if piece == ".git"
+        %w[.gitignore .ignore].each do |name|
+          source = path(base.empty? ? name : File.join(base, name))
+          rules = rules.add(File.read(source), base: base) if File.file?(source)
+        end
+        base = base.empty? ? piece : "#{base}/#{piece}"
+        return true if rules.ignored?(base, directory: index < pieces.length - 1 || directory)
+      end
+      false
+    end
+
+    def search(pattern, **options, &block) = Search.new(self).call(pattern, **options, &block)
+    def watcher(**options) = Watcher.new(self, **options)
+
+    # Each file is replaced atomically; a concurrent write aborts that file.
+    def replace(pattern, replacement, **options)
+      expression = pattern.is_a?(Regexp) ? pattern : Regexp.new(Regexp.escape(pattern))
+      searcher = Search.new(self)
+      paths = searcher.call(expression, **options).map(&:path).uniq
+      paths.to_h do |relative|
+        absolute = path(relative)
+        before = File.stat(absolute)
+        text = File.read(absolute, encoding: "UTF-8")
+        count, updated = Canopus.with_regexp_timeout(expression) do
+          [text.scan(expression).length, text.gsub(expression, replacement)]
+        end
+        Tempfile.create([".canopus-", ".tmp"], File.dirname(absolute)) do |temp|
+          temp.binmode
+          temp.chmod(before.mode & 0o777)
+          temp.write(updated)
+          temp.flush
+          temp.fsync
+          temp.close
+          current = File.stat(absolute)
+          unless [before.ino, before.size, before.mtime, before.ctime] == [current.ino, current.size, current.mtime, current.ctime]
+            raise IOError, "file changed during replacement: #{relative}"
+          end
+          File.rename(temp.path, absolute)
+        end
+        [relative, count]
+      end
+    end
+  end
+end
+
+require_relative "project/ignore_matcher"
+require_relative "project/search"
+require_relative "project/watcher"

data/lib/canopus/regexp_compat.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+unless Regexp.method_defined?(:timeout)
+  require "timeout"
+
+  class Regexp
+    TimeoutError = Class.new(StandardError)
+    TIMEOUTS = ObjectSpace::WeakMap.new
+
+    def timeout = TIMEOUTS[self]
+  end
+
+  Regexp.singleton_class.prepend(Module.new do
+    def new(*arguments, timeout: nil)
+      expression = super(*arguments)
+      Regexp::TIMEOUTS[expression] = timeout
+      expression
+    end
+  end)
+
+  module Canopus
+    def self.with_regexp_timeout(expression)
+      expression.timeout ? Timeout.timeout(expression.timeout, Regexp::TimeoutError) { yield } : yield
+    end
+  end
+else
+  module Canopus
+    def self.with_regexp_timeout(_expression) = yield
+  end
+end

data/lib/canopus/save_conflict.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Canopus
+  class SaveConflict < Error; end
+end

data/lib/canopus/selection.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Canopus
+  Selection = Data.define(:id, :anchor, :head, :goal) do
+    def start = [anchor, head].min
+    def end = [anchor, head].max
+    def range = self.start...self.end
+    def empty? = anchor == head
+    def reversed? = head < anchor
+  end
+end