mbeditor 0.13.0 → 0.14.0

data/app/services/mbeditor/archive_service.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "rubygems/package"
+require "stringio"
+require "zlib"
+
+module Mbeditor
+  # Bundles a selection of workspace files and directories into one .tar.gz,
+  # for the explorer's multi-select download.
+  #
+  # tar.gz rather than zip because Gem::Package::TarWriter and Zlib ship with
+  # Ruby; a zip writer would mean adding rubyzip to a gemspec that declares
+  # only rails and sprockets-rails.
+  #
+  # Callers pass paths that have already been through resolve_path. The walk
+  # re-checks every entry it discovers anyway — see #collect.
+  class ArchiveService
+    # Same caps as #import (EditorsController::IMPORT_MAX_FILES /
+    # IMPORT_MAX_TOTAL_BYTES). Deliberately a refusal, not a truncation: an
+    # archive that is silently short looks complete when it is opened.
+    MAX_FILES = 100
+    MAX_TOTAL_BYTES = 50 * 1024 * 1024
+
+    class LimitExceededError < StandardError; end
+
+    def initialize(workspace_root)
+      @root = workspace_root.to_s.chomp("/")
+      @matcher = ExclusionMatcher.new(Mbeditor.configuration.excluded_paths, root: @root)
+    end
+
+    # +full_paths+ are absolute paths inside the workspace. Returns the
+    # .tar.gz bytes; 50 MB compresses fine in memory, so there is no streaming
+    # response to keep alive.
+    def build(full_paths)
+      entries = collect(full_paths)
+      raise "Nothing to archive" if entries.empty?
+
+      if entries.length > MAX_FILES
+        raise LimitExceededError, "Too many files (#{entries.length}). Limit is #{MAX_FILES}."
+      end
+
+      total = entries.sum { |e| e[:size] }
+      if total > MAX_TOTAL_BYTES
+        raise LimitExceededError,
+              "Selection is too large (#{human(total)}). Limit is #{human(MAX_TOTAL_BYTES)}."
+      end
+
+      write(entries)
+    end
+
+    private
+
+    # Breadth-first walk of the selection, yielding regular files only.
+    def collect(full_paths)
+      entries = []
+      seen_dirs = {}
+      queue = full_paths.dup
+
+      until queue.empty?
+        path = queue.shift
+        # Containment is re-checked on every entry the walk discovers, not just
+        # on the paths the request named. A directory being inside the
+        # workspace says nothing about what turns up underneath it, and this is
+        # the only check standing between a name on disk and bytes in a file
+        # the user downloads.
+        next unless contained?(path)
+
+        rel = relative(path)
+        next if rel.start_with?("/") || rel.split("/").include?("..")
+        next if !rel.empty? && @matcher.excluded?(rel)
+
+        st = safe_lstat(path)
+        next if st.nil?
+
+        # Symlinks are skipped, which is deliberately stricter than
+        # ApplicationController#resolve_path, which follows them. That policy
+        # is about opening the one file a developer named and put in the tree.
+        # A recursive walk is a different question: following a link here would
+        # copy bytes from outside the workspace into a file the user
+        # downloads, and a link back up the tree loops forever. Neither is
+        # something the developer asked for by selecting a folder.
+        next if st.symlink?
+
+        if st.directory?
+          # Cycle guard even so — the symlink skip already rules out the usual
+          # loop, but a bind mount or a hard-linked directory does not need one.
+          key = [st.dev, st.ino]
+          next if seen_dirs.key?(key)
+
+          seen_dirs[key] = true
+          queue.concat(Dir.children(path).sort.map { |name| File.join(path, name) })
+        elsif st.file?
+          entries << { path: path, name: rel, mode: st.mode & 0o777, size: st.size }
+        end
+      end
+
+      entries
+    end
+
+    def write(entries)
+      buffer = StringIO.new(+"".b)
+      Zlib::GzipWriter.wrap(buffer) do |gz|
+        Gem::Package::TarWriter.new(gz) do |tar|
+          entries.each do |entry|
+            # Read first, then declare the size from the bytes in hand:
+            # TarWriter bounds the entry to the size it was given, so a file
+            # that changed since it was stat-ed would otherwise write a short
+            # or over-long member.
+            data = File.binread(entry[:path])
+            tar.add_file_simple(entry[:name], entry[:mode], data.bytesize) { |io| io.write(data) }
+          end
+        end
+      end
+      buffer.string
+    end
+
+    def contained?(path)
+      full = File.expand_path(path)
+      full == @root || full.start_with?("#{@root}/")
+    end
+
+    def relative(path)
+      path == @root ? "" : path.delete_prefix("#{@root}/")
+    end
+
+    # A file can vanish between listing its directory and stat-ing it.
+    def safe_lstat(path)
+      File.lstat(path)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def human(bytes)
+      ActiveSupport::NumberHelper.number_to_human_size(bytes)
+    end
+  end
+end

data/app/services/mbeditor/collaboration_doc_store.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require "digest"
+require "fileutils"
+require "tmpdir"
+
 module Mbeditor
   # Thread-safe, in-memory cache of opaque Yjs bytes per active file. It never
   # interprets the bytes (no Ruby CRDT): it only buffers the latest snapshot plus
@@ -29,24 +33,97 @@ module Mbeditor
 
     # Hard cap on buffered deltas per room; a long editing session on one file
     # would otherwise grow it without limit. Over the cap the oldest go, which
-    # costs a late joiner an incomplete replay — it then waits for the next
-    # snapshot instead of syncing from the buffer.
+    # costs a late joiner an incomplete replay — the client detects the missing
+    # dependencies and asks a peer for a full snapshot instead of attaching to
+    # the partial replay (see CollaborationService#_maybeAttach).
     MAX_DELTAS = 500
 
+    # A seed claim left behind by a crashed process would wedge the room forever
+    # (every later opener defers to content that never comes). Claims older than
+    # this are treated as abandoned and may be taken over. It matches GRACE_TTL
+    # because a room that has been idle that long is itself evictable, so the
+    # claim and the room it guards expire together.
+    CLAIM_TTL = GRACE_TTL
+
+    # Grant the right to seed a room's shared document from disk, to exactly one
+    # client. Without this every client that finds the room empty inserts the whole
+    # file at offset 0, and Yjs merges those inserts into two concatenated copies —
+    # the file appended to itself. That happens whenever two editor tabs attach to
+    # a room the server has no state for: a fresh room, or one that was evicted
+    # while both were idle. The claim is per room, so the loser waits for the
+    # winner's content instead of inventing its own.
+    #
+    # The in-memory flag only covers one process. With a cross-process cable
+    # adapter (redis/postgres/solid_cable) and more than one Puma worker, two
+    # clients can reach two different processes whose stores are both empty, so
+    # the claim is also serialized through a lock file under tmp/ (shared by
+    # every process of the app). `File::CREAT | File::EXCL` is the atomic test,
+    # so exactly one process wins even in the race.
+    def claim_seed(path, now: monotonic)
+      MUTEX.synchronize do
+        room = touch(path, now)
+        next false if room[:seed_claimed] || room[:snapshot] || !room[:deltas].empty?
+
+        next false unless acquire_claim(path, now)
+
+        room[:seed_claimed] = true
+      end
+    end
+
+    # Subscriber accounting. A room with a live subscriber must never be evicted:
+    # eviction empties the server's copy while clients still hold content, and the
+    # next client to attach would be granted a seed it must not perform.
+    def join(path, now: monotonic)
+      MUTEX.synchronize { touch(path, now)[:subscribers] += 1 }
+      nil
+    end
+
+    def leave(path, now: monotonic)
+      MUTEX.synchronize do
+        room = rooms[path]
+        next unless room
+
+        room[:subscribers] -= 1 if room[:subscribers] > 0
+        # The claimer can leave before it ever seeds (a tab closed during the
+        # handshake). Releasing the claim on an empty, empty-handed room keeps the
+        # next opener from deferring to content that will never arrive.
+        if room[:subscribers].zero? && room[:snapshot].nil? && room[:deltas].empty?
+          room[:seed_claimed] = false
+          release_claim(path)
+        end
+      end
+      nil
+    end
+
+    # Record a delta and return the room-local sequence number assigned to it.
+    # The sequence lets a client tell the server how far its full-state snapshot
+    # reaches, so replace_snapshot can keep exactly the deltas the snapshot does
+    # not already contain (see #97).
     def record_update(path, bytes, now: monotonic)
       MUTEX.synchronize do
         room = touch(path, now)
-        room[:deltas] << bytes
+        room[:seq] += 1
+        room[:deltas] << { seq: room[:seq], bytes: bytes }
         room[:deltas].shift while room[:deltas].size > MAX_DELTAS
+        room[:seq]
       end
-      nil
     end
 
-    def replace_snapshot(path, bytes, now: monotonic)
+    # Swap the cached snapshot. Deltas up to +applied_seq+ are already folded into
+    # the snapshot and are dropped; anything recorded after that point stays, so a
+    # delta that crossed the snapshot in flight is not lost from the buffer.
+    # +applied_seq+ nil (legacy/unknown) clears the buffer as before.
+    def replace_snapshot(path, bytes, applied_seq: nil, now: monotonic)
       MUTEX.synchronize do
         room = touch(path, now)
         room[:snapshot] = bytes
-        room[:deltas] = []
+        if applied_seq
+          room[:deltas] = room[:deltas].select { |d| d[:seq] > applied_seq }
+          room[:snapshot_seq] = applied_seq
+        else
+          room[:deltas] = []
+          room[:snapshot_seq] = room[:seq]
+        end
       end
       nil
     end
@@ -56,9 +133,15 @@ module Mbeditor
         room = rooms[path]
         if room
           room[:last_activity] = now
-          result = { snapshot: room[:snapshot], deltas: room[:deltas].dup }
+          result = {
+            snapshot: room[:snapshot],
+            deltas: room[:deltas].map { |d| d[:bytes] },
+            delta_seqs: room[:deltas].map { |d| d[:seq] },
+            snapshot_seq: room[:snapshot_seq],
+            seq: room[:seq]
+          }
         else
-          result = { snapshot: nil, deltas: [] }
+          result = { snapshot: nil, deltas: [], delta_seqs: [], snapshot_seq: 0, seq: 0 }
         end
         maybe_sweep(now)
         result
@@ -74,6 +157,7 @@ module Mbeditor
       MUTEX.synchronize do
         @rooms = {}
         @last_sweep = nil
+        clear_claims
       end
       nil
     end
@@ -85,7 +169,8 @@ module Mbeditor
 
     def touch(path, now)
       new_room = !rooms.key?(path)
-      room = (rooms[path] ||= { snapshot: nil, deltas: [] })
+      room = (rooms[path] ||= { snapshot: nil, deltas: [], subscribers: 0, seed_claimed: false,
+                                seq: 0, snapshot_seq: 0 })
       room[:last_activity] = now
       maybe_sweep(now)
       evict_lru if new_room && rooms.size > ROOM_CAP
@@ -105,19 +190,102 @@ module Mbeditor
     private_class_method :maybe_sweep
 
     def evict_idle(now, grace)
-      rooms.delete_if { |_path, room| (now - room[:last_activity]) > grace }
+      rooms.delete_if do |path, room|
+        evict = room[:subscribers].zero? && (now - room[:last_activity]) > grace
+        release_claim(path) if evict
+        evict
+      end
     end
     private_class_method :evict_idle
 
+    # Only idle rooms are evictable, per the invariant on join: emptying a room
+    # clients still hold content for gets the next opener granted a seed it must
+    # not perform. With every room subscribed the cap is simply exceeded.
     def evict_lru
-      oldest = rooms.min_by { |_path, room| room[:last_activity] }
-      rooms.delete(oldest.first) if oldest
+      oldest = rooms.reject { |_path, room| room[:subscribers].positive? }
+                    .min_by { |_path, room| room[:last_activity] }
+      return unless oldest
+
+      release_claim(oldest.first)
+      rooms.delete(oldest.first)
     end
     private_class_method :evict_lru
 
+    # ── cross-process seed claim ────────────────────────────────────────────
+    #
+    # The workspace tmp/ directory is shared by every process of the app (the
+    # same tree WorkspaceRootResolver points at), so a claim file there is
+    # visible to all of them without new infrastructure.
+
+    def claim_dir
+      root = begin
+        WorkspaceRootResolver.call.to_s
+      rescue StandardError
+        nil
+      end
+      root = Rails.root.to_s if (root.nil? || root.empty?) && defined?(Rails) && Rails.respond_to?(:root)
+      root = Dir.tmpdir if root.nil? || root.empty?
+      File.join(root, "tmp", "mbeditor_collab_claims")
+    end
+    private_class_method :claim_dir
+
+    def claim_path(path)
+      File.join(claim_dir, Digest::SHA256.hexdigest(path.to_s))
+    end
+    private_class_method :claim_path
+
+    # Atomically take the claim, or refuse when another process holds a live one.
+    # Returns true only when this call created the file.
+    def acquire_claim(path, now)
+      file = claim_path(path)
+      if File.exist?(file)
+        return false if (Time.now - File.mtime(file)) <= CLAIM_TTL
+
+        # Abandoned by a crashed/evicted process; reclaim it.
+        begin
+          File.delete(file)
+        rescue Errno::ENOENT
+          nil
+        end
+      end
+
+      FileUtils.mkdir_p(claim_dir)
+      begin
+        File.open(file, File::WRONLY | File::CREAT | File::EXCL) { |f| f.write(now.to_s) }
+        true
+      rescue Errno::EEXIST
+        false
+      end
+    rescue SystemCallError, IOError
+      # No writable tmp/ (e.g. a read-only checkout). Fall back to the in-memory
+      # claim rather than crashing the channel; cross-process serialization is
+      # lost, but the alternative is no collaboration at all.
+      true
+    end
+    private_class_method :acquire_claim
+
+    def release_claim(path)
+      File.delete(claim_path(path))
+    rescue SystemCallError
+      nil
+    end
+    private_class_method :release_claim
+
+    def clear_claims
+      Dir.glob(File.join(claim_dir, "*")).each do |file|
+        File.delete(file)
+      rescue SystemCallError
+        nil
+      end
+    rescue StandardError
+      nil
+    end
+    private_class_method :clear_claims
+
     def monotonic
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
     private_class_method :monotonic
   end
 end
+

data/app/services/mbeditor/duplicate_content_scanner.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "find"
+
+module Mbeditor
+  # Finds files that look like they were written back to disk with their own
+  # content appended to themselves.
+  #
+  # The corruption this detects came from the collaborative buffer: two clients
+  # that each found an empty shared document seeded it from disk, and Yjs merged
+  # both inserts at offset 0 into two concatenated copies. That bug is fixed
+  # (CollaborationDocStore.claim_seed), but files saved while it was live are
+  # still on disk, so there has to be a way to find them.
+  #
+  # Two signals, cheapest first:
+  #
+  #   * :exact          — the file is byte-for-byte X + X. What a duplicated save
+  #                       produces before anyone edits it again.
+  #   * :repeated_block — the first ANCHOR_LINES lines of the file occur again
+  #                       later, verbatim. Survives edits made after the
+  #                       corruption, which is the common case for a file noticed
+  #                       days later.
+  #
+  # Report-only. Nothing here writes.
+  class DuplicateContentScanner
+    ANCHOR_LINES  = 10
+    MIN_LINES     = 20 # below this, a repeat is unremarkable
+    MAX_FILE_SIZE = 5 * 1024 * 1024
+
+    Finding = Struct.new(:path, :reason, :line, :lines, keyword_init: true)
+
+    def self.check(content)
+      return nil if content.nil? || content.empty?
+
+      lines = content.lines
+      return nil if lines.size < MIN_LINES
+
+      half = content.bytesize / 2
+      if content.bytesize.even? && content.byteslice(0, half) == content.byteslice(half, half)
+        return { reason: :exact, line: (lines.size / 2) + 1 }
+      end
+
+      at = repeated_anchor_line(lines)
+      at ? { reason: :repeated_block, line: at } : nil
+    end
+
+    # Line number (1-based) of a later verbatim recurrence of the file's opening
+    # block, or nil. The anchor starts at the first non-blank line so a leading
+    # blank does not shift it out of alignment with its copy.
+    def self.repeated_anchor_line(lines)
+      start = lines.index { |l| !l.strip.empty? }
+      return nil if start.nil? || (start + ANCHOR_LINES) > lines.size
+
+      anchor = lines[start, ANCHOR_LINES]
+      # A block of identical lines (padding, a long table) repeats for boring
+      # reasons; require the anchor itself to carry some variety.
+      return nil if anchor.uniq.size < 3
+
+      idx = ((start + 1)..(lines.size - ANCHOR_LINES)).find { |i| lines[i, ANCHOR_LINES] == anchor }
+      idx && (idx + 1)
+    end
+    private_class_method :repeated_anchor_line
+
+    def initialize(root, excluded: Mbeditor.configuration.excluded_paths)
+      @root = File.expand_path(root.to_s)
+      @matcher = ExclusionMatcher.new(excluded, root: @root)
+    end
+
+    def call
+      findings = []
+      Find.find(@root) do |path|
+        rel = path == @root ? "" : path.delete_prefix("#{@root}/")
+        if File.directory?(path)
+          Find.prune if !rel.empty? && @matcher.excluded?(rel)
+          next
+        end
+        next if @matcher.excluded?(rel)
+        next unless File.file?(path) && File.size(path).between?(1, MAX_FILE_SIZE)
+
+        content = read_text(path)
+        next unless content
+
+        hit = self.class.check(content)
+        next unless hit
+
+        findings << Finding.new(path: rel, reason: hit[:reason], line: hit[:line], lines: content.lines.size)
+      end
+      findings.sort_by { |f| [f.reason == :exact ? 0 : 1, f.path] }
+    end
+
+    private
+
+    # nil for anything that is not readable UTF-8 text — a NUL byte is the same
+    # binary test the rest of the editor uses.
+    def read_text(path)
+      content = File.read(path, encoding: "UTF-8")
+      return nil unless content.valid_encoding?
+      return nil if content.include?("\0")
+
+      content
+    rescue SystemCallError
+      nil
+    end
+  end
+end

data/app/services/mbeditor/editor_state_service.rb

@@ -14,7 +14,6 @@ module Mbeditor
     # is acquired non-blocking with a bounded retry and gives up with a clear
     # error rather than hanging the worker.
     DEFAULT_LOCK_TIMEOUT = 5.0
-    LOCK_RETRY_INTERVAL  = 0.01
 
     def initialize(workspace_root, lock_timeout: DEFAULT_LOCK_TIMEOUT)
       @root = workspace_root
@@ -22,13 +21,13 @@ module Mbeditor
     end
 
     def read_state
-      read_json(workspace_path)
+      locked_json_file(workspace_path).read
     rescue JSON::ParserError, Errno::ENOENT
       {}
     end
 
     def read_branch_state(branch)
-      read_json(branch_states_path)[branch] || {}
+      locked_json_file(branch_states_path).read[branch] || {}
     rescue JSON::ParserError, Errno::ENOENT
       {}
     end
@@ -37,16 +36,15 @@ module Mbeditor
       raise InvalidBranchError, "Invalid branch name" unless branch.match?(SAFE_BRANCH_NAME)
       payload_json = state.to_json
       raise PayloadTooLargeError, "State payload too large" if payload_json.bytesize > STATE_MAX_BYTES
-      path = branch_states_path
-      FileUtils.mkdir_p(path.dirname)
-      with_lock(path) do
-        existing = read_json(path)
+      file = locked_json_file(branch_states_path)
+      file.with_lock do
+        existing = file.read
         # Auto-save fires on a timer even with no changes; skip the full-file
         # rewrite when this branch's entry is already identical.
         next if existing[branch] == JSON.parse(payload_json)
 
         existing[branch] = state
-        atomic_write(path, existing.to_json)
+        file.write(existing)
       end
       nil
     end
@@ -54,10 +52,11 @@ module Mbeditor
     def prune_branch_states(active_branches:)
       path = branch_states_path
       return [] unless File.exist?(path)
+      file = locked_json_file(path)
       pruned = []
-      with_lock(path) do
+      file.with_lock do
         all = begin
-          read_json(path)
+          file.read
         rescue JSON::ParserError => e
           Rails.logger.error("[mbeditor] EditorStateService#prune_branch_states: discarding corrupt branch_states JSON at #{path}: #{e.message}")
           {}
@@ -65,7 +64,7 @@ module Mbeditor
         pruned = all.keys - active_branches
         if pruned.any?
           pruned.each { |b| all.delete(b) }
-          atomic_write(path, all.to_json)
+          file.write(all)
         end
       end
       pruned
@@ -74,51 +73,15 @@ module Mbeditor
     def write_state(state)
       payload = state.to_json
       raise PayloadTooLargeError, "State payload too large" if payload.bytesize > STATE_MAX_BYTES
-      path = workspace_path
-      FileUtils.mkdir_p(path.dirname)
-      with_lock(path) { atomic_write(path, payload) }
+      file = locked_json_file(workspace_path)
+      file.with_lock { file.write(payload) }
       nil
     end
 
     private
 
-    # Readers take no lock at all: every write lands by rename, so a read sees
-    # either the whole previous file or the whole new one — never the empty
-    # window a truncate-then-write leaves, which readers swallowed as {}.
-    def read_json(path)
-      return {} unless File.exist?(path)
-
-      raw = File.read(path)
-      raw.empty? ? {} : JSON.parse(raw)
-    end
-
-    def atomic_write(path, payload)
-      tmp = "#{path}.tmp"
-      File.write(tmp, payload)
-      File.rename(tmp, path)
-    end
-
-    # The lock sits on a sidecar file, not on the state file: the state file is
-    # replaced by rename, so a lock held on the inode it had before the write
-    # would exclude nobody afterwards.
-    def with_lock(path)
-      File.open("#{path}.lock", File::RDWR | File::CREAT) do |f|
-        lock_exclusive!(f)
-        yield
-      end
-    end
-
-    # Acquire an exclusive lock without blocking forever. Retries the
-    # non-blocking flock until @lock_timeout elapses, then raises so the caller
-    # fails fast instead of pinning a worker on a stuck holder.
-    def lock_exclusive!(file)
-      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @lock_timeout
-      until file.flock(File::LOCK_EX | File::LOCK_NB)
-        if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
-          raise LockTimeoutError, "could not acquire state lock within #{@lock_timeout}s"
-        end
-        sleep LOCK_RETRY_INTERVAL
-      end
+    def locked_json_file(path)
+      LockedJsonFile.new(path, lock_timeout: @lock_timeout, error_class: LockTimeoutError)
     end
 
     def workspace_path