lutaml-store 0.2.4 → 0.3.1

data/lib/lutaml/store/adapter/filesystem.rb

@@ -3,13 +3,46 @@
 require "fileutils"
 require "digest"
 require "json"
+require "monitor"
+require "securerandom"
 
 module Lutaml
   module Store
     module Adapter
+      # One data file per key, with an optional `.meta` file beside it.
+      #
+      # File names percent-encode every byte outside `[A-Za-z0-9._-]` and a
+      # leading ".", so distinct keys never share a file and no name is hidden
+      # or climbs out of the root. The empty key is named "%". A name longer
+      # than MAX_NAME_BYTES becomes "~" + SHA-256 of the key, and the `.meta`
+      # file keeps the key. Files written by 0.3.0 (lossy "_" names) are still
+      # read, and are moved to the new name on the next write.
+      #
+      # Writers hold an exclusive lock on `<root>/.lock` (plus a per-root
+      # Monitor for threads); readers hold a shared lock. Files are written
+      # to a unique temp file and renamed into place.
       class FileSystem < Base
         DEFAULT_EXTENSION = ".data"
         METADATA_EXTENSION = ".meta"
+        LOCK_FILE = ".lock"
+        MAX_NAME_BYTES = 200
+        DIGEST_PREFIX = "~"
+        JSON_FORMAT = "json"
+        UNSAFE_BYTE = /[^A-Za-z0-9._-]/n
+        LEGACY_UNSAFE_CHAR = /[^a-zA-Z0-9._-]/
+        EMPTY_KEY_NAME = "%"
+        HELD_LOCKS = :lutaml_store_filesystem_locks
+
+        @monitors = {}
+        @monitors_guard = Mutex.new
+
+        class << self
+          # One Monitor per lock file, shared by every adapter instance in
+          # the process, so two instances on one root exclude each other.
+          def monitor_for(lock_path)
+            @monitors_guard.synchronize { @monitors[lock_path] ||= ::Monitor.new }
+          end
+        end
 
         def initialize(config = {})
           super
@@ -18,137 +51,134 @@ module Lutaml
           @create_directories = @config.fetch(:create_directories, true)
           @integrity_enabled = @config.fetch(:integrity_checks, true)
           @integrity_algorithm = @config.fetch(:integrity_algorithm, "sha256")
+          @lock_path = File.expand_path(File.join(@root_path, LOCK_FILE))
+          @monitor = self.class.monitor_for(@lock_path)
 
           setup_directory_structure
         end
 
         def get(key)
-          file_path = path_for_key(key)
-          return nil unless File.exist?(file_path)
+          status, result = with_lock(:sh) { read_verified(key) }
+          return result unless status == :corrupt
 
-          data = file_safe_read(file_path)
-          metadata = read_metadata(key)
-
-          begin
-            verify_data_integrity(data, metadata)
-          rescue Integrity::IntegrityError => e
-            repaired_data = repair_corruption(key)
-            return repaired_data if repaired_data
+          # Repair writes, so it runs under the exclusive lock, never the shared one.
+          with_lock(:ex) do
+            status, result = read_verified(key)
+            next result unless status == :corrupt
 
-            raise e
+            repair_corruption(key) || raise(result)
           end
-
-          data
         end
 
         def set(key, value)
-          file_path = path_for_key(key)
-          ensure_directory_exists(File.dirname(file_path))
-
-          full_metadata = wrap_with_integrity(value)
-          file_safe_write(file_path, value)
-          write_metadata(key, full_metadata) if full_metadata.any?
+          content, format = encode_value(value)
+          with_lock(:ex) { write_entry(key, content, format) }
           value
         end
 
-        def delete(key)
-          file_path = path_for_key(key)
-          metadata_path = metadata_path_for_key(key)
+        # Atomic read-modify-write: the exclusive lock is held across the
+        # read, the block and the write, for threads and processes.
+        def update(key)
+          with_lock(:ex) do
+            new_value = yield(get(key))
+            set(key, new_value)
+            new_value
+          end
+        end
 
-          return nil unless File.exist?(file_path)
+        def transaction(&block)
+          with_lock(:ex, &block)
+        end
 
-          value = file_safe_read(file_path)
-          File.delete(file_path)
-          File.delete(metadata_path) if File.exist?(metadata_path)
+        def delete(key)
+          with_lock(:ex) do
+            file_path = existing_path_for_key(key)
+            next nil unless file_path
 
-          cleanup_empty_directories(File.dirname(file_path))
+            value = decode_value(file_safe_read(file_path), read_metadata(metadata_path_for(file_path))[:format])
+            remove_entry(file_path)
+            remove_legacy_entry(key)
 
-          value
+            value
+          end
         end
 
         def exists?(key)
-          File.exist?(path_for_key(key))
+          !existing_path_for_key(key).nil?
         end
 
         def keys
           return [] unless Dir.exist?(@root_path)
 
-          Dir.glob(File.join(@root_path, "**", "*#{@extension}")).filter_map do |file_path|
-            key_from_path(file_path) if File.file?(file_path)
+          with_lock(:sh) do
+            data_files.filter_map { |file_path| key_from_path(file_path) }
           end
         end
 
         def all
-          result = {}
-
-          Dir.glob(File.join(@root_path, "**", "*#{@extension}")).each do |file_path|
-            next unless File.file?(file_path)
-
-            key = key_from_path(file_path)
+          keys.each_with_object({}) do |key, result|
             value = get(key)
-            result[key] = value if value
+            result[key] = value unless value.nil?
           end
-
-          result
         end
 
         def clear
           return 0 unless Dir.exist?(@root_path)
 
-          count = 0
-          Dir.glob(File.join(@root_path, "**", "*#{@extension}")).each do |file_path|
-            if File.file?(file_path)
+          with_lock(:ex) do
+            count = 0
+            data_files.each do |file_path|
               File.delete(file_path)
               count += 1
             end
-          end
 
-          Dir.glob(File.join(@root_path, "**", "*#{METADATA_EXTENSION}")).each do |file_path|
-            File.delete(file_path) if File.file?(file_path)
-          end
+            Dir.glob(File.join(@root_path, "**", "*#{METADATA_EXTENSION}")).each do |file_path|
+              File.delete(file_path) if File.file?(file_path)
+            end
 
-          cleanup_empty_directories(@root_path, preserve_root: true)
+            cleanup_empty_directories(@root_path, preserve_root: true)
 
-          count
+            count
+          end
         end
 
         def size
           return 0 unless Dir.exist?(@root_path)
 
-          Dir.glob(File.join(@root_path, "**", "*#{@extension}")).count { |path| File.file?(path) }
+          data_files.size
         end
 
         def close; end
 
         def verify_integrity
           corrupted_keys = []
+          all_keys = keys
 
-          keys.each do |key|
+          all_keys.each do |key|
             get(key)
           rescue Integrity::IntegrityError
             corrupted_keys << key
           end
 
           {
-            total_keys: keys.size,
+            total_keys: all_keys.size,
             corrupted_keys: corrupted_keys,
             integrity_ok: corrupted_keys.empty?
           }
         end
 
         def repair_corruption(key, backup_data = nil)
-          file_path = path_for_key(key)
-          return nil unless File.exist?(file_path)
+          with_lock(:ex) do
+            file_path = existing_path_for_key(key)
+            next nil unless file_path
 
-          corrupted_data = file_safe_read(file_path)
-          repaired_data = Integrity.repair_data(corrupted_data, backup_data)
+            format = read_metadata(metadata_path_for(file_path))[:format]
+            repaired_data = Integrity.repair_data(file_safe_read(file_path), backup_data)
+            next nil unless Integrity.valid_data?(repaired_data)
 
-          if Integrity.valid_data?(repaired_data)
-            set(key, repaired_data)
-            return repaired_data
+            write_entry(key, repaired_data, format)
+            decode_value(repaired_data, format)
           end
-
-          nil
         end
 
         def stats
@@ -167,31 +197,197 @@ module Lutaml
           FileUtils.mkdir_p(@root_path) unless Dir.exist?(@root_path)
         end
 
+        # ── Locking ──
+
+        # Re-entrant for the thread that already holds a lock on this root.
+        # A shared lock cannot be upgraded: flock would deadlock on its own fd.
+        def with_lock(mode, &block)
+          held = held_lock_mode
+          if held
+            raise BackendError, "Cannot take an exclusive lock inside a shared lock" if mode == :ex && held == :sh
+
+            return yield
+          end
+
+          if mode == :ex
+            @monitor.synchronize { flock_lock_file(File::LOCK_EX, :ex, &block) }
+          else
+            flock_lock_file(File::LOCK_SH, :sh, &block)
+          end
+        end
+
+        def flock_lock_file(operation, mode)
+          # Nothing to read and nothing to protect before the root exists.
+          return yield if mode == :sh && !Dir.exist?(@root_path)
+
+          ensure_directory_exists(@root_path)
+          File.open(@lock_path, File::RDWR | File::CREAT, 0o644) do |lock_file|
+            lock_file.flock(operation)
+            held_locks[@lock_path] = { mode: mode, pid: Process.pid, file: lock_file }
+            yield
+          ensure
+            held_locks.delete(@lock_path)
+          end
+        end
+
+        # A lock entry copied into a forked child is not the child's lock.
+        # The child closes its copy of the lock file, so the parent's flock
+        # is released when the parent closes its own descriptor.
+        def held_lock_mode
+          entry = held_locks[@lock_path]
+          return nil unless entry
+          return entry[:mode] if entry[:pid] == Process.pid
+
+          held_locks.delete(@lock_path)
+          entry[:file].close unless entry[:file].closed?
+          nil
+        end
+
+        # Per thread, not per fiber: an Enumerator or Fiber that runs inside
+        # a locked block re-enters the lock instead of waiting on itself.
+        def held_locks
+          Thread.current.thread_variable_get(HELD_LOCKS) ||
+            Thread.current.thread_variable_set(HELD_LOCKS, {})
+        end
+
+        # ── Entries ──
+
+        # Returns [:ok, value], [:missing, nil] or [:corrupt, error].
+        def read_verified(key)
+          file_path = existing_path_for_key(key)
+          return [:missing, nil] unless file_path
+
+          data = file_safe_read(file_path)
+          metadata = read_metadata(metadata_path_for(file_path))
+
+          begin
+            verify_data_integrity(data, metadata)
+          rescue Integrity::IntegrityError => e
+            return [:corrupt, e]
+          end
+
+          [:ok, decode_value(data, metadata[:format])]
+        end
+
+        def write_entry(key, content, format)
+          file_path = path_for_key(key)
+          metadata_path = metadata_path_for(file_path)
+          ensure_directory_exists(File.dirname(file_path))
+
+          metadata = {}
+          metadata[:integrity] = create_integrity_metadata(content) if @integrity_enabled
+          metadata[:format] = format if format
+          metadata[:key] = key.to_s if File.basename(file_path).start_with?(DIGEST_PREFIX)
+
+          file_safe_write(file_path, content)
+          if metadata.any?
+            file_safe_write(metadata_path, JSON.generate(metadata))
+          elsif File.exist?(metadata_path)
+            File.delete(metadata_path)
+          end
+          remove_legacy_entry(key)
+        end
+
+        def remove_entry(file_path)
+          metadata_path = metadata_path_for(file_path)
+          File.delete(file_path) if File.exist?(file_path)
+          File.delete(metadata_path) if File.exist?(metadata_path)
+          cleanup_empty_directories(File.dirname(file_path))
+        end
+
+        def remove_legacy_entry(key)
+          legacy_path = legacy_path_for_key(key)
+          remove_entry(legacy_path) if legacy_path && File.exist?(legacy_path)
+        end
+
+        def encode_value(value)
+          return [value, nil] if value.is_a?(String)
+
+          [JSON.generate(value), JSON_FORMAT]
+        end
+
+        def decode_value(content, format)
+          return content unless format == JSON_FORMAT
+
+          JSON.parse(content)
+        rescue JSON::ParserError
+          content
+        end
+
+        # ── Paths ──
+
+        def data_files
+          Dir.glob(File.join(@root_path, "**", "*#{@extension}")).select { |path| File.file?(path) }
+        end
+
         def path_for_key(key)
-          safe_key = sanitize_key(key)
+          shard_path(file_name_for_key(key))
+        end
 
-          if safe_key.length >= 2
-            subdir = safe_key[0, 2]
-            File.join(@root_path, subdir, "#{safe_key}#{@extension}")
+        def shard_path(name)
+          if name.length >= 2
+            File.join(@root_path, name[0, 2], "#{name}#{@extension}")
           else
-            File.join(@root_path, "#{safe_key}#{@extension}")
+            File.join(@root_path, "#{name}#{@extension}")
           end
         end
 
-        def metadata_path_for_key(key)
-          data_path = path_for_key(key)
-          data_path.sub(@extension, METADATA_EXTENSION)
+        # The current path, or the 0.3.0 path when only that one exists.
+        def existing_path_for_key(key)
+          file_path = path_for_key(key)
+          return file_path if File.exist?(file_path)
+
+          legacy_path = legacy_path_for_key(key)
+          legacy_path if legacy_path && File.exist?(legacy_path)
+        end
+
+        # 0.3.0 replaced unsafe characters with "_". Nil when that name is
+        # the current one, or when it would leave the root ("..").
+        def legacy_path_for_key(key)
+          name = key.to_s.gsub(LEGACY_UNSAFE_CHAR, "_")
+          return nil if name.empty? || name.start_with?(".")
+
+          legacy_path = shard_path(name)
+          legacy_path unless legacy_path == path_for_key(key)
+        end
+
+        def metadata_path_for(data_path)
+          "#{data_path.delete_suffix(@extension)}#{METADATA_EXTENSION}"
         end
 
         def key_from_path(file_path)
-          relative_path = file_path.sub(@root_path, "").sub(%r{^/}, "")
-          File.basename(relative_path, @extension)
+          name = File.basename(file_path, @extension)
+          return decode_key(name) unless name.start_with?(DIGEST_PREFIX)
+
+          read_metadata(metadata_path_for(file_path))[:key]
         end
 
-        def sanitize_key(key)
-          key.to_s.gsub(/[^a-zA-Z0-9._-]/, "_")
+        def file_name_for_key(key)
+          encoded = encode_key(key)
+          return encoded if encoded.bytesize <= MAX_NAME_BYTES
+
+          "#{DIGEST_PREFIX}#{Digest::SHA256.hexdigest(key.to_s)}"
+        end
+
+        def encode_key(key)
+          raw = key.to_s
+          return EMPTY_KEY_NAME if raw.empty?
+
+          encoded = raw.b.gsub(UNSAFE_BYTE) { |byte| format("%%%02X", byte.ord) }
+          encoded = "%2E#{encoded[1..]}" if encoded.start_with?(".")
+          encoded.force_encoding(Encoding::UTF_8)
+        end
+
+        def decode_key(name)
+          return "" if name == EMPTY_KEY_NAME
+
+          decoded = name.b.gsub(/%([0-9A-F]{2})/n) { Regexp.last_match(1).hex.chr }
+          decoded.force_encoding(Encoding::UTF_8)
+          decoded.valid_encoding? ? decoded : decoded.b
         end
 
+        # ── Integrity and metadata ──
+
         def create_integrity_metadata(data)
           return {} unless @integrity_enabled
 
@@ -205,32 +401,16 @@ module Lutaml
           Integrity.verify_integrity_metadata(data, metadata[:integrity])
         end
 
-        def wrap_with_integrity(data, user_metadata = {})
-          metadata = user_metadata.dup
-          metadata[:integrity] = create_integrity_metadata(data) if @integrity_enabled
-          metadata
-        end
-
-        def write_metadata(key, metadata)
-          return unless metadata.any?
-
-          metadata_path = metadata_path_for_key(key)
-          ensure_directory_exists(File.dirname(metadata_path))
-
-          File.write(metadata_path, JSON.generate(metadata))
-        end
-
-        def read_metadata(key)
-          metadata_path = metadata_path_for_key(key)
+        def read_metadata(metadata_path)
           return {} unless File.exist?(metadata_path)
 
-          begin
-            JSON.parse(File.read(metadata_path), symbolize_names: true)
-          rescue JSON::ParserError
-            {}
-          end
+          JSON.parse(File.read(metadata_path), symbolize_names: true)
+        rescue JSON::ParserError, Errno::ENOENT
+          {}
         end
 
+        # ── Files ──
+
         def ensure_directory_exists(dir_path)
           return if Dir.exist?(dir_path)
 
@@ -249,38 +429,33 @@ module Lutaml
         end
 
         def file_safe_read(file_path)
-          File.open(file_path, "rb") do |file|
-            file.flock(File::LOCK_SH)
-            file.read
-          end
+          content = File.binread(file_path)
+          utf8 = content.dup.force_encoding(Encoding::UTF_8)
+          utf8.valid_encoding? ? utf8 : content
         rescue StandardError => e
           raise BackendError, "Failed to read file #{file_path}: #{e.message}"
         end
 
+        # The temp name is unique per call, so concurrent writers never share
+        # a temp file; rename makes the new content visible atomically.
         def file_safe_write(file_path, content)
-          temp_path = "#{file_path}.tmp.#{Process.pid}"
+          temp_path = "#{file_path}.tmp.#{Process.pid}.#{SecureRandom.hex(6)}"
 
           File.open(temp_path, "wb") do |file|
-            file.flock(File::LOCK_EX)
             file.write(content)
             file.fsync
           end
 
           File.rename(temp_path, file_path)
         rescue StandardError => e
-          File.delete(temp_path) if File.exist?(temp_path)
+          FileUtils.rm_f(temp_path)
           raise BackendError, "Failed to write file #{file_path}: #{e.message}"
         end
 
         def calculate_disk_usage
           return 0 unless Dir.exist?(@root_path)
 
-          total_size = 0
-          Dir.glob(File.join(@root_path, "**", "*#{@extension}")).each do |file_path|
-            total_size += File.size(file_path) if File.file?(file_path)
-          end
-
-          total_size
+          data_files.sum { |file_path| File.size(file_path) }
         end
       end
     end

data/lib/lutaml/store/adapter/memory.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
+require "monitor"
+
 module Lutaml
   module Store
     module Adapter
       # In-memory key-value adapter optimized for read-heavy workloads.
       # Writes are synchronized; reads are lock-free using snapshot copies.
+      # The write lock is a re-entrant Monitor so #update can call #set.
       class Memory < Base
         def initialize(config = {})
           super
           @data = {}
-          @write_mutex = Mutex.new
+          @write_mutex = ::Monitor.new
           @read_snapshot = {}.freeze
           @snapshot_stale = true
           @ttl_enabled = @config.fetch(:ttl_enabled, false)
@@ -72,6 +75,18 @@ module Lutaml
           value
         end
 
+        # Atomic read-modify-write across threads. Keeps the key's expiry.
+        def update(key)
+          @write_mutex.synchronize do
+            old_value = get(key)
+            expires_at = @ttl_data&.[](key)
+            new_value = yield(old_value)
+            set(key, new_value)
+            @ttl_data[key] = expires_at if expires_at
+            new_value
+          end
+        end
+
         def delete(key)
           @write_mutex.synchronize do
             existed = @data.key?(key)

data/lib/lutaml/store/adapter/sqlite.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "monitor"
 
 module Lutaml
   module Store
@@ -19,6 +20,7 @@ module Lutaml
           @db_path = @config[:path] || raise(ConfigurationError, "SQLite adapter requires :path config")
           @table_name = @config[:table_name] || DEFAULT_TABLE_NAME
           @timeout = @config[:timeout] || 30_000
+          @update_lock = ::Monitor.new
 
           setup_database
         end
@@ -178,14 +180,37 @@ module Lutaml
           results
         end
 
+        # Atomic read-modify-write. BEGIN IMMEDIATE takes the database write
+        # lock before the read, so no other process can write in between;
+        # the Monitor keeps threads that share this connection apart.
+        def update(key, &block)
+          @update_lock.synchronize do
+            if @db.transaction_active?
+              update_in_transaction(key, &block)
+            else
+              @db.transaction(:immediate) { update_in_transaction(key, &block) }
+            end
+          end
+        rescue SQLite3::Exception => e
+          raise BackendError, "Update failed: #{e.message}"
+        end
+
+        # Holds the update lock too, so a concurrent #update from another
+        # thread cannot run inside this transaction.
         def transaction(&block)
-          @db.transaction(&block)
+          @update_lock.synchronize { @db.transaction(&block) }
         rescue SQLite3::Exception => e
           raise BackendError, "Transaction failed: #{e.message}"
         end
 
         private
 
+        def update_in_transaction(key)
+          new_value = yield(get(key))
+          set(key, new_value)
+          new_value
+        end
+
         def setup_database
           @db = SQLite3::Database.new(@db_path)
           @db.busy_timeout = @timeout

data/lib/lutaml/store/basic_store.rb

@@ -57,6 +57,17 @@ module Lutaml
         end
       end
 
+      # Atomic read-modify-write through the adapter. The block gets the
+      # adapter's current value, not the cached one.
+      def update(key, &block)
+        with_monitoring(:set) do
+          value = @adapter.update(key, &block)
+          @cache&.set(key, value)
+          emit_event(:set, key: key, value: value)
+          value
+        end
+      end
+
       def delete(key)
         with_monitoring(:delete) do
           result = @adapter.delete(key)