net-connector 0.4.0 → 0.4.2

data/lib/net/connector/netdisco/worker.rb

@@ -21,37 +21,29 @@ module Net
           worker_count = [tasks.size, @concurrency].min
           worker_count.times { queue << nil }
           callback_errors = Queue.new
-          threads = Array.new(worker_count) do
-            Thread.new do
-              Thread.current.report_on_exception = false
-              while (task = queue.pop)
-                index, device = task
-                started_at = Time.now.utc
-                begin
-                  on_start&.call(device)
-                rescue StandardError => error
-                  callback_errors << { host: device.host, error_type: error.class.name }.freeze
-                end
-                result = begin
-                           yield device
-                         rescue StandardError => error
-                           on_error.call(device, error)
-                         end
-                result = result.with(started_at: started_at, finished_at: Time.now.utc)
-                outcomes[index] = result
-                Array(on_result).each do |callback|
-                  begin
-                    callback.call(result)
-                  rescue StandardError => error
-                    callback_errors << { host: device.host, error_type: error.class.name }.freeze
+          finished = Queue.new
+          threads = []
+          completed = false
+          begin
+            # 创建线程也属于批次生命周期；后续创建失败时要关闭已有任务。
+            worker_count.times do
+              threads << Thread.new do
+                Thread.current.report_on_exception = false
+                while (task = queue.pop)
+                  index, device = task
+                  result = run_task(device, on_error, on_start, callback_errors) { yield device }
+                  outcomes[index] = result
+                  Array(on_result).each do |callback|
+                    notify(callback, result, host: device.host, errors: callback_errors)
                   end
                 end
+              ensure
+                finished << Thread.current
               end
             end
-          end
-          completed = false
-          begin
-            threads.each(&:value)
+            # 按完成顺序观察失败，避免后启动线程的中断被前面的慢设备阻塞。
+            # 结果仍写回清单槽位，完成通知不会改变公开结果的顺序。
+            worker_count.times { finished.pop.value }
             completed = true
           ensure
             # 中断或某个 worker 抛出非 StandardError 时，不让其他设备任务
@@ -69,6 +61,30 @@ module Net
           end
           Array.new(callback_errors.size) { callback_errors.pop }.freeze
         end
+
+        private
+
+        # 普通设备故障转换为结果，中断仍交给批次统一停止其余线程。
+        def run_task(device, on_error, on_start, callback_errors)
+          started_at = Time.now.utc
+          started = monotonic
+          notify(on_start, device, host: device.host, errors: callback_errors) unless on_start.nil?
+          result = begin
+                     yield
+                   rescue StandardError => error
+                     on_error.call(device, error)
+                   end
+          result.with(started_at: started_at, finished_at: Time.now.utc, duration_ms: ((monotonic - started) * 1000).round)
+        end
+
+        def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        # 回调故障不覆盖设备结果，也不把异常消息中的凭据写入报告。
+        def notify(callback, value, host:, errors:)
+          callback.call(value)
+        rescue StandardError => error
+          errors << { host: host, error_type: error.class.name }.freeze
+        end
       end
     end
   end

data/lib/net/connector/operations/backup_lock.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "../engine/errors"
+
+module Net
+  module Connector
+    class BackupBusy < Error; end
+
+    module Operations
+      # 路径锁在设备采集之前取得；锁文件长期保留，避免等待者落在已经删除的 inode 上。
+      class BackupLock
+        STORAGE_KEY = :net_connector_backup_delegation
+        private_constant :STORAGE_KEY
+
+        attr_reader :path
+
+        def self.lock_path(path)
+          destination = File.expand_path(path)
+          directory = File.realpath(File.dirname(destination))
+          # 锁名先折叠 Unicode 与大小写，避免大小写不敏感文件系统上的别名绕过互斥。
+          # 在区分大小写的文件系统上可能保守地合并两个锁，但不会改写目标文件名。
+          filename = File.basename(destination).b.force_encoding(Encoding::UTF_8)
+          filename = filename.valid_encoding? ? filename.unicode_normalize(:nfc).downcase(:fold).unicode_normalize(:nfc) : filename.b.downcase
+          File.join(directory, ".net-connector-#{Digest::SHA256.hexdigest(filename)}.lock")
+        end
+
+        def self.synchronize(path, **options, &block) = new(path, **options).synchronize(&block)
+
+        def initialize(path, timeout: 0, host: nil)
+          raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
+          unless timeout.is_a?(Numeric) && timeout.real? && timeout.finite? && timeout >= 0 && timeout.to_f.finite?
+            raise ArgumentError, "lock_timeout must be a nonnegative finite number"
+          end
+
+          @path = File.expand_path(path).freeze
+          @lock_path = self.class.lock_path(@path).freeze
+          @timeout, @host = timeout.to_f, host
+        end
+
+        def synchronize
+          # Fleet 只向下层备份借用一次；消费后清除，采集回调中的递归备份仍须竞争锁。
+          # Thread#[] 是 Fiber 局部存储；fork 后不能消费父进程的借用记录。
+          if Thread.current[STORAGE_KEY] == [@lock_path, Process.pid]
+            Thread.current[STORAGE_KEY] = nil
+            return yield self
+          end
+
+          file = nil
+          begin
+            file = open_lock
+            verify_lock!(file)
+            acquire(file)
+            verify_lock!(file)
+            @owner = [Process.pid, Fiber.current]
+            yield self
+          ensure
+            @owner = nil
+            file&.close
+          end
+        end
+
+        # 仅持有实际 flock 的同一 Fiber 可授权一次内层采集，避免把路径锁变成任意可重入锁。
+        def delegate
+          raise ThreadError, "backup path delegation requires lock ownership" unless @owner == [Process.pid, Fiber.current]
+
+          previous = Thread.current[STORAGE_KEY]
+          begin
+            Thread.current[STORAGE_KEY] = [@lock_path, Process.pid]
+            yield
+          ensure
+            Thread.current[STORAGE_KEY] = previous
+          end
+        end
+
+        private
+
+        def open_lock
+          File.open(@lock_path, File::RDWR | File::CREAT | File::NOFOLLOW | File::NONBLOCK, 0o600)
+        rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO
+          raise ArgumentError, "backup lock must be a private regular file", cause: nil
+        end
+
+        def verify_lock!(file)
+          stat = file.stat
+          unless stat.file? && stat.uid == Process.euid && stat.nlink == 1 && (stat.mode & 0o7777) == 0o600
+            raise ArgumentError, "backup lock must be an owned regular file with mode 0600 and one link"
+          end
+          current = File.lstat(@lock_path)
+          unless current.file? && current.dev == stat.dev && current.ino == stat.ino
+            raise ArgumentError, "backup lock path changed while acquiring ownership"
+          end
+        rescue Errno::ENOENT
+          raise ArgumentError, "backup lock path changed while acquiring ownership", cause: nil
+        end
+
+        def acquire(file)
+          deadline = monotonic + @timeout
+          loop do
+            acquired = file.flock(File::LOCK_EX | File::LOCK_NB)
+            return if acquired && (@timeout.zero? || monotonic < deadline)
+
+            remaining = deadline - monotonic
+            if remaining <= 0
+              raise BackupBusy.new("backup path already belongs to another operation", host: @host, phase: :backup), cause: nil
+            end
+            wait([remaining, 0.05].min)
+          end
+        end
+
+        def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        def wait(seconds) = sleep(seconds)
+      end
+    end
+  end
+end

data/lib/net/connector/operations/local_backup.rb

@@ -2,6 +2,8 @@
 
 require "digest"
 require_relative "private_file"
+require_relative "backup_lock"
+require_relative "safe_file"
 
 module Net
   module Connector
@@ -15,19 +17,47 @@ module Net
       def changed? = [:created, :changed].include?(change)
     end
 
+    # 只携带库自身的提交回执和配置元数据，不携带正文或原始文件系统异常。
+    class BackupPersistenceError < Error
+      attr_reader :backup, :receipt, :underlying_type
+
+      def initialize(backup:, write_error:, host: nil)
+        unless backup.instance_of?(Backup) && Operations::PrivateFile.receipt_error?(write_error) &&
+               write_error.receipt.committed? && write_error.receipt.path == backup.path
+          raise ArgumentError, "backup persistence error requires a matching committed backup receipt"
+        end
+
+        @backup, @receipt = backup, write_error.receipt
+        @underlying_type = write_error.underlying_type
+        code = if write_error.instance_of?(Operations::PrivateFile::DirectorySyncUnsupported)
+                 :backup_durability_unsupported
+               elsif receipt.durable?
+                 :backup_finalize_failed
+               else
+                 :backup_persistence_unconfirmed
+               end
+        super(write_error.message, code: code, host: host, phase: :backup)
+      end
+    end
+
     module Operations
       class LocalBackup
         # 保存执行配置采集的设备对象。
         def initialize(device) = @device = device
 
         # 采集配置、识别变更并原子写入私有文件。
-        def call(path:)
-          raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
+        def call(path:, lock_timeout: 0)
+          host = @device.host if @device.respond_to?(:host)
+          BackupLock.synchronize(path, timeout: lock_timeout, host: host) { |lock| collect_and_write(lock.path) }
+        end
+
+        private
 
+        def collect_and_write(destination)
           contents = @device.running_config.value!
-          destination = File.expand_path(path)
           digest = Digest::SHA256.hexdigest(contents)
-          previous_digest = Digest::SHA256.file(destination).hexdigest if File.file?(destination) && !File.symlink?(destination)
+          previous = SafeFile.fingerprint(destination, missing: true, replace_symlink: true)
+          previous_digest = previous&.sha256
           change = if previous_digest.nil?
                      :created
                    elsif previous_digest == digest
@@ -36,12 +66,22 @@ module Net
                      :changed
                    end
 
-          if change != :unchanged || (File.stat(destination).mode & 0o777) != 0o600
-            PrivateFile.write(destination, contents)
+          backup = Backup.new(path: destination, bytes: contents.bytesize,
+                              sha256: digest, collected_at: Time.now.utc, change: change,
+                              previous_sha256: previous_digest)
+          if change != :unchanged || previous.mode != 0o600 || !SafeFile.same_entry?(destination, previous)
+            write_backup(backup, contents)
           end
-          Backup.new(path: destination, bytes: contents.bytesize,
-                     sha256: digest, collected_at: Time.now.utc, change: change,
-                     previous_sha256: previous_digest)
+          backup
+        end
+
+        def write_backup(backup, contents)
+          PrivateFile.write_receipt(backup.path, contents)
+        rescue PrivateFile::WriteError => error
+          raise unless PrivateFile.receipt_error?(error) && error.receipt.committed?
+
+          host = @device.host if @device.respond_to?(:host)
+          raise BackupPersistenceError.new(backup: backup, write_error: error, host: host), cause: nil
         end
       end
     end

data/lib/net/connector/operations/parse_output.rb

@@ -21,11 +21,19 @@ module Net
           @index = index
         end
 
-        # 显式模板按路径读取；未指定时按厂商和命令从索引选择模板。
-        def call(text, template: nil, vendor: nil, command: nil, host: nil)
+        # 解析按 UTF-8 字节处理副本，先验证原文，避免回车或转义序列隐藏非法字节。
+        # 拓扑选模板和计数也使用这份副本，但仍保留未经终端渲染的完整证据。
+        def utf8(text, host: nil)
           raise ArgumentError, "text must be a String" unless text.is_a?(String)
 
-          text = TerminalRenderer.render(text).force_encoding(Encoding::UTF_8)
+          text = text.b.force_encoding(Encoding::UTF_8)
+          invalid_encoding!(host) unless text.valid_encoding?
+          text
+        end
+
+        # 显式模板按路径读取；未指定时按厂商和命令从索引选择模板。
+        def call(text, template: nil, vendor: nil, command: nil, host: nil)
+          text = TerminalRenderer.render(utf8(text, host: host), strict_utf8: true).force_encoding(Encoding::UTF_8)
           if template
             raise ArgumentError, "template must be a nonempty String" unless template.is_a?(String) && !template.empty?
 
@@ -41,6 +49,8 @@ module Net
             end
             table.parse(text, attributes: attributes).to_hashes
           end
+        rescue Encoding::InvalidByteSequenceError
+          invalid_encoding!(host)
         rescue TextFSM::IndexError
           raise ParsingError.new("TextFSM template index is invalid",
                                  code: :template_invalid, host: host, phase: :parse), cause: nil
@@ -54,6 +64,13 @@ module Net
           raise ParsingError.new("unable to read TextFSM template",
                                  code: :template_unreadable, host: host, phase: :parse), cause: nil
         end
+
+        private
+
+        def invalid_encoding!(host)
+          raise ParsingError.new("device output is not valid UTF-8", code: :invalid_output_encoding,
+                                 host: host, phase: :parse), cause: nil
+        end
       end
     end
   end

data/lib/net/connector/operations/private_file.rb

@@ -5,19 +5,108 @@ require "tempfile"
 module Net
   module Connector
     module Operations
-      # 将配置写入同目录的私有临时文件，再原子替换目标文件。
+      # 文件同步和目录项同步分别记录；原子可见不等于断电后仍能恢复新目录项。
       module PrivateFile
-        # 写入完整内容并同步到磁盘，避免覆盖失败留下半份配置。
+        Receipt = Data.define(:path, :state, :phase) do
+          def initialize(path:, state:, phase:)
+            unless path.is_a?(String) && !path.empty? && %i[not_committed committed durable].include?(state) &&
+                   %i[resolve_parent temporary_file write file_sync rename directory_open directory_sync cleanup complete].include?(phase)
+              raise ArgumentError, "invalid file write receipt"
+            end
+            super(path: path.dup.freeze, state: state, phase: phase)
+          end
+
+          def committed? = state != :not_committed
+          def durable? = state == :durable
+        end
+
+        class WriteError < IOError
+          attr_reader :receipt, :code, :underlying_type
+
+          def initialize(receipt:, underlying_type:)
+            raise ArgumentError, "receipt must be a file write receipt" unless receipt.instance_of?(Receipt)
+
+            @receipt = receipt
+            @underlying_type = underlying_type.dup.freeze
+            @code, message = if receipt.durable?
+                               [:file_finalize_failed, "file replacement is durable; finalization failed"]
+                             elsif receipt.committed?
+                               [:file_persistence_unconfirmed, "file replacement is committed; durability is unconfirmed"]
+                             else
+                               [:file_write_failed, "file write failed before replacement was committed"]
+                             end
+            super(message)
+          end
+
+          def inspect = "#<#{self.class} state=#{receipt.state} phase=#{receipt.phase}>"
+        end
+
+        class PersistenceError < WriteError; end
+
+        class DirectorySyncUnsupported < WriteError
+          def initialize(**options)
+            super
+            @code = :directory_sync_unsupported
+          end
+
+          def message = "file replacement is committed; directory synchronization is unsupported"
+          alias to_s message
+        end
+
+        UNSUPPORTED_DIRECTORY_SYNC = [Errno::EINVAL, Errno::ENOSYS, Errno::ENOTSUP, Errno::EOPNOTSUPP,
+                                      NotImplementedError].uniq.freeze
+        private_constant :UNSUPPORTED_DIRECTORY_SYNC
+
+        # 外部子类可重写消息或回执；只有本写入器定义的具体错误类型可提供完成事实。
+        def self.receipt_error?(error)
+          [WriteError, PersistenceError, DirectorySyncUnsupported].any? { |type| error.instance_of?(type) }
+        end
+
+        # 保留原路径返回值；失败时可从异常 receipt 判断目录项是否已经替换。
         def self.write(path, contents)
-          Tempfile.create([".net-connector-", ".tmp"], File.dirname(path)) do |file|
+          write_receipt(path, contents)
+          path
+        end
+
+        def self.write_receipt(path, contents)
+          raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
+
+          receipt = Receipt.new(path: File.expand_path(path), state: :not_committed, phase: :resolve_parent)
+          directory = File.realpath(File.dirname(receipt.path))
+          destination = File.join(directory, File.basename(receipt.path))
+          receipt = receipt.with(phase: :temporary_file)
+          Tempfile.create([".net-connector-", ".tmp"], directory) do |file|
             file.binmode
             file.chmod(0o600)
+            receipt = receipt.with(phase: :write)
             file.write(contents)
+            receipt = receipt.with(phase: :file_sync)
             file.flush
             file.fsync
-            File.rename(file.path, path)
+            receipt = receipt.with(phase: :rename)
+            File.rename(file.path, destination)
+            receipt = receipt.with(state: :committed, phase: :directory_open)
+            File.open(directory, File::RDONLY | File::NOFOLLOW) do |parent|
+              raise IOError, "parent is not a directory" unless parent.stat.directory?
+
+              receipt = receipt.with(phase: :directory_sync)
+              parent.fsync
+              receipt = receipt.with(state: :durable, phase: :cleanup)
+            end
           end
-          path
+          receipt.with(phase: :complete)
+        rescue StandardError, NotImplementedError => error
+          raise unless receipt
+
+          unsupported = receipt.phase == :directory_sync && UNSUPPORTED_DIRECTORY_SYNC.any? { |type| error.is_a?(type) }
+          klass = if unsupported
+                    DirectorySyncUnsupported
+                  elsif receipt.committed?
+                    PersistenceError
+                  else
+                    WriteError
+                  end
+          raise klass.new(receipt: receipt, underlying_type: error.class.name.to_s), cause: nil
         end
       end
     end

data/lib/net/connector/operations/safe_file.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Net
+  module Connector
+    module Operations
+      # 对同一 FD 做类型检查和读取；NOFOLLOW 只保护最后一个路径分量，目录由调用方保护。
+      module SafeFile
+        Fingerprint = Data.define(:path, :sha256, :mode, :dev, :ino)
+
+        def self.open(path, missing: false, replace_symlink: false)
+          file = open_handle(path, missing: missing, replace_symlink: replace_symlink)
+          return nil unless file
+
+          begin
+            stat = file.stat
+            raise ArgumentError, "saved configuration is not a regular file" unless stat.file?
+
+            file.binmode
+            yield file, stat
+          ensure
+            file.close
+          end
+        end
+
+        def self.fingerprint(path, **options)
+          self.open(path, **options) { |file, stat| fingerprint_io(path, file, stat) }
+        end
+
+        def self.fingerprint_io(path, file, stat)
+          digest = Digest::SHA256.new
+          while (chunk = file.read(64 * 1024))
+            digest << chunk
+          end
+          Fingerprint.new(path: path.dup.freeze, sha256: digest.hexdigest.freeze,
+                          mode: stat.mode & 0o7777, dev: stat.dev, ino: stat.ino)
+        end
+
+        # 未变内容可保留 mtime，但不能把已替换的目录项误认成刚读取的旧文件。
+        def self.same_entry?(path, fingerprint)
+          stat = File.lstat(path)
+          stat.file? && stat.dev == fingerprint.dev && stat.ino == fingerprint.ino
+        rescue Errno::ENOENT
+          false
+        end
+
+        def self.open_handle(path, missing:, replace_symlink:)
+          File.open(path, File::RDONLY | File::NOFOLLOW | File::NONBLOCK)
+        rescue Errno::ENOENT
+          raise unless missing
+        rescue Errno::ELOOP
+          raise ArgumentError, "saved configuration is not a regular file", cause: nil unless replace_symlink
+        rescue Errno::ENXIO
+          raise ArgumentError, "saved configuration is not a regular file", cause: nil
+        end
+
+        private_class_method :open_handle
+      end
+    end
+  end
+end

data/lib/net/connector/operations/saved_config/legacy_index.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative "../../engine/errors"
+require_relative "../safe_file"
+
+module Net
+  module Connector
+    class SavedConfigChanged < Error; end
+
+    module Operations
+      class SavedConfig
+        # 每批只发布一份只读目录快照。记录文件身份而不读取正文；不跨批次缓存。
+        class LegacyIndex
+          Entry = Data.define(:name, :identity, :regular)
+          EMPTY = [].freeze
+
+          def initialize(directory)
+            @directory = directory
+            @mutex = Mutex.new
+            @entries = nil
+            @failed = false
+          end
+
+          def candidates(filename)
+            @mutex.synchronize do
+              raise IOError, "unable to index saved configurations", cause: nil if @failed
+
+              @entries ||= snapshot
+              @entries.fetch(filename, EMPTY)
+            rescue IOError, SystemCallError
+              @failed = true
+              raise IOError, "unable to index saved configurations", cause: nil
+            end
+          end
+
+          # 目录快照不是锁：同一 FD 读取前后及最终目录项都必须仍匹配快照。
+          # 非合作写入者仍可在检查之后修改文件，调用方必须保护整个备份目录。
+          def open(entry)
+            path = File.join(@directory, entry.name)
+            verify!(entry, File.lstat(path))
+            SafeFile.open(path) do |file, stat|
+              verify!(entry, stat)
+              value = yield path, file, stat
+              verify!(entry, file.stat)
+              verify!(entry, File.lstat(path))
+              value
+            end
+          rescue IOError, SystemCallError
+            changed!
+          rescue ArgumentError
+            raise unless entry.regular
+
+            changed!
+          end
+
+          private
+
+          def snapshot
+            entries = {}
+            Dir.children(@directory).each do |name|
+              filename = canonical_suffix(name)
+              next unless filename
+
+              (entries[filename] ||= []) << snapshot_entry(name)
+            end
+            entries.each_value(&:freeze)
+            entries.freeze
+          end
+
+          # 下划线仅还原地址部分，IPv6 zone 中的下划线保持原义；合法性仍由 IPAddr 判断。
+          def canonical_suffix(name)
+            suffix = name.b[/-([0-9a-f_.]+(?:%[a-z0-9_.-]+)?)\.txt\z/in, 1]
+            return unless suffix
+
+            address, zone = suffix.split("%", 2)
+            host = address.tr("_", ":")
+            host += "%#{zone}" if zone
+            SavedConfig.filename(host).freeze
+          rescue ArgumentError
+            nil
+          end
+
+          def snapshot_entry(name)
+            stat = File.lstat(File.join(@directory, name))
+            Entry.new(name: name.freeze, identity: identity(stat), regular: stat.file?)
+          rescue SystemCallError
+            # 列表与 lstat 之间消失的候选仍占据该身份，不能把后来出现的文件当作旧快照。
+            Entry.new(name: name.freeze, identity: nil, regular: false)
+          end
+
+          def identity(stat)
+            [stat.dev, stat.ino, stat.mode, stat.size, stat.mtime.to_r, stat.ctime.to_r].freeze
+          end
+
+          def verify!(entry, stat)
+            changed! unless entry.identity == identity(stat)
+          end
+
+          def changed!
+            raise SavedConfigChanged.new("legacy saved configuration changed or became unavailable after indexing",
+                                         code: :saved_config_changed, phase: :backup), cause: nil
+          end
+        end
+
+        private_constant :LegacyIndex
+      end
+    end
+  end
+end