sandcrawler 0.1.0

data/lib/sandcrawler/acquisition/picotool.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "open3"
+require "tempfile"
+require "digest"
+require "json"
+require "time"
+
+module Sandcrawler
+  module Acquisition
+    class Picotool
+      COMMANDS = %w[version info save].freeze
+      FLASH_BASE = 0x10000000
+      BLOCK_SIZE = 4096
+      MAX_OUTPUT_BYTES = 1024 * 1024
+
+      def initialize(executable: nil, log: nil, timeout: 120)
+        unless timeout.is_a?(Numeric) && timeout.positive? && timeout.finite?
+          raise UsageError, "Picotool timeout must be finite and positive"
+        end
+
+        @executable = executable || ENV["PICOTOOL"] || "picotool"
+        @log = log
+        @timeout = timeout
+      end
+
+      def run(command, *arguments)
+        validate_command(command, arguments)
+        stdout, stderr, status = capture(command, arguments)
+        return stdout.b if status.success?
+
+        detail = [stdout.b, stderr.b].reject(&:empty?).join("\n").strip
+        missing = detail.match?(/no .*devices|device.*not found|unable to connect|no accessible/i)
+        error = missing ? DeviceNotFoundError : Error
+        raise error, "picotool #{command}: #{detail}"
+      rescue Errno::ENOENT
+        raise DeviceNotFoundError, "picotool not found; install picotool 2.x or use --picotool PATH"
+      end
+
+      def version
+        run("version").strip
+      end
+
+      def info
+        raw = run("info", "-a")
+        match = raw.match(/flash\s+size\s*:\s*(\d+(?:\.\d+)?)\s*([KMGT](?:i?B|bytes?)?|bytes?)?/i)
+        multiplier = match && { "K" => 1024, "M" => 1024**2, "G" => 1024**3, "T" => 1024**4 }.fetch(
+          match[2].to_s[0]&.upcase, 1
+        )
+        { raw: raw, chip: raw[/\bRP(?:2040|2350)\b/i]&.upcase,
+          flash_size: match && (match[1].to_f * multiplier).to_i }
+      end
+
+      def dump(out, passes: 2, force: false)
+        raise UsageError, "--passes must be an integer of at least 2" unless passes.is_a?(Integer) && passes >= 2
+
+        tool_version = version
+        device = info
+        output = Extractor.new(out, source: { type: "picotool", executable: @executable }, force: force, log: @log)
+        records = []
+        passes.times { records << save_pass(output, records.length + 1, device[:flash_size]) }
+        if records.map { |record| record[:sha256] }.uniq.length > 1
+          records << save_pass(output, records.length + 1, device[:flash_size])
+        end
+        validate_sizes!(records, device[:flash_size])
+        data, unstable = combine(output, records)
+        result = output.write(path: "dump.bin", data: data, category: nil, status: unstable.empty? ? "ok" : "unstable")
+        verify_output!(result)
+        File.chmod(0o444, File.join(output.out, result[:output]))
+        unless unstable.empty?
+          output.warning("UNSTABLE_BLOCKS",
+                         "No strict majority for block offsets: #{unstable.join(', ')}")
+        end
+        metadata = {
+          tool: "sandcrawler", tool_version: VERSION, created_at: Time.now.iso8601,
+          picotool: { version: tool_version, info: device[:raw] },
+          device: device.except(:raw), passes: records,
+          result: { file: result[:output], sha256: result[:sha256], unstable_blocks: unstable }
+        }
+        metadata = Manifest.json_safe(metadata)
+        verify_output!(output.write(path: "dump.bin.sha256", data: "#{result[:sha256]}  #{result[:output]}\n",
+                                    category: nil))
+        verify_output!(output.write(path: "dump.json", data: "#{JSON.pretty_generate(metadata)}\n", category: nil))
+        output.finish
+        metadata
+      rescue Error, IOError, SystemCallError => e
+        if output
+          output.warning("DUMP_FAILED", e.message)
+          output.finish
+        end
+        raise
+      end
+
+      private
+
+      def capture(command, arguments)
+        Open3.popen3(@executable, command, *arguments) do |input, output, errors, process|
+          input.close
+          readers = [output, errors].map do |stream|
+            Thread.new do
+              text = stream.binmode.read(MAX_OUTPUT_BYTES + 1) || "".b
+              stop_process(process) if text.bytesize > MAX_OUTPUT_BYTES
+              text
+            end
+          end
+          unless process.join(@timeout)
+            stop_process(process)
+            raise IntegrityError, "picotool #{command} timed out after #{@timeout} seconds"
+          end
+          unless readers.all? { |reader| reader.join(1) }
+            raise IntegrityError, "picotool #{command} did not close its output streams"
+          end
+
+          captured = readers.map(&:value)
+          if captured.any? { |text| text.bytesize > MAX_OUTPUT_BYTES }
+            raise IntegrityError, "picotool #{command} exceeded the 1 MiB output limit"
+          end
+
+          [*captured, process.value]
+        ensure
+          readers&.each { |reader| reader.kill if reader.alive? }
+        end
+      end
+
+      def stop_process(process)
+        Process.kill(Gem.win_platform? ? "KILL" : "TERM", process.pid)
+        Process.kill("KILL", process.pid) unless process.join(1)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def validate_command(command, args)
+        raise SafetyError, "picotool command is not read-only: #{command}" unless COMMANDS.include?(command)
+        raise SafetyError, "Picotool arguments must be strings" unless args.all?(String)
+
+        allowed = case command
+                  when "version" then args.empty?
+                  when "info" then args.empty? || args == ["-a"]
+                  when "save"
+                    valid_all = args.size == 5 && args[0, 2] == %w[-a -v] && safe_filename?(args[2])
+                    valid_range = args.size == 7 && args[0] == "-r" && args[3] == "-v" &&
+                                  args[1, 2].all? { |a| a.match?(/\A0x[0-9a-f]+\z/i) } && safe_filename?(args[4])
+                    (valid_all || valid_range) && args[-2, 2] == %w[-t bin]
+                  end
+        raise SafetyError, "Unsupported picotool arguments; reboot and write options are forbidden" unless allowed
+      end
+
+      def safe_filename?(path)
+        path.is_a?(String) && !path.empty? && !path.start_with?("-") && !path.include?("\0")
+      end
+
+      def save_pass(output, number, flash_size)
+        Tempfile.create(["sandcrawler-pass", ".bin"]) do |temp|
+          path = temp.path
+          temp.close
+          begin
+            run("save", "-a", "-v", path, "-t", "bin")
+          rescue Error => e
+            preserve_failed_pass(output, path, number, "all", e)
+            raise if e.is_a?(DeviceNotFoundError) || !flash_size&.positive?
+
+            File.truncate(path, 0)
+            begin
+              run("save", "-r", format("0x%x", FLASH_BASE), format("0x%x", FLASH_BASE + flash_size), "-v", path, "-t",
+                  "bin")
+            rescue Error => range_error
+              preserve_failed_pass(output, path, number, "range", range_error)
+              raise
+            end
+          end
+          record = File.open(path, "rb") do |file|
+            output.write(path: "pass#{number}.bin", data: file, category: nil)
+          end
+          verify_output!(record)
+          File.chmod(0o444, File.join(output.out, record[:output]))
+          { file: record[:output], sha256: record[:sha256], size: record[:size] }
+        end
+      end
+
+      def preserve_failed_pass(output, path, number, mode, error)
+        output.warning("SAVE_FAILED", "Pass #{number} (#{mode}): #{error.message}")
+        return unless File.file?(path) && File.size(path).positive?
+
+        record = File.open(path, "rb") do |file|
+          output.write(path: "pass#{number}-#{mode}-failed.bin", data: file, category: nil,
+                       status: "acquisition_error", error: error.message)
+        end
+        verify_output!(record)
+        File.chmod(0o444, File.join(output.out, record[:output]))
+      end
+
+      def verify_output!(record)
+        return if record[:output] && %w[ok unstable acquisition_error].include?(record[:status])
+
+        raise IntegrityError, "Could not preserve #{record[:path]}"
+      end
+
+      def validate_sizes!(records, expected)
+        sizes = records.map { |record| record[:size] }
+        return if sizes.uniq.size == 1 && sizes.first.positive? && (!expected || sizes.first == expected)
+
+        raise IntegrityError,
+              "Dump lengths #{sizes.inspect} do not match#{expected ? " flash size #{expected}" : ' each other'}"
+      end
+
+      def combine(output, records)
+        images = records.map { |record| File.open(File.join(output.out, record[:file]), "rb") }
+        data = +"".b
+        unstable = []
+        (0...records.first[:size]).step(BLOCK_SIZE) do |offset|
+          blocks = images.map { |file| file.read(BLOCK_SIZE) }
+          expected = [BLOCK_SIZE, records.first[:size] - offset].min
+          unless blocks.all? { |block| block&.bytesize == expected }
+            raise IntegrityError, "Preserved pass length changed at offset #{offset}"
+          end
+
+          winner, count = blocks.tally.max_by { |_block, votes| votes }
+          unstable << offset unless count > records.length / 2
+          data << winner
+        end
+        [data, unstable]
+      ensure
+        images&.each(&:close)
+      end
+    end
+  end
+end

data/lib/sandcrawler/acquisition/serial_protocol.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "io/wait"
+
+module Sandcrawler
+  module Acquisition
+    module SerialProtocol
+      MARKER = "@@SANDCRAWLER|"
+      ANSI = %r{\e\[[0-?]*[ -/]*[@-~]}
+      MAX_RESPONSE_BYTES = 40 * 1024 * 1024
+      MAX_RESPONSE_RECORDS = 65_537
+
+      private
+
+      def decode_transfer(records)
+        data = +"".b
+        sequence = 0
+        ended = false
+        records.each do |record|
+          kind, first, second = record
+          unless record.size == 3 && %w[DATA END].include?(kind)
+            raise IntegrityError, "Unexpected serial transfer record"
+          end
+
+          if kind == "DATA"
+            unless !ended && first == sequence.to_s && second&.match?(/\A(?:[0-9a-f]{2}){1,256}\z/i)
+              raise IntegrityError, "Missing, duplicate or invalid serial chunk"
+            end
+
+            data << [second].pack("H*")
+            sequence += 1
+          elsif kind == "END"
+            unless !ended && first == data.bytesize.to_s && second == (data.bytes.sum % 65_536).to_s
+              raise IntegrityError, "Serial byte count or checksum mismatch"
+            end
+
+            ended = true
+          end
+        end
+        raise IntegrityError, "Serial END marker missing" unless ended
+
+        data
+      end
+
+      def decode_listing(records)
+        ending = records.last
+        entries = records[0...-1]
+        unless ending&.size == 3 && ending[0] == "LISTEND" && ending[1] == entries.size.to_s &&
+               entries.all? { |record| record.size == 3 && record[0] == "LIST" }
+          raise IntegrityError, "Serial directory count or terminator mismatch"
+        end
+
+        checksum = entries.sum { |record| record.drop(1).join("|").bytes.sum } % 65_536
+        raise IntegrityError, "Serial directory checksum mismatch" unless ending[2] == checksum.to_s
+
+        names = entries.map { |record| record[2] }
+        raise IntegrityError, "Duplicate serial directory entry" unless names.uniq.size == names.size
+
+        entries
+      end
+
+      def request(expected:)
+        marker = "#{MARKER}#{SecureRandom.hex(8)}|"
+        code = yield marker
+        error_marker = SerialTemplates.literal("#{marker}ERROR|")
+        done_marker = SerialTemplates.literal("#{marker}DONE")
+        @io.write("begin;#{code};rescue=>e;puts #{error_marker}+e.to_s;end;puts #{done_marker}\n")
+        records = []
+        response_bytes = 0
+        deadline = monotonic + @timeout
+        absolute_deadline = monotonic + 3600
+        loop do
+          line = read_line([deadline, absolute_deadline].min).gsub(ANSI, "").strip
+          next unless line.start_with?(marker)
+
+          record = line.delete_prefix(marker).split("|", -1)
+          break if record == ["DONE"]
+
+          allowed = [expected, "ERROR", { "DATA" => "END", "LIST" => "LISTEND" }[expected]]
+          raise IntegrityError, "Unexpected serial response record" unless allowed.include?(record[0])
+
+          response_bytes += line.bytesize
+          if response_bytes > MAX_RESPONSE_BYTES || records.size >= MAX_RESPONSE_RECORDS
+            raise IntegrityError, "Serial response exceeds the Pico transfer budget"
+          end
+
+          records << record
+          deadline = monotonic + @timeout
+        end
+        error = records.find { |record| record[0] == "ERROR" }
+        raise Error, "R2P2: #{error.drop(1).join('|')}" if error
+
+        records
+      end
+
+      def wait_prompt
+        deadline = monotonic + @timeout
+        loop do
+          if @buffer.gsub(ANSI, "").match?(/[>$#] ?\z/)
+            @buffer.clear
+            return
+          end
+          read_more(deadline)
+        end
+      end
+
+      def read_line(deadline)
+        loop do
+          index = @buffer.index("\n")
+          return @buffer.slice!(0, index + 1) if index
+
+          read_more(deadline)
+        end
+      end
+
+      def read_more(deadline)
+        remaining = deadline - monotonic
+        unless remaining.positive? && @io.to_io.wait_readable(remaining)
+          raise IntegrityError, "Serial response timed out"
+        end
+
+        chunk = @io.read_nonblock(4096, exception: false)
+        raise IOError, "Serial connection closed" if chunk.nil?
+        return if chunk == :wait_readable
+
+        @buffer << chunk
+        raise FormatError, "Serial line exceeds 1 MiB" if @buffer.bytesize > 1_048_576
+      end
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/sandcrawler/acquisition/serial_shell.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "serial_templates"
+require_relative "serial_protocol"
+
+module Sandcrawler
+  module Acquisition
+    class SerialShell
+      include SerialProtocol
+
+      def self.ports
+        if Gem.win_platform?
+          stdout, _stderr, status = Open3.capture3(
+            "powershell.exe", "-NoProfile", "-NonInteractive", "-Command",
+            "[System.IO.Ports.SerialPort]::GetPortNames()"
+          )
+          return status.success? ? stdout.lines.map(&:strip).grep(/\ACOM\d+\z/i).uniq.sort : []
+        end
+
+        Dir.glob(%w[/dev/ttyACM* /dev/ttyUSB* /dev/cu.usbmodem* /dev/cu.usbserial*])
+      rescue Errno::ENOENT
+        []
+      end
+
+      def initialize(port: nil, io: nil, baud: 115_200, timeout: 10, log: nil)
+        raise UsageError, "Serial timeout must be positive" unless timeout.is_a?(Numeric) && timeout.positive?
+        raise UsageError, "Serial baud rate must be a positive integer" unless baud.is_a?(Integer) && baud.positive?
+
+        @port = port
+        @io = io
+        @baud = baud
+        @timeout = timeout
+        @log = log
+        @buffer = +"".b
+      end
+
+      def copy(out, force: false, root: "/")
+        unless root.start_with?("/") && !root.include?("\0")
+          raise UsageError,
+                "Serial root must be an absolute path without NUL"
+        end
+
+        output = Extractor.new(out, source: { type: "serial", port: @port, root: root }, force: force, log: @log)
+        connect
+        @io.write("\x03\n")
+        wait_prompt
+        @io.write("irb\n")
+        wait_prompt
+        @entered_irb = true
+        capabilities = probe(root)
+        output.warning("SERIAL_CAPABILITIES", capabilities.inspect)
+        copy_tree(output, root.sub(%r{/+\z}, ""), capabilities)
+        output.finish
+      rescue Error, SystemCallError, IOError => e
+        output&.warning("SERIAL_FAILED", e.message)
+        output&.finish
+        raise
+      ensure
+        begin
+          @io&.write("exit\n") if @entered_irb
+        rescue IOError, SystemCallError
+          nil
+        end
+        @io&.close if @owns_io
+      end
+
+      private
+
+      def connect
+        return if @io
+        unless @port
+          raise UsageError,
+                "Select CDC0 with --port PORT (Windows: --port COM3). Candidates: #{self.class.ports.join(', ')}"
+        end
+
+        require "serialport"
+        @io = SerialPort.new(@port, @baud, 8, 1, SerialPort::NONE)
+        @owns_io = true
+      rescue LoadError
+        raise UsageError, "Serial support needs the optional dependency: gem install serialport"
+      rescue Errno::ENOENT, Errno::ENODEV
+        raise DeviceNotFoundError, "Serial port not found: #{@port}"
+      end
+
+      def probe(root)
+        records = request(expected: "CAP") { |marker| SerialTemplates.capabilities(root, marker) }
+        cap = records.find { |record| record[0] == "CAP" }
+        unless records.size == 1 && cap&.size == 4 && %w[entries
+                                                         open].include?(cap[1]) && %w[directory
+                                                                                      stat].include?(cap[2]) &&
+               %w[bytes each_byte].include?(cap[3])
+          raise FormatError, "R2P2 does not provide the required read-only file APIs"
+        end
+
+        { list: cap[1], directory: cap[2], bytes: cap[3] }
+      end
+
+      def copy_tree(output, root, capabilities)
+        queue = [[root.empty? ? "/" : root, ""]]
+        visited = {}
+        until queue.empty?
+          remote, relative = queue.shift
+          next if visited[remote]
+
+          visited[remote] = true
+          begin
+            records = request(expected: "LIST") { |marker| SerialTemplates.listing(remote, capabilities, marker) }
+            decode_listing(records).each do |record|
+              name = decode_name(record[2])
+              path = [relative, name].reject(&:empty?).join("/")
+              full_path = "#{remote.sub(%r{/+\z}, '')}/#{name}"
+              if record[1] == "D"
+                queue << [full_path, path] if output.directory(path: path)
+              elsif record[1] == "F"
+                copy_file(output, full_path, path, capabilities)
+              else
+                raise FormatError, "Invalid serial entry type"
+              end
+            end
+          rescue Error, IOError, SystemCallError => e
+            output.warning("LIST_FAILED", "#{remote}: #{e.message}")
+          end
+        end
+      end
+
+      def decode_name(hex)
+        raise FormatError, "Invalid filename hex" unless hex&.match?(/\A(?:[0-9a-f]{2})+\z/i)
+
+        name = [hex].pack("H*").force_encoding(Encoding::UTF_8)
+        unless name.valid_encoding? && !%w[. ..].include?(name) && !name.match?(%r{[/\\\x00]})
+          raise SafetyError, "Unsafe name in serial directory response"
+        end
+
+        name
+      end
+
+      def copy_file(output, remote, relative, capabilities)
+        attempts = 0
+        begin
+          attempts += 1
+          records = request(expected: "READCAP") { |marker| SerialTemplates.read_capability(remote, marker) }
+          cap = records.find { |record| record[0] == "READCAP" }
+          unless records.size == 1 && cap&.size == 2 && %w[read getc].include?(cap[1])
+            raise FormatError, "No supported file reader"
+          end
+
+          records = request(expected: "DATA") do |marker|
+            SerialTemplates.transfer(remote, capabilities, cap[1], marker)
+          end
+          data = decode_transfer(records)
+          output.write(path: relative, data: data, original_name: File.basename(relative))
+        rescue Error, IOError, SystemCallError => e
+          retry if attempts < 3
+
+          output.warning("TRANSFER_FAILED", "#{remote}: #{e.message}")
+          output.write(path: relative, data: "".b, status: "transfer_error", error: e.message)
+        end
+      end
+    end
+  end
+end

data/lib/sandcrawler/acquisition/serial_templates.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Sandcrawler
+  module Acquisition
+    module SerialTemplates
+      module_function
+
+      def literal(path)
+        path.b.dump
+      end
+
+      def capabilities(root, marker)
+        [
+          "r=#{literal(root)}",
+          'begin;Dir.entries(r);a="entries";rescue;Dir.open(r){|d|d.read};a="open";end',
+          'begin;File.directory?(r);b="directory";rescue;File.stat(r).directory?;b="stat";end',
+          'begin;"A".bytes.each{|x|x};c="bytes";rescue;"A".each_byte{|x|x};c="each_byte";end',
+          "puts #{literal("#{marker}CAP|")}+a+\"|\"+b+\"|\"+c"
+        ].join(";")
+      end
+
+      def listing(path, capabilities, marker)
+        listing = if capabilities[:list] == "entries"
+                    "names=Dir.entries(r)"
+                  else
+                    "names=[];Dir.open(r){|d|while n=d.read;names<<n;end}"
+                  end
+        directory = if capabilities[:directory] == "directory"
+                      'File.directory?(p+"/"+n)'
+                    else
+                      'File.stat(p+"/"+n).directory?'
+                    end
+        prefix = path.sub(%r{/+\z}, "")
+        iterator = capabilities[:bytes] == "bytes" ? "record.bytes.each" : "record.each_byte"
+        "r=#{literal(path)};p=#{literal(prefix)};#{listing};count=0;sum=0;names.each{|n|if n!=\".\" && n!=\"..\";" \
+          "s=n;#{hex(capabilities)};record=(#{directory} ? \"D\" : \"F\")+\"|\"+h;" \
+          "#{iterator}{|b|sum=(sum+b)%65536};count+=1;puts #{literal("#{marker}LIST|")}+record;end};" \
+          "puts #{literal("#{marker}LISTEND|")}+count.to_s+\"|\"+sum.to_s"
+      end
+
+      def read_capability(path, marker)
+        "f=File.open(#{literal(path)},\"r\");begin;begin;f.read(1);m=\"read\";rescue;f.getc;m=\"getc\";end;" \
+          "puts #{literal("#{marker}READCAP|")}+m;ensure;f.close;end"
+      end
+
+      def transfer(path, capabilities, read_mode, marker)
+        read = if read_mode == "read"
+                 "s=f.read(256)"
+               else
+                 's="";256.times{c=f.getc;break if c.nil?;s+=(c.is_a?(Integer) ? c.chr : c)}'
+               end
+        "f=File.open(#{literal(path)},\"r\");begin;i=0;n=0;sum=0;loop{#{read};break if s.nil? || s.empty?;" \
+          "#{hex(capabilities, checksum: true)};puts #{literal("#{marker}DATA|")}+i.to_s+\"|\"+h;" \
+          "i+=1};puts #{literal("#{marker}END|")}+n.to_s+\"|\"+sum.to_s;ensure;f.close;end"
+      end
+
+      def hex(capabilities, checksum: false)
+        iterator = capabilities[:bytes] == "bytes" ? "s.bytes.each" : "s.each_byte"
+        "h=\"\";#{iterator}{|b|v=b.to_s(16);h+=(v.length==1 ? \"0\"+v : v)#{';n+=1;sum=(sum+b)%65536' if checksum}}"
+      end
+    end
+  end
+end