sandcrawler 0.1.0

data/lib/sandcrawler/fat/volume_locator.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "volume"
+
+module Sandcrawler
+  module FAT
+    class VolumeLocator
+      Candidate = Struct.new(:offset, :boot_sector, :score, :score_details, :boot_sector_offset, keyword_init: true) do
+        def to_h
+          boot_sector.to_h.merge(offset: offset, size: boot_sector.byte_size, score: score,
+                                 score_details: score_details, boot_sector_offset: boot_sector_offset || offset,
+                                 warnings: warnings)
+        end
+
+        def warnings
+          return [] unless boot_sector_offset && boot_sector_offset != offset
+
+          [{ code: "BACKUP_BOOT_SECTOR",
+             detail: "Boot sector at #{boot_sector_offset} implies volume at #{offset}; " \
+                     "FAT copies match and the root entry is plausible" }]
+        end
+      end
+
+      def initialize(source)
+        @source = source
+      end
+
+      def candidates
+        @candidates ||= begin
+          found = {}
+          (0..(@source.size - 512)).step(512) do |offset|
+            bytes = @source.read_at(offset, 512)
+            candidate = inspect_boot(offset, bytes)
+            backup = inspect_backup(offset, bytes, candidate)
+            if backup
+              found[backup.offset] ||= backup
+            elsif candidate
+              found[offset] = candidate
+            end
+            next if candidate || backup || bytes.byteslice(510, 2) != "\x55\xAA".b
+
+            mbr_offsets(bytes).each do |partition_offset|
+              next if found.key?(partition_offset) || partition_offset + 512 > @source.size
+
+              candidate = inspect_boot(partition_offset, @source.read_at(partition_offset, 512))
+              found[partition_offset] = candidate if candidate
+            end
+          end
+          found.values.sort_by { |candidate| [-candidate.score, candidate.offset] }
+        end
+      end
+
+      private
+
+      def inspect_backup(offset, bytes, candidate)
+        return if candidate&.score_details&.key?(:media)
+
+        boot = candidate&.boot_sector || BootSector.new(bytes)
+        return unless boot.fat_type == "FAT32" && boot.backup_sector.between?(1, boot.reserved_sectors - 1)
+
+        base = offset - (boot.backup_sector * boot.bytes_per_sector)
+        return if base.negative?
+
+        recovered = inspect_boot(base, bytes)
+        return unless recovered && %i[media fat_copies root_entry].all? { |key| recovered.score_details.key?(key) }
+
+        recovered.boot_sector_offset = offset
+        recovered
+      rescue FormatError, OutOfRangeError
+        nil
+      end
+
+      def inspect_boot(offset, bytes)
+        boot = BootSector.new(bytes, available_size: @source.size - offset)
+        details = {}
+        fat_offset = offset + (boot.reserved_sectors * boot.bytes_per_sector)
+        details[:media] = 2 if @source.read_at(fat_offset, 1).getbyte(0) == boot.media
+        if boot.fat_count == 2
+          size = boot.fat_sectors * boot.bytes_per_sector
+          details[:fat_copies] = 2 if @source.read_at(fat_offset, size) == @source.read_at(fat_offset + size, size)
+        end
+        root = if boot.root_cluster
+                 offset + boot.cluster_offset(boot.root_cluster)
+               else
+                 offset + ((boot.reserved_sectors + (boot.fat_count * boot.fat_sectors)) * boot.bytes_per_sector)
+               end
+        entry = @source.read_at(root, 32)
+        if entry.getbyte(0).positive? && entry.getbyte(0) != 255 && (entry.getbyte(11) & 0xC0).zero?
+          details[:root_entry] = 2
+        end
+        details[:r2p2_label] = 3 if boot.label.include?("R2P2")
+        Candidate.new(offset: offset, boot_sector: boot, score: details.values.sum, score_details: details)
+      rescue FormatError, OutOfRangeError
+        nil
+      end
+
+      def mbr_offsets(bytes)
+        4.times.flat_map do |index|
+          entry = bytes.byteslice(446 + (index * 16), 16)
+          next [] unless [0x01, 0x04, 0x06, 0x0B, 0x0C, 0x0E].include?(entry.getbyte(4))
+
+          start = entry.byteslice(8, 4).unpack1("V")
+          start.positive? ? [start * 512, start * 4096] : []
+        end
+      end
+    end
+  end
+end

data/lib/sandcrawler/image/raw_image.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "source"
+
+module Sandcrawler
+  module Image
+    class RawImage < Source
+      def initialize(path)
+        @io = File.open(path, "rb")
+        @size = @io.size
+        @lock = Mutex.new
+      end
+
+      def read_at(offset, length)
+        check_range(offset, length)
+        return "".b if length.zero?
+
+        data = begin
+          @io.pread(length, offset)
+        rescue NoMethodError, NotImplementedError, Errno::ENOSYS
+          @lock.synchronize do
+            @io.seek(offset)
+            @io.read(length)
+          end
+        end
+        raise OutOfRangeError, "Image changed or ended during read" unless data&.bytesize == length
+
+        data.b
+      rescue EOFError
+        raise OutOfRangeError, "Image ended during read"
+      end
+
+      def close
+        @io.close unless @io.closed?
+      end
+    end
+  end
+end

data/lib/sandcrawler/image/slice.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "source"
+
+module Sandcrawler
+  module Image
+    class Slice < Source
+      def initialize(source, offset, length)
+        unless offset.is_a?(Integer) && length.is_a?(Integer) &&
+               offset >= 0 && length >= 0 && offset + length <= source.size
+          raise OutOfRangeError, "Slice outside image"
+        end
+
+        @source = source
+        @offset = offset
+        @size = length
+      end
+
+      def read_at(offset, length)
+        check_range(offset, length)
+        @source.read_at(@offset + offset, length)
+      end
+    end
+  end
+end

data/lib/sandcrawler/image/source.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Sandcrawler
+  module Image
+    def self.open(path)
+      File.extname(path).downcase == ".uf2" ? Uf2Image.new(path) : RawImage.new(path)
+    end
+
+    class Source
+      attr_reader :size
+
+      def check_range(offset, length)
+        unless offset.is_a?(Integer) && length.is_a?(Integer) && offset >= 0 && length >= 0 && offset + length <= size
+          raise OutOfRangeError, "Read outside image: offset=#{offset}, length=#{length}, size=#{size}"
+        end
+      end
+
+      def close; end
+
+      def warnings
+        []
+      end
+    end
+  end
+end

data/lib/sandcrawler/image/uf2_image.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require_relative "source"
+
+module Sandcrawler
+  module Image
+    class Uf2Image < Source
+      FLASH_BASE = 0x10000000
+      FLASH_SIZE = 16 * 1024 * 1024
+      attr_reader :warnings
+
+      def initialize(path)
+        @blocks = []
+        @warnings = []
+        @numbers = {}
+        File.open(path, "rb") do |io|
+          raise FormatError, "UF2 size must be a multiple of 512" unless io.size.positive? && (io.size % 512).zero?
+
+          parse_block(io.read(512)) until io.eof?
+        end
+        raise FormatError, "UF2 has no flash payload" if @blocks.empty?
+
+        @blocks.sort_by!(&:first)
+        @blocks.each_cons(2) do |left, right|
+          overlap = left[0] + left[1].bytesize - right[0]
+          next unless overlap.positive?
+          raise FormatError, "Overlapping UF2 flash payloads" unless left == right
+        end
+        @blocks.uniq!
+        if @numbers.size < @total
+          @warnings << { code: "UF2_INCOMPLETE", detail: "UF2 contains #{@numbers.size} of #{@total} declared blocks" }
+        end
+        @size = @blocks.map { |offset, bytes| offset + bytes.bytesize }.max
+        return unless @blocks.sum { |_, bytes| bytes.bytesize } != size
+
+        @warnings << { code: "UF2_GAPS",
+                       detail: "Missing UF2 ranges are filled with 0xFF; they are not verified erased flash" }
+      end
+
+      def read_at(offset, length)
+        check_range(offset, length)
+        result = "\xFF".b * length
+        index = @blocks.bsearch_index { |start, bytes| start + bytes.bytesize > offset } || @blocks.length
+        while index < @blocks.length
+          start, bytes = @blocks[index]
+          break if start >= offset + length
+
+          first = [start, offset].max
+          last = [start + bytes.bytesize, offset + length].min
+          result[first - offset, last - first] = bytes.byteslice(first - start, last - first) if last > first
+          index += 1
+        end
+        result
+      end
+
+      private
+
+      def parse_block(block)
+        raise FormatError, "UF2 block was truncated during read" unless block&.bytesize == 512
+
+        magic1, magic2, flags, address, length, number, total = block.unpack("V7")
+        unless magic1 == 0x0A324655 && magic2 == 0x9E5D5157 && block[508, 4].unpack1("V") == 0x0AB16F30
+          raise FormatError, "Invalid UF2 magic"
+        end
+        raise FormatError, "UF2 file containers are unsupported" unless (flags & 0x1000).zero?
+        unless length.between?(1, 476) && total.positive? && number < total
+          raise FormatError, "Invalid UF2 payload or block number"
+        end
+        raise FormatError, "Inconsistent UF2 block count" if @total && total != @total
+        raise FormatError, "Conflicting UF2 block number #{number}" if @numbers[number] && @numbers[number] != block
+
+        @total = total
+        @numbers[number] = block
+        return unless (flags & 1).zero?
+
+        offset = address - FLASH_BASE
+        raise FormatError, "UF2 payload outside Pico XIP flash" unless offset >= 0 && offset + length <= FLASH_SIZE
+
+        @blocks << [offset, block.byteslice(32, length)]
+      end
+    end
+  end
+end

data/lib/sandcrawler/log.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Sandcrawler
+  class Log
+    attr_reader :lines
+
+    def self.escape(message)
+      message.to_s.b.force_encoding(Encoding::UTF_8).scrub.gsub(/[\u0000-\u001f\u007f-\u009f]/) do |character|
+        format("\\u%04x", character.ord)
+      end
+    end
+
+    def initialize(io: $stderr, verbose: false, quiet: false)
+      @io = io
+      @verbose = verbose
+      @quiet = quiet
+      @lines = []
+    end
+
+    def info(message)
+      record("INFO", message, !@quiet)
+    end
+
+    def warn(message)
+      record("WARN", message, true)
+    end
+
+    def error(message)
+      record("ERROR", message, true)
+    end
+
+    def debug(message)
+      record("DEBUG", message, @verbose && !@quiet)
+    end
+
+    private
+
+    def record(level, message, display)
+      line = "#{level}: #{self.class.escape(message)}"
+      @lines << line
+      @io.puts(line) if display
+    end
+  end
+end

data/lib/sandcrawler/manifest.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Sandcrawler
+  class Manifest
+    attr_reader :data
+
+    def initialize(source:, volume: nil)
+      @data = { tool: "sandcrawler", tool_version: VERSION, created_at: Time.now.iso8601,
+                source: source, volume: volume, files: [], directories: [], warnings: [] }
+    end
+
+    def self.json_safe(value)
+      case value
+      when Hash
+        value.each_with_object({}) do |(key, item), result|
+          result[key] = json_safe(item)
+          next unless item.is_a?(String) && item.b != result[key].b
+
+          hex_key = key.is_a?(Symbol) ? :"#{key}_hex" : "#{key}_hex"
+          result[hex_key] = item.b.unpack1("H*")
+        end
+      when Array then value.map { |item| json_safe(item) }
+      when String then value.dup.force_encoding(Encoding::UTF_8).scrub
+      else value
+      end
+    end
+
+    def warning(code, detail)
+      item = { code: code.to_s.dup.force_encoding(Encoding::UTF_8).scrub,
+               detail: detail.to_s.dup.force_encoding(Encoding::UTF_8).scrub }
+      @data[:warnings] << item unless @data[:warnings].include?(item)
+      item
+    end
+  end
+end

data/lib/sandcrawler/path_sanitizer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Sandcrawler
+  class PathSanitizer
+    RESERVED = /\A(?:CON|PRN|AUX|NUL|COM[1-9¹²³]|LPT[1-9¹²³])(?:\.|\z)/i
+
+    def initialize
+      @directories = {}
+      @used = Hash.new { |hash, key| hash[key] = {} }
+      %w[manifest.json sandcrawler.log].each { |name| @used[""][name] = true }
+    end
+
+    def self.component(name)
+      text = name.to_s.b.force_encoding(Encoding::UTF_8).scrub("_")
+      text = text.gsub(%r{[\x00-\x1f\x7f<>:"/\\|?*]}, "_").sub(/[. ]+\z/, "")
+      text = "_" if text.empty? || %w[. ..].include?(text)
+      text = "_#{text}" if RESERVED.match?(text)
+      if text.bytesize > 180
+        suffix = Digest::SHA256.hexdigest(text)[0, 12]
+        text = text.byteslice(0, 160).scrub("") + "~#{suffix}"
+      end
+      text
+    end
+
+    def path(original, directory: false, namespace: nil)
+      text = original.to_s.b
+      parts = text.tr("\\", "/").sub(%r{\A[A-Za-z]:/}, "").split("/")
+      parts.reject! { |part| part.empty? || %w[. ..].include?(part) }
+      parts = ["_"] if parts.empty?
+      parts.unshift(namespace) if namespace
+      parent = ""
+      parts.each_with_index do |part, index|
+        is_directory = index < parts.size - 1 || directory
+        key = [parent, part]
+        name = is_directory && @directories[key]
+        unless name
+          name = available_name(parent, self.class.component(part))
+          @directories[key] = name if is_directory
+        end
+        parent = parent.empty? ? name : "#{parent}/#{name}"
+      end
+      parent
+    end
+
+    private
+
+    def available_name(parent, name)
+      candidate = name
+      number = 1
+      while @used[parent][candidate.unicode_normalize(:nfc).downcase]
+        number += 1
+        extension = File.extname(name)
+        candidate = "#{File.basename(name, extension)}~#{number}#{extension}"
+      end
+      @used[parent][candidate.unicode_normalize(:nfc).downcase] = true
+      candidate
+    end
+  end
+end

data/lib/sandcrawler/recovery/carver.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "../fat/volume"
+
+module Sandcrawler
+  module Recovery
+    class Carver
+      def initialize(source, block_size: 4096, offset: 0, length: nil)
+        @source = source
+        @block_size = block_size
+        @offset = offset
+        @length = length || (source.size - offset)
+        unless block_size.is_a?(Integer) && block_size.positive? &&
+               offset >= 0 && @length >= 0 && offset + @length <= source.size
+          raise UsageError, "Invalid carving range or block size"
+        end
+      end
+
+      def classify(bytes)
+        return "zero" if bytes.empty? || bytes.bytes.all?(&:zero?)
+        return "erased" if bytes.bytes.all? { |byte| byte == 255 }
+
+        text = bytes.dup.force_encoding("UTF-8")
+        return "binary" unless text.valid_encoding?
+
+        printable = text.each_char.count { |char| char.match?(/[[:print:]\t\r\n]/) }
+        printable.to_f / text.length >= 0.95 ? "text" : "binary"
+      end
+
+      def each
+        return enum_for(__method__) unless block_given?
+
+        data = "".b
+        start = nil
+        (@offset...(@offset + @length)).step(@block_size) do |offset|
+          length = [@block_size, @offset + @length - offset].min
+          begin
+            bytes = @source.read_at(offset, length)
+          rescue Error, IOError, SystemCallError => e
+            yield fragment(start, data) if start
+            yield unreadable_fragment(offset, length, e)
+            start = nil
+            data = "".b
+            next
+          end
+          if classify(bytes) == "text"
+            start ||= offset
+            data << bytes
+          elsif start
+            yield fragment(start, data)
+            start = nil
+            data = "".b
+          end
+        end
+        yield fragment(start, data) if start
+      end
+
+      private
+
+      def unreadable_fragment(offset, length, error)
+        entry = FAT::DirEntry.recovered(path: format("%08x.txt", offset), size: length)
+        warnings = [{ code: "READ_ERROR", detail: "Source offset #{offset}, length #{length}: #{error.message}" }]
+        result = FAT::Volume::ReadResult.new(data: "".b, status: "read_error", warnings: warnings)
+        [entry, result]
+      end
+
+      def fragment(offset, data)
+        entry = FAT::DirEntry.recovered(path: format("%08x.txt", offset), size: data.bytesize)
+        warnings = [{ code: "CARVED_TEXT",
+                      detail: "Text fragment at source offset #{offset}; original boundaries are unknown" }]
+        result = FAT::Volume::ReadResult.new(data: data, status: "carved", warnings: warnings)
+        [entry, result]
+      end
+    end
+  end
+end

data/lib/sandcrawler/recovery/orphans.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "../fat/volume"
+
+module Sandcrawler
+  module Recovery
+    class Orphans
+      def initialize(volume)
+        @volume = volume
+      end
+
+      def each
+        return enum_for(__method__) unless block_given?
+
+        @volume.each_entry.to_a
+        allocation = @volume.allocation_table
+        remaining = Set.new((2..(@volume.boot_sector.cluster_count + 1)).select do |cluster|
+          allocation.allocated?(cluster) && !@volume.referenced_clusters.include?(cluster)
+        end)
+        pointed_to = Set.new(remaining.filter_map do |cluster|
+          allocation[cluster] if remaining.include?(allocation[cluster])
+        end)
+        roots = (remaining - pointed_to).to_a + remaining.to_a
+        roots.each do |first|
+          next unless remaining.include?(first)
+
+          chain = allocation.chain(first)
+          clusters = chain.clusters.take_while { |cluster| remaining.include?(cluster) }
+          warnings = chain.warnings + [
+            { code: "ORPHAN_CHAIN", detail: "Chain has no directory entry; original name and file length are unknown" }
+          ]
+          data = "".b
+          status = chain.status == "ok" ? "recovered" : chain.status
+          clusters.each do |cluster|
+            remaining.delete(cluster)
+            data << @volume.read_cluster(cluster)
+          rescue Error, IOError, SystemCallError => e
+            status = "read_error"
+            warnings << { code: "READ_ERROR", detail: "Chain #{first}, cluster #{cluster}: #{e.message}" }
+            break
+          end
+          text = data.dup.force_encoding("UTF-8")
+          extension = text.valid_encoding? && text.match?(/\b(?:def|end|require|class|module|puts)\b/) ? "rb" : "bin"
+          entry = FAT::DirEntry.recovered(path: "CHAIN_#{first}.#{extension}", size: data.bytesize,
+                                          first_cluster: first)
+          result = FAT::Volume::ReadResult.new(data: data, status: status, warnings: warnings)
+          yield entry, result
+        end
+      end
+    end
+  end
+end

data/lib/sandcrawler/recovery/undelete.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "../fat/volume"
+
+module Sandcrawler
+  module Recovery
+    class Undelete
+      def initialize(volume)
+        @volume = volume
+      end
+
+      def each
+        return enum_for(__method__) unless block_given?
+
+        @volume.each_entry(deleted: true).to_a.each do |entry|
+          next unless entry.deleted? && !entry.directory?
+
+          warnings = [{ code: "DELETED_GUESS",
+                        detail: "Contiguous free clusters assumed; verify recovered content manually" }]
+          data = "".b
+          status = "recovered"
+          count = (entry.size + @volume.boot_sector.cluster_size - 1) / @volume.boot_sector.cluster_size
+          count.times do |index|
+            cluster = entry.first_cluster + index
+            unless cluster.between?(2, @volume.boot_sector.cluster_count + 1) &&
+                   @volume.allocation_table[cluster].zero? && !@volume.referenced_clusters.include?(cluster)
+              status = "partial"
+              warnings << { code: "DELETED_OVERLAP", detail: "Stopped at occupied or invalid cluster #{cluster}" }
+              break
+            end
+            data << @volume.read_cluster(cluster)
+          rescue Error, IOError, SystemCallError => e
+            status = "read_error"
+            warnings << { code: "READ_ERROR", detail: "#{entry.path}, cluster #{cluster}: #{e.message}" }
+            break
+          end
+          yield entry, FAT::Volume::ReadResult.new(data: data.byteslice(0, [entry.size, data.bytesize].min),
+                                                   status: status, warnings: warnings)
+        end
+      end
+    end
+  end
+end

data/lib/sandcrawler/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Sandcrawler
+  VERSION = "0.1.0"
+end

data/lib/sandcrawler.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "sandcrawler/version"
+require_relative "sandcrawler/errors"
+require_relative "sandcrawler/log"
+require_relative "sandcrawler/manifest"
+require_relative "sandcrawler/path_sanitizer"
+require_relative "sandcrawler/extractor"
+require_relative "sandcrawler/image/source"
+require_relative "sandcrawler/image/raw_image"
+require_relative "sandcrawler/image/uf2_image"
+require_relative "sandcrawler/image/slice"
+require_relative "sandcrawler/fat/boot_sector"
+require_relative "sandcrawler/fat/allocation_table"
+require_relative "sandcrawler/fat/dir_entry"
+require_relative "sandcrawler/fat/lfn_assembler"
+require_relative "sandcrawler/fat/volume"
+require_relative "sandcrawler/fat/volume_locator"
+require_relative "sandcrawler/recovery/undelete"
+require_relative "sandcrawler/recovery/orphans"
+require_relative "sandcrawler/recovery/carver"
+require_relative "sandcrawler/acquisition/picotool"
+require_relative "sandcrawler/acquisition/mounted_volume"
+require_relative "sandcrawler/acquisition/serial_shell"
+require_relative "sandcrawler/cli"