sandcrawler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +157 -0
- data/docs/findings.md +49 -0
- data/docs/releasing.md +28 -0
- data/docs/verification.md +80 -0
- data/exe/sandcrawler +5 -0
- data/lib/sandcrawler/acquisition/mounted_volume.rb +81 -0
- data/lib/sandcrawler/acquisition/picotool.rb +227 -0
- data/lib/sandcrawler/acquisition/serial_protocol.rb +136 -0
- data/lib/sandcrawler/acquisition/serial_shell.rb +162 -0
- data/lib/sandcrawler/acquisition/serial_templates.rb +63 -0
- data/lib/sandcrawler/cli/devices.rb +109 -0
- data/lib/sandcrawler/cli/offline.rb +199 -0
- data/lib/sandcrawler/cli.rb +149 -0
- data/lib/sandcrawler/errors.rb +11 -0
- data/lib/sandcrawler/extractor.rb +157 -0
- data/lib/sandcrawler/fat/allocation_table.rb +139 -0
- data/lib/sandcrawler/fat/boot_sector.rb +142 -0
- data/lib/sandcrawler/fat/dir_entry.rb +80 -0
- data/lib/sandcrawler/fat/lfn_assembler.rb +60 -0
- data/lib/sandcrawler/fat/volume.rb +198 -0
- data/lib/sandcrawler/fat/volume_locator.rb +108 -0
- data/lib/sandcrawler/image/raw_image.rb +38 -0
- data/lib/sandcrawler/image/slice.rb +25 -0
- data/lib/sandcrawler/image/source.rb +25 -0
- data/lib/sandcrawler/image/uf2_image.rb +83 -0
- data/lib/sandcrawler/log.rb +44 -0
- data/lib/sandcrawler/manifest.rb +37 -0
- data/lib/sandcrawler/path_sanitizer.rb +61 -0
- data/lib/sandcrawler/recovery/carver.rb +76 -0
- data/lib/sandcrawler/recovery/orphans.rb +52 -0
- data/lib/sandcrawler/recovery/undelete.rb +43 -0
- data/lib/sandcrawler/version.rb +5 -0
- data/lib/sandcrawler.rb +25 -0
- metadata +81 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sandcrawler
|
|
4
|
+
class CLI
|
|
5
|
+
module Devices
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def picotool
|
|
9
|
+
Acquisition::Picotool.new(executable: @options[:picotool], log: @log)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def run_doctor(_args)
|
|
13
|
+
@log.warn("Preserve a BOOTSEL dump before using flash_nuke.uf2 or reinstalling firmware.")
|
|
14
|
+
result = { ruby: RUBY_VERSION, platform: RUBY_PLATFORM, serialport: !Gem::Specification.find_all_by_name("serialport").empty? }
|
|
15
|
+
begin
|
|
16
|
+
tool = picotool
|
|
17
|
+
result[:picotool] = { version: tool.version }
|
|
18
|
+
begin
|
|
19
|
+
result[:device] = tool.info
|
|
20
|
+
rescue Error => e
|
|
21
|
+
result[:device] = { available: false, detail: e.message }
|
|
22
|
+
end
|
|
23
|
+
rescue Error => e
|
|
24
|
+
result[:picotool] = { available: false, detail: e.message }
|
|
25
|
+
end
|
|
26
|
+
if RUBY_PLATFORM.include?("linux")
|
|
27
|
+
result[:usb_permissions] =
|
|
28
|
+
"picotool info checks access; install the upstream udev rules if permission is denied"
|
|
29
|
+
end
|
|
30
|
+
emit(result) { result.each { |key, value| @stdout.puts("#{key}: #{value}") } }
|
|
31
|
+
0
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def run_dump(_args)
|
|
35
|
+
result = picotool.dump(@options[:out], passes: @options[:passes], force: @options[:force])
|
|
36
|
+
emit(result) { @stdout.puts("Preserved dump in #{@options[:out]}: #{result.dig(:result, :sha256)}") }
|
|
37
|
+
result.dig(:result, :unstable_blocks).to_a.empty? ? 0 : 5
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def run_copy(args)
|
|
41
|
+
result = Acquisition::MountedVolume.new(args.first).copy(@options[:out], force: @options[:force], log: @log)
|
|
42
|
+
emit(result) { @stdout.puts("Copied #{result[:files].size} files to #{@options[:out]}") }
|
|
43
|
+
result_code(result)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def run_serial(_args)
|
|
47
|
+
port = @options[:port] || select_port
|
|
48
|
+
shell = Acquisition::SerialShell.new(port: port, log: @log)
|
|
49
|
+
result = shell.copy(@options[:out], force: @options[:force], root: @options[:root] || "/")
|
|
50
|
+
emit(result) { @stdout.puts("Transferred #{result[:files].size} files to #{@options[:out]}") }
|
|
51
|
+
result_code(result)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def select_port
|
|
55
|
+
ports = Acquisition::SerialShell.ports
|
|
56
|
+
raise DeviceNotFoundError, "No serial ports found; pass --port for R2P2 CDC0" if ports.empty?
|
|
57
|
+
|
|
58
|
+
@stderr.puts("Choose R2P2 CDC0 (shell), not CDC1/CDC2:")
|
|
59
|
+
ports.each_with_index { |port, i| @stderr.puts("#{i + 1}: #{port}") }
|
|
60
|
+
index = number(prompt("Port number")) - 1
|
|
61
|
+
raise UsageError, "Invalid serial port selection" unless index.between?(0, ports.size - 1)
|
|
62
|
+
|
|
63
|
+
ports[index]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def run_auto(_args)
|
|
67
|
+
base = @options[:out]
|
|
68
|
+
raise SafetyError, "Output is a symlink: #{base}" if File.symlink?(base)
|
|
69
|
+
if File.exist?(base) && (!File.directory?(base) || (!@options[:force] && !Dir.empty?(base)))
|
|
70
|
+
raise SafetyError, "Output must be an empty directory (or use --force): #{base}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
route = prompt("Recovery route: [a] BOOTSEL (recommended), [b] mounted volume, [c] serial",
|
|
74
|
+
default: "a").downcase
|
|
75
|
+
raise UsageError, "Choose a, b, or c" unless %w[a b c].include?(route)
|
|
76
|
+
|
|
77
|
+
@options[:out] = File.join(base, "acquisition") if route == "a"
|
|
78
|
+
code = case route
|
|
79
|
+
when "a"
|
|
80
|
+
dump = picotool.dump(@options[:out], passes: @options[:passes], force: @options[:force])
|
|
81
|
+
unless dump.dig(:result, :unstable_blocks).to_a.empty?
|
|
82
|
+
emit(dump) { @stdout.puts("Dump contains unstable blocks; inspect dump.json before recovery") }
|
|
83
|
+
return 5
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
image = File.join(@options[:out], "dump.bin")
|
|
87
|
+
@options[:out] = File.join(base, "recovered")
|
|
88
|
+
run_extract([image])
|
|
89
|
+
when "b"
|
|
90
|
+
run_copy([prompt("Mounted R2P2 directory")])
|
|
91
|
+
when "c"
|
|
92
|
+
run_serial([])
|
|
93
|
+
end
|
|
94
|
+
code
|
|
95
|
+
ensure
|
|
96
|
+
@stderr.puts("Check the dump and recovered files before erasing or reinstalling R2P2. Keep dump.bin.")
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def prompt(message, default: nil)
|
|
100
|
+
@stderr.print("#{message}: ")
|
|
101
|
+
input = @stdin.gets
|
|
102
|
+
raise UsageError, "Interactive input required; use an explicit command for unattended operation" unless input
|
|
103
|
+
|
|
104
|
+
value = input.strip
|
|
105
|
+
value.empty? && default ? default : value
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sandcrawler
|
|
4
|
+
class CLI
|
|
5
|
+
module Offline
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def run_scan(args)
|
|
9
|
+
image = Image.open(args.first)
|
|
10
|
+
candidates = locate(image)
|
|
11
|
+
emit(candidates.map(&:to_h)) do
|
|
12
|
+
candidates.each do |candidate|
|
|
13
|
+
line = format("0x%<offset>08X score=%<score>s %<geometry>s",
|
|
14
|
+
offset: candidate.offset, score: candidate.score, geometry: candidate.boot_sector.to_h)
|
|
15
|
+
@stdout.puts(line)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
0
|
|
19
|
+
ensure
|
|
20
|
+
image&.close
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def run_ls(args)
|
|
24
|
+
image = Image.open(args.first)
|
|
25
|
+
volume = open_volume(image)
|
|
26
|
+
entries = volume.each_entry(deleted: @options[:deleted]).to_a
|
|
27
|
+
emit(entries.map(&:to_h)) do
|
|
28
|
+
entries.each do |entry|
|
|
29
|
+
prefix = @options[:long] ? "#{entry.size.to_s.rjust(10)} #{entry.mtime&.iso8601 || '-'} " : ""
|
|
30
|
+
name = Log.escape(entry.path)
|
|
31
|
+
@stdout.puts("#{prefix}#{'[deleted] ' if entry.deleted?}#{name}#{'/' if entry.directory?}")
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
volume.warnings.each { |warning| @log.warn(warning.to_s) }
|
|
35
|
+
volume.warnings.empty? ? 0 : 4
|
|
36
|
+
ensure
|
|
37
|
+
image&.close
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def run_extract(args)
|
|
41
|
+
path = args.first
|
|
42
|
+
protect_input(path)
|
|
43
|
+
protect_input(@options[:reference]) if @options[:reference]
|
|
44
|
+
image = Image.open(path)
|
|
45
|
+
source = { type: "image", path: File.expand_path(path), sha256: Digest::SHA256.file(path).hexdigest,
|
|
46
|
+
size: File.size(path), image_size: image.size }
|
|
47
|
+
if @options[:reference]
|
|
48
|
+
source[:reference_image] = { path: File.expand_path(@options[:reference]),
|
|
49
|
+
sha256: Digest::SHA256.file(@options[:reference]).hexdigest }
|
|
50
|
+
end
|
|
51
|
+
volume = volume_for_extraction(image)
|
|
52
|
+
output = Extractor.new(@options[:out], source: source, volume: volume&.metadata,
|
|
53
|
+
force: @options[:force], log: @log)
|
|
54
|
+
if @options[:reference]
|
|
55
|
+
output.warning("REFERENCE_GEOMETRY",
|
|
56
|
+
"Geometry borrowed from #{@options[:reference]}; verify firmware and board match")
|
|
57
|
+
end
|
|
58
|
+
output.warning("MANUAL_GEOMETRY", @options[:geometry].inspect) if @options[:geometry]
|
|
59
|
+
Array(image.respond_to?(:warnings) ? image.warnings : []).each { |w| record_warning(output, w) }
|
|
60
|
+
if volume
|
|
61
|
+
extract_volume(volume, output)
|
|
62
|
+
else
|
|
63
|
+
output.warning("NO_FAT_VOLUME", "No usable FAT boot sector; carving the image without filesystem metadata")
|
|
64
|
+
end
|
|
65
|
+
carve(image, volume, output) if @options[:carve]
|
|
66
|
+
verify_source(source, output)
|
|
67
|
+
verify_source(source[:reference_image], output) if source[:reference_image]
|
|
68
|
+
manifest = output.finish
|
|
69
|
+
emit(manifest) { @stdout.puts("Recovered #{manifest[:files].size} files to #{output.out}") }
|
|
70
|
+
volume ? result_code(manifest) : 4
|
|
71
|
+
rescue Error, IOError, SystemCallError => e
|
|
72
|
+
if output
|
|
73
|
+
output.warning("EXTRACTION_FAILED", e.message)
|
|
74
|
+
output.finish
|
|
75
|
+
end
|
|
76
|
+
raise
|
|
77
|
+
ensure
|
|
78
|
+
image&.close
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def verify_source(source, output)
|
|
82
|
+
return if Digest::SHA256.file(source[:path]).hexdigest == source[:sha256]
|
|
83
|
+
|
|
84
|
+
output.warning("SOURCE_CHANGED", "Source changed during recovery: #{source[:path]}")
|
|
85
|
+
raise IntegrityError, "Source changed during recovery; saved results need a fresh acquisition"
|
|
86
|
+
rescue IOError, SystemCallError => e
|
|
87
|
+
output.warning("SOURCE_CHANGED", "Cannot verify source after recovery: #{e.message}")
|
|
88
|
+
raise IntegrityError, "Source could not be verified after recovery"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def volume_for_extraction(image)
|
|
92
|
+
open_volume(image)
|
|
93
|
+
rescue FormatError
|
|
94
|
+
raise unless @options[:carve] && !@options[:geometry] && !@options[:reference] && !@options[:offset]
|
|
95
|
+
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def extract_volume(volume, output)
|
|
100
|
+
volume.each_entry do |entry|
|
|
101
|
+
if entry.directory?
|
|
102
|
+
output.directory(path: entry.path, mtime: entry.mtime, attributes: entry.attributes,
|
|
103
|
+
first_cluster: entry.first_cluster)
|
|
104
|
+
else
|
|
105
|
+
write_result(output, entry, volume.read(entry), "files")
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
if @options[:deleted]
|
|
109
|
+
Recovery::Undelete.new(volume).each { |entry, result| write_result(output, entry, result, "deleted") }
|
|
110
|
+
end
|
|
111
|
+
if @options[:orphans]
|
|
112
|
+
Recovery::Orphans.new(volume).each { |entry, result| write_result(output, entry, result, "lost+found") }
|
|
113
|
+
end
|
|
114
|
+
volume.warnings.each { |warning| record_warning(output, warning) }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def write_result(output, entry, result, category)
|
|
118
|
+
output.write(path: entry.path, original_name: entry.name, data: result.data, category: category,
|
|
119
|
+
status: result.status, mtime: entry.mtime, attributes: entry.attributes,
|
|
120
|
+
first_cluster: entry.first_cluster, expected_size: entry.size, warnings: result.warnings)
|
|
121
|
+
result.warnings.each { |warning| record_warning(output, warning) }
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def record_warning(output, warning)
|
|
125
|
+
if warning.is_a?(Hash)
|
|
126
|
+
output.warning(warning[:code] || warning["code"], warning[:detail] || warning["detail"])
|
|
127
|
+
else
|
|
128
|
+
output.warning("RECOVERY_WARNING", warning.to_s)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def carve(image, volume, output)
|
|
133
|
+
boot = volume&.boot_sector
|
|
134
|
+
options = if boot
|
|
135
|
+
{ offset: volume.offset + (boot.first_data_sector * boot.bytes_per_sector),
|
|
136
|
+
block_size: boot.cluster_size, length: boot.cluster_count * boot.cluster_size }
|
|
137
|
+
else
|
|
138
|
+
{}
|
|
139
|
+
end
|
|
140
|
+
Recovery::Carver.new(image, **options).each { |entry, result| write_result(output, entry, result, "carved") }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def open_volume(image)
|
|
144
|
+
candidate = locate(image).first
|
|
145
|
+
raise FormatError, "No FAT volume found; try --reference-image, --geometry, or extract --carve" unless candidate
|
|
146
|
+
|
|
147
|
+
volume = FAT::Volume.new(image, offset: candidate.offset, boot_sector: candidate.boot_sector,
|
|
148
|
+
fat: @options[:fat], codepage: @options[:codepage])
|
|
149
|
+
volume.warnings.concat(candidate.warnings) if candidate.respond_to?(:warnings)
|
|
150
|
+
volume
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def locate(image)
|
|
154
|
+
if @options[:geometry]
|
|
155
|
+
offset = @options[:offset] || @options[:geometry][:offset] || 0
|
|
156
|
+
begin
|
|
157
|
+
boot = FAT::BootSector.from_geometry(@options[:geometry], available_size: image.size - offset)
|
|
158
|
+
rescue FormatError => e
|
|
159
|
+
raise UsageError, e.message
|
|
160
|
+
end
|
|
161
|
+
return [manual_candidate(offset, boot)]
|
|
162
|
+
end
|
|
163
|
+
source = @options[:reference] ? Image.open(@options[:reference]) : image
|
|
164
|
+
if @options.key?(:offset)
|
|
165
|
+
offset = @options[:offset]
|
|
166
|
+
boot = FAT::BootSector.new(source.read_at(offset, 512), available_size: image.size - offset)
|
|
167
|
+
return [manual_candidate(offset, boot)]
|
|
168
|
+
end
|
|
169
|
+
candidates = FAT::VolumeLocator.new(source).candidates
|
|
170
|
+
candidates.each { |candidate| @log.debug(candidate.to_h.inspect) }
|
|
171
|
+
candidates
|
|
172
|
+
ensure
|
|
173
|
+
source&.close if source && source != image
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def manual_candidate(offset, boot)
|
|
177
|
+
Struct.new(:offset, :boot_sector, :score) do
|
|
178
|
+
def to_h
|
|
179
|
+
boot_sector.to_h.merge(offset: offset, score: score)
|
|
180
|
+
end
|
|
181
|
+
end.new(offset, boot, 0)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def protect_input(path)
|
|
185
|
+
input = File.realpath(path)
|
|
186
|
+
parent = File.expand_path(@options[:out])
|
|
187
|
+
tail = []
|
|
188
|
+
until File.exist?(parent) || File.symlink?(parent)
|
|
189
|
+
tail.unshift(File.basename(parent))
|
|
190
|
+
parent = File.dirname(parent)
|
|
191
|
+
end
|
|
192
|
+
destination = File.join(File.realpath(parent), *tail)
|
|
193
|
+
return unless input == destination || input.start_with?(destination + File::SEPARATOR)
|
|
194
|
+
|
|
195
|
+
raise SafetyError, "Output must not contain the source image: #{path}"
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
require "json"
|
|
5
|
+
require_relative "cli/offline"
|
|
6
|
+
require_relative "cli/devices"
|
|
7
|
+
|
|
8
|
+
module Sandcrawler
|
|
9
|
+
class CLI
|
|
10
|
+
include Offline
|
|
11
|
+
include Devices
|
|
12
|
+
|
|
13
|
+
COMMANDS = %w[doctor dump scan ls extract copy serial auto].freeze
|
|
14
|
+
|
|
15
|
+
def initialize(stdout: $stdout, stderr: $stderr, stdin: $stdin)
|
|
16
|
+
@stdout = stdout
|
|
17
|
+
@stderr = stderr
|
|
18
|
+
@stdin = stdin
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def run(arguments)
|
|
22
|
+
args = arguments.dup
|
|
23
|
+
return @stdout.puts(VERSION) || 0 if args == ["--version"]
|
|
24
|
+
return @stdout.puts(help) || 0 if args.empty? || %w[-h --help].include?(args.first)
|
|
25
|
+
|
|
26
|
+
command = args.shift
|
|
27
|
+
raise UsageError, "Unknown command: #{command}" unless COMMANDS.include?(command)
|
|
28
|
+
|
|
29
|
+
@options = { passes: 2, codepage: "IBM437" }
|
|
30
|
+
parser = option_parser(command)
|
|
31
|
+
parser.parse!(args)
|
|
32
|
+
return @stdout.puts(parser) || 0 if @options[:help]
|
|
33
|
+
|
|
34
|
+
validate_arguments(command, args)
|
|
35
|
+
@log = Log.new(io: @stderr, verbose: @options[:verbose], quiet: @options[:quiet])
|
|
36
|
+
send("run_#{command}", args)
|
|
37
|
+
rescue UsageError, OptionParser::ParseError => e
|
|
38
|
+
@stderr.puts("#{Log.escape(e.message)}\nUse sandcrawler --help for usage.")
|
|
39
|
+
2
|
|
40
|
+
rescue DeviceNotFoundError => e
|
|
41
|
+
@stderr.puts(Log.escape(e.message))
|
|
42
|
+
3
|
|
43
|
+
rescue IntegrityError => e
|
|
44
|
+
@stderr.puts(Log.escape(e.message))
|
|
45
|
+
5
|
|
46
|
+
rescue Error, SystemCallError, IOError => e
|
|
47
|
+
@stderr.puts(Log.escape(e.message))
|
|
48
|
+
1
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def help
|
|
54
|
+
<<~TEXT
|
|
55
|
+
sandcrawler #{VERSION} — read-only R2P2 file recovery
|
|
56
|
+
Usage: sandcrawler COMMAND [IMAGE|MOUNTPOINT] [options]
|
|
57
|
+
doctor Check environment and optional acquisition tools
|
|
58
|
+
dump -o DIR Preserve BOOTSEL flash with repeated reads
|
|
59
|
+
scan IMAGE List all FAT volume candidates
|
|
60
|
+
ls IMAGE List files (--long, --deleted)
|
|
61
|
+
extract IMAGE -o DIR Recover files (--deleted, --orphans, --carve)
|
|
62
|
+
copy MOUNTPOINT -o DIR Copy a mounted volume without following symlinks
|
|
63
|
+
serial -o DIR Transfer files from CDC0 (--port PORT)
|
|
64
|
+
auto -o DIR Choose a route interactively and recover
|
|
65
|
+
Use COMMAND --help for options. Dump before using flash_nuke.uf2.
|
|
66
|
+
TEXT
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def option_parser(command)
|
|
70
|
+
OptionParser.new do |parser|
|
|
71
|
+
parser.banner = "Usage: sandcrawler #{command} [IMAGE|MOUNTPOINT] [options]"
|
|
72
|
+
parser.on("-o", "--output DIR", "Output directory") { |v| @options[:out] = v }
|
|
73
|
+
parser.on("--offset N", "Volume byte offset (decimal or 0xHEX)") { |v| @options[:offset] = number(v) }
|
|
74
|
+
parser.on("--geometry VALUES", "bps,spc,rsvd,fats,fatsz,root[,total,offset,root_cluster]") do |v|
|
|
75
|
+
@options[:geometry] = geometry(v)
|
|
76
|
+
end
|
|
77
|
+
parser.on("--reference-image FILE") { |v| @options[:reference] = v }
|
|
78
|
+
parser.on("--fat N", %w[1 2]) { |v| @options[:fat] = v.to_i }
|
|
79
|
+
parser.on("--codepage NAME") do |v|
|
|
80
|
+
@options[:codepage] = Encoding.find(v).name
|
|
81
|
+
rescue ArgumentError
|
|
82
|
+
raise UsageError, "Unknown codepage: #{v}"
|
|
83
|
+
end
|
|
84
|
+
parser.on("--passes N", Integer) { |v| @options[:passes] = v }
|
|
85
|
+
parser.on("--picotool PATH") { |v| @options[:picotool] = v }
|
|
86
|
+
parser.on("--port PORT") { |v| @options[:port] = v }
|
|
87
|
+
parser.on("--root PATH", "Serial source directory (default /)") { |v| @options[:root] = v }
|
|
88
|
+
%i[deleted orphans carve long json verbose quiet force].each do |option|
|
|
89
|
+
parser.on("--#{option}") { @options[option] = true }
|
|
90
|
+
end
|
|
91
|
+
parser.on("-h", "--help") { @options[:help] = true }
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def validate_arguments(command, args)
|
|
96
|
+
count = %w[scan ls extract copy].include?(command) ? 1 : 0
|
|
97
|
+
raise UsageError, "#{command} expects #{count} input path(s)" unless args.size == count
|
|
98
|
+
if %w[dump extract copy serial auto].include?(command) && !@options[:out]
|
|
99
|
+
raise UsageError, "#{command} requires -o DIR"
|
|
100
|
+
end
|
|
101
|
+
raise UsageError, "--passes must be at least 2" if @options[:passes] < 2
|
|
102
|
+
return unless @options[:geometry] && @options[:reference]
|
|
103
|
+
|
|
104
|
+
raise UsageError, "Choose --geometry or --reference-image"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def number(value)
|
|
108
|
+
result = Integer(value, value.start_with?("0x", "0X") ? 16 : 10)
|
|
109
|
+
raise ArgumentError if result.negative?
|
|
110
|
+
|
|
111
|
+
result
|
|
112
|
+
rescue ArgumentError
|
|
113
|
+
raise UsageError, "Invalid nonnegative integer: #{value}"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def geometry(value)
|
|
117
|
+
allowed = %w[bps spc rsvd fats fatsz root total totsec offset root_cluster media]
|
|
118
|
+
fields = value.split(",", -1).each_with_object({}) do |field, result|
|
|
119
|
+
key, number = field.split("=", 2)
|
|
120
|
+
raise UsageError, "Invalid geometry field: #{field}" unless allowed.include?(key) && number
|
|
121
|
+
raise UsageError, "Duplicate geometry field: #{key}" if result.key?(key.to_sym)
|
|
122
|
+
|
|
123
|
+
result[key.to_sym] = self.number(number)
|
|
124
|
+
end
|
|
125
|
+
missing = %i[bps spc rsvd fats fatsz root] - fields.keys
|
|
126
|
+
raise UsageError, "Missing geometry fields: #{missing.join(', ')}" unless missing.empty?
|
|
127
|
+
raise UsageError, "Specify either total or totsec, not both" if fields.key?(:total) && fields.key?(:totsec)
|
|
128
|
+
|
|
129
|
+
fields
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def emit(value)
|
|
133
|
+
if @options[:json]
|
|
134
|
+
@stdout.puts(JSON.pretty_generate(Manifest.json_safe(value)))
|
|
135
|
+
elsif !@options[:quiet]
|
|
136
|
+
yield if block_given?
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def result_code(manifest)
|
|
141
|
+
loss_codes = %w[DIRECTORY_DAMAGED DIRECTORY_ERROR DIRECTORY_LOOP DIRECTORY_WRITE_ERROR COPY_FAILED LIST_FAILED
|
|
142
|
+
FILE_ERROR UF2_INCOMPLETE FAT_ERASED_ENTRY CHAIN_ERROR CHAIN_FREE CHAIN_TOO_LONG CROSS_LINK
|
|
143
|
+
DUPLICATE_ENTRY]
|
|
144
|
+
partial = manifest[:files].any? { |file| file[:status] != "ok" }
|
|
145
|
+
partial ||= manifest[:warnings].any? { |warning| loss_codes.include?(warning[:code]) }
|
|
146
|
+
partial ? 4 : 0
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sandcrawler
|
|
4
|
+
class Error < StandardError; end
|
|
5
|
+
class UsageError < Error; end
|
|
6
|
+
class SafetyError < Error; end
|
|
7
|
+
class DeviceNotFoundError < Error; end
|
|
8
|
+
class IntegrityError < Error; end
|
|
9
|
+
class OutOfRangeError < Error; end
|
|
10
|
+
class FormatError < Error; end
|
|
11
|
+
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "tempfile"
|
|
5
|
+
require "json"
|
|
6
|
+
require "digest"
|
|
7
|
+
|
|
8
|
+
module Sandcrawler
|
|
9
|
+
class Extractor
|
|
10
|
+
CATEGORIES = [nil, "files", "deleted", "lost+found", "carved"].freeze
|
|
11
|
+
attr_reader :out, :manifest
|
|
12
|
+
|
|
13
|
+
def initialize(out, source:, volume: nil, force: false, log: nil)
|
|
14
|
+
@out = File.expand_path(out)
|
|
15
|
+
raise SafetyError, "Output is a symlink: #{@out}" if File.symlink?(@out)
|
|
16
|
+
if File.exist?(@out) && (!File.directory?(@out) || (!force && !Dir.empty?(@out)))
|
|
17
|
+
raise SafetyError, "Output must be an empty directory (or use --force): #{@out}"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
FileUtils.mkdir_p(@out)
|
|
21
|
+
@out = File.realpath(@out)
|
|
22
|
+
@manifest = Manifest.new(source: source, volume: volume)
|
|
23
|
+
@paths = PathSanitizer.new
|
|
24
|
+
@log = log || Log.new
|
|
25
|
+
@directory_times = []
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def write(path:, data:, category: "files", status: "ok", mtime: nil, original_name: nil, **metadata)
|
|
29
|
+
validate_category(category)
|
|
30
|
+
entry = metadata.merge(path: path, original_name: original_name || path.to_s.b.split("/").last,
|
|
31
|
+
status: status, mtime: mtime&.iso8601)
|
|
32
|
+
%i[path original_name].each do |key|
|
|
33
|
+
text = entry[key].to_s.dup.force_encoding(Encoding::UTF_8)
|
|
34
|
+
entry[:"#{key}_hex"] = text.b.unpack1("H*") unless text.valid_encoding?
|
|
35
|
+
entry[key] = text.scrub
|
|
36
|
+
end
|
|
37
|
+
relative = output_path(path, category, status)
|
|
38
|
+
target = safe_target(relative)
|
|
39
|
+
Tempfile.create([".sandcrawler-", ".tmp"], File.dirname(target), binmode: true) do |file|
|
|
40
|
+
begin
|
|
41
|
+
data.is_a?(String) ? file.write(data.b) : IO.copy_stream(data, file)
|
|
42
|
+
rescue IOError, SystemCallError => e
|
|
43
|
+
raise unless file.size.positive?
|
|
44
|
+
|
|
45
|
+
warning("FILE_ERROR", "#{path}: #{e.message}; preserving readable prefix")
|
|
46
|
+
entry[:status] = "read_error"
|
|
47
|
+
relative = output_path(path, category, "read_error")
|
|
48
|
+
target = safe_target(relative)
|
|
49
|
+
end
|
|
50
|
+
if entry[:status] == "ok" && entry[:expected_size] && file.size != entry[:expected_size]
|
|
51
|
+
warning("SIZE_MISMATCH", "#{path}: expected #{entry[:expected_size]} bytes, recovered #{file.size}")
|
|
52
|
+
entry[:status] = file.size < entry[:expected_size] ? "truncated" : "read_error"
|
|
53
|
+
relative = output_path(path, category, entry[:status])
|
|
54
|
+
target = safe_target(relative)
|
|
55
|
+
end
|
|
56
|
+
commit_file(file, target)
|
|
57
|
+
end
|
|
58
|
+
entry.merge!(output: relative, size: File.size(target), sha256: Digest::SHA256.file(target).hexdigest)
|
|
59
|
+
restore_time(target, mtime) if mtime
|
|
60
|
+
@manifest.data[:files] << entry
|
|
61
|
+
entry
|
|
62
|
+
rescue IOError, SystemCallError => e
|
|
63
|
+
warning("FILE_ERROR", "#{path}: #{e.message}")
|
|
64
|
+
entry ||= { path: path, original_name: original_name }
|
|
65
|
+
entry.merge!(status: "read_error", output: nil, sha256: nil)
|
|
66
|
+
@manifest.data[:files] << entry
|
|
67
|
+
entry
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def directory(path:, category: "files", mtime: nil, **metadata)
|
|
71
|
+
validate_category(category)
|
|
72
|
+
entry = metadata.merge(path: path, original_name: path.to_s.b.split("/").last,
|
|
73
|
+
mtime: mtime&.iso8601, status: "ok")
|
|
74
|
+
relative = @paths.path(path, namespace: category, directory: true)
|
|
75
|
+
target = safe_target("#{relative}/.directory-check")
|
|
76
|
+
@directory_times << [File.dirname(target), mtime] if mtime
|
|
77
|
+
@manifest.data[:directories] << entry.merge(output: relative)
|
|
78
|
+
relative
|
|
79
|
+
rescue IOError, SystemCallError => e
|
|
80
|
+
warning("DIRECTORY_WRITE_ERROR", "#{path}: #{e.message}")
|
|
81
|
+
@manifest.data[:directories] << entry.merge(status: "write_error", output: nil)
|
|
82
|
+
nil
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def warning(code, detail)
|
|
86
|
+
@log.warn("#{code}: #{detail}")
|
|
87
|
+
@manifest.warning(code, detail)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def finish
|
|
91
|
+
@directory_times.reverse_each do |path, time|
|
|
92
|
+
raise SafetyError, "Output became a symlink: #{path}" if File.symlink?(path)
|
|
93
|
+
|
|
94
|
+
File.utime(time, time, path)
|
|
95
|
+
rescue SystemCallError => e
|
|
96
|
+
warning("TIMESTAMP_ERROR", e.message)
|
|
97
|
+
end
|
|
98
|
+
result = Manifest.json_safe(@manifest.data)
|
|
99
|
+
@log.info("Manifest: files=#{result[:files].size}, directories=#{result[:directories].size}, " \
|
|
100
|
+
"warnings=#{result[:warnings].size}")
|
|
101
|
+
atomic_write(safe_target("manifest.json")) { |file| file.write("#{JSON.pretty_generate(result)}\n") }
|
|
102
|
+
atomic_write(safe_target("sandcrawler.log")) { |file| file.write("#{@log.lines.join("\n")}\n") }
|
|
103
|
+
result
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
def validate_category(category)
|
|
109
|
+
raise UsageError, "Unknown output category: #{category}" unless CATEGORIES.include?(category)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def output_path(path, category, status)
|
|
113
|
+
suffix = %w[truncated chain_error partial read_error transfer_error].include?(status) ? ".partial" : ""
|
|
114
|
+
@paths.path("#{path}#{suffix}", namespace: category)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def restore_time(target, time)
|
|
118
|
+
File.utime(time, time, target)
|
|
119
|
+
rescue SystemCallError => e
|
|
120
|
+
warning("TIMESTAMP_ERROR", e.message)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def safe_target(relative)
|
|
124
|
+
raise SafetyError, "Output became a symlink: #{@out}" if File.symlink?(@out)
|
|
125
|
+
|
|
126
|
+
current = @out
|
|
127
|
+
components = relative.split("/")
|
|
128
|
+
components.each_with_index do |component, index|
|
|
129
|
+
current = File.join(current, component)
|
|
130
|
+
raise SafetyError, "Refusing output symlink: #{current}" if File.symlink?(current)
|
|
131
|
+
next if index == components.size - 1
|
|
132
|
+
|
|
133
|
+
begin
|
|
134
|
+
Dir.mkdir(current)
|
|
135
|
+
rescue Errno::EEXIST
|
|
136
|
+
raise SafetyError, "Refusing output symlink: #{current}" if File.symlink?(current)
|
|
137
|
+
end
|
|
138
|
+
raise SafetyError, "Not a directory: #{current}" unless File.directory?(current)
|
|
139
|
+
end
|
|
140
|
+
current
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def atomic_write(target)
|
|
144
|
+
Tempfile.create([".sandcrawler-", ".tmp"], File.dirname(target), binmode: true) do |file|
|
|
145
|
+
yield file
|
|
146
|
+
commit_file(file, target)
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def commit_file(file, target)
|
|
151
|
+
file.flush
|
|
152
|
+
file.close
|
|
153
|
+
safe_target(target.delete_prefix(@out + File::SEPARATOR))
|
|
154
|
+
File.rename(file.path, target)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|