sandcrawler 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +5 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +157 -0
  5. data/docs/findings.md +49 -0
  6. data/docs/releasing.md +28 -0
  7. data/docs/verification.md +80 -0
  8. data/exe/sandcrawler +5 -0
  9. data/lib/sandcrawler/acquisition/mounted_volume.rb +81 -0
  10. data/lib/sandcrawler/acquisition/picotool.rb +227 -0
  11. data/lib/sandcrawler/acquisition/serial_protocol.rb +136 -0
  12. data/lib/sandcrawler/acquisition/serial_shell.rb +162 -0
  13. data/lib/sandcrawler/acquisition/serial_templates.rb +63 -0
  14. data/lib/sandcrawler/cli/devices.rb +109 -0
  15. data/lib/sandcrawler/cli/offline.rb +199 -0
  16. data/lib/sandcrawler/cli.rb +149 -0
  17. data/lib/sandcrawler/errors.rb +11 -0
  18. data/lib/sandcrawler/extractor.rb +157 -0
  19. data/lib/sandcrawler/fat/allocation_table.rb +139 -0
  20. data/lib/sandcrawler/fat/boot_sector.rb +142 -0
  21. data/lib/sandcrawler/fat/dir_entry.rb +80 -0
  22. data/lib/sandcrawler/fat/lfn_assembler.rb +60 -0
  23. data/lib/sandcrawler/fat/volume.rb +198 -0
  24. data/lib/sandcrawler/fat/volume_locator.rb +108 -0
  25. data/lib/sandcrawler/image/raw_image.rb +38 -0
  26. data/lib/sandcrawler/image/slice.rb +25 -0
  27. data/lib/sandcrawler/image/source.rb +25 -0
  28. data/lib/sandcrawler/image/uf2_image.rb +83 -0
  29. data/lib/sandcrawler/log.rb +44 -0
  30. data/lib/sandcrawler/manifest.rb +37 -0
  31. data/lib/sandcrawler/path_sanitizer.rb +61 -0
  32. data/lib/sandcrawler/recovery/carver.rb +76 -0
  33. data/lib/sandcrawler/recovery/orphans.rb +52 -0
  34. data/lib/sandcrawler/recovery/undelete.rb +43 -0
  35. data/lib/sandcrawler/version.rb +5 -0
  36. data/lib/sandcrawler.rb +25 -0
  37. metadata +81 -0
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sandcrawler
4
+ class CLI
5
+ module Devices
6
+ private
7
+
8
+ def picotool
9
+ Acquisition::Picotool.new(executable: @options[:picotool], log: @log)
10
+ end
11
+
12
+ def run_doctor(_args)
13
+ @log.warn("Preserve a BOOTSEL dump before using flash_nuke.uf2 or reinstalling firmware.")
14
+ result = { ruby: RUBY_VERSION, platform: RUBY_PLATFORM, serialport: !Gem::Specification.find_all_by_name("serialport").empty? }
15
+ begin
16
+ tool = picotool
17
+ result[:picotool] = { version: tool.version }
18
+ begin
19
+ result[:device] = tool.info
20
+ rescue Error => e
21
+ result[:device] = { available: false, detail: e.message }
22
+ end
23
+ rescue Error => e
24
+ result[:picotool] = { available: false, detail: e.message }
25
+ end
26
+ if RUBY_PLATFORM.include?("linux")
27
+ result[:usb_permissions] =
28
+ "picotool info checks access; install the upstream udev rules if permission is denied"
29
+ end
30
+ emit(result) { result.each { |key, value| @stdout.puts("#{key}: #{value}") } }
31
+ 0
32
+ end
33
+
34
+ def run_dump(_args)
35
+ result = picotool.dump(@options[:out], passes: @options[:passes], force: @options[:force])
36
+ emit(result) { @stdout.puts("Preserved dump in #{@options[:out]}: #{result.dig(:result, :sha256)}") }
37
+ result.dig(:result, :unstable_blocks).to_a.empty? ? 0 : 5
38
+ end
39
+
40
+ def run_copy(args)
41
+ result = Acquisition::MountedVolume.new(args.first).copy(@options[:out], force: @options[:force], log: @log)
42
+ emit(result) { @stdout.puts("Copied #{result[:files].size} files to #{@options[:out]}") }
43
+ result_code(result)
44
+ end
45
+
46
+ def run_serial(_args)
47
+ port = @options[:port] || select_port
48
+ shell = Acquisition::SerialShell.new(port: port, log: @log)
49
+ result = shell.copy(@options[:out], force: @options[:force], root: @options[:root] || "/")
50
+ emit(result) { @stdout.puts("Transferred #{result[:files].size} files to #{@options[:out]}") }
51
+ result_code(result)
52
+ end
53
+
54
+ def select_port
55
+ ports = Acquisition::SerialShell.ports
56
+ raise DeviceNotFoundError, "No serial ports found; pass --port for R2P2 CDC0" if ports.empty?
57
+
58
+ @stderr.puts("Choose R2P2 CDC0 (shell), not CDC1/CDC2:")
59
+ ports.each_with_index { |port, i| @stderr.puts("#{i + 1}: #{port}") }
60
+ index = number(prompt("Port number")) - 1
61
+ raise UsageError, "Invalid serial port selection" unless index.between?(0, ports.size - 1)
62
+
63
+ ports[index]
64
+ end
65
+
66
+ def run_auto(_args)
67
+ base = @options[:out]
68
+ raise SafetyError, "Output is a symlink: #{base}" if File.symlink?(base)
69
+ if File.exist?(base) && (!File.directory?(base) || (!@options[:force] && !Dir.empty?(base)))
70
+ raise SafetyError, "Output must be an empty directory (or use --force): #{base}"
71
+ end
72
+
73
+ route = prompt("Recovery route: [a] BOOTSEL (recommended), [b] mounted volume, [c] serial",
74
+ default: "a").downcase
75
+ raise UsageError, "Choose a, b, or c" unless %w[a b c].include?(route)
76
+
77
+ @options[:out] = File.join(base, "acquisition") if route == "a"
78
+ code = case route
79
+ when "a"
80
+ dump = picotool.dump(@options[:out], passes: @options[:passes], force: @options[:force])
81
+ unless dump.dig(:result, :unstable_blocks).to_a.empty?
82
+ emit(dump) { @stdout.puts("Dump contains unstable blocks; inspect dump.json before recovery") }
83
+ return 5
84
+ end
85
+
86
+ image = File.join(@options[:out], "dump.bin")
87
+ @options[:out] = File.join(base, "recovered")
88
+ run_extract([image])
89
+ when "b"
90
+ run_copy([prompt("Mounted R2P2 directory")])
91
+ when "c"
92
+ run_serial([])
93
+ end
94
+ code
95
+ ensure
96
+ @stderr.puts("Check the dump and recovered files before erasing or reinstalling R2P2. Keep dump.bin.")
97
+ end
98
+
99
+ def prompt(message, default: nil)
100
+ @stderr.print("#{message}: ")
101
+ input = @stdin.gets
102
+ raise UsageError, "Interactive input required; use an explicit command for unattended operation" unless input
103
+
104
+ value = input.strip
105
+ value.empty? && default ? default : value
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,199 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sandcrawler
4
+ class CLI
5
+ module Offline
6
+ private
7
+
8
+ def run_scan(args)
9
+ image = Image.open(args.first)
10
+ candidates = locate(image)
11
+ emit(candidates.map(&:to_h)) do
12
+ candidates.each do |candidate|
13
+ line = format("0x%<offset>08X score=%<score>s %<geometry>s",
14
+ offset: candidate.offset, score: candidate.score, geometry: candidate.boot_sector.to_h)
15
+ @stdout.puts(line)
16
+ end
17
+ end
18
+ 0
19
+ ensure
20
+ image&.close
21
+ end
22
+
23
+ def run_ls(args)
24
+ image = Image.open(args.first)
25
+ volume = open_volume(image)
26
+ entries = volume.each_entry(deleted: @options[:deleted]).to_a
27
+ emit(entries.map(&:to_h)) do
28
+ entries.each do |entry|
29
+ prefix = @options[:long] ? "#{entry.size.to_s.rjust(10)} #{entry.mtime&.iso8601 || '-'} " : ""
30
+ name = Log.escape(entry.path)
31
+ @stdout.puts("#{prefix}#{'[deleted] ' if entry.deleted?}#{name}#{'/' if entry.directory?}")
32
+ end
33
+ end
34
+ volume.warnings.each { |warning| @log.warn(warning.to_s) }
35
+ volume.warnings.empty? ? 0 : 4
36
+ ensure
37
+ image&.close
38
+ end
39
+
40
+ def run_extract(args)
41
+ path = args.first
42
+ protect_input(path)
43
+ protect_input(@options[:reference]) if @options[:reference]
44
+ image = Image.open(path)
45
+ source = { type: "image", path: File.expand_path(path), sha256: Digest::SHA256.file(path).hexdigest,
46
+ size: File.size(path), image_size: image.size }
47
+ if @options[:reference]
48
+ source[:reference_image] = { path: File.expand_path(@options[:reference]),
49
+ sha256: Digest::SHA256.file(@options[:reference]).hexdigest }
50
+ end
51
+ volume = volume_for_extraction(image)
52
+ output = Extractor.new(@options[:out], source: source, volume: volume&.metadata,
53
+ force: @options[:force], log: @log)
54
+ if @options[:reference]
55
+ output.warning("REFERENCE_GEOMETRY",
56
+ "Geometry borrowed from #{@options[:reference]}; verify firmware and board match")
57
+ end
58
+ output.warning("MANUAL_GEOMETRY", @options[:geometry].inspect) if @options[:geometry]
59
+ Array(image.respond_to?(:warnings) ? image.warnings : []).each { |w| record_warning(output, w) }
60
+ if volume
61
+ extract_volume(volume, output)
62
+ else
63
+ output.warning("NO_FAT_VOLUME", "No usable FAT boot sector; carving the image without filesystem metadata")
64
+ end
65
+ carve(image, volume, output) if @options[:carve]
66
+ verify_source(source, output)
67
+ verify_source(source[:reference_image], output) if source[:reference_image]
68
+ manifest = output.finish
69
+ emit(manifest) { @stdout.puts("Recovered #{manifest[:files].size} files to #{output.out}") }
70
+ volume ? result_code(manifest) : 4
71
+ rescue Error, IOError, SystemCallError => e
72
+ if output
73
+ output.warning("EXTRACTION_FAILED", e.message)
74
+ output.finish
75
+ end
76
+ raise
77
+ ensure
78
+ image&.close
79
+ end
80
+
81
+ def verify_source(source, output)
82
+ return if Digest::SHA256.file(source[:path]).hexdigest == source[:sha256]
83
+
84
+ output.warning("SOURCE_CHANGED", "Source changed during recovery: #{source[:path]}")
85
+ raise IntegrityError, "Source changed during recovery; saved results need a fresh acquisition"
86
+ rescue IOError, SystemCallError => e
87
+ output.warning("SOURCE_CHANGED", "Cannot verify source after recovery: #{e.message}")
88
+ raise IntegrityError, "Source could not be verified after recovery"
89
+ end
90
+
91
+ def volume_for_extraction(image)
92
+ open_volume(image)
93
+ rescue FormatError
94
+ raise unless @options[:carve] && !@options[:geometry] && !@options[:reference] && !@options[:offset]
95
+
96
+ nil
97
+ end
98
+
99
+ def extract_volume(volume, output)
100
+ volume.each_entry do |entry|
101
+ if entry.directory?
102
+ output.directory(path: entry.path, mtime: entry.mtime, attributes: entry.attributes,
103
+ first_cluster: entry.first_cluster)
104
+ else
105
+ write_result(output, entry, volume.read(entry), "files")
106
+ end
107
+ end
108
+ if @options[:deleted]
109
+ Recovery::Undelete.new(volume).each { |entry, result| write_result(output, entry, result, "deleted") }
110
+ end
111
+ if @options[:orphans]
112
+ Recovery::Orphans.new(volume).each { |entry, result| write_result(output, entry, result, "lost+found") }
113
+ end
114
+ volume.warnings.each { |warning| record_warning(output, warning) }
115
+ end
116
+
117
+ def write_result(output, entry, result, category)
118
+ output.write(path: entry.path, original_name: entry.name, data: result.data, category: category,
119
+ status: result.status, mtime: entry.mtime, attributes: entry.attributes,
120
+ first_cluster: entry.first_cluster, expected_size: entry.size, warnings: result.warnings)
121
+ result.warnings.each { |warning| record_warning(output, warning) }
122
+ end
123
+
124
+ def record_warning(output, warning)
125
+ if warning.is_a?(Hash)
126
+ output.warning(warning[:code] || warning["code"], warning[:detail] || warning["detail"])
127
+ else
128
+ output.warning("RECOVERY_WARNING", warning.to_s)
129
+ end
130
+ end
131
+
132
+ def carve(image, volume, output)
133
+ boot = volume&.boot_sector
134
+ options = if boot
135
+ { offset: volume.offset + (boot.first_data_sector * boot.bytes_per_sector),
136
+ block_size: boot.cluster_size, length: boot.cluster_count * boot.cluster_size }
137
+ else
138
+ {}
139
+ end
140
+ Recovery::Carver.new(image, **options).each { |entry, result| write_result(output, entry, result, "carved") }
141
+ end
142
+
143
+ def open_volume(image)
144
+ candidate = locate(image).first
145
+ raise FormatError, "No FAT volume found; try --reference-image, --geometry, or extract --carve" unless candidate
146
+
147
+ volume = FAT::Volume.new(image, offset: candidate.offset, boot_sector: candidate.boot_sector,
148
+ fat: @options[:fat], codepage: @options[:codepage])
149
+ volume.warnings.concat(candidate.warnings) if candidate.respond_to?(:warnings)
150
+ volume
151
+ end
152
+
153
+ def locate(image)
154
+ if @options[:geometry]
155
+ offset = @options[:offset] || @options[:geometry][:offset] || 0
156
+ begin
157
+ boot = FAT::BootSector.from_geometry(@options[:geometry], available_size: image.size - offset)
158
+ rescue FormatError => e
159
+ raise UsageError, e.message
160
+ end
161
+ return [manual_candidate(offset, boot)]
162
+ end
163
+ source = @options[:reference] ? Image.open(@options[:reference]) : image
164
+ if @options.key?(:offset)
165
+ offset = @options[:offset]
166
+ boot = FAT::BootSector.new(source.read_at(offset, 512), available_size: image.size - offset)
167
+ return [manual_candidate(offset, boot)]
168
+ end
169
+ candidates = FAT::VolumeLocator.new(source).candidates
170
+ candidates.each { |candidate| @log.debug(candidate.to_h.inspect) }
171
+ candidates
172
+ ensure
173
+ source&.close if source && source != image
174
+ end
175
+
176
+ def manual_candidate(offset, boot)
177
+ Struct.new(:offset, :boot_sector, :score) do
178
+ def to_h
179
+ boot_sector.to_h.merge(offset: offset, score: score)
180
+ end
181
+ end.new(offset, boot, 0)
182
+ end
183
+
184
+ def protect_input(path)
185
+ input = File.realpath(path)
186
+ parent = File.expand_path(@options[:out])
187
+ tail = []
188
+ until File.exist?(parent) || File.symlink?(parent)
189
+ tail.unshift(File.basename(parent))
190
+ parent = File.dirname(parent)
191
+ end
192
+ destination = File.join(File.realpath(parent), *tail)
193
+ return unless input == destination || input.start_with?(destination + File::SEPARATOR)
194
+
195
+ raise SafetyError, "Output must not contain the source image: #{path}"
196
+ end
197
+ end
198
+ end
199
+ end
@@ -0,0 +1,149 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+ require "json"
5
+ require_relative "cli/offline"
6
+ require_relative "cli/devices"
7
+
8
+ module Sandcrawler
9
+ class CLI
10
+ include Offline
11
+ include Devices
12
+
13
+ COMMANDS = %w[doctor dump scan ls extract copy serial auto].freeze
14
+
15
+ def initialize(stdout: $stdout, stderr: $stderr, stdin: $stdin)
16
+ @stdout = stdout
17
+ @stderr = stderr
18
+ @stdin = stdin
19
+ end
20
+
21
+ def run(arguments)
22
+ args = arguments.dup
23
+ return @stdout.puts(VERSION) || 0 if args == ["--version"]
24
+ return @stdout.puts(help) || 0 if args.empty? || %w[-h --help].include?(args.first)
25
+
26
+ command = args.shift
27
+ raise UsageError, "Unknown command: #{command}" unless COMMANDS.include?(command)
28
+
29
+ @options = { passes: 2, codepage: "IBM437" }
30
+ parser = option_parser(command)
31
+ parser.parse!(args)
32
+ return @stdout.puts(parser) || 0 if @options[:help]
33
+
34
+ validate_arguments(command, args)
35
+ @log = Log.new(io: @stderr, verbose: @options[:verbose], quiet: @options[:quiet])
36
+ send("run_#{command}", args)
37
+ rescue UsageError, OptionParser::ParseError => e
38
+ @stderr.puts("#{Log.escape(e.message)}\nUse sandcrawler --help for usage.")
39
+ 2
40
+ rescue DeviceNotFoundError => e
41
+ @stderr.puts(Log.escape(e.message))
42
+ 3
43
+ rescue IntegrityError => e
44
+ @stderr.puts(Log.escape(e.message))
45
+ 5
46
+ rescue Error, SystemCallError, IOError => e
47
+ @stderr.puts(Log.escape(e.message))
48
+ 1
49
+ end
50
+
51
+ private
52
+
53
+ def help
54
+ <<~TEXT
55
+ sandcrawler #{VERSION} — read-only R2P2 file recovery
56
+ Usage: sandcrawler COMMAND [IMAGE|MOUNTPOINT] [options]
57
+ doctor Check environment and optional acquisition tools
58
+ dump -o DIR Preserve BOOTSEL flash with repeated reads
59
+ scan IMAGE List all FAT volume candidates
60
+ ls IMAGE List files (--long, --deleted)
61
+ extract IMAGE -o DIR Recover files (--deleted, --orphans, --carve)
62
+ copy MOUNTPOINT -o DIR Copy a mounted volume without following symlinks
63
+ serial -o DIR Transfer files from CDC0 (--port PORT)
64
+ auto -o DIR Choose a route interactively and recover
65
+ Use COMMAND --help for options. Dump before using flash_nuke.uf2.
66
+ TEXT
67
+ end
68
+
69
+ def option_parser(command)
70
+ OptionParser.new do |parser|
71
+ parser.banner = "Usage: sandcrawler #{command} [IMAGE|MOUNTPOINT] [options]"
72
+ parser.on("-o", "--output DIR", "Output directory") { |v| @options[:out] = v }
73
+ parser.on("--offset N", "Volume byte offset (decimal or 0xHEX)") { |v| @options[:offset] = number(v) }
74
+ parser.on("--geometry VALUES", "bps,spc,rsvd,fats,fatsz,root[,total,offset,root_cluster]") do |v|
75
+ @options[:geometry] = geometry(v)
76
+ end
77
+ parser.on("--reference-image FILE") { |v| @options[:reference] = v }
78
+ parser.on("--fat N", %w[1 2]) { |v| @options[:fat] = v.to_i }
79
+ parser.on("--codepage NAME") do |v|
80
+ @options[:codepage] = Encoding.find(v).name
81
+ rescue ArgumentError
82
+ raise UsageError, "Unknown codepage: #{v}"
83
+ end
84
+ parser.on("--passes N", Integer) { |v| @options[:passes] = v }
85
+ parser.on("--picotool PATH") { |v| @options[:picotool] = v }
86
+ parser.on("--port PORT") { |v| @options[:port] = v }
87
+ parser.on("--root PATH", "Serial source directory (default /)") { |v| @options[:root] = v }
88
+ %i[deleted orphans carve long json verbose quiet force].each do |option|
89
+ parser.on("--#{option}") { @options[option] = true }
90
+ end
91
+ parser.on("-h", "--help") { @options[:help] = true }
92
+ end
93
+ end
94
+
95
+ def validate_arguments(command, args)
96
+ count = %w[scan ls extract copy].include?(command) ? 1 : 0
97
+ raise UsageError, "#{command} expects #{count} input path(s)" unless args.size == count
98
+ if %w[dump extract copy serial auto].include?(command) && !@options[:out]
99
+ raise UsageError, "#{command} requires -o DIR"
100
+ end
101
+ raise UsageError, "--passes must be at least 2" if @options[:passes] < 2
102
+ return unless @options[:geometry] && @options[:reference]
103
+
104
+ raise UsageError, "Choose --geometry or --reference-image"
105
+ end
106
+
107
+ def number(value)
108
+ result = Integer(value, value.start_with?("0x", "0X") ? 16 : 10)
109
+ raise ArgumentError if result.negative?
110
+
111
+ result
112
+ rescue ArgumentError
113
+ raise UsageError, "Invalid nonnegative integer: #{value}"
114
+ end
115
+
116
+ def geometry(value)
117
+ allowed = %w[bps spc rsvd fats fatsz root total totsec offset root_cluster media]
118
+ fields = value.split(",", -1).each_with_object({}) do |field, result|
119
+ key, number = field.split("=", 2)
120
+ raise UsageError, "Invalid geometry field: #{field}" unless allowed.include?(key) && number
121
+ raise UsageError, "Duplicate geometry field: #{key}" if result.key?(key.to_sym)
122
+
123
+ result[key.to_sym] = self.number(number)
124
+ end
125
+ missing = %i[bps spc rsvd fats fatsz root] - fields.keys
126
+ raise UsageError, "Missing geometry fields: #{missing.join(', ')}" unless missing.empty?
127
+ raise UsageError, "Specify either total or totsec, not both" if fields.key?(:total) && fields.key?(:totsec)
128
+
129
+ fields
130
+ end
131
+
132
+ def emit(value)
133
+ if @options[:json]
134
+ @stdout.puts(JSON.pretty_generate(Manifest.json_safe(value)))
135
+ elsif !@options[:quiet]
136
+ yield if block_given?
137
+ end
138
+ end
139
+
140
+ def result_code(manifest)
141
+ loss_codes = %w[DIRECTORY_DAMAGED DIRECTORY_ERROR DIRECTORY_LOOP DIRECTORY_WRITE_ERROR COPY_FAILED LIST_FAILED
142
+ FILE_ERROR UF2_INCOMPLETE FAT_ERASED_ENTRY CHAIN_ERROR CHAIN_FREE CHAIN_TOO_LONG CROSS_LINK
143
+ DUPLICATE_ENTRY]
144
+ partial = manifest[:files].any? { |file| file[:status] != "ok" }
145
+ partial ||= manifest[:warnings].any? { |warning| loss_codes.include?(warning[:code]) }
146
+ partial ? 4 : 0
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sandcrawler
4
+ class Error < StandardError; end
5
+ class UsageError < Error; end
6
+ class SafetyError < Error; end
7
+ class DeviceNotFoundError < Error; end
8
+ class IntegrityError < Error; end
9
+ class OutOfRangeError < Error; end
10
+ class FormatError < Error; end
11
+ end
@@ -0,0 +1,157 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "tempfile"
5
+ require "json"
6
+ require "digest"
7
+
8
+ module Sandcrawler
9
+ class Extractor
10
+ CATEGORIES = [nil, "files", "deleted", "lost+found", "carved"].freeze
11
+ attr_reader :out, :manifest
12
+
13
+ def initialize(out, source:, volume: nil, force: false, log: nil)
14
+ @out = File.expand_path(out)
15
+ raise SafetyError, "Output is a symlink: #{@out}" if File.symlink?(@out)
16
+ if File.exist?(@out) && (!File.directory?(@out) || (!force && !Dir.empty?(@out)))
17
+ raise SafetyError, "Output must be an empty directory (or use --force): #{@out}"
18
+ end
19
+
20
+ FileUtils.mkdir_p(@out)
21
+ @out = File.realpath(@out)
22
+ @manifest = Manifest.new(source: source, volume: volume)
23
+ @paths = PathSanitizer.new
24
+ @log = log || Log.new
25
+ @directory_times = []
26
+ end
27
+
28
+ def write(path:, data:, category: "files", status: "ok", mtime: nil, original_name: nil, **metadata)
29
+ validate_category(category)
30
+ entry = metadata.merge(path: path, original_name: original_name || path.to_s.b.split("/").last,
31
+ status: status, mtime: mtime&.iso8601)
32
+ %i[path original_name].each do |key|
33
+ text = entry[key].to_s.dup.force_encoding(Encoding::UTF_8)
34
+ entry[:"#{key}_hex"] = text.b.unpack1("H*") unless text.valid_encoding?
35
+ entry[key] = text.scrub
36
+ end
37
+ relative = output_path(path, category, status)
38
+ target = safe_target(relative)
39
+ Tempfile.create([".sandcrawler-", ".tmp"], File.dirname(target), binmode: true) do |file|
40
+ begin
41
+ data.is_a?(String) ? file.write(data.b) : IO.copy_stream(data, file)
42
+ rescue IOError, SystemCallError => e
43
+ raise unless file.size.positive?
44
+
45
+ warning("FILE_ERROR", "#{path}: #{e.message}; preserving readable prefix")
46
+ entry[:status] = "read_error"
47
+ relative = output_path(path, category, "read_error")
48
+ target = safe_target(relative)
49
+ end
50
+ if entry[:status] == "ok" && entry[:expected_size] && file.size != entry[:expected_size]
51
+ warning("SIZE_MISMATCH", "#{path}: expected #{entry[:expected_size]} bytes, recovered #{file.size}")
52
+ entry[:status] = file.size < entry[:expected_size] ? "truncated" : "read_error"
53
+ relative = output_path(path, category, entry[:status])
54
+ target = safe_target(relative)
55
+ end
56
+ commit_file(file, target)
57
+ end
58
+ entry.merge!(output: relative, size: File.size(target), sha256: Digest::SHA256.file(target).hexdigest)
59
+ restore_time(target, mtime) if mtime
60
+ @manifest.data[:files] << entry
61
+ entry
62
+ rescue IOError, SystemCallError => e
63
+ warning("FILE_ERROR", "#{path}: #{e.message}")
64
+ entry ||= { path: path, original_name: original_name }
65
+ entry.merge!(status: "read_error", output: nil, sha256: nil)
66
+ @manifest.data[:files] << entry
67
+ entry
68
+ end
69
+
70
+ def directory(path:, category: "files", mtime: nil, **metadata)
71
+ validate_category(category)
72
+ entry = metadata.merge(path: path, original_name: path.to_s.b.split("/").last,
73
+ mtime: mtime&.iso8601, status: "ok")
74
+ relative = @paths.path(path, namespace: category, directory: true)
75
+ target = safe_target("#{relative}/.directory-check")
76
+ @directory_times << [File.dirname(target), mtime] if mtime
77
+ @manifest.data[:directories] << entry.merge(output: relative)
78
+ relative
79
+ rescue IOError, SystemCallError => e
80
+ warning("DIRECTORY_WRITE_ERROR", "#{path}: #{e.message}")
81
+ @manifest.data[:directories] << entry.merge(status: "write_error", output: nil)
82
+ nil
83
+ end
84
+
85
+ def warning(code, detail)
86
+ @log.warn("#{code}: #{detail}")
87
+ @manifest.warning(code, detail)
88
+ end
89
+
90
+ def finish
91
+ @directory_times.reverse_each do |path, time|
92
+ raise SafetyError, "Output became a symlink: #{path}" if File.symlink?(path)
93
+
94
+ File.utime(time, time, path)
95
+ rescue SystemCallError => e
96
+ warning("TIMESTAMP_ERROR", e.message)
97
+ end
98
+ result = Manifest.json_safe(@manifest.data)
99
+ @log.info("Manifest: files=#{result[:files].size}, directories=#{result[:directories].size}, " \
100
+ "warnings=#{result[:warnings].size}")
101
+ atomic_write(safe_target("manifest.json")) { |file| file.write("#{JSON.pretty_generate(result)}\n") }
102
+ atomic_write(safe_target("sandcrawler.log")) { |file| file.write("#{@log.lines.join("\n")}\n") }
103
+ result
104
+ end
105
+
106
+ private
107
+
108
+ def validate_category(category)
109
+ raise UsageError, "Unknown output category: #{category}" unless CATEGORIES.include?(category)
110
+ end
111
+
112
+ def output_path(path, category, status)
113
+ suffix = %w[truncated chain_error partial read_error transfer_error].include?(status) ? ".partial" : ""
114
+ @paths.path("#{path}#{suffix}", namespace: category)
115
+ end
116
+
117
+ def restore_time(target, time)
118
+ File.utime(time, time, target)
119
+ rescue SystemCallError => e
120
+ warning("TIMESTAMP_ERROR", e.message)
121
+ end
122
+
123
+ def safe_target(relative)
124
+ raise SafetyError, "Output became a symlink: #{@out}" if File.symlink?(@out)
125
+
126
+ current = @out
127
+ components = relative.split("/")
128
+ components.each_with_index do |component, index|
129
+ current = File.join(current, component)
130
+ raise SafetyError, "Refusing output symlink: #{current}" if File.symlink?(current)
131
+ next if index == components.size - 1
132
+
133
+ begin
134
+ Dir.mkdir(current)
135
+ rescue Errno::EEXIST
136
+ raise SafetyError, "Refusing output symlink: #{current}" if File.symlink?(current)
137
+ end
138
+ raise SafetyError, "Not a directory: #{current}" unless File.directory?(current)
139
+ end
140
+ current
141
+ end
142
+
143
+ def atomic_write(target)
144
+ Tempfile.create([".sandcrawler-", ".tmp"], File.dirname(target), binmode: true) do |file|
145
+ yield file
146
+ commit_file(file, target)
147
+ end
148
+ end
149
+
150
+ def commit_file(file, target)
151
+ file.flush
152
+ file.close
153
+ safe_target(target.delete_prefix(@out + File::SEPARATOR))
154
+ File.rename(file.path, target)
155
+ end
156
+ end
157
+ end