shell_helpers 0.1.0 → 0.6.0

data/lib/shell_helpers/sysutils.rb

@@ -0,0 +1,427 @@
+require 'shell_helpers/export'
+require 'shell_helpers/sh'
+require 'shell_helpers/pathname'
+require 'shellwords'
+
+module ShellHelpers
+
+	module SysUtils
+		extend self
+		SysError=Class.new(StandardError)
+
+		# wrap 'stat'
+		def stat_file(file)
+			require 'time'
+			opts=%w(a b B f F g G h i m n N o s u U w x y z)
+			stats=Run.run_simple("stat --format='#{opts.map{|o| "%#{o}\n"}.join}' #{file.shellescape}", chomp: :lines)
+			r={}
+			r[:access]=stats[0]
+			r[:blocknumber]=stats[1].to_i
+			r[:blocksize]=stats[2].to_i
+			r[:rawmode]=stats[3]
+			r[:filetype]=stats[4]
+			r[:gid]=stats[5].to_i
+			r[:group]=stats[6]
+			r[:hardlinks]=stats[7].to_i
+			r[:inode]=stats[8].to_i
+			r[:mountpoint]=stats[9]
+			r[:filename]=stats[10]
+			r[:quotedfilename]=stats[11]
+			r[:optimalsize]=stats[12]
+			r[:size]=stats[13].to_i
+			r[:uid]=stats[14].to_i
+			r[:user]=stats[15]
+			r[:birthtime]  = begin Time.parse(stats[16]) rescue nil end
+			r[:accesstime] = begin Time.parse(stats[17]) rescue nil end
+			r[:changedtime]= begin Time.parse(stats[18]) rescue nil end
+			r[:statustime] = begin Time.parse(stats[19]) rescue nil end
+			r
+		end
+		# wrap stat --file-system
+		def stat_filesystem(file, up: true)
+			if up
+				file=Pathname.new(file)
+				file.ascend.each do |f|
+					return stat_filesystem(f, up: false) if f.exist?
+				end
+			end
+			opts=%w(a b c d f i l n s S T)
+			stats=Run.run_simple("stat --file-system --format='#{opts.map{|o| "%#{o}\n"}.join}' #{file.shellescape}", chomp: :lines)
+			stats=stats.each_line.map {|l| l.chomp}
+			r={}
+			r[:userfreeblocks]=stats[0].to_i
+			r[:totalblocks]=stats[1].to_i
+			r[:totalnodes]=stats[2].to_i
+			r[:freenodes]=stats[3].to_i
+			r[:freeblocks]=stats[4].to_i
+			r[:fsid]=stats[5]
+			r[:maxlength]=stats[6].to_i
+			r[:name]=stats[7]
+			r[:blocksize]=stats[8].to_i
+			r[:innerblocksize]=stats[9].to_i
+			r[:fstype]=stats[10]
+			r
+		end
+
+		def parse_blkid(output)
+			devs={}
+			r=[]
+			convert=lambda do |h|
+				h[:type] && h[:fstype]=h.delete(:type)
+				name=h[:devname]
+				devs[name]=h
+			end
+			output=output.each_line if output.is_a?(String)
+			output.each do |l|
+				l=l.chomp
+				if l.empty?
+					convert.(Export.import_parse(r))
+					r=[]
+				else
+					r<<l
+				end
+			end
+			convert.(Export.import_parse(r)) unless r.empty?
+			devs
+		end
+
+		# output should be the result of `blkid -o export ...`
+		# return a list of things like
+		#  {:devname=>"/dev/sda2",
+		#   :label=>"swap",
+		#   :uuid=>"82af0d2f-5ef6-418a-8656-bdfe843f19e1",
+		#   :type=>"swap",
+		#   :partlabel=>"swap",
+		#   :partuuid=>"f4eef373-0803-4701-bd47-b968c44065a6"}
+		def blkid(*args, sudo: false)
+			# get devname, (part)label/uuid, fstype
+			fsoptions=Run.run_simple("blkid -o export #{args.shelljoin}", fail_mode: :empty, chomp: true, sudo: sudo)
+			parse_blkid(fsoptions)
+		end
+
+		# use lsblk to get infos about devices
+		def lsblk(sudo: false)
+			# get devname, mountpoint, (part)label/uuid, (part/dev/fs)type
+			fsoptions=Run.run_simple("lsblk -l -J -o NAME,MOUNTPOINT,LABEL,UUID,PARTLABEL,PARTUUID,PARTTYPE,TYPE,FSTYPE", fail_mode: :empty, chomp: true, sudo: sudo)
+			require 'json'
+			json=JSON.parse(fsoptions)
+			fs={}
+			json["blockdevices"]&.each do |props|
+				r={}
+				props.each do |k,v|
+					k=k.to_sym
+					k=:devtype if k==:type
+					if k==:name
+						k=:devname
+						v="/dev/#{v}"
+					end
+					r[k]=v unless v.nil?
+				end
+				fs[r[:devname]]=r
+			end
+			fs
+		end
+
+		# use findmnt to get infos about mount points
+		def findmnt(sudo: false)
+			# get devname, mountpoint, mountoptions, (part)label/uuid, fsroot
+			# only looks at mounted devices (but in comparison to lsblk also show
+			# virtual mounts and bind mounts)
+			fsoptions=SH::Run.run_simple("findmnt --raw -o SOURCE,TARGET,FSTYPE,OPTIONS,LABEL,UUID,PARTLABEL,PARTUUID,FSROOT", fail_mode: :empty, chomp: true, sudo: sudo)
+			fs={}
+			fsoptions.each_line.to_a[1..-1]&.each do |l|
+				#two '	' means a missing option, so we want to split on / /, not on ' '
+				source,target,fstype,options,label,uuid,partlabel,partuuid,fsroot=l.chomp.split(/ /)
+				next unless source=~%r(^/dev/) #skip non dev mountpoints
+				options=options.split(',')
+				fs[source]={mountpoint: target, devname: source, fstype: fstype, mountoptions: options, label: label, uuid: uuid, partlabel: partlabel, partuuid: partuuid, fsroot: fsroot}
+			end
+			fs
+		end
+		
+		def fs_infos(mode: :devices)
+			return findmnt if mode == :mount
+			return lsblk.merge(findmnt) if mode == :all
+			# :devname, :devtype, :mountpoint, [:mountoptions], :label, :uuid, :partlabel, :partuuid, :parttype, :fstype, [:fsroot]
+			lsblk
+		end
+
+		def refresh_blkid_cache
+			Sh.sh("blkid", sudo: true)
+		end
+
+		# find devices matching props
+		def find_devices(props, method: :all)
+			props=props.clone
+			return [{devname: props[:devname]}] unless props[:devname].nil?
+			# name is both for label and partlabel
+			if props.key?(:name)
+				props[:label] = props[:name] unless props.key?(:label)
+				props[:partlabel] = props[:name] unless props.key?(:partlabel)
+			end
+
+			if method==:blkid
+				# Warning, since 'blkid' can only test one label, we cannot check
+				# that all parameters are valid
+				# search from most discriminant to less discriminant
+				%i(uuid label partuuid partlabel).each do |key|
+					if (label=props[key])
+						return parse_blkid(%x/blkid -o export -t #{key.to_s.upcase}=#{label.shellescape}/).values
+					end
+				end
+				# unfortunately `blkid PARTTYPE=...` does not work, so we need to parse
+				# ourselves
+				if props[:parttype]
+					find_devices(props, method: :all)
+				end
+			else
+				fs=fs_infos
+				# here we check all parameters
+				# however, if none are defined, this return true, so we check that at least one is defined
+				return [] unless %i(uuid label partuuid partlabel parttype).any? {|k| props[k]}
+				return fs.keys.select do |k|
+					fsprops=fs[k]
+					# the fsinfos should have one of this parameters defined
+					next false unless %i(uuid label partuuid partlabel parttype).any? {|k| fsprops[k]}
+					next false if (disk=props[:disk]) && !fsprops[:devname].start_with?(disk.to_s)
+					%i(uuid label partuuid partlabel parttype).all? do |key|
+						ptype=props[key]
+						ptype=partition_type(ptype) if key==:parttype and ptype.is_a?(Symbol)
+						!ptype or !fsprops[key] or ptype==fsprops[key]
+					end
+				end.map {|k| fs[k]}
+			end
+			return []
+		end
+
+		# like find_devices but warn out if the result is of length > 1
+		def find_device(props)
+			devs=find_devices(props)
+			devs=yield(devs) if block_given?
+			devs=[devs].flatten
+			warn "Device #{props} not found" if devs.empty?
+			warn "Several devices for #{props} found: #{devs.map {|d| d&.fetch(:devname)}}" if devs.length >1
+			return devs.first&.fetch(:devname)
+		end
+
+		def mount(paths, mkpath: true, abort_on_error: true, sort: true)
+			paths=paths.values if paths.is_a?(Hash)
+			paths=paths.select {|p| p[:mountpoint]}
+			# sort so that the mounts are in correct order
+			paths=paths.sort { |p1, p2| Pathname.new(p1[:mountpoint]) <=> Pathname.new(p2[:mountpoint]) } if sort
+			close=lambda do
+				umount(paths, sort: sort)
+			end
+			paths.each do |path|
+				dev=find_device(path)
+				raise SysError.new("Device #{path} not found") unless dev
+				options=path[:mountoptions]||[]
+				options=options.split(',') if options.is_a?(String)
+				options<<"subvol=#{path[:subvol].shellescape}" if path[:subvol]
+				#options=options.join(',') if options.is_a?(Array)
+				mntpoint=Pathname.new(path[:mountpoint])
+				mntpoint.sudo_mkpath if mkpath
+				cmd="mount #{(fs=path[:fstype]) && "-t #{fs.shellescape}"} #{options.empty? ? "" : "-o #{options.join(',').shellescape}"} #{dev.shellescape} #{mntpoint.shellescape}"
+				abort_on_error ? Sh.sh!(cmd, sudo: true) : Sh.sh(cmd, sudo: true)
+			end
+			if block_given?
+				begin
+					yield paths
+				ensure
+					close.call
+				end
+			end
+			return paths, close
+		end
+
+		def umount(paths, sort: true)
+			paths=paths.values if paths.is_a?(Hash)
+			paths=paths.select {|p| p[:mountpoint]}
+			paths=paths.sort { |p1, p2| Pathname.new(p1[:mountpoint]) <=> Pathname.new(p2[:mountpoint]) } if sort
+			paths.reverse.each do |path|
+				mntpoint=path[:mountpoint]
+				Sh.sh("umount #{mntpoint.shellescape}", sudo: true)
+			end
+		end
+
+		def partition_type(type, mode: :guid)
+			if mode==:symbol
+				%i(boot swap home x86_root x86-64_root arm64_root arm32_root linux).each do |symb|
+					%i(hexa guid).each do |mode|
+						partition_type(symb, mode: mode) == type.downcase and return symb
+					end
+				end
+			end
+			case type
+			when :boot
+				mode == :hexa ? "ef00" : "c12a7328-f81f-11d2-ba4b-00a0c93ec93b"
+			when :swap
+				mode == :hexa ? "8200" : "0657fd6d-a4ab-43c4-84e5-0933c84b4f4f"
+			when :home
+				mode == :hexa ? "8302" : "933ac7e1-2eb4-4f13-b844-0e14e2aef915"
+			when :x86_root
+				mode == :hexa ? "8303" : "44479540-f297-41b2-9af7-d131d5f0458a"
+			when :"x86-64_root"
+				mode == :hexa ? "8304" : "4f68bce3-e8cd-4db1-96e7-fbcaf984b709"
+			when :arm64_root
+				mode == :hexa ? "8305" : "b921b045-1df0-41c3-af44-4c6f280d3fae"
+			when :arm32_root
+				mode == :hexa ? "8307" : "69dad710-2ce4-4e3c-b16c-21a1d49abed3"
+			when :linux
+				mode == :hexa ? "8300" : "0fc63daf-8483-4772-8e79-3d69d8477de4"
+			end
+		end
+
+		def partition_infos(device, sudo: false)
+			parts = Run.run_simple("partx -o NR --show #{device.shellescape}", sudo: sudo) { return nil }
+			infos=[]
+			nums=parts.each_line.count - 1
+			(1..nums).each do |i|
+				infos[i-1]={}
+				part_options=Run.run_simple("sgdisk -i#{i} #{device.shellescape}", chomp: true, sudo: sudo)
+				part_options.match(/^Partition name: '(.*)'/) do |m|
+					infos[i-1][:partlabel]=m[1]
+				end
+				part_options.match(/^Attribute flags: (.*)/) do |m|
+					infos[i-1][:partattributes]=m[1]
+				end
+				part_options.match(/^Partition unique GUID: (.*)/) do |m|
+					infos[i-1][:partuuid]=m[1].downcase
+				end
+				part_options.match(/^Partition GUID code: (\S*)/) do |m|
+					infos[i-1][:parttype]=m[1].downcase
+				end
+			end
+			infos
+		end
+
+		#options: check => check that no partitions exist first
+		def make_partitions(partitions, check: true, partprobe: true)
+			partitions=partitions.values if partitions.is_a?(Hash)
+			done=[]
+			disk_partitions=partitions.group_by {|p| p[:disk]}
+			disk_partitions.each do |disk, dpartitions|
+				next if disk.nil?
+				if check
+					partinfos=blkid(disk, sudo: true)
+					# gpt partitions: PTUUID="652121ab-7935-403c-8b87-65a149a415ac" PTTYPE="gpt"
+					# dos partitions: PTUUID="17a4a006" PTTYPE="dos"
+					# others: PTTYPE="PMBR"
+					unless partinfos.empty?
+						raise SysError("Disk #{disk} is not empty: #{partinfos}") if check==:raise
+						warn "Disk #{disk} is not empty: #{partinfos}, skipping..."
+						next
+					end
+				end
+				opts=[]
+				dpartitions.each do |partition|
+					next unless %i(partnum partstart partlength partlabel partattributes parttype).any? {|k| partition.key?(k)}
+					num=partition[:partnum]&.to_i || 0
+					start=partition[:partstart] || 0
+					length=partition[:partlength] || 0
+					name=partition[:partlabel] || partition[:name]
+					attributes=partition[:partattributes]
+					type=partition[:parttype]
+					attributes=2 if type==:boot
+					type=partition_type(type, mode: :hexa) if type.is_a?(Symbol)
+					uuid=partition[:partuuid]
+					alignment=partition[:partalignment]
+					opts += ["-n", "#{num}:#{start}:#{length}"]
+					opts += ["-c", "#{num}:#{name}"] if name
+					opts += ["-t", "#{num}:#{type}"] if type
+					opts += ["-u", "#{num}:#{uuid}"] if uuid
+					opts << "--attributes=#{num}:set:#{attributes}" if attributes
+					opts << ["--set-alignment=#{alignment}"] if alignment
+				end
+				unless opts.empty?
+					Sh.sh!("sgdisk #{opts.shelljoin} #{disk.shellescape}", sudo: true)
+					done << disk
+				end
+			end
+			SH.sh("partprobe #{done.shelljoin}", sudo: true) unless done.empty? or !partprobe
+			done
+		end
+
+		def zap_partitions(disk)
+			# Zap (destroy) the GPT and MBR data  structures  and  then  exit.
+			Sh.sh("sgdisk --zap-all #{disk.shellescape}", sudo: true)
+		end
+		def wipefs(disk)
+			# wipe all signatures
+			Sh.sh("wipefs -a #{disk.shellescape}", sudo: true)
+		end
+
+		def make_fs(fs, check: true)
+			fs=fs.values if fs.is_a?(Hash)
+			fs.each do |partfs|
+				dev=SH.find_device(partfs)
+				if dev and (fstype=partfs[:fstype])
+					opts=partfs[:fsoptions]||[]
+					bin="mkfs.#{fstype.to_s.shellescape}"
+					bin="mkswap" if fstype.to_s=="swap"
+					label=partfs[:label]||partfs[:name]
+					if label
+						labelkey="-L"
+						labelkey="-n" if fstype.to_s=="vfat"
+						opts+=[labelkey, label]
+					end
+					if check
+						diskinfos=blkid(dev, sudo: true)
+						unless diskinfos.dig(dev,:fstype).nil?
+							raise SysError("Device #{dev} already has a filesystem: #{diskinfos[dev]}") if check==:raise
+							warn "Device #{dev} already has a filesystem: #{diskinfos[dev]}"
+							next
+						end
+					end
+					SH.sh("#{bin} #{opts.shelljoin} #{dev.shellescape}", sudo: true)
+				end
+			end
+		end
+
+		def make_raw_image(name, size="1G")
+			raw=Pathname.new(name)
+			raw.touch
+			rawfs=stat_filesystem(raw)
+			raw.chattr("+C") if rawfs[:fstype]=="btrfs"
+			Sh.sh("fallocate -l #{size} #{raw.shellescape}")
+			raw
+		end
+
+		def make_btrfs_subvolume(dir, check: true)
+			if check and dir.directory?
+				raise SysError("Subvolume already exists at #{dir}") if check==:raise
+				warn "Subvolume already exists at #{dir}, skipping..."
+			else
+				SH.sh("btrfs subvolume create #{dir.shellescape}", sudo: true)
+				dir
+			end
+		end
+		def make_dir_or_subvolume(dir)
+			dir=Pathname.new(dir)
+			return :directory if dir.directory?
+			fstype=stat_filesystem(dir, up: true)
+			if fstype[:fstype]=="btrfs"
+				make_btrfs_subvolume(dir)
+				return :subvol
+			else
+				dir.sudo_mkpath
+				return :directory
+			end
+		end
+
+		def losetup(img)
+			disk = Run.run_simple("losetup -f --show #{img.shellescape}", sudo: true, chomp: true, error_mode: :nil)
+			close=lambda do
+				SH.sh("losetup -d #{disk.shellescape}", sudo: true) if disk
+			end
+			if block_given?
+				begin
+					yield disk
+				ensure
+					close.call
+				end
+			end
+			return disk, close
+		end
+	end
+end

data/lib/shell_helpers/utils.rb

@@ -1,95 +1,38 @@
 require 'shellwords'
-require 'dr/ruby_ext/core_ext'
-require_relative 'pathname'
-require_relative 'parser'
+require 'shell_helpers/pathname'
+require 'dr/base/uri'
 
-module SH
-	module ShellExport
-		extend self
+module ShellHelpers
 
-		#export a value for SHELL consumption
-		def export_value(v)
-			case v
-				when String
-					return v.shellescape
-				when Array
-					return "(#{v.map {|i| i.to_s.shellescape}.join(' ')})"
-				when Hash
-					return "(#{v.map {|k,v| k.to_s.shellescape+" "+v.to_s.shellescape}.join(' ')})"
-				when nil
-					return ""
-				else
-					return v.to_s.shellescape
-			end
+	module ExtendSSHKit
+		def backend(&b)
+			local? ? SSHKit::Backend::Local.new(&b) : SSHKit.config.backend.new(self, &b)
 		end
-
-		#from {ploum: plim} return something like
-		#ploum=plim
-		#that can be evaluated by the shell
-		def export_hash(hash, local: false, export: false, prefix:"")
-			r=""
-			r+="local #{hash.keys.map {|s| s.to_s.upcase}.join(" ")}\n" if local
-			hash.each do |k,v|
-				name=prefix+k.to_s.upcase
-				r+="typeset -A #{name};\n" if Hash === v
-				r+=name+"="+export_value(v)+";\n"
-			end
-			r+="export #{hash.keys.map {|k| prefix+k.to_s.upcase}.join(" ")}\n" if export
-			return r
+		def connect(&b)
+			backend(&b).run
 		end
+	end
 
-		#export_variable("ploum","plam") yields ploum="plam"
-		def export_variable(name, value, local: false, export: false)
-			r=""
-			#name=name.upcase
-			r+="local #{name}\n" if local
-			r+="typeset -A #{name};\n" if Hash === value
-			r+=name+"="+export_value(value)+";\n"
-			r+="export #{name}\n" if export
-			return r
-		end
+	module Utils
+		extend self
 
-		#export_parse(hash,"name:value")
-		#will output name=$(hash[value])
-		#special cases: when value = '/' we return the full hash
-		#		when value ends by /, we return the splitted hash (and name serves
-		#		as a prefix)
-		#Ex: Numenor ~ $ ./mine/00COMPUTERS.rb --export=//
-		#		 HOSTNAME=Numenor;
-		#		 HOSTTYPE=perso;
-		#		 HOMEPATH=/home/dams;...
-		#		 Numenor ~ $ ./mine/00COMPUTERS.rb --export=syst/
-		#		 LAPTOP=true;
-		#		 ARCH=i686;...
-		#Remark: in name:value, we don't put name in uppercase
-		#But in split hash mode, we put the keys in uppercase (to prevent collisions)
-		def export_parse(hash,s)
-			r=""
-			args=Parser.parse_string(s)
-			args[:values].each do |k,v|
-				name=k
-				if !v
-					v=name
-					#in split mode, don't use prefix if name gave the value
-					name= name.size==1? "all" : "" if name[name.size-1]=="/"
-				end
-				if v.size>1 && v[v.size-1]=='/'
-					all=true
-					v=v[0...v.size-1]
-				end
-				value=hash.keyed_value(v)
-				if all
-					r+=export_hash(value, local: args[:opts][k]["local"], export: args[:opts][k]["export"], prefix: name)
-				else
-					r+=export_variable(name,value, local: args[:opts][k]["local"], export: args[:opts][k]["export"])
-				end
+		def eval_shell(r, shell: :puts)
+			return r if r.nil? or r.empty?
+			case (shell||"").to_sym
+			when :puts
+				puts r
+			when :eval
+				r+=";" if r && !r.end_with?(';')
+				print r
+			when :exec
+				require 'shell_helpers/sh'
+				return ShLog.sh(r)
+			when :exec_quiet
+				require 'shell_helpers/sh'
+				return Sh.sh(r)
 			end
 			return r
 		end
-	end
-
-	module ShellUtils
-		extend self
 
 		class << self
 			attr_accessor :orig_stdin, :orig_stdout, :orig_stderr
@@ -99,45 +42,68 @@ module SH
 		@orig_stderr=$stderr
 
 		#An improved find from Find::find that takes in the block the absolute and relative name of the files (+the directory where the relative file is from), and has filter options
-		def find(*paths, filter: nil, follow_symlink: false, depth: nil)
-			block_given? or return enum_for(__method__, *paths, filter: filter, follow_symlink: follow_symlink, depth: depth)
-
-			paths.collect!{|d| raise Errno::ENOENT unless File.exist?(d); Pathname.new(d.dup)}
-			paths.collect!{|d| [ d, Pathname.new('.'), d ]}
-			while filedata = paths.shift
-				file, filerel, fileabs = *filedata
-				catch(:prune) do #use throw(:prune) to skip a path
-					Dir.chdir(fileabs) do
-						#if the filter is true, we don't yield the file
-						case filter
-						when Proc
-							filter.call(file,filerel,fileabs)
-						when Array
-							filter.all? do |test|
-								case test
-								when :directory? #special case
-									file.directory? && !file.symlink?
-								else
-									file.send(test)
-								end
+		#Returns ::Pathname, except when the value is a SH::Pathname where it
+		#returns a SH::Pathname
+		def find(*bases, filter: nil, prune: nil, follow_symlink: false, depth: false, max_depth: nil, chdir: false)
+			block_given? or return enum_for(__method__, *bases, filter: filter, follow_symlink: follow_symlink, depth: depth, max_depth: max_depth, chdir: chdir)
+			bases.collect!{|d| raise Errno::ENOENT unless File.exist?(d); d.dup}.each do |base|
+				klass=base.is_a?(::Pathname) ? base.class : ::Pathname
+				base=klass.new(base)
+
+				test_filter=lambda do |filter,*files|
+					case filter
+					when Proc
+						filter.call(*files)
+					when Array
+						file=files.first
+						filter.any? do |test|
+							case test
+							when :directory? #special case
+								file.directory? && !file.symlink?
+							else
+								file.send(test)
 							end
-						end or yield file.dup.taint, filerel.dup.taint, fileabs.dup.taint
-						if file.directory? and (depth.nil? or filerel.each_filename.size <= depth) then
-							next if !follow_symlink && file.symlink?
-							file.children(false).sort.reverse_each do |f|
-								fj = file + f
-								f = filerel + f
-								paths.unshift [fj.untaint,f.untaint,fileabs]
+						end
+					end
+				end
+
+				yield_files=lambda do |*files|
+					unless test_filter.(filter,*files)
+						files.map! {|f| f.dup.taint}
+						if chdir
+							Dir.chdir(base) do
+								yield *files, base
+							end
+						else
+							yield *files, base
+						end
+					end
+				end
+
+				do_find=lambda do |*files|
+					file,filerel=*files
+					catch(:prune) do #use throw(:prune) to skip a path (recursively)
+						unless test_filter.(prune,*files)
+							yield_files.(*files) unless depth
+							if file.directory? and (max_depth.nil? or (filerel.to_s=="." and max_depth>0) or filerel.each_filename.to_a.size < max_depth)
+								next if !follow_symlink && file.symlink?
+								file.children(false).sort.reverse_each do |f|
+									fj = file + f
+									f = filerel + f
+									do_find.(fj.untaint,f.untaint)
+								end
+								yield_files.(*files) if depth
 							end
 						end
 					end
 				end
+				do_find.call(base, klass.new('.'))
 			end
 		end
 
 		#all output is sent to the pager
-		def run_pager(opt=nil)
-			return unless $stdout.tty? and opt != :never
+		def run_pager(*args, launch: :tty, default_less_env: "-FRX")
+			return unless $stdout.tty? and launch != :never
 			read, write = IO.pipe
 
 			unless Kernel.fork # Child process
@@ -154,16 +120,17 @@ module SH
 			write.close
 
 			#ENV['LESS'] = 'FSRX' # Don't page if the input is short enough
-			lessenv=ENV['LESS']
-			lessenv="-FRX" if lessenv.empty?
-			lessenv+="F" unless lessenv.match(/F/) or opt == :always
-			lessenv+="R" unless lessenv.match(/R/)
-			lessenv+="X" unless lessenv.match(/X/)
-			ENV['LESS']=lessenv
+			less_env=ENV['LESS']
+			less_env=default_less_env if less_env.empty?
+			less_env+="F" unless less_env.match(/F/) or launch == :always
+			less_env+="R" unless less_env.match(/R/)
+			less_env+="X" unless less_env.match(/X/)
+			ENV['LESS']=less_env
 
 			Kernel.select [$stdin] # Wait until we have input before we start the pager
 			pager = ENV['PAGER'] || 'less'
-			exec pager rescue exec "/bin/sh", "-c", pager
+			run=args.unshift(pager)
+			exec *run rescue exec "/bin/sh", "-c", *run
 		end
 
 		#inside run_pager, escape from the pager
@@ -222,5 +189,135 @@ module SH
 			nil
 		end
 
+		def find_file(file,path)
+			path.each do |dir|
+				dir=Pathname.new(dir)
+				path=dir+file
+				return path if path.file?
+			end
+			return nil
+		end
+
+		def find_files(pattern,path)
+			path.map { |dir| Pathname.glob(dir+pattern) }.flatten
+		end
+
+		def rsync(*files, out, default_opts: "-vcz", preserve: true, partial: true, keep_dirlinks: false, sudo: false, backup: false, relative: false, delete: false, clean_out: false, expected: 23, chown: nil, sshcommand: nil, exclude: [], **opts)
+			require 'shell_helpers/sh'
+			rsync_opts=[*opts.delete(:rsync_opts)] || []
+			rsync_opts << default_opts
+			rsync_opts << "-a" if preserve
+			rsync_opts << "-P" if partial #--partial --progress
+			rsync_opts+=%w(--no-owner --no-group) if preserve==:nochown
+			rsync_opts+=["--chown", chown] if chown
+			#on dest: do not replace a symlink to a directory with the real directory
+			#use --copy-dirlinks for the same usage on source
+			rsync_opts << "--keep-dirlinks" if keep_dirlinks
+			exclude.each do |ex|
+				rsync_opts += ["--exclude", ex.shellescape]
+			end
+			if relative
+				rsync_opts << "--relative"
+				rsync_opts << "--no-implied-dirs"
+			end
+			rsync_opts << "--delete" if delete
+			if clean_out
+				out=Pathname.new(out)
+				out.rmtree
+				out.mkpath
+			end
+			opts[:log]||=true
+			opts[:log_level_execute]||=:info
+			if backup
+				rsync_opts << "--backup"
+				rsync_opts << (backup.to_s[-1]=="/" ? "--backup-dir=#{backup}" : "--suffix=#{backup}") unless backup==true
+			end
+			if sshcommand
+				rsync_opts << "-e"
+				rsync_opts << sshcommand.shellescape
+			end
+			rsync_opts+=opts.delete(:rsync_late_opts)||[]
+			Sh.sh( (sudo ? ["sudo"] : [])+["rsync"]+rsync_opts+files.map(&:to_s)+[out.to_s], expected: expected, **opts)
+			#expected: rsync error code 23 is some files/attrs were not transferred
+		end
+
+		# host can be of the form user@host:port
+		# warning this is different from standard ssh syntax of user@host:path
+		def ssh(host, *commands, mode: :exec, ssh_command: 'ssh',
+			ssh_options: [], ssh_Ooptions: [],
+			port: nil, forward: nil, x11: nil, user: nil, path: nil, parse: true,
+			pty: nil,
+			**opts)
+
+			#sshkit has a special setting for :local
+			host=host.to_s unless mode==:sshkit and host.is_a?(Symbol)
+			parse and host.is_a?(String) and host.match(/^(?:(.*)@)?(.*?)(?::(\d*))?$/) do |m|
+				user||=m[1]
+				host=m[2]
+				port||=m[3]
+			end
+			unless mode==:net_ssh or mode==:sshkit
+				ssh_command, *command_options= ssh_command.shellsplit
+				ssh_options=command_options+ssh_options
+				ssh_options += ["-p", port.to_s] if port
+				ssh_options += ["-W", forward] if forward
+				if x11 == :trusted
+					ssh_options << "-Y"
+				elsif x11
+					ssh_options << "-X"
+				end
+				ssh_options << "-T" if pty==false
+				ssh_options << "-t" if pty==true
+				ssh_options += ssh_Ooptions.map {|o| ["-o", o]}.flatten
+			else #net_ssh options needs to be a hash
+				ssh_options={} if ssh_options.is_a?(Array)
+				ssh_options[:port]=port if port
+			end
+			case mode
+			when :system,:spawn,:capture,:exec
+				host="#{user}@#{host}" if user
+				Sh.sh([ssh_command]+ssh_options+[host]+commands, mode: mode, **opts)
+			when :net_ssh
+				require 'net/ssh'
+				user=nil;
+				Net::SSH.start(host, user, ssh_options)
+			when :sshkit
+				require 'sshkit'
+				host=SSHKit::Host.new(host)
+				host.extend(ExtendSSHKit)
+				host.port=port if port
+				host.user=user if user
+				host.ssh_options=ssh_options
+				host
+			when :uri
+				URI::Ssh::Generic.build(scheme: 'ssh', userinfo: user, host: host, path: path, port: port) #, query: ssh_options.join('&'))
+			else
+				# return options
+				{ ssh_command: ssh_command,
+				  ssh_options: ssh_options,
+				  ssh_command_options: ([ssh_command]+ssh_options).shelljoin,
+				  user: user,
+				  host: host,
+				  hostssh: user ? "#{user}@#{host}" : host,
+				  command: commands }
+			end
+		end
+
+		def capture_stdout
+			old_stdout = $stdout
+			$stdout = StringIO.new('','w')
+			if block_given?
+				begin
+					yield
+					output=$stdout.string
+				ensure
+					$stdout = old_stdout
+				end
+				return output
+			else
+				return old_stdout
+			end
+		end
+
 	end
 end