zsteg 0.0.0 → 0.0.1

data/lib/zsteg/file_cmd.rb

@@ -8,6 +8,7 @@ module ZSteg
       'data',
       'empty',
       'Sendmail frozen configuration',
+      'DBase 3 data file',
       'DOS executable',
       'Dyalog APL',
       '8086 relocatable',
@@ -18,9 +19,50 @@ module ZSteg
       'very short file',
       'International EBCDIC text',
       'lif file',
-      'AmigaOS bitmap font'
+      'AmigaOS bitmap font',
+      'a python script text executable' # common false positive
     ]
 
+    MIN_DATA_SIZE = 5
+
+    class Result < Struct.new(:title, :data)
+      COLORMAP_TEXT = {
+        /DBase 3 data/i               => :gray
+      }
+      COLORMAP_WORD = {
+        /bitmap|jpeg|pdf|zip|rar|7-?z/i => :bright_red,
+      }
+
+      def to_s
+        if title[/UTF-8 Unicode text/i]
+          begin
+            t = data.force_encoding("UTF-8").encode("UTF-32LE").encode("UTF-8")
+          rescue
+            t = data.force_encoding('binary')
+          end
+          return "utf8: " + t
+        end
+        COLORMAP_TEXT.each do |re,color|
+          return colorize(color) if title[re]
+        end
+        title.downcase.split.each do |word|
+          COLORMAP_WORD.each do |re,color|
+            return colorize(color) if title.index(re) == 0
+          end
+        end
+        colorize(:yellow)
+      end
+
+      def colorize color
+        if color == :gray
+          # gray whole string
+          "file: #{title}".send(color)
+        else
+          "file: " + title.send(color)
+        end
+      end
+    end
+
     def start!
       @stdin, @stdout, @stderr, @wait_thr = Open3.popen3("file -n -b -f -")
     end
@@ -39,6 +81,27 @@ module ZSteg
       check_file @tempfile.path
     end
 
+    # checks data and resurns Result, if any
+    def data2result data
+      return if data.size < MIN_DATA_SIZE
+
+      title = check_data data
+      return unless title
+
+      if title[/UTF-8 Unicode text/i]
+        begin
+          t = data.force_encoding("UTF-8")
+        rescue
+          t = data.force_encoding('binary')
+        end
+        if t.size >= Checker::MIN_TEXT_LENGTH
+          ZSteg::Result::UnicodeText.new(t,0)
+        end
+      else
+        Result.new(title,data)
+      end
+    end
+
     def stop!
       @stdin.close
       @stdout.close

data/lib/zsteg/mask_cli.rb

@@ -0,0 +1,268 @@
+require 'optparse'
+require 'set'
+require 'digest/md5'
+
+module ZSteg
+  class MaskCLI
+    DEFAULT_ACTIONS = %w'mask'
+
+    COMMON_MASKS = [
+      0b0000_0001, 0b0000_0011, 0b0000_0111, 0b0000_1111,
+                   0b0000_0010, 0b0000_0100, 0b0000_1000,
+      0b0001_0000, 0b0010_0000, 0b0100_0000, 0b1000_0000,
+    ]
+
+    CHANNELS = [:r, :g, :b, :a]
+
+    def initialize argv = ARGV
+      @argv = argv
+      @wasfiles = Set.new
+      @cache = {}
+    end
+
+    def run
+      @actions = []
+      @options = {
+        :verbose   => 0,
+        :masks     => Hash.new{|k,v| k[v] = [] },
+        :normalize => true
+      }
+      optparser = OptionParser.new do |opts|
+        opts.banner = "Usage: zsteg-mask [options] filename.png [param_string]"
+        opts.separator ""
+
+        opts.on("-m", "--mask M", "apply mask to all channels",
+                "mask: 0-255 OR 0x00-0xff OR 00000000-11111111",
+                "OR 'all' for all common masks"
+        ){ |x| @options[:masks][:all] << parse_mask(x) }
+
+        opts.on("-R", "--red M", "red channel mask"){ |x|
+          @options[:masks][:r] << parse_mask(x) }
+
+        opts.on("-G", "--green M", "green channel mask"){ |x|
+          @options[:masks][:g] << parse_mask(x) }
+
+        opts.on("-B", "--blue M", "blue channel mask"){ |x|
+          @options[:masks][:b] << parse_mask(x) }
+
+        opts.on("-A", "--alpha M", "alpha channel mask"){ |x|
+          @options[:masks][:a] << parse_mask(x) }
+
+        opts.separator ""
+
+        opts.on "-a", "--all", "try all common masks (default)" do
+          @options[:try_all] = true
+        end
+
+        opts.separator ""
+
+        opts.on "-N", "--[no-]normalize", "normalize color value after applying mask",
+          "(default: normalize)" do |x|
+          @options[:normalize] = x
+        end
+
+        opts.on "-O", "--outfile FILENAME", "output single result to specified file" do |x|
+          @options[:outfile] = x
+        end
+
+        opts.on "-D", "--dir DIRNAME", "output multiple results to specified dir" do |x|
+          @options[:dir] = x
+        end
+
+        opts.separator ""
+        opts.on "-v", "--verbose", "Run verbosely (can be used multiple times)" do |v|
+          @options[:verbose] += 1
+        end
+        opts.on "-q", "--quiet", "Silent any warnings (can be used multiple times)" do |v|
+          @options[:verbose] -= 1
+        end
+        opts.on "-C", "--[no-]color", "Force (or disable) color output (default: auto)" do |x|
+          Sickill::Rainbow.enabled = x
+        end
+      end
+
+      if (argv = optparser.parse(@argv)).empty?
+        puts optparser.help
+        return
+      end
+
+      # default :all mask if none specified
+      if @options[:masks].empty?
+        @options[:try_all] = true
+      end
+
+      @actions = DEFAULT_ACTIONS if @actions.empty?
+
+      argv.each do |arg|
+        if arg[','] && !File.exist?(arg)
+          @options.merge!(decode_param_string(arg))
+          argv.delete arg
+        end
+      end
+
+      argv.each_with_index do |fname,idx|
+        if argv.size > 1 && @options[:verbose] >= 0
+          puts if idx > 0
+          puts "[.] #{fname}".green
+        end
+        next unless @image=load_image(@fname=fname)
+
+        @actions.each do |action|
+          if action.is_a?(Array)
+            self.send(*action) if self.respond_to?(action.first)
+          else
+            self.send(action) if self.respond_to?(action)
+          end
+        end
+      end
+    rescue Errno::EPIPE
+      # output interrupt, f.ex. when piping output to a 'head' command
+      # prevents a 'Broken pipe - <STDOUT> (Errno::EPIPE)' message
+    end
+
+    def parse_mask x
+      case x
+      when /0x/i
+        x.to_i(16)
+      when /^[01]{8}$/
+        x.to_i(2)
+      when /^\d{1,3}$/
+        x.to_i
+      when /^all$/
+        COMMON_MASKS
+      else raise "invalid mask #{x.inspect}"
+      end
+    end
+
+    def load_image fname
+      if File.directory?(fname)
+        puts "[?] #{fname} is a directory".yellow
+      else
+        ZPNG::Image.load(fname)
+      end
+    rescue ZPNG::Exception, Errno::ENOENT
+      puts "[!] #{$!.inspect}".red
+    end
+
+    ###########################################################################
+    # actions
+
+    def mask
+      masks = @options[:masks]
+      masks.each{ |k,v| v.flatten!; v.uniq! }
+
+      if @options[:try_all]
+        # try all common masks
+        masks = masks[:all] || []
+        masks = COMMON_MASKS if masks.empty?
+        masks.each{ |x| run_masker x,x,x,x    }
+        masks.each{ |x| run_masker x,0,0,0xff }
+        masks.each{ |x| run_masker 0,x,0,0xff }
+        masks.each{ |x| run_masker 0,0,x,0xff }
+        if @image.alpha_used?
+          masks.each{ |x| run_masker 0,0,0,x    }
+        end
+
+      elsif CHANNELS.all?{ |c| !masks[c] || masks[c].empty? }
+        # no specific channels
+        masks[:all].each do |x|
+          run_masker x,x,x,x
+        end
+
+      else
+        # specific channels
+        CHANNELS.each{ |x| masks[x] = [x==:a ? 0xff : 0] if !masks[x] || masks[x].empty? }
+        masks[:r].each do |r|
+          masks[:g].each do |g|
+            masks[:b].each do |b|
+              if @image.alpha_used?
+                masks[:a].each do |a|
+                  run_masker r,g,b,a
+                end
+              else
+                run_masker r,g,b,0xff
+              end
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def _all_pixels_same img
+      sl0 = img.scanlines.first
+      return false if sl0.pixels.to_a.uniq.size != 1
+
+      db0 = sl0.decoded_bytes
+      img.scanlines[1..-1].each do |sl|
+        return false if sl.decoded_bytes != db0
+      end
+      true
+    end
+
+    def run_masker r,g,b,a
+      params = @options.dup
+      params[:masks] = params[:masks].merge( :r => r, :g => g, :b => b, :a => a)
+      fname,color = @options[:outfile],nil
+      fname,color = masks2fname(params[:masks]) unless fname
+
+      print "[.] #{fname.send(color||:to_s)} .. "
+
+      raise "already written to #{fname}" if @wasfiles.include?(fname)
+      @wasfiles << fname
+
+      dst = Masker.new(@image, params).mask
+
+      if _all_pixels_same(dst)
+        puts "all pixels = #{dst[0,0].inspect}".gray
+        return
+      end
+
+      data = dst.export
+
+      md5 = Digest::MD5.hexdigest(data)
+      if @cache[md5]
+        puts "same as #{File.basename(@cache[md5])}".gray
+        return
+      end
+      @cache[md5] = fname
+
+      File.open(fname, "wb"){ |f| f<<data }
+      printf "%6d bytes\n".green, File.size(fname)
+    end
+
+    def masks2fname masks
+      masks = masks.dup.delete_if{ |k,v| !CHANNELS.include?(k) }
+      bname = @fname.sub(/#{Regexp.escape(File.extname(@fname))}$/,'')
+      color = nil
+      raise "TODO" if masks.values.all?(&:nil?)
+      if masks.values.uniq.size == 1
+        tail = "%08b" % masks.values.first
+      else
+        a = []
+        masks.each do |c,mask|
+          a << "%s%08b" % [c,mask] if mask && mask != 0
+        end
+        raise "TODO" if a.empty?
+        a -= ['a11111111'] if a.size > 1 # fully opaque alpha is OK
+        if a.size == 1
+          color =
+            case a[0][0,1]
+            when 'r'; :red
+            when 'g'; :green
+            when 'b'; :blue
+            when 'a'; :gray
+            else nil
+            end
+        end
+        tail = a.join(".")
+      end
+
+      fname = [bname, tail, "png"].join('.')
+      fname = File.join(@options[:dir], File.basename(fname)) if @options[:dir]
+      [fname, color]
+    end
+
+  end
+end

data/lib/zsteg/masker.rb

@@ -0,0 +1,52 @@
+require 'set'
+
+module ZSteg
+  class Masker
+    def initialize image, params = {}
+      @src = image.is_a?(ZPNG::Image) ? image : ZPNG::Image.load(image)
+      @masks = params[:masks] || {}
+      @mask = params[:mask] || @masks[:all]
+      @normalize = params[:normalize]
+      [:r, :g, :b, :a].each{ |x| @masks[x] ||= @mask }
+    end
+
+    def mask params = {}
+      dst = ZPNG::Image.new :width => @src.width, :height => @src.height
+      rm, gm, bm, am = @masks[:r], @masks[:g], @masks[:b], @masks[:a]
+      rd, gd, bd, ad = rm==0?1:rm, gm==0?1:gm, bm==0?1:bm, am==0?1:am
+      # duplicate loops for performance reason
+      if @normalize
+        if rm == 0 && gm == 0 && bm == 0 && am != 0
+          # alpha2grayscale
+          @src.each_pixel do |c,x,y|
+            c.r = c.g = c.b = (c.a & am) * 255 / ad
+            c.a = 0xff
+            dst[x,y] = c
+          end
+        else
+          # normal operation
+          @src.each_pixel do |c,x,y|
+            #TODO: c.to_depth(8)
+            # further possible optimizations:
+            # a) precalculate (255 / Xm)
+            c.r = (c.r & rm) * 255 / rd
+            c.g = (c.g & gm) * 255 / gd
+            c.b = (c.b & bm) * 255 / bd
+            c.a = (c.a & am) * 255 / ad
+            dst[x,y] = c
+          end
+        end
+      else
+        @src.each_pixel do |c,x,y|
+          #TODO: c.to_depth(8)
+          c.r &= rm
+          c.g &= gm
+          c.b &= bm
+          c.a &= am
+          dst[x,y] = c
+        end
+      end
+      dst
+    end
+  end
+end

data/lib/zsteg/result.rb

@@ -17,13 +17,30 @@ module ZSteg
       end
 
       def to_s
-        super.sub(/^<Result::/,'').sub(/>$/,'').red
+        super.sub(/^<Result::/,'').sub(/>$/,'').bright_red
       end
     end
 
     class Text < Struct.new(:text, :offset)
+      def one_char?
+        (text =~ /\A(.)\1+\Z/m) == 0
+      rescue # invalid byte sequence in UTF-8
+        text.chars.to_a.uniq.size == 1 # ~10x slower than regexp
+      end
+
       def to_s
-        "text: ".gray + (offset == 0 ? text.inspect.red : text.inspect)
+        "text: ".gray +
+          if one_char?
+            "[#{text[0].inspect} repeated #{text.size} times]".gray
+          elsif offset == 0
+            # first byte of data is also first char of text
+            text.inspect.bright_red
+          elsif text.size > 10 && text[' '] && text =~ /\A[a-z0-9 .,:!_-]+\Z/i
+            # text is ASCII with spaces
+            text.inspect.bright_red
+          else
+            text.inspect
+          end
       end
     end
 
@@ -33,9 +50,15 @@ module ZSteg
     # part of data is text
     class PartialText < Text; end
 
+    # unicode text
+    class UnicodeText < Text; end
+
     class Zlib < Struct.new(:data, :offset)
+      MAX_SHOW_SIZE = 100
       def to_s
-        "zlib: data=#{data.inspect.red}, offset=#{offset}"
+        x = data
+        x=x[0,MAX_SHOW_SIZE] + "..." if x.size > MAX_SHOW_SIZE
+        "zlib: data=#{x.inspect.bright_red}, offset=#{offset}, size=#{data.size}"
       end
     end
 
@@ -45,34 +68,6 @@ module ZSteg
       end
     end
 
-    class FileCmd < Struct.new(:title, :data)
-      COLORMAP = {
-        /bitmap|jpeg|pdf|zip|rar|7z/i => :red,
-        /DBase 3 data/i               => :gray
-      }
-
-      def to_s
-        if title[/UTF-8 Unicode text/i]
-          begin
-            t = data.force_encoding("UTF-8").encode("UTF-32LE").encode("UTF-8")
-          rescue
-            t = data.force_encoding('binary')
-          end
-          return "utf8: " + t
-        end
-        COLORMAP.each do |re,color|
-          if title[re]
-            if color == :gray
-              return "file: #{title}".send(color)
-            else
-              return "file: " + title.send(color)
-            end
-          end
-        end
-        "file: " + title.yellow
-      end
-    end
-
     class Camouflage < Struct.new(:hidden_data_len, :host_orig_len)
       def initialize(data)
         self.hidden_data_len = (data[0x1a,4] || '').unpack('V').first
@@ -83,7 +78,7 @@ module ZSteg
       end
 
       def to_s
-        super.red
+        super.bright_red
       end
     end
   end