filecamo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc867e4a005a911f06b95fc7a483ab8321a517c3
+  data.tar.gz: 8229c95f127d9e938ff75ea31c24a817a30099d3
+SHA512:
+  metadata.gz: 2a1af293cb02a0e77998337a2c75e348f0578c3abcedf93c421ea7de885ec207141112f2465dc35ed52e7be08ed8d32f042002ae10c5c4a2d37b73341996d8fc
+  data.tar.gz: 3f3a3363e9d00aa1006e3174ff700d643449bdf13541a892ba65f274004736d1fa3ecb9fc07ff700afd8b2afe87cf741ff9580a5db37898ae0d53072d9e86952

data/README.md

@@ -0,0 +1 @@
+# filecamo

data/bin/filecamo

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/filecamo'
+
+BN = File.basename($0, '.rb')
+
+cmd = ARGV.shift
+if cmd == 'gen'
+  if i = ARGV.index('-d')
+    ARGV.delete_at(i)
+    dst_path = ARGV.delete_at(i)
+  end
+
+  if ARGV.size  < 2
+    bin, bm = BetterBytes::BINARY
+    dec, dm = BetterBytes::DECIMAL
+    
+    $stderr.puts <<EOF
+
+usage: #{BN} gen [-d <destination_path>] <minimum_size> <maximum_size> <count> <depth> [<percent_text>]
+
+     The size MAY end with a unit label:
+       * BINARY labels (#{bm} multiplier):  #{bin.join(', ')}
+       * DECIMAL labels (#{dm} multiplier): #{dec.join(', ')}
+
+     The count will determine the number of files to generate.
+
+     The depth will determine the hierarchy of the files. The number of files will be randomly
+     distributed among the tree.
+
+     The percent text optional value indicates how many files should be text instead of binary.
+     Default is none (i.e. all files have random binary content).
+
+EOF
+    exit 1
+  end
+
+  min, max, count, depth, percent_text = ARGV
+  generator = Filecamo::Generator.new
+  generator.generate(min, max, count, depth,
+                     percent_text: percent_text, destination_path: dst_path) do |path, len|
+    puts "#{BetterBytes.humanize(len, precision: 6.1)} => #{path}"
+  end
+  generator.wait do |txt_count, bin_count|
+    $stdout.printf "\rWaiting for #{txt_count} txt and #{bin_count} bin jobs "\
+                   "to complete..." + (' '*10)
+    $stdout.flush
+  end
+  $stdout.puts
+  generator.kill
+
+elsif cmd == 'muck'
+  if ARGV.size < 3
+    $stderr.puts <<EOF
+
+usage: #{BN} muck <percent_select> <percent_change> <path> [<path>...]
+
+     The percent select should be a value from 1 to 100 indicating how likely a given file will be
+     modified.
+
+     The percent change should be a value from 1 to 100 indicating (roughly) how many lines should
+     be added.
+
+     Paths can be filenames or directories to recurse and will be descended breadth-first. Any file
+     entry starting with '.' is skipped (ignored).
+
+EOF
+    exit 1
+  end
+
+  percent_select, percent_change, *paths = ARGV
+  mucker = Filecamo::TextMucker.new(" #{BN}: ")
+  mucker.muck(percent_select, percent_change, paths) do |fn, lang, line_nums|
+    puts "Modified #{fn} (#{lang}): #{line_nums.join(',')}"
+  end
+
+else
+  $stderr.puts <<EOF
+
+usage: #{BN} { gen | muck } ...
+
+EOF
+  exit 1
+end

data/filecamo.gempec

@@ -0,0 +1,15 @@
+Gem::Specification.new do |s|
+  s.name        = 'filecamo'
+  s.version     = File.readlines('lib/filecamo.rb').grep(/VERSION/){|v|v.match(/'([^']+)'/)[1]}[0]
+  s.summary     = 'File content generator and manipulator.'
+  s.description = 'Generate and manipulate entire directory trees of either binary or text file content.'
+  s.authors     = ['Brad Robel-Forrest']
+  s.email       = 'brad@bitpony.com'
+  s.files         = `git ls-files -z`.split("\x0")
+  s.executables = ['filecamo']
+  s.homepage    = 'https://github.com/bradrf/filecamo#readme'
+  s.license     = 'MIT'
+  s.add_runtime_dependency 'better_bytes', '~> 0.0.1'
+  s.add_runtime_dependency 'literate_randomizer', '~> 0.4.0'
+  s.add_runtime_dependency 'ruby-filemagic', '~> 0.7.1'
+end

data/lib/filecamo.rb

@@ -0,0 +1,6 @@
+module Filecamo
+  VERSION = '0.0.1'
+end
+
+require_relative './filecamo/generator'
+require_relative './filecamo/text_mucker'

data/lib/filecamo/generator.rb

@@ -0,0 +1,121 @@
+require 'logger'
+require 'thread'
+require 'thwait'
+require 'pathname'
+require 'better_bytes'
+require 'literate_randomizer'
+
+module Filecamo
+  class Generator
+    def initialize(logger: Logger.new($stdout),
+                   txt_workers: {count: 4, queue_size: 5000},
+                   bin_workers: {count: 4, queue_size: 5000})
+      @gen = Random.new
+      @logger = logger
+      @words_generated = {}
+
+      @txt_work_q = SizedQueue.new(txt_workers[:queue_size])
+      @txt_workers = start_workers(:txt, @txt_work_q, txt_workers[:count]) do |file, len|
+        while len > 0
+          line = (LiterateRandomizer.sentence + $/)
+          line.slice!(len..-1)
+          len -= file.write(line)
+        end
+      end
+
+      @bin_work_q = SizedQueue.new(bin_workers[:queue_size])
+      @bin_workers = start_workers(:bin, @bin_work_q, bin_workers[:count]) do |file, len|
+        while len > 0
+          len -= file.write(@gen.bytes(len < 32768 ? len : 32768))
+        end
+      end
+    end
+
+    def generate(min, max, count, depth, percent_text: 0, destination_path: nil, &block)
+      min = BetterBytes.dehumanize(min)
+      max = BetterBytes.dehumanize(max)
+      count = count.to_i
+      depth = depth.to_i
+      percent_text = percent_text.to_f / 100
+      dst_pn = Pathname.new(destination_path || '.')
+      feed_workers(min, max, count, depth, percent_text, dst_pn, &block)
+    end
+
+    def wait(sleep_interval: 0.3)
+      until @txt_work_q.empty? && @bin_work_q.empty?
+        block_given? and
+          yield(@txt_work_q.size, @bin_work_q.size)
+        sleep(sleep_interval)
+      end
+    end
+
+    def kill
+      (@txt_workers + @bin_workers).each{|th| th.kill}
+      ThreadsWait.all_waits(@txt_workers + @bin_workers) do |th|
+        th.join
+      end
+    end
+
+    ######################################################################
+    private
+
+    def start_workers(name, queue, count)
+      count.times.map do
+        Thread.new do
+          begin
+            loop do
+              len, fn = queue.pop
+              fn.open('wb') do |file|
+                yield file, len
+              end
+            end
+          rescue Exception => ex
+            @logger.fatal "#{Thread.current} failed (#{name})"
+            @logger.fatal ex
+          end
+        end
+      end
+    end
+
+    def feed_workers(min, max, count, depth, percent_text, dst_pn)
+      paths = {}
+
+      count.times do |i|
+        d = Random.rand(depth)
+        if d < 1
+          pn = Pathname.new('')
+        else
+          pn = nil
+          pns = paths[d] and pn = pns[@gen.rand(pns.size)]
+          unless pn
+            pn = Pathname.new(gen_name)
+            d.times{ pn += gen_name }
+            (paths[d] ||= []) << pn
+          end
+        end
+
+        fn = dst_pn + pn + (gen_name + (@gen.rand > percent_text ? '.bin' : '.txt'))
+        fn.parent.mkpath
+        len = @gen.rand(min..max).round
+
+        block_given? and
+          yield(fn, len)
+
+        if fn.extname == '.txt'
+          @txt_work_q.push([len, fn])
+        else
+          @bin_work_q.push([len, fn])
+        end
+      end
+    end
+
+    def gen_name
+      # guarantee the same capitalization is used for the same word
+      word = LiterateRandomizer.word
+      lword = word.downcase
+      prev_word = @words_generated[lword] and
+        return prev_word
+      return @words_generated[lword] = word
+    end
+  end
+end

data/lib/filecamo/text_mucker.rb

@@ -0,0 +1,139 @@
+require 'logger'
+require 'filemagic'
+require 'literate_randomizer'
+
+module Filecamo
+  class TextMucker
+    MAX_FILE_SIZE = 128 * 1024
+
+    LANG_MARKS = {
+      csharp: '//',
+      python: '#',
+      ruby: '#',
+      shell: '#',
+      js: '#',
+      plain: '#',
+    }
+
+    # todo: fix parsable types, somethign like the following
+    # (figure out how to insert, perhaps deep iterater and counters?)
+    # (figure out types to insert, (i.e. would be bad to insert string into number array))
+    LANG_PARSERS = {
+      json: ->(fn){}
+    }
+
+    def initialize(comment_prefix, logger: Logger.new($stdout))
+      @marks = LANG_MARKS.clone
+      @marks.each_value{|m| m << comment_prefix}
+      @logger = logger
+      @magic = FileMagic.new
+      @mime = FileMagic.mime
+    end
+
+    def muck(percent_select, percent_lines, paths)
+      select_chance = percent_select.to_f / 100
+      lines_chance = percent_lines.to_f / 100
+
+      paths.each do |path|
+        path[0] == '.' and next
+        if File.directory?(path)
+          paths.concat(Dir.entries(path).map{|e| e[0] == '.' ? nil : File.join(path,e)}.compact)
+          next
+        end
+
+        fn = path
+        fn_size = File.size(fn)
+
+        # todo: support working with large files by reading next line
+        if fn_size > MAX_FILE_SIZE
+          @logger.debug "Skipping #{fn} by size: #{file.size}"
+          break
+        end
+
+        lang = case File.extname(fn)
+               when '.cs' then :csharp
+               when '.py' then :python
+               when '.js' then :js
+               when '.json' then :json
+               when '.yaml','meta' then :yaml
+               when '.html' then :html
+               when '.txt' then :plain
+               else
+                 case m = @mime.file(fn)
+                 when /python/ then :python
+                 when /ruby/ then :ruby
+                 when /shell/ then :shell
+                 when /plain/
+                   case g = @magic.file(fn)
+                   when /python/ then :python
+                   when /ruby/ then :ruby
+                   when /node/ then :js
+                   else
+                     :plain
+                   end
+                 else
+                   @logger.debug "Skipping #{fn} by mime type: #{m}"
+                   next
+                 end
+               end
+
+        if Random.rand > select_chance
+          @logger.debug "Skipping #{fn} by chance"
+          next
+        end
+
+        new_lines = {}
+        new_bytes_needed = (fn_size * lines_chance).floor
+        while new_bytes_needed > 0
+          offset = Random.rand(fn_size)
+          new_line = get_line_for(lang)
+          new_lines[offset] = new_line
+          new_bytes_needed -= new_line.bytesize
+        end
+        new_lines = new_lines.sort
+
+        body = ''
+        line_nums = []
+
+        File.open(fn) do |file|
+          line_num = 0
+          while !file.eof? && line = file.readline
+            body << line
+            line_num += 1
+            new_lines.empty? and next # read remainder of file
+            offset = new_lines[0][0]
+            if file.pos >= offset # add a line as soon as passed the offset
+              offset, new_line = new_lines.shift
+              line_nums << (line_num+=1)
+              body << new_line
+            end
+          end
+
+          # concat any remaining lines
+          if !new_lines.empty?
+            body[-1] == $/ or body << $/
+            new_lines.each do |offset, new_line|
+              line_num += 1
+              line_nums << line_num
+              body << new_line
+            end
+          end
+        end
+
+        # todo: use same charset as mime type indicates when writing!
+        File.open(fn, 'wb') {|f| f.write(body)}
+
+        block_given? and yield(fn, lang, line_nums)
+      end
+    end
+
+    ######################################################################
+    private
+
+    def get_line_for(lang)
+      mark = @marks[lang] or return ''
+      # todo: match line endings of file!
+      return mark + LiterateRandomizer.sentence + $/
+    end
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: filecamo
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brad Robel-Forrest
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: better_bytes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: literate_randomizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: ruby-filemagic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+description: Generate and manipulate entire directory trees of either binary or text
+  file content.
+email: brad@bitpony.com
+executables:
+- filecamo
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/filecamo
+- filecamo.gempec
+- lib/filecamo.rb
+- lib/filecamo/generator.rb
+- lib/filecamo/text_mucker.rb
+homepage: https://github.com/bradrf/filecamo#readme
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: File content generator and manipulator.
+test_files: []