feldtruby 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. data/.autotest +23 -0
  2. data/.gemtest +0 -0
  3. data/History.txt +4 -0
  4. data/Manifest.txt +44 -0
  5. data/README.md +63 -0
  6. data/README.txt +59 -0
  7. data/Rakefile +19 -0
  8. data/TODO +6 -0
  9. data/lib/feldtruby/array/basic_stats.rb +88 -0
  10. data/lib/feldtruby/array/count_by.rb +7 -0
  11. data/lib/feldtruby/array.rb +34 -0
  12. data/lib/feldtruby/file/file_change_watcher.rb +88 -0
  13. data/lib/feldtruby/file/tempfile.rb +25 -0
  14. data/lib/feldtruby/float.rb +17 -0
  15. data/lib/feldtruby/math/rand.rb +5 -0
  16. data/lib/feldtruby/net/html_doc_getter.rb +31 -0
  17. data/lib/feldtruby/optimize/differential_evolution.rb +186 -0
  18. data/lib/feldtruby/optimize/max_steps_termination_criterion.rb +24 -0
  19. data/lib/feldtruby/optimize/objective.rb +302 -0
  20. data/lib/feldtruby/optimize/optimizer.rb +145 -0
  21. data/lib/feldtruby/optimize/random_search.rb +9 -0
  22. data/lib/feldtruby/optimize/search_space.rb +69 -0
  23. data/lib/feldtruby/optimize/stdout_logger.rb +138 -0
  24. data/lib/feldtruby/optimize.rb +28 -0
  25. data/lib/feldtruby/string/to_iso.rb +7 -0
  26. data/lib/feldtruby/time.rb +22 -0
  27. data/lib/feldtruby/vector.rb +14 -0
  28. data/lib/feldtruby/visualization/circos.rb +25 -0
  29. data/lib/feldtruby/word_counter.rb +100 -0
  30. data/lib/feldtruby.rb +6 -0
  31. data/test/helper.rb +7 -0
  32. data/test/test_array.rb +71 -0
  33. data/test/test_array_basic_stats.rb +130 -0
  34. data/test/test_array_count_by.rb +13 -0
  35. data/test/test_float.rb +20 -0
  36. data/test/test_html_doc_getter.rb +16 -0
  37. data/test/test_optimize.rb +55 -0
  38. data/test/test_optimize_differential_evolution.rb +42 -0
  39. data/test/test_optimize_objective.rb +157 -0
  40. data/test/test_optimize_populationbasedoptimizer.rb +24 -0
  41. data/test/test_optimize_random_search.rb +46 -0
  42. data/test/test_optimize_search_space.rb +97 -0
  43. data/test/test_time.rb +27 -0
  44. data/test/test_vector.rb +98 -0
  45. data/test/test_word_counter.rb +57 -0
  46. metadata +149 -0
data/.autotest ADDED
@@ -0,0 +1,23 @@
1
+ # -*- ruby -*-
2
+
3
+ require 'autotest/restart'
4
+
5
+ # Autotest.add_hook :initialize do |at|
6
+ # at.extra_files << "../some/external/dependency.rb"
7
+ #
8
+ # at.libs << ":../some/external"
9
+ #
10
+ # at.add_exception 'vendor'
11
+ #
12
+ # at.add_mapping(/dependency.rb/) do |f, _|
13
+ # at.files_matching(/test_.*rb$/)
14
+ # end
15
+ #
16
+ # %w(TestA TestB).each do |klass|
17
+ # at.extra_class_map[klass] = "test/test_misc.rb"
18
+ # end
19
+ # end
20
+
21
+ # Autotest.add_hook :run_command do |at|
22
+ # system "rake build"
23
+ # end
data/.gemtest ADDED
File without changes
data/History.txt ADDED
@@ -0,0 +1,4 @@
1
+ === 0.2.0 / 2012-12-21
2
+
3
+ * First public release since we need it in some other projects and local install is messy...
4
+
data/Manifest.txt ADDED
@@ -0,0 +1,44 @@
1
+ .autotest
2
+ History.txt
3
+ Manifest.txt
4
+ README.txt
5
+ README.md
6
+ Rakefile
7
+ TODO
8
+ lib/feldtruby.rb
9
+ lib/feldtruby/array.rb
10
+ lib/feldtruby/array/basic_stats.rb
11
+ lib/feldtruby/array/count_by.rb
12
+ lib/feldtruby/file/file_change_watcher.rb
13
+ lib/feldtruby/file/tempfile.rb
14
+ lib/feldtruby/float.rb
15
+ lib/feldtruby/math/rand.rb
16
+ lib/feldtruby/net/html_doc_getter.rb
17
+ lib/feldtruby/optimize.rb
18
+ lib/feldtruby/optimize/differential_evolution.rb
19
+ lib/feldtruby/optimize/max_steps_termination_criterion.rb
20
+ lib/feldtruby/optimize/objective.rb
21
+ lib/feldtruby/optimize/optimizer.rb
22
+ lib/feldtruby/optimize/random_search.rb
23
+ lib/feldtruby/optimize/search_space.rb
24
+ lib/feldtruby/optimize/stdout_logger.rb
25
+ lib/feldtruby/string/to_iso.rb
26
+ lib/feldtruby/time.rb
27
+ lib/feldtruby/vector.rb
28
+ lib/feldtruby/visualization/circos.rb
29
+ lib/feldtruby/word_counter.rb
30
+ test/helper.rb
31
+ test/test_array.rb
32
+ test/test_array_basic_stats.rb
33
+ test/test_array_count_by.rb
34
+ test/test_float.rb
35
+ test/test_optimize.rb
36
+ test/test_optimize_differential_evolution.rb
37
+ test/test_optimize_objective.rb
38
+ test/test_optimize_populationbasedoptimizer.rb
39
+ test/test_optimize_random_search.rb
40
+ test/test_optimize_search_space.rb
41
+ test/test_time.rb
42
+ test/test_vector.rb
43
+ test/test_html_doc_getter.rb
44
+ test/test_word_counter.rb
data/README.md ADDED
@@ -0,0 +1,63 @@
1
+ feldtruby
2
+ =========
3
+ Robert Feldt's Common Ruby Code lib. I will gradually collect the many generally useful Ruby tidbits I have laying around and clean them up into here. Don't want to rewrite these things again and again... So far this collects a number of generally useful additions to the standard Ruby classes/libs and then includes a simple optimization framework (FeldtRuby::Optimize).
4
+
5
+ email: robert.feldt ((a)) gmail.com
6
+
7
+ Contents
8
+ --------
9
+ ### Time
10
+ Time.timestamp() # Get a timestamp string back with the current time
11
+
12
+ ### Array
13
+
14
+ Basic calc/statistics: sum, mean, average, stdev, variance, rms, weighted_sum, weighted_mean,
15
+ sum_of_abs, sum_of_abs_deviations
16
+
17
+ [1,2,3].swap!(0,2) => [3, 2, 1] # destructive swap of two elements
18
+ [1,2,5].distance_between_elements => [1, 3]
19
+ [15, 1, 7, 0].ranks => [1, 3, 2, 0]
20
+ [[2.3, :a], [1.7, :b]].ranks_by {|v| v[0]} => [[1, 2.3, :a], [2, 1.7, :b]]
21
+
22
+ ### Float
23
+ 1.456.round_to_decimals(2) => 1.46 (round to given num of decimals)
24
+
25
+ ### FileChangeWatcher
26
+ Watch for file changes in given paths then call hooks with the updated files.
27
+
28
+ ### Kernel
29
+ rand_int(top) # random integer in range 0...top
30
+
31
+ ### Optimize
32
+ A simple optimization framework with classes:
33
+
34
+ * Objective (single or multi-objective optimization critera)
35
+ * SearchSpace (capture constraints for optimization values/parameters)
36
+ * RandomSearcher (random search for optimal values)
37
+ * DifferentialEvolution (effective numerical optimization with evolutionary algorithm)
38
+
39
+ but also support for different type of logging etc. Setting up an optimization can
40
+ be quite involved but there is a simple wrapper method, with good defaults, for
41
+ numerical optimization using DE:
42
+
43
+ # Optimizing with the Rosenbrock function on [0, 2], see:
44
+ # http://en.wikipedia.org/wiki/Rosenbrock_function
45
+ require 'feldtruby/optimize'
46
+ xbest, ybest = FeldtRuby::Optimize.optimize(0, 2) {|x, y|
47
+ (1 - x)**2 + 100*(y - x*x)**2
48
+ }
49
+
50
+ Contributing to feldtruby
51
+ -------------------------
52
+ * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
53
+ * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
54
+ * Fork the project.
55
+ * Start a feature/bugfix branch.
56
+ * Commit and push until you are happy with your contribution.
57
+ * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
58
+ * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
59
+
60
+ Copyright
61
+ ------------
62
+ Copyright (c) 2012 Robert Feldt. See LICENSE.txt for
63
+ further details.
data/README.txt ADDED
@@ -0,0 +1,59 @@
1
+ = feldtruby
2
+
3
+ * https://github.com/robertfeldt/feldtruby
4
+
5
+ == DESCRIPTION:
6
+
7
+ Robert Feldt's Common Ruby Code lib. I will gradually collect the many generally useful Ruby tidbits I have laying around and clean them up into here. Don't want to rewrite these things again and again... So far this collects a number of generally useful additions to the standard Ruby classes/libs and then includes a simple optimization framework (FeldtRuby::Optimize).
8
+
9
+ email: robert.feldt ((a)) gmail.com
10
+
11
+ == FEATURES/PROBLEMS:
12
+
13
+ * FIX (list of features or problems)
14
+
15
+ == SYNOPSIS:
16
+
17
+ FIX (code sample of usage)
18
+
19
+ == REQUIREMENTS:
20
+
21
+ * FIX (list of requirements)
22
+
23
+ == INSTALL:
24
+
25
+ * FIX (sudo gem install, anything else)
26
+
27
+ == DEVELOPERS:
28
+
29
+ After checking out the source, run:
30
+
31
+ $ rake newb
32
+
33
+ This task will install any missing dependencies, run the tests/specs,
34
+ and generate the RDoc.
35
+
36
+ == LICENSE:
37
+
38
+ (The MIT License)
39
+
40
+ Copyright (c) 2012 FIX
41
+
42
+ Permission is hereby granted, free of charge, to any person obtaining
43
+ a copy of this software and associated documentation files (the
44
+ 'Software'), to deal in the Software without restriction, including
45
+ without limitation the rights to use, copy, modify, merge, publish,
46
+ distribute, sublicense, and/or sell copies of the Software, and to
47
+ permit persons to whom the Software is furnished to do so, subject to
48
+ the following conditions:
49
+
50
+ The above copyright notice and this permission notice shall be
51
+ included in all copies or substantial portions of the Software.
52
+
53
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
54
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
55
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
56
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
57
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
58
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
59
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/Rakefile ADDED
@@ -0,0 +1,19 @@
1
+ # -*- ruby -*-
2
+
3
+ require 'rubygems'
4
+ require 'hoe'
5
+
6
+ # Hoe.plugin :compiler
7
+ # Hoe.plugin :gem_prelude_sucks
8
+ # Hoe.plugin :inline
9
+ # Hoe.plugin :racc
10
+ # Hoe.plugin :rcov
11
+ # Hoe.plugin :rubyforge
12
+
13
+ Hoe.plugin :gemcutter
14
+
15
+ Hoe.spec 'feldtruby' do
16
+ developer 'Robert Feldt', 'robert.feldt@gmail.com'
17
+
18
+ license 'MIT' # this should match the license in the README
19
+ end
data/TODO ADDED
@@ -0,0 +1,6 @@
1
+ * Add Pareto front calculations and saving by the stats logger and/or in collab with the Objective!?
2
+ - In one way it is easier for the Objective to keep track of these things since it already saves the global mins and maxs.
3
+ - OTOH it may be more logical that the optimizer or the stats object keeps track of this.
4
+
5
+ * Consider if the quality of the candidate vectors should be be attached to them instead of being handled separately.
6
+ - We can add instance vars to any Ruby object with "special tricks"... ;)
@@ -0,0 +1,88 @@
1
+ module BasicStatistics
2
+ def sum
3
+ self.inject(0) {|s, e| s+e}
4
+ end
5
+
6
+ def mean
7
+ return 0 if self.length == 0
8
+ self.sum / self.length.to_f
9
+ end
10
+
11
+ def average; mean(); end
12
+
13
+ def median
14
+ sorted = self.sort
15
+ if self.length % 2 == 0
16
+ mid = self.length / 2
17
+ (sorted[mid-1] + sorted[mid])/2.0
18
+ else
19
+ sorted[self.length/2.0]
20
+ end
21
+ end
22
+
23
+ def variance
24
+ return 0 if self.length == 0
25
+ avg = self.mean
26
+ self.map {|e| (e-avg)**2}.sum / self.length.to_f
27
+ end
28
+
29
+ def stdev
30
+ Math.sqrt( self.variance )
31
+ end
32
+
33
+ def root_mean_square
34
+ Math.sqrt( self.map {|v| v**2}.mean )
35
+ end
36
+
37
+ def rms; self.root_mean_square(); end
38
+
39
+ def rms_from_scalar(scalar)
40
+ Math.sqrt( self.map {|v| (v-scalar)**2}.mean )
41
+ end
42
+
43
+ # Weighted sum of elements
44
+ def weighted_sum(weights = nil)
45
+ if weights
46
+ raise "Not same num of weights (#{weights.length}) as num of elements (#{self.length})" if self.length != weights.length
47
+ self.zip(weights).map {|e,w| e*w}.sum
48
+ else
49
+ self.sum
50
+ end
51
+ end
52
+
53
+ # Weighted mean of elements
54
+ def weighted_mean(weights = nil)
55
+ if weights
56
+ self.weighted_sum(weights) / weights.sum.to_f
57
+ else
58
+ self.mean
59
+ end
60
+ end
61
+
62
+ def sum_of_abs_deviations(fromValue = 0.0)
63
+ sum = 0.0
64
+ self.each {|v| sum += (v-fromValue).abs}
65
+ sum
66
+ end
67
+
68
+ def sum_of_abs
69
+ sum = 0.0
70
+ self.each {|v| sum += v.abs}
71
+ sum
72
+ end
73
+
74
+ def sum_squared_error(b)
75
+ sum = 0.0
76
+ self.each_with_index {|e,i| d = e-b[i]; sum += d*d}
77
+ sum
78
+ end
79
+
80
+ # Return summary stats for an array of numbers.
81
+ def summary_stats
82
+ "%.3f (min = %.1f, max = %.1f, median = %.1f, stdev = %.2f)" % [mean, self.min, self.max, median, stdev]
83
+ end
84
+ end
85
+
86
+ class Array
87
+ include BasicStatistics
88
+ end
@@ -0,0 +1,7 @@
1
+ class Array
2
+ def count_by(&proc)
3
+ counts = Hash.new(0)
4
+ self.each {|e| counts[proc.call(e)] += 1}
5
+ counts
6
+ end
7
+ end
@@ -0,0 +1,34 @@
1
+ require 'feldtruby/array/basic_stats'
2
+
3
+ class Array
4
+ # Calculate the distance between the elements.
5
+ def distance_between_elements
6
+ return nil if self.length == 0
7
+ self[0...-1].zip(self[1..-1]).map {|x,y| y-x}
8
+ end
9
+
10
+ # Swap two elements given their indices. Assumes both indices are in range.
11
+ def swap!(index1, index2)
12
+ self[index1], self[index2] = self[index2], self[index1]
13
+ end
14
+
15
+ # Rank of values in array from 1..length
16
+ def ranks
17
+ ranks = Array.new(length)
18
+ self.each_with_index.map {|e,i| [i, e]}.sort_by {|v| v.last}.each_with_index.map {|v,i| ranks[v.first]=length-i}
19
+ ranks
20
+ end
21
+
22
+ # Prepend (or append) ranks after sorting by the value supplied from a block
23
+ def ranks_by(prependRanks = true, &mapToValueUsedForRanking)
24
+ res = Array.new(length)
25
+ sorted_with_indices = self.each_with_index.map {|e,i| [i, e]}.sort_by {|v| mapToValueUsedForRanking.call(v.last)}
26
+ sorted_with_indices.each_with_index.map do |v,index|
27
+ orig_index, orig_element = v
28
+ rank = length - index
29
+ new_element = prependRanks ? ([rank] + orig_element) : (orig_element + [rank])
30
+ res[orig_index] = new_element
31
+ end
32
+ res
33
+ end
34
+ end
@@ -0,0 +1,88 @@
1
+ # Watch directories for changes and provide callback hooks for when files change.
2
+ # Partly inspired by and built on code from Autotest/Zentest by Ryan Davis and Eric Hodel.
3
+ #
4
+ require 'find'
5
+
6
+ class String
7
+ def starts_with?(str)
8
+ self[0, str.length] == str
9
+ end
10
+ end
11
+
12
+ # Watch for file changes in given paths then call hooks with the updated files.
13
+ class FileChangeWatcher
14
+ attr_accessor :find_directories, :last_mtime, :sleep_time, :exclusions
15
+
16
+ def initialize(specifiedDirectories = ["."], sleepTime = 5*60, excludeFilesRegexps = [], &runWhenFilesUpdated)
17
+ specified_directories = specifiedDirectories.reject { |path| path.starts_with?("-") }
18
+ self.find_directories = specified_directories.empty? ? ['.'] : specified_directories
19
+ self.exclusions = excludeFilesRegexps
20
+ self.sleep_time = sleepTime
21
+ self.last_mtime = nil # Ensure we run first time when started
22
+ @hooks = Hash.new { |h,k| h[k] = [] }
23
+ add_hook :updated, &runWhenFilesUpdated if runWhenFilesUpdated
24
+ end
25
+
26
+ # Find the files to process, ignoring temporary files, source
27
+ # configuration management files, etc., and return a Hash mapping
28
+ # filename to modification time.
29
+ def find_files
30
+ result = {}
31
+ targets = self.find_directories
32
+ targets.each do |target|
33
+ Find.find target do |f|
34
+ next if test ?d, f
35
+ next if f =~ /(swp|~|rej|orig)$/ # temporary/patch files
36
+ next if f =~ /^\.\/tmp/ # temporary dir, used by isolate
37
+ next if f =~ /\/\.?#/ # Emacs autosave/cvs merge files
38
+ filename = f.sub(/^\.\//, '')
39
+ result[filename] = File.stat(filename).mtime rescue next
40
+ end
41
+ end
42
+ result
43
+ end
44
+
45
+ def exclude_matched_files(files, exclusionRegexps)
46
+ files.reject {|f| exclusionRegexps.any? {|exre| exre.match(f)}}
47
+ end
48
+
49
+ # Find files that has changed since last time.
50
+ # Call updated hook if any found.
51
+ def find_updated_files files = find_files
52
+ hook :checkingChanges, files
53
+ updated = self.last_mtime.nil? ? files : files.select { |filename, mtime| self.last_mtime < mtime }
54
+ updated = exclude_matched_files(updated, self.exclusions)
55
+
56
+ unless updated.empty? then
57
+ self.last_mtime = Time.now
58
+ hook :updated, updated
59
+ end
60
+ end
61
+
62
+ # Add the supplied block to the available hooks, with the given
63
+ # name.
64
+ def add_hook name, &block
65
+ # New hooks added in front
66
+ @hooks[name] = [block] + @hooks[name]
67
+ end
68
+
69
+ # Call the event hook named +name+, passing in optional args
70
+ # depending on the hook itself.
71
+ #
72
+ # Returns false if no hook handled the event.
73
+ #
74
+ def hook name, *args
75
+ @hooks[name].any? { |plugin| plugin[self, *args] }
76
+ end
77
+
78
+ def wait_for_changes
79
+ hook :waiting
80
+ Kernel.sleep self.sleep_time until find_updated_files
81
+ end
82
+ end
83
+
84
+ if __FILE__ == $0
85
+ dirs_to_watch = ARGV.length > 0 ? ARGV : ["."]
86
+ watcher = FileChangeWatcher.new(dirs_to_watch) {|fwc, ufs| puts ufs.inspect}
87
+ watcher.wait_for_changes
88
+ end
@@ -0,0 +1,25 @@
1
+ # Find a unique temp file name starting with _basename_
2
+ # and having a file suffix _suffix_, then supply it to a block.
3
+ # After the block has executed make sure there is no tempfile
4
+ # with that name, if there is delete it.
5
+ def File.with_tempfile(suffix = ".temp", basename = "temp")
6
+ tempfilename = basename + rand(1e7).to_s + suffix
7
+ while Dir.exist?(tempfilename)
8
+ tempfilename = basename + rand(1e7).to_s + suffix
9
+ end
10
+ begin
11
+ yield tempfilename
12
+ ensure
13
+ File.delete(tempfilename) if Dir.exist?(tempfilename)
14
+ end
15
+ end
16
+
17
+ # Find a unique temp file name in the same way as for with_tempfile
18
+ # but now save a string to that file before supplying the filename
19
+ # to the block.
20
+ def File.with_tempfile_containing(contents, suffix = ".temp", basename = "temp")
21
+ File.with_tempfile do |tempfilename|
22
+ File.open(tempfilename, "w") {|fh| fh.write(contents)}
23
+ yield tempfilename
24
+ end
25
+ end
@@ -0,0 +1,17 @@
1
+ require 'feldtruby'
2
+
3
+ class Numeric
4
+ def round_to_decimals(numDecimals = 2)
5
+ factor = 10**numDecimals
6
+ (self * factor).round / factor.to_f
7
+ end
8
+
9
+ def protected_division_with(denom)
10
+ return 0.0 if denom == 0
11
+ self / denom
12
+ end
13
+
14
+ def ratio_diff_vs(other)
15
+ (self - other).protected_division_with(other)
16
+ end
17
+ end
@@ -0,0 +1,5 @@
1
+ module Kernel
2
+ def rand_int(top)
3
+ (top * rand()).floor
4
+ end
5
+ end
@@ -0,0 +1,31 @@
1
+ require 'feldtruby'
2
+ require 'nokogiri'
3
+ require 'open-uri'
4
+
5
+ # Fetch html pages from a site but ensure some time between subsequent get operations.
6
+ # To minimize the risk that we "annoy" the site operators.
7
+ class FeldtRuby::HtmlDocGetter
8
+ def initialize(minTimeBetweenGets = 1.0, maxRandomDelayBetweenGets = 3.0)
9
+ @min_delay = minTimeBetweenGets
10
+ @delta_delay = maxRandomDelayBetweenGets - @min_delay
11
+ @delay_until = Time.now - 1.0 # Ensure no wait the first time
12
+ end
13
+ def get(url)
14
+ wait_until_delay_passed()
15
+ begin
16
+ open(url).read
17
+ ensure
18
+ set_new_delay
19
+ end
20
+ end
21
+ def get_html_doc(url)
22
+ Nokogiri::HTML(get(url))
23
+ end
24
+ def wait_until_delay_passed
25
+ now = Time.now
26
+ sleep(@delay_until - now) if now < @delay_until
27
+ end
28
+ def set_new_delay
29
+ @delay_until = Time.now + (@min_delay + rand() * @delta_delay)
30
+ end
31
+ end