evoasm 0.0.2.pre7 → 0.1.0.pre2

data/Makefile

@@ -3,4 +3,4 @@ all:
 clean:
 	rake clean
 debug:
-	rake compile -- --enable-debug
+	rake compile -- --debug

data/README.md

@@ -9,24 +9,32 @@ It will then try to come up with a short program (in the form of machine code) t
 by means of genetic programming.
 *Evoasm* contains a JIT that executes the generated machine code on the fly.
 
-Currently, the only supported architecture is **x86_64**.
+Currently, the only supported architecture is **x86-64**.
 
-**NOTE:** *Evoasm* is in a very early stage.
+## Features
+
+* Fast JIT
+* Non-linear control flow
+* [x86-64](https://github.com/evoasm/evoasm-gen/blob/master/data/tables/x64.csv) up to AVX2 (no FPU)
+* Lightweight backend [C library](https://github.com/evoasm/libevoasm) with no third-party dependencies
+* Ruby bindings
 
 ## Installation
 
-    $ gem install evoasm --pre
+    $ gem install evoasm
     
 ### Requirements
 
-* Ruby 2.3 (MRI)
+* Ruby (MRI >= 2.3, JRuby >= 9.1.2)
 * [Capstone](http://www.capstone-engine.org/) for disassembling (*optional*).
+* [Graphviz](http://www.graphviz.org/) (libgraphviz) for visualizing programs (*optional*).
+* [Gnuplot](http://gnuplot.sourceforge.net) for visualizing loss functions (*optional*)
 * POSIX-compliant OS (Linux and Mac OS X should both work).
 
-## Usage
-
-Please see [Getting Started](https://github.com/furunkel/evoasm/wiki/Getting-Started).
+## Documentation
 
+Please see the [API documentation](https://evoasm.github.io/evoasm/doc/) or
+have a look at the [examples](https://evoasm.github.io/evoasm/doc/file.SymbolicRegression.html).
 
 ## Contributing
 
@@ -38,6 +46,6 @@ Please see [Getting Started](https://github.com/furunkel/evoasm/wiki/Getting-Sta
 
 ## License
 
-[MPL-2.0][license]
+[AGPL-3.0][license]
 
-[license]: https://github.com/furunkel/evoasm/blob/master/LICENSE.txt
+[license]: https://github.com/furunkel/evoasm/blob/master/LICENSE.md

data/Rakefile

@@ -1,128 +1,60 @@
 require 'bundler/gem_tasks'
-
 require 'rake/testtask'
-require 'rake/extensiontask'
 
-require 'evoasm/tasks/gen_task'
-require 'evoasm/tasks/template_task'
+require 'evoasm/gen'
 
-Rake::ExtensionTask.new('evoasm_ext')
+import 'ext/evoasm_ext/compile.rake'
 
-Rake::TestTask.new do |t|
-  t.libs.push 'lib'
-  t.pattern = "test/**/*_test.rb"
-end
 
-Evoasm::Tasks::GenTask.new
+namespace :test do
+  Rake::TestTask.new :unit do |t|
+    t.libs.push 'lib', 'test/unit', 'test/helpers'
+    t.pattern = "test/unit/**/*_test.rb"
+    t.verbose = true
+  end
 
-begin
-  require 'evoasm/scrapers'
-  Evoasm::Scrapers::X64.new do |t|
-    t.output_filename = Evoasm::Tasks::GenTask::X64_TABLE_FILENAME
+  Rake::TestTask.new :integration do |t|
+    t.libs.push 'lib', 'test/integration', 'test/helpers'
+    t.pattern = "test/integration/**/*_test.rb"
+    t.verbose = true
   end
-rescue LoadError
 end
 
-directory 'lib' => ['evoasm:gen', 'evoasm:templates']
+task :test => ['test:unit', 'test:integration']
+task :default => :test
 
-task :console do
-  sh "pry --gem"
-end
-
-=begin
-Evoasm::Tasks::TemplateTask.new do |t|
-  t.source = %w(evoasm-edge.h.tmpl)
-  t.target = %w(evoasm-asg-edge.h)
-  t.subs = {
-    t: 'evoasm_sym',
-    s: 'evoasm_asg_edge',
-    w: 32,
-    includes: <<~EOL
-      #include "evoasm-sym.h"
-    EOL
-  }
-end
 
-Evoasm::Tasks::TemplateTask.new do |t|
-  t.source = %w(evoasm-node.h.tmpl)
-  t.target = %w(evoasm-asg-node.h)
-  t.subs = {
-    t: 'evoasm_token',
-    l: 'token',
-    s: 'evoasm_asg_node',
-    w: 32,
-    includes: <<~EOL
-      #include "evoasm-token.h"
-    EOL
-  }
+require 'evoasm/gen'
+Evoasm::Gen::GenTask.new 'lib/evoasm/libevoasm' do |t|
+  t.file_types = %i(ruby_ffi)
 end
 
-Evoasm::Tasks::TemplateTask.new do |t|
-  t.source = %w(evoasm-free-list.c.tmpl evoasm-free-list.h.tmpl)
-  t.target = %w(evoasm-asg-edge-list.c evoasm-asg-edge-list.h)
-  t.subs = {
-    s: 'evoasm_asg_edge_list',
-    e: 'evoasm_asg_edge',
-    includes: '#include "gen/evoasm-asg-edge.h"',
-    embed: 0,
-    w: 32,
-    eql: <<~EOL
-      return a->dir == b->dir &&
-             a->node_idx == b->node_idx &&
-             a->label == b->label;
-    EOL
-  }
+begin
+  require 'evoasm/scrapers'
+  Evoasm::Scrapers::X64.new do |t|
+    t.output_filename = Evoasm::Tasks::GenTask::X64_TABLE_FILENAME
+  end
+rescue LoadError
 end
 
-Evoasm::Tasks::TemplateTask.new do |t|
-  t.source = %w(evoasm-free-list.c.tmpl evoasm-free-list.h.tmpl)
-  t.target = %w(evoasm-asg-node-list.c evoasm-asg-node-list.h)
-  t.subs = {
-    s: 'evoasm_asg_node_list',
-    e: 'evoasm_asg_node',
-    w: 32,
-    includes: '#include "evoasm-asg-node.h"',
-    embed: 0,
-  }
-end
+namespace :yard do
+  require 'yard'
+  YARD::Rake::YardocTask.new :build do |t|
+    t.files   = %w(lib/**/*.rb - docs/**/*.md)
+    t.options = %w(--asset docs/examples:examples)
+  end
 
-Evoasm::Tasks::TemplateTask.new do |t|
-  t.source = %w(evoasm-graph.c.tmpl evoasm-graph.h.tmpl)
-  t.target = %w(evoasm-asg.c evoasm-asg.h)
-  t.subs = {
-    s: 'evoasm_asg',
-    el: 'evoasm_asg_edge_list',
-    nl: 'evoasm_asg_node_list',
-    e: 'evoasm_asg_edge',
-    n: 'evoasm_asg_node',
-    l: 'evoasm_sym',
-    w: 32,
-    includes: <<~END,
-      #include "evoasm-asg-edge-list.h"
-      #include "evoasm-asg-node-list.h"
-      #include "evoasm-sym.h"
-    END
-    edge_eql: <<~EOL,
-      return a->dir == b->dir &&
-             a->node_index == b->node_index &&
-             a->index == b->index;
-    EOL
-    embed: 0,
-  }
-end
+  desc "Push YARD documentation to GitHub Pages"
+  task :push => :build do
+    cp_r 'doc', Dir.tmpdir
+    sh 'git checkout gh-pages'
+    rm_r 'doc'
+    mv File.join(Dir.tmpdir, 'doc'), 'doc'
+    sh 'git commit doc -m "Update documentation"'
+    sh 'git push origin gh-pages'
+    sh 'git checkout master'
+  end
 
-Evoasm::Tasks::TemplateTask.new do |t|
-  t.source = %w(evoasm-free-list.c.tmpl evoasm-free-list.h.tmpl)
-  t.target = %w(evoasm-page-list.c evoasm-page-list.h)
-  t.subs = {
-    s: 'evoasm_page_list',
-    e: 'evoasm_page',
-    w: 32,
-    includes: '#include "evoasm-page.h"',
-    embed: 0,
-  }
 end
 
-=end
 
-task 'evoasm:templates' => Evoasm::Tasks::TemplateTask.all

data/bin/gdb

@@ -1,2 +1,2 @@
 #!/bin/sh
-gdb -ex 'handle SIGFPE noprint nostop' --args $(rbenv which ruby) -rbundler/setup $@
+gdb -ex 'handle SIGFPE noprint pass' --args $(rbenv which ruby) -rbundler/setup $@

data/bin/gdb_loop

@@ -0,0 +1,4 @@
+#!/bin/sh
+while true; do
+  gdb -ex 'handle SIGFPE noprint nostop' -ex run -ex quit --args $(rbenv which ruby) -rbundler/setup $@
+done

data/docs/FindingInstructions.md

@@ -0,0 +1,17 @@
+# @title Finding Instructions
+# Finding Instructions
+
+*Evoasm* can be used to find instructions that exhibit a certain behavior.
+Let's assume we want to find an instruction that counts the number of 1s or the number 
+of trailing 0s in a binary number.
+Is there an instruction for that? Let's find out.
+
+{include:file:docs/examples/bit_insts.rb}
+
+Depending on your CPU model this might output the following:
+
+```
+popcnt
+
+bsf
+```

data/docs/JIT.md

@@ -0,0 +1,14 @@
+# @title Just-in-time Compilation
+# Just-in-time Compilation
+
+Using *Evoasm*'s {Evoasm::Buffer Buffer} class, it is possible to do simple just-in-time compilation.
+
+{include:file:docs/examples/jit.rb}
+
+The second block executes a division-by-zero, causing an exception to be thrown.
+The expected output is thus:
+
+```
+Result: 3
+Execution failed with exception `de'
+```

data/docs/SymbolicRegression.md

@@ -0,0 +1,102 @@
+# @title Symbolic Regression
+# Symbolic Regression
+
+A classical application of genetic programming is [symbolic regression](https://en.wikipedia.org/wiki/Symbolic_regression).
+Symbolic regression is the task of finding a mathematical expression that approximates a given set of data points as closely as possible.
+
+## Example Data
+
+For purposes of illustration, assume we are given the following
+table of data points, sampled from the function
+<math xmlns='http://www.w3.org/1998/Math/MathML'>
+  <mi>y</mi>
+  <mo>=</mo>
+  <msqrt>
+    <msup>
+      <mi>x</mi>
+      <mn>3</mn>
+    </msup>
+    <mo>+</mo>
+    <mrow>
+      <mn>2</mn>
+      <mi>x</mi>
+    </mrow>
+  </msqrt>
+</math>.
+
+| x  | y |
+| ------- | ----- |
+| 0.0 | 0.0 |
+| 0.5 | 1.0606601717798212 |
+| 1.0 | 1.7320508075688772 |
+| 1.5 | 2.5248762345905194 |
+| 2.0 | 3.4641016151377544 |
+| 2.5 | 4.541475531146237  |
+| 3.0 | 5.744562646538029  |
+| 3.5 | 7.0622234459127675 |
+| 4.0 | 8.48528137423857   |
+| 4.5 | 10.00624804809475  |
+| 5.0 | 11.61895003862225  |
+
+Using only the points in this table, we now want *Evoasm*
+to come up with a program that, given *x*, will output the corresponding
+*y* for all *x* (i.e. not only those listed in the table).
+
+## Finding a Solution Program
+
+Here is how it is done:
+ 
+{include:file:docs/examples/sym_reg.rb}
+
+On my machine, *Evoasm* will find a solution in less than a second:
+
+```
+0x555556b9e270:	vmulsd	xmm2, xmm0, xmm2
+0x555556b9e274:	vfmadd213sd	xmm2, xmm1, xmm0
+0x555556b9e279:	vaddsd	xmm3, xmm3, xmm2
+0x555556b9e27d:	vfnmadd231sd	xmm2, xmm2, xmm2
+0x555556b9e282:	vfnmadd231sd	xmm0, xmm1, xmm0
+0x555556b9e287:	vfmadd132sd	xmm0, xmm0, xmm0
+0x555556b9e28c:	vsqrtsd	xmm1, xmm1, xmm3
+0x555556b9e290:	addsd	xmm0, xmm2
+0x555556b9e294:	vaddsd	xmm3, xmm3, xmm2
+0x555556b9e298:	vmulsd	xmm0, xmm2, xmm2
+```
+
+## Examining the Solution
+You can now experiment with the found program.
+Use the {Evoasm::Program#run Program#run} method to run the found program with arbitrary input.
+
+```ruby
+program.run 1.0  # => [1.7320508075688772]
+# test for values not given in table
+program.run 10.0 # => [31.937438845342623]
+```
+
+## Intron Elimination
+The solutions found by *Evoasm* will usually contain large 
+portions of noneffective code (so-called introns).
+
+Introns can be removed using the `eliminate_introns!` method.
+This will considerably shorten the size of the solution:
+
+```ruby
+program.eliminate_introns.disassembly format: true
+```
+gives
+
+```
+0x555556ba59f0:	vmulsd	xmm2, xmm0, xmm2
+0x555556ba59f4:	vfmadd213sd	xmm2, xmm1, xmm0
+0x555556ba59f9:	vaddsd	xmm3, xmm3, xmm2
+0x555556ba59fd:	vsqrtsd	xmm1, xmm1, xmm3
+```
+
+For comparison, here is what GCC 6.2 outputs with `-O3 -march=core-avx2`
+```
+vmulsd  %xmm0, %xmm0, %xmm2
+vaddsd  %xmm0, %xmm0, %xmm1
+vfmadd132sd     %xmm2, %xmm1, %xmm0
+vsqrtsd %xmm0, %xmm1, %xmm1
+```        
+

data/docs/Visualization.md

@@ -0,0 +1,29 @@
+# @title Visualization
+# Visualization
+
+*Evoasm* provides means to visualize loss functions and found programs.
+
+{include:file:docs/examples/vis.rb}
+
+
+## Loss Functions
+
+Loss functions can be visualized by using {Evoasm::Population#plot} which
+will plot the loss function using [Gnuplot](https://gnuplot.sourceforge.net/).
+If a filename is provided, the loss function graph is saved to file. Otherwise,
+a window will open.
+
+Each column represents a deme. The first row shows program losses, the following
+rows kernel losses.
+
+![Loss functions](examples/loss.gif)
+
+
+## Programs
+
+In a similar fashion, programs can be visualized using [Graphviz](https://www.graphviz.org/).
+Use {Evoasm::Program#to_gv} to obtain a [GV::Graph](http://www.rubydoc.info/github/furunkel/gv/GV/Graph), which
+can be saved to a file using [GV::Graph#save](http://www.rubydoc.info/github/furunkel/gv/GV/Graph#save-instance_method).
+
+![Function](examples/program.png)
+

data/docs/examples/bit_insts.rb

@@ -0,0 +1,44 @@
+require 'evoasm'
+require 'evoasm/x64'
+
+count_1s = {
+  0b0 => 0,
+  0b1 => 1,
+  0b100 => 1,
+  0b101 => 2,
+  0b111 => 3,
+  0b1000 => 1
+}
+
+parameters = Evoasm::Population::Parameters.new do |p|
+  p.instructions = Evoasm::X64.instruction_names(:gp, :rflags)
+  p.deme_size = 1024
+  p.deme_count = 1
+  p.kernel_size = 1
+  p.program_size = 1
+  p.parameters = %i(reg0 reg1 reg2 reg3)
+end
+
+parameters.examples = count_1s
+
+population = Evoasm::Population.new parameters
+program, loss = population.run
+
+puts "#{program.disassemble.first[1]}"
+
+puts
+
+count_trailing_0s = {
+  0b100 => 2,
+  0b1 => 0,
+  0b10 => 1,
+  0b101 => 0,
+  0b10000 => 4
+}
+
+parameters.examples = count_trailing_0s
+
+population = Evoasm::Population.new parameters
+program, loss = population.run
+
+puts "#{program.disassemble.first[1]}"