webruby 0.1.0

data/modules/emscripten/src/relooper/Relooper.h

@@ -0,0 +1,250 @@
+/*
+This is an optimized C++ implemention of the Relooper algorithm originally
+developed as part of Emscripten. This implementation includes optimizations
+added since the original academic paper [1] was published about it, and is
+written in an LLVM-friendly way with the goal of inclusion in upstream
+LLVM.
+
+[1] Alon Zakai. 2011. Emscripten: an LLVM-to-JavaScript compiler. In Proceedings of the ACM international conference companion on Object oriented programming systems languages and applications companion (SPLASH '11). ACM, New York, NY, USA, 301-312. DOI=10.1145/2048147.2048224 http://doi.acm.org/10.1145/2048147.2048224
+*/
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+#ifdef __cplusplus
+
+#include <map>
+#include <deque>
+#include <set>
+
+struct Block;
+struct Shape;
+
+// Info about a branching from one block to another
+struct Branch {
+  enum FlowType {
+    Direct = 0, // We will directly reach the right location through other means, no need for continue or break
+    Break = 1,
+    Continue = 2
+  };
+  Shape *Ancestor; // If not NULL, this shape is the relevant one for purposes of getting to the target block. We break or continue on it
+  Branch::FlowType Type; // If Ancestor is not NULL, this says whether to break or continue
+  bool Labeled; // If a break or continue, whether we need to use a label
+  const char *Condition; // The condition for which we branch. For example, "my_var == 1". Conditions are checked one by one. One of the conditions should have NULL as the condition, in which case it is the default
+  const char *Code; // If provided, code that is run right before the branch is taken. This is useful for phis
+
+  Branch(const char *ConditionInit, const char *CodeInit=NULL);
+  ~Branch();
+
+  // Prints out the branch
+  void Render(Block *Target, bool SetLabel);
+};
+
+typedef std::set<Block*> BlockSet;
+typedef std::map<Block*, Branch*> BlockBranchMap;
+
+// Represents a basic block of code - some instructions that end with a
+// control flow modifier (a branch, return or throw).
+struct Block {
+  // Branches become processed after we finish the shape relevant to them. For example,
+  // when we recreate a loop, branches to the loop start become continues and are now
+  // processed. When we calculate what shape to generate from a set of blocks, we ignore
+  // processed branches.
+  // Blocks own the Branch objects they use, and destroy them when done.
+  BlockBranchMap BranchesOut;
+  BlockSet BranchesIn;
+  BlockBranchMap ProcessedBranchesOut;
+  BlockSet ProcessedBranchesIn;
+  Shape *Parent; // The shape we are directly inside
+  int Id; // A unique identifier
+  const char *Code; // The string representation of the code in this block. Owning pointer (we copy the input)
+  bool IsCheckedMultipleEntry; // If true, we are a multiple entry, so reaching us requires setting the label variable
+
+  Block(const char *CodeInit);
+  ~Block();
+
+  void AddBranchTo(Block *Target, const char *Condition, const char *Code=NULL);
+
+  // Prints out the instructions code and branchings
+  void Render(bool InLoop);
+
+  // INTERNAL
+  static int IdCounter;
+};
+
+// Represents a structured control flow shape, one of
+//
+//  Simple: No control flow at all, just instructions. If several
+//          blocks, then 
+//
+//  Multiple: A shape with more than one entry. If the next block to
+//            be entered is among them, we run it and continue to
+//            the next shape, otherwise we continue immediately to the
+//            next shape.
+//
+//  Loop: An infinite loop.
+//
+//  Emulated: Control flow is managed by a switch in a loop. This
+//            is necessary in some cases, for example when control
+//            flow is not known until runtime (indirect branches,
+//            setjmp returns, etc.)
+//
+
+class SimpleShape;
+class LabeledShape;
+class MultipleShape;
+class LoopShape;
+
+struct Shape {
+  int Id; // A unique identifier. Used to identify loops, labels are Lx where x is the Id.
+  Shape *Next; // The shape that will appear in the code right after this one
+  Shape *Natural; // The shape that control flow gets to naturally (if there is Next, then this is Next)
+
+  enum ShapeType {
+    Simple,
+    Multiple,
+    Loop
+  };
+  ShapeType Type;
+
+  Shape(ShapeType TypeInit) : Id(Shape::IdCounter++), Next(NULL), Type(TypeInit) {}
+  virtual ~Shape() {}
+
+  virtual void Render(bool InLoop) = 0;
+
+  static SimpleShape *IsSimple(Shape *It) { return It && It->Type == Simple ? (SimpleShape*)It : NULL; }
+  static MultipleShape *IsMultiple(Shape *It) { return It && It->Type == Multiple ? (MultipleShape*)It : NULL; }
+  static LoopShape *IsLoop(Shape *It) { return It && It->Type == Loop ? (LoopShape*)It : NULL; }
+  static LabeledShape *IsLabeled(Shape *It) { return IsMultiple(It) || IsLoop(It) ? (LabeledShape*)It : NULL; }
+
+  // INTERNAL
+  static int IdCounter;
+};
+
+struct SimpleShape : public Shape {
+  Block *Inner;
+
+  SimpleShape() : Shape(Simple), Inner(NULL) {}
+  void Render(bool InLoop) {
+    Inner->Render(InLoop);
+    if (Next) Next->Render(InLoop);
+  }
+};
+
+typedef std::map<Block*, Shape*> BlockShapeMap;
+
+// A shape that may be implemented with a labeled loop.
+struct LabeledShape : public Shape {
+  bool Labeled; // If we have a loop, whether it needs to be labeled
+
+  LabeledShape(ShapeType TypeInit) : Shape(TypeInit), Labeled(false) {}
+};
+
+struct MultipleShape : public LabeledShape {
+  BlockShapeMap InnerMap; // entry block -> shape
+  int NeedLoop; // If we have branches, we need a loop. This is a counter of loop requirements,
+                // if we optimize it to 0, the loop is unneeded
+
+  MultipleShape() : LabeledShape(Multiple), NeedLoop(0) {}
+
+  void RenderLoopPrefix();
+  void RenderLoopPostfix();
+
+  void Render(bool InLoop);
+};
+
+struct LoopShape : public LabeledShape {
+  Shape *Inner;
+
+  LoopShape() : LabeledShape(Loop), Inner(NULL) {}
+  void Render(bool InLoop);
+};
+
+/*
+struct EmulatedShape : public Shape {
+  std::deque<Block*> Blocks;
+  void Render(bool InLoop);
+};
+*/
+
+// Implements the relooper algorithm for a function's blocks.
+//
+// Usage:
+//  1. Instantiate this struct.
+//  2. Call AddBlock with the blocks you have. Each should already
+//     have its branchings in specified (the branchings out will
+//     be calculated by the relooper).
+//  3. Call Render().
+//
+// Implementation details: The Relooper instance has
+// ownership of the blocks and shapes, and frees them when done.
+struct Relooper {
+  std::deque<Block*> Blocks;
+  std::deque<Shape*> Shapes;
+  Shape *Root;
+
+  Relooper();
+  ~Relooper();
+
+  void AddBlock(Block *New);
+
+  // Calculates the shapes
+  void Calculate(Block *Entry);
+
+  // Renders the result.
+  void Render();
+
+  // Sets the global buffer all printing goes to. Must call this or MakeOutputBuffer.
+  static void SetOutputBuffer(char *Buffer, int Size);
+
+  // Creates an output buffer. Must call this or SetOutputBuffer.
+  static void MakeOutputBuffer(int Size);
+
+  // Sets asm.js mode on or off (default is off)
+  static void SetAsmJSMode(int On);
+};
+
+typedef std::map<Block*, BlockSet> BlockBlockSetMap;
+
+#if DEBUG
+struct Debugging {
+  static void Dump(BlockSet &Blocks, const char *prefix=NULL);
+  static void Dump(Shape *S, const char *prefix=NULL);
+};
+#endif
+
+#endif // __cplusplus
+
+// C API - useful for binding to other languages
+
+#ifdef _WIN32
+  #ifdef RELOOPERDLL_EXPORTS
+    #define RELOOPERDLL_API __declspec(dllexport)
+  #else
+    #define RELOOPERDLL_API __declspec(dllimport)
+  #endif
+#else
+  #define RELOOPERDLL_API
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+RELOOPERDLL_API void  rl_set_output_buffer(char *buffer, int size);
+RELOOPERDLL_API void  rl_make_output_buffer(int size);
+RELOOPERDLL_API void  rl_set_asm_js_mode(int on);
+RELOOPERDLL_API void *rl_new_block(const char *text);
+RELOOPERDLL_API void  rl_delete_block(void *block);
+RELOOPERDLL_API void  rl_block_add_branch_to(void *from, void *to, const char *condition, const char *code);
+RELOOPERDLL_API void *rl_new_relooper();
+RELOOPERDLL_API void  rl_delete_relooper(void *relooper);
+RELOOPERDLL_API void  rl_relooper_add_block(void *relooper, void *block);
+RELOOPERDLL_API void  rl_relooper_calculate(void *relooper, void *entry);
+RELOOPERDLL_API void  rl_relooper_render(void *relooper);
+
+#ifdef __cplusplus
+}
+#endif
+

data/modules/emscripten/src/relooper/doit.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+
+echo "relooper"
+g++ Relooper.cpp -c -g
+g++ Relooper.cpp -c -g -DDEBUG -o RelooperDebug.o
+
+echo "test"
+g++ test.cpp -c -o test.o -g
+g++ Relooper.o test.o -o test
+
+echo "test 2"
+gcc test2.c -c -o test2.o -g
+g++ Relooper.o test2.o -o test2
+
+echo "test 3"
+gcc test3.c -c -o test3.o -g
+g++ Relooper.o test3.o -o test3
+
+echo "test debug"
+gcc test_debug.cpp -c -o test_debug.o -g -DDEBUG
+g++ RelooperDebug.o test_debug.o -o test_debug
+
+echo "test dead"
+gcc test_dead.cpp -c -o test_dead.o -g
+g++ Relooper.o test_dead.o -o test_dead
+
+echo "test 4"
+g++ test4.cpp -c -o test4.o -g
+g++ Relooper.o test4.o -o test4
+
+echo "test 5"
+g++ test5.cpp -c -o test5.o -g
+g++ Relooper.o test5.o -o test5
+
+echo "test 6"
+g++ test6.cpp -c -o test6.o -g
+g++ Relooper.o test6.o -o test6
+
+echo "test inf"
+g++ test_inf.cpp -c -o test_inf.o -g
+g++ Relooper.o test_inf.o -o test_inf
+
+echo "test fuzz1"
+g++ test_fuzz1.cpp -c -o test_fuzz1.o -g
+g++ Relooper.o test_fuzz1.o -o test_fuzz1
+
+echo "test fuzz2"
+g++ test_fuzz2.cpp -c -o test_fuzz2.o -g
+g++ Relooper.o test_fuzz2.o -o test_fuzz2
+
+echo "test fuzz3"
+g++ test_fuzz3.cpp -c -o test_fuzz3.o -g
+g++ Relooper.o test_fuzz3.o -o test_fuzz3
+
+echo "test fuzz4"
+g++ test_fuzz4.cpp -c -o test_fuzz4.o -g
+g++ Relooper.o test_fuzz4.o -o test_fuzz4
+
+echo "test fuzz5"
+g++ test_fuzz5.cpp -c -o test_fuzz5.o -g
+g++ Relooper.o test_fuzz5.o -o test_fuzz5
+
+echo "test fuzz6"
+g++ test_fuzz6.cpp -c -o test_fuzz6.o -g
+g++ Relooper.o test_fuzz6.o -o test_fuzz6
+
+echo
+echo "============================="
+echo
+

data/modules/emscripten/src/relooper/emscripten/glue.js

@@ -0,0 +1,57 @@
+
+  var RBUFFER_SIZE = 20*1024*1024;
+  var rbuffer = _malloc(RBUFFER_SIZE);
+  _rl_set_output_buffer(rbuffer, RBUFFER_SIZE);
+
+  var TBUFFER_SIZE = 10*1024*1024;
+  var tbuffer = _malloc(TBUFFER_SIZE);
+
+  var RelooperGlue = {};
+  RelooperGlue['init'] = function() {
+    this.r = _rl_new_relooper();
+  },
+  RelooperGlue['addBlock'] = function(text) {
+    assert(this.r);
+    assert(text.length+1 < TBUFFER_SIZE);
+    writeStringToMemory(text, tbuffer);
+    var b = _rl_new_block(tbuffer);
+    _rl_relooper_add_block(this.r, b);
+    return b;
+  };
+  RelooperGlue['addBranch'] = function(from, to, condition, code) {
+    assert(this.r);
+    if (condition) {
+      assert(condition.length+1 < TBUFFER_SIZE/2);
+      writeStringToMemory(condition, tbuffer);
+      condition = tbuffer;
+    } else {
+      condition = 0; // allow undefined, null, etc. as inputs
+    }
+    if (code) {
+      assert(code.length+1 < TBUFFER_SIZE/2);
+      writeStringToMemory(code, tbuffer + TBUFFER_SIZE/2);
+      code = tbuffer + TBUFFER_SIZE/2;
+    } else {
+      code = 0; // allow undefined, null, etc. as inputs
+    }
+    _rl_block_add_branch_to(from, to, condition, code);
+  };
+  RelooperGlue['render'] = function(entry) {
+    assert(this.r);
+    assert(entry);
+    _rl_relooper_calculate(this.r, entry);
+    _rl_relooper_render(this.r);
+    var ret = Pointer_stringify(rbuffer);
+    _rl_delete_relooper(this.r);
+    this.r = 0;
+    return ret;
+  };
+  RelooperGlue['setDebug'] = function(debug) {
+    _rl_set_debugging(+!!debug);
+  };
+  RelooperGlue['setAsmJSMode'] = function(on) {
+    _rl_set_asm_js_mode(on);
+  };
+
+  Module['Relooper'] = RelooperGlue;
+

data/modules/emscripten/src/relooper/emscripten/test.js

@@ -0,0 +1,44 @@
+// js -m -n -e "load('relooper.js')" test.js
+
+function test() {
+  print("-- If shape --\n");
+
+  //Relooper.setDebug(1);
+
+  {
+    Relooper.init();
+
+    var b_a = Relooper.addBlock("// block A\n");
+    var b_b = Relooper.addBlock("// block B\n");
+    var b_c = Relooper.addBlock("// block C\n");
+
+    Relooper.addBranch(b_a, b_b, "check == 10", "atob();");
+    Relooper.addBranch(b_a, b_c, 0, "atoc();");
+
+    Relooper.addBranch(b_b, b_c, 0, "btoc();");
+
+    var output = Relooper.render(b_a);
+    print(output);
+  }
+
+  {
+    Relooper.init();
+
+    var b_a = Relooper.addBlock("// block A\n");
+    var b_b = Relooper.addBlock("// block B\n");
+    var b_c = Relooper.addBlock("// block C\n");
+
+    Relooper.addBranch(b_a, b_b, "check == fee()");
+    Relooper.addBranch(b_a, b_c, 0, 0);
+
+    Relooper.addBranch(b_c, b_b);
+
+    var output = Relooper.render(b_a);
+    print(output);
+  }
+}
+
+test();
+
+// TODO: wrap the relooper itself
+

data/modules/emscripten/src/relooper/fuzzer.py

@@ -0,0 +1,116 @@
+
+import random, subprocess, difflib
+
+while True:
+  # Random decisions
+  num = random.randint(2, 250)
+  density = random.random() * random.random()
+  decisions = [random.randint(1, num*20) for x in range(num*3)]
+  branches = [0]*num
+  defaults = [0]*num
+  for i in range(num):
+    b = set([])
+    bs = random.randint(1, max(1, round(density*random.random()*(num-1))))
+    for j in range(bs):
+      b.add(random.randint(1, num-1))
+    b = list(b)
+    defaults[i] = random.choice(b)
+    b.remove(defaults[i])
+    branches[i] = b
+  print num, density
+
+  for temp in ['fuzz', 'fuzz.fast.js', 'fuzz.slow.js', 'fuzz.cpp']:
+    try:
+      os.unlink(temp)
+    except:
+      pass
+
+  # parts
+  entry = '''print('entry'); var label; var state; var decisions = %s; var index = 0; function check() { if (index == decisions.length) throw 'HALT'; return decisions[index++] }''' % str(decisions)
+
+  slow = entry + '\n'
+  for i in range(len(branches[0])):
+    if i > 0: slow += 'else '
+    b = branches[0]
+    slow += 'if (state %% %d == %d) { label = %d; }\n' % (len(b)+1, i, b[i]) # TODO: split range 1-n into these options
+  if len(branches[0]): slow += 'else '
+  slow += 'label = %d;\n' % defaults[0]
+
+  slow += '''
+while(1) switch(label) {
+'''
+
+  fast = '''
+
+#include <stdlib.h>
+#include "Relooper.h"
+
+int main() {
+  char *buffer = (char*)malloc(10*1024*1024);
+  Relooper::SetOutputBuffer(buffer, 10*1024*1024);
+'''
+
+  for i in range(1, num):
+    slow += '  case %d: print(%d); state = check(); \n' % (i, i)
+    b = branches[i]
+    for j in range(len(b)):
+      slow += '    if (state %% %d == %d) { label = %d; break }\n' % (len(b)+1, j, b[j]) # TODO: split range 1-n into these options
+    slow += '    label = %d; break\n' % defaults[i]
+
+  for i in range(num):
+    if i == 0:
+      fast += '''
+  Block *b%d = new Block("%s");
+''' % (i, entry)
+    else:
+      fast += '''  Block *b%d = new Block("print(%d); state = check();%s");
+''' % (i, i, '// ' + ('.' * int(random.expovariate(0.5/num))))
+
+  for i in range(num):
+    b = branches[i]
+    for j in range(len(b)):
+      fast += '''  b%d->AddBranchTo(b%d, "state %% %d == %d");
+''' % (i, b[j], len(b)+1, j)
+    fast += '''  b%d->AddBranchTo(b%d, NULL);
+''' % (i, defaults[i])
+
+  fast += '''
+  Relooper r;
+'''
+
+  for i in range(num):
+    fast += '''  r.AddBlock(b%d);
+''' % i
+
+  fast += '''
+  r.Calculate(b0);
+  printf("\\n\\n");
+  r.Render();
+
+  puts(buffer);
+
+  return 1;
+}
+'''
+
+  slow += '}'
+
+  open('fuzz.slow.js', 'w').write(slow)
+  open('fuzz.cpp', 'w').write(fast)
+  print '_'
+  slow_out = subprocess.Popen(['mozjs', '-m', '-n', 'fuzz.slow.js'], stdout=subprocess.PIPE).communicate()[0]
+
+  print '.'
+  subprocess.call(['g++', 'fuzz.cpp', 'Relooper.o', '-o', 'fuzz', '-g'])
+  print '*'
+  subprocess.call(['./fuzz'], stdout=open('fuzz.fast.js', 'w'))
+  print '-'
+  fast_out = subprocess.Popen(['mozjs', '-m', '-n', 'fuzz.fast.js'], stdout=subprocess.PIPE).communicate()[0]
+  print
+
+  if slow_out != fast_out:
+    print ''.join([a.rstrip()+'\n' for a in difflib.unified_diff(slow_out.split('\n'), fast_out.split('\n'), fromfile='slow', tofile='fast')])
+    assert False
+
+  #break
+

data/modules/emscripten/src/relooper/ministring.h

@@ -0,0 +1,35 @@
+
+// Tiny implementation of strings. Avoids linking in all of std::string
+
+#include <stdlib.h>
+#include <string.h>
+
+class ministring {
+  int used;
+  char *buffer;
+  int bufferSize;
+public:
+  ministring() : used(0), buffer(NULL), bufferSize(0) {}
+  ~ministring() { if (buffer) free(buffer); }
+
+  char *c_str() { return buffer; }
+  int size() { return used; }
+
+  void clear() {
+    used = 0; // keep the buffer alive as an optimization, just resize
+  }
+
+  ministring& operator+=(const char *s) {
+    int len = strlen(s);
+    if (used + len + 2 > bufferSize) {
+      // try to avoid frequent reallocations
+      bufferSize = 2*(bufferSize + len);
+      bufferSize += 1024 - bufferSize % 1024;
+      buffer = (char*)(buffer ? realloc(buffer, bufferSize) : malloc(bufferSize));
+    }
+    strcpy(buffer + used, s);
+    used += len;
+    return *this;
+  }
+};
+