flok 0.0.38 → 0.0.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 847b5b5b82356f175e5193de663c69487bc543fc
-  data.tar.gz: 06b557dd275ee22398619033cc94b893eb815cc8
+  metadata.gz: 069696218ad132364486c4d0256e3f274f8e19e3
+  data.tar.gz: 4d2f67fe000dbeb7baf2077632ede49358f5dc6e
 SHA512:
-  metadata.gz: 2dd662522a021063c44cfdbea4db37ea388f7d82f42108be93cd2b68507286b21650e524c9a2f99d2025781a375ae6d133383bb767e77cfa5af9642d37467f36
-  data.tar.gz: eed43cf8edd7d656d0364770c73468658bff40759621629f3e127627327ab7c9f03a3d3bf4200c079a5293297216fac924efb35bd259bb474bd1e97c6687ce26
+  metadata.gz: 127c0e54aa11cf77024c208600cff1027350272b39d57f8abd28f6e6030a8765483f3a732cdb7cdc39d221ba0ead994228b811e4f0da87efec5d68b9506ab516
+  data.tar.gz: 8c2cd9a6c868e0ffbfc8359817cea534221ab0a8e623fc75f70c64e49233716a78ee68cc5e968624efb0e35041648d4d0281b9a5501b2c3fa501673361054ccb

data/app/drivers/chrome/src/dispatch.js

@@ -7,6 +7,13 @@
 //Here is an example with two successive calls
 //  [2, 'mul', 3, 4, 1, 'print', 'hello world']
 function if_dispatch(qq) {
+  if (qq[0] == 'i') {
+    qq.shift();
+    if_dispatch_call_int_end = true
+  } else {
+    if_dispatch_call_int_end = false
+  }
+
   //If debug socket is attached, forward events to it
   //and do not process the events
   <% if @mods.include? "debug" %>
@@ -22,13 +29,32 @@ function if_dispatch(qq) {
     //The very first thing is the queue type
     var queueType = q.shift();
 
-    //Where there is still things left on the queue
-    while (q.length > 0) {
-      //Grab the first thing off the queue, this is the arg count
-      var argc = q.shift();
+    //Main queue events are run synchronously on w.r.t to this thread of execution
+    //Asynchronous events are dispatched individually
+    if (queueType === 0) {
+      //Where there is still things left on the queue
+      while (q.length > 0) {
+        //Grab the first thing off the queue, this is the arg count
+        var argc = q.shift();
 
-      //Grab the next thing and look that up in the function table. Pass args left
-      this[q.shift()].apply(null, q.splice(0, argc));
+        //Grab the next thing and look that up in the function table. Pass args left
+        this[q.shift()].apply(null, q.splice(0, argc));
+      }
+    } else {
+        //Dispatch asynchronous queue events
+        while (q.length > 0) {
+          //Grab the next thing and look that up in the function table. Pass args left
+          function({
+            var argc = q.shift();
+            var q0 = q.shift();
+            var q1 = q.splice(0, argc);
+            async_call = function() {
+              this[q0].apply(null, q1);
+            }
+
+            setTimeout(async_call, 0);
+          })();
+        }
     }
   }
 
@@ -36,6 +62,12 @@ function if_dispatch(qq) {
   <% if @mods.include? "debug" %>
     }
   <% end %>
+
+
+  if (if_dispatch_call_int_end) {
+    if_dispatch_call_int_end = false;
+    int_dispatch([])
+  }
 }
 
 function ping() {
@@ -50,3 +82,6 @@ function ping2(arg1, arg2) {
   int_dispatch([1, "pong2", arg1])
   int_dispatch([2, "pong2", arg1, arg2])
 }
+
+function ping_nothing() {
+}

data/app/drivers/chrome/src/persist.js

@@ -14,16 +14,7 @@ function if_per_del_ns(ns) {
 }
 
 function if_per_get(s, ns, key) {
-  function async() {
-    var _ns = store.namespace(ns);
-    var res = _ns.get(key);
-    int_dispatch([2, "int_per_get_res", s, res]);
-  }
-  setTimeout(async, 0);
-}
-
-function if_per_get_sync(s, ns, key) {
   var _ns = store.namespace(ns);
   var res = _ns.get(key);
-  int_dispatch([2, "int_per_get_res", s, res]);
+  int_dispatch([2, "int_per_get_res", s, ns, res]);
 }

data/app/kern/dispatch.js

@@ -30,6 +30,9 @@ function int_dispatch(q) {
   //Now push all of what we can back
   var dump = [];
 
+  //Add 'i' to start
+  var incomplete = false;
+
   //Send main queue
   if (main_q.length > 0) {
     var out = [0];
@@ -43,20 +46,21 @@ function int_dispatch(q) {
   if (net_q.length > 0 && net_q_rem > 0) {
     //Always pick the minimum between the amount remaining and the q length
     var n = net_q.length < net_q_rem ? net_q.length : net_q_rem;
+    if (n != net_q.length) { incomplete = true; }
 
     var out = [1];
     var piece = net_q.splice(0, n);
     for (var i = 0; i < piece.length; ++i) {
       out.push.apply(out, piece[i]);
     }
-    dump.push(out);
 
-    net_q_rem -= n;
+    dump.push(out);
   }
 
   if (disk_q.length > 0 && disk_q_rem > 0) {
     //Always pick the minimum between the amount remaining and the q length
     var n = disk_q.length < disk_q_rem ? disk_q.length : disk_q_rem;
+    if (n != disk_q.length) { incomplete = true; }
 
     var out = [2];
     var piece = disk_q.splice(0, n);
@@ -64,13 +68,12 @@ function int_dispatch(q) {
       out.push.apply(out, piece[i]);
     }
     dump.push(out);
-
-    disk_q_rem -= n;
   }
 
   if (cpu_q.length > 0 && cpu_q_rem > 0) {
     //Always pick the minimum between the amount remaining and the q length
     var n = cpu_q.length < cpu_q_rem ? cpu_q.length : cpu_q_rem;
+    if (n != cpu_q.length) { incomplete = true; }
 
     var out = [3];
     var piece = cpu_q.splice(0, n);
@@ -78,13 +81,12 @@ function int_dispatch(q) {
       out.push.apply(out, piece[i]);
     }
     dump.push(out);
-
-    cpu_q_rem -= n;
   }
 
   if (gpu_q.length > 0 && gpu_q_rem > 0) {
     //Always pick the minimum between the amount remaining and the q length
     var n = gpu_q.length < gpu_q_rem ? gpu_q.length : gpu_q_rem;
+    if (n != gpu_q.length) { incomplete = true; }
 
     var out = [4];
     var piece = gpu_q.splice(0, n);
@@ -92,19 +94,9 @@ function int_dispatch(q) {
       out.push.apply(out, piece[i]);
     }
     dump.push(out);
-
-    gpu_q_rem -= n;
   }
 
-  //Send async queue
-  if (async_q.length > 0) {
-    var out = [5];
-    for (var i = 0; i < async_q.length; ++i) {
-      out.push.apply(out, async_q[i]);
-    }
-    dump.push(out);
-    async_q = [];
-  }
+  if (incomplete) { dump.unshift("i"); }
 
   if (dump.length != 0) {
     if_dispatch(dump);
@@ -135,8 +127,6 @@ function ping3(arg1) {
     SEND("cpu", "pong3");
   } else if (arg1 == "gpu") {
     SEND("gpu", "pong3");
-  } else if (arg1 == "async") {
-    SEND("async", "pong3");
   }
 }
 
@@ -151,8 +141,6 @@ function ping4(arg1) {
     SEND("cpu", "pong4");
   } else if (arg1 == "gpu") {
     SEND("gpu", "pong4");
-  } else if (arg1 == "async") {
-    SEND("async", "pong4");
   }
 }
 
@@ -166,7 +154,6 @@ function ping4_int(arg1) {
     ++cpu_q_rem;
   } else if (arg1 == "gpu") {
     ++gpu_q_rem;
-  } else if (arg1 == "async") {
   }
 }
 
@@ -176,7 +163,6 @@ net_q = [];
 disk_q = [];
 cpu_q = [];
 gpu_q = [];
-async_q = [];
 
 //Each queue has a max # of things that can be en-queued
 //These are decremented when the message is sent (not just queued)
@@ -185,3 +171,11 @@ net_q_rem = 5;
 disk_q_rem = 5;
 cpu_q_rem = 5;
 gpu_q_rem = 5;
+
+<% if @debug %>
+function spec_dispatch_q(queue, count) {
+  for (var i = 0; i < count; ++i) {
+    queue.push([0, "spec"]);
+  }
+}
+<% end %>

data/app/kern/gen_id.js

@@ -0,0 +1,8 @@
+//Generate a random seed
+gen_seed = Math.floor(Math.random() * 1000000000);
+gen_seed_s = Math.floor(Math.random() * 1000000000);
+
+function gen_id() {
+  gen_seed += 1;
+  return crc32(gen_seed, gen_seed_s).toString();
+}

data/app/kern/macro.rb

@@ -1,24 +1,26 @@
-#Process one js code file at a time
-def macro_process text
-  out = StringIO.new
+module Flok
+  #Process one js code file at a time
+  def self.macro_process text
+    out = StringIO.new
 
-  text.split("\n").each do |l|
-    #Send macro
-    if l =~ /SEND/
-      l.strip!
-      l.gsub!(/SEND\(/, "")
-      l.gsub! /\)$/, ""
-      l.gsub! /\);$/, ""
-      o = l.split(",").map{|e| e.strip}
+    text.split("\n").each do |l|
+      #Send macro
+      if l =~ /SEND/
+        l.strip!
+        l.gsub!(/SEND\(/, "")
+        l.gsub! /\)$/, ""
+        l.gsub! /\);$/, ""
+        o = l.split(",").map{|e| e.strip}
 
-      queue_name = o.shift.gsub(/"/, "")
+        queue_name = o.shift.gsub(/"/, "")
 
-      res = %{#{queue_name}_q.push([#{o.count-1}, #{o.join(", ")}])}
-      out.puts res
-    else
-      out.puts l
+        res = %{#{queue_name}_q.push([#{o.count-1}, #{o.join(", ")}])}
+        out.puts res
+      else
+        out.puts l
+      end
     end
-  end
 
-  return out.string
+    return out.string
+  end
 end

data/app/kern/pagers/pg_spec0.js

@@ -0,0 +1,20 @@
+<% if @debug %>
+function pg_spec0_init(ns, options) {
+  pg_spec0_watchlist = [];
+  pg_spec0_unwatchlist = [];
+  pg_spec0_init_params = {ns: ns, options: options};
+  pg_spec0_ns = ns;
+}
+
+function pg_spec0_watch(id, page) {
+  pg_spec0_watchlist.push({id: id, page: page});
+}
+
+function pg_spec0_unwatch(id) {
+  pg_spec0_unwatchlist.push(id);
+}
+
+function pg_spec0_write(page) {
+  vm_cache_write(pg_spec0_ns, page);
+}
+<% end %>

data/app/kern/services/vm.rb

@@ -7,31 +7,91 @@ service :vm do
       <% end %>
     };
 
+    vm_dirty = {
+      <% @options[:pagers].each do |p| %>
+        <%= p[:namespace] %>: {},
+      <% end %>
+    };
+
+    vm_bp_to_nmap = {};
+
     //Notification listeners, converts ns+key to an array of base pointers
     vm_notify_map = {};
 
     //Cache
-    function vm_cache_write(ns, key, value) {
-      vm_cache[ns][key] = value;
-    }
+    function vm_cache_write(ns, page) {
+      var old = vm_cache[ns][page._id];
+      if (old && old._hash == page._hash) { return; }
+
+      vm_dirty[ns][page._id] = page;
+      vm_cache[ns][page._id] = page;
 
-    //Notification of a change
-    function vm_notify(ns, key) {
       var a = vm_notify_map[ns];
       if (a) {
-        var b = a[key];
+        var b = a[page._id];
 
         if (b) {
           for (var i = 0; i < b.length; ++i) {
-            <% @options[:pagers].each do |p| %>
-              if (ns === "<%= p[:namespace] %>") {
-                <%= p[:name] %>_read(ns, b[i], key);
-              }
-            <% end %>
+            int_event(b, "read_res", page);
           }
         }
       }
     }
+
+    function vm_rehash_page(page) {
+      var z = 0;
+
+      //head and next are optional
+      if (page._head) { var z = crc32(0, page._head) }
+      if (page._next) { z = crc32(z, page._next) }
+
+      z = crc32(z, page._id)
+
+      var e = page.entries;
+      for (var i = 0; i < e.length; ++i) {
+        z = crc32(z, e[i]._sig);
+      }
+
+      page._hash = z.toString();
+    }
+
+    function vm_pageout() {
+      <% @options[:pagers].each do |p| %>
+        //Get id_to_page mappings
+        var id_to_page = vm_dirty["<%= p[:namespace] %>"];
+        if (id_to_page) {
+          var ids = Object.keys(id_to_page);
+
+          //For each mapping, write the page
+          for (var i = 0; i < ids.length; ++i) {
+            var p = id_to_page[ids[i]];
+            SEND("main", "if_per_set", "<%= p[:namespace] %>", ids[i], p);
+          }
+        <% end %>
+        }
+
+      //Clear dirty list
+      vm_dirty = {
+        <% @options[:pagers].each do |p| %>
+          <%= p[:namespace] %>: {},
+        <% end %>
+      };
+    }
+
+    //Part of the persist module
+    //res is page
+    function int_per_get_res(s, ns, res) {
+      //If there is already a cached entry, a pager beat us to it
+      //ignore this for now because the pager should be more up to
+      //date
+      if (vm_cache[ns][res._id]) { return };
+
+      vm_cache_write(ns, res);
+    }
+
+    <% if @debug %>
+      vm_write_list = [];
+    <% end %>
   }
 
   on_wakeup %{
@@ -44,7 +104,7 @@ service :vm do
 
     //Call init functions
     <% @options[:pagers].each do |p| %>
-      <%= p[:name] %>_init(<%= (p[:options] || {}).to_json %>);
+      <%= p[:name] %>_init("<%= p[:namespace] %>", <%= (p[:options] || {}).to_json %>);
     <% end %>
   }
 
@@ -55,6 +115,35 @@ service :vm do
   }
 
   on_disconnect %{
+    //We need to remove all the entries in vm_notify_map, but we only
+    //get an array of bp for each array in vm_notify_map[ns][key]...
+    //So we use the inverted lookup of vm_bp_to_nmap[bp][ns][key] to get a pointer
+    //to vm_notify_map[ns][key] and associated index. We then delete all the
+    //entries out of vm_notify_map
+
+    //Foreach namespace
+    var nss = Object.keys(vm_bp_to_nmap[bp]);
+    for (var i = 0; i < nss.length; ++i) {
+      //Namespace node
+      var nn = vm_bp_to_nmap[bp][nss[i]];
+
+      //Get all keys (which are ids)
+      var nnk = Object.keys(nn);
+
+      for (var x = 0; x < nnk.length; ++x) {
+        //Array contains [node (pointer to vm_notify_map[ns][key]), index] where index points to base pointer of this
+        //controller in the array
+        var arr = nn[nnk[i]][0]
+        var idx = nn[nnk[i]][1]
+
+        //Remove
+        arr.splice(idx, 1);
+      }
+
+    }
+
+    //Now we just clean up vm_bp_to_nmap because it's no longer used
+    delete vm_bp_to_nmap[bp];
   }
 
   on "read_sync", %{
@@ -76,27 +165,19 @@ service :vm do
     int_event(bp, "read_sync_res", res);
   }
 
-  on "read", %{
+  on "write", %{
     <% raise "No pagers given in options for vm" unless @options[:pagers] %>
 
-    var cres = vm_cache[params.ns][params.key]; 
-    if (cres != undefined) {
-      int_event(bp, "read_res", {key: params.key, value: cres});
-    }
+    //We are going to fix the _hash on the page
+    vm_rehash_page(params.page);
 
-    <% @options[:pagers].each do |p| %>
-      if (params.ns === "<%= p[:namespace] %>") {
-        <%= p[:name] %>_read(params.ns, bp, params.key);
-      }
+    <% if @debug %>
+      vm_write_list.push(params.page);
     <% end %>
-  }
-
-  on "write", %{
-    <% raise "No pagers given in options for vm" unless @options[:pagers] %>
 
     <% @options[:pagers].each do |p| %>
       if (params.ns === "<%= p[:namespace] %>") {
-        <%= p[:name] %>_write(params.key, params.value);
+        <%= p[:name] %>_write(params.page);
       }
     <% end %>
   }
@@ -104,6 +185,9 @@ service :vm do
   on "watch", %{
     <% raise "No pagers given in options for vm" unless @options[:pagers] %>
 
+    //Cache entry
+    var cache_entry = vm_cache[params.ns][params.id];
+
     //Ensure map exists
     ////////////////////////////////////////////////
     var a = vm_notify_map[params.ns];
@@ -112,18 +196,57 @@ service :vm do
       vm_notify_map[params.ns] = a;
     }
 
-    var b = a[params.key];
+    var b = a[params.id];
     if (!b) {
       b = [];
-      a[params.key] = b;
+      a[params.id] = b;
     }
 
+    <% if @debug %>
+      var midx = vm_notify_map[params.ns][params.id].indexOf(bp)
+      if (midx != -1) {
+        throw "Multiple calls to watch for the ns: " +  params.ns + " and id: " + params.id
+      }
+    <% end %>
     b.push(bp)
     ////////////////////////////////////////////////
 
+    //Add to vm_bp_to_nmap
+    ////////////////////////////////////////////////
+    //Construct 
+    if (vm_bp_to_nmap[bp] === undefined) { vm_bp_to_nmap[bp] = {}; }
+    if (vm_bp_to_nmap[bp][params.ns] === undefined) { vm_bp_to_nmap[bp][params.ns] = {}; }
+
+    //Add reverse mapping, length-1 because it was just pushed
+    vm_bp_to_nmap[bp][params.ns][params.id] = [b, b.length-1];
+
+    //If cache exists, then signal controller *now* while we wait for the pager
+    if (cache_entry) {
+      int_event(bp, "read_res", cache_entry);
+    }
+
+    //Send a request now for disk read for sync
+    if (!cache_entry && params.sync) {
+      SEND("main", "if_per_get", "vm", params.ns, params.id);
+    }
+
+    //Do not signal pager if there is a watch request already in place
+    //as pager already knows; if it's equal to 1, this is the 'first'
+    //watch to go through as we have no info on it but just added it
+    if (vm_notify_map[params.ns][params.id].length > 1) { return; }
+
+    //While we're waiting for the pager try loading from disk, if this
+    //disk request is slower than the pager response, that's ok...
+    //the disk response will double check to see if the cache got set
+    //somewhere and not set it itself.
+    if (!cache_entry && !params.sync) {
+      SEND("disk", "if_per_get", "vm", params.ns, params.id);
+    }
+
+    //Now load the appropriate pager
     <% @options[:pagers].each do |p| %>
       if (params.ns === "<%= p[:namespace] %>") {
-        <%= p[:name] %>_watch(params.ns, params.key);
+        <%= p[:name] %>_watch(params.id, cache_entry);
       }
     <% end %>
   }
@@ -131,11 +254,34 @@ service :vm do
   on "unwatch", %{
     <% raise "No pagers given in options for vm" unless @options[:pagers] %>
 
+    var midx = vm_notify_map[params.ns][params.id].indexOf(bp)
+    vm_notify_map[params.ns][params.id].splice(midx, 1);
+
+    delete vm_bp_to_nmap[bp][params.ns][params.id];
+
     <% @options[:pagers].each do |p| %>
       if (params.ns === "<%= p[:namespace] %>") {
-        <%= p[:name] %>_unwatch(params.ns, params.key);
+        <%= p[:name] %>_unwatch(params.id);
       }
     <% end %>
   }
 
+  on "unwatch", %{
+    <% raise "No pagers given in options for vm" unless @options[:pagers] %>
+
+    var midx = vm_notify_map[params.ns][params.id].indexOf(bp)
+    vm_notify_map[params.ns][params.id].splice(midx, 1);
+
+    delete vm_bp_to_nmap[bp][params.ns][params.id];
+
+    <% @options[:pagers].each do |p| %>
+      if (params.ns === "<%= p[:namespace] %>") {
+        <%= p[:name] %>_unwatch(params.id);
+      }
+    <% end %>
+  }
+
+  every 20.seconds, %{
+    vm_pageout();
+  }
 end