flok 0.0.82 → 0.0.83

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebf963c82063ec89074d43114c4416bcc3505df8
-  data.tar.gz: ece62f398cf5e92ce976597f1fee070678bf13d4
+  metadata.gz: 7bd8fa8fdb50bddf42da16826b0046fb59193ce7
+  data.tar.gz: f65050b61273c66c424aeae68568cf0eb7272512
 SHA512:
-  metadata.gz: 5b3367ebb243879208147f3b944558fb880e9774c00338e390b0d947cc41ca088a1638ca3db3915deab288bc2477b7955ad39a8095e66ce4942c71f54a2a98e1
-  data.tar.gz: e85b5607da669fcac1a3e795541ca7b37cf9f59295c187989bd367b2db26d2701dfcf69b28bbc22c82ffad4114d5e87c95f6ec552da50c5b48f489d7e66a9841
+  metadata.gz: d70b9cdc7e87a2b068b55b255a112838a04bac27dca48d5d05b1d6e5023d71a05930856dfe3a256161f4b557a1fbb13996cd7d561c7b45808882256d39057c34
+  data.tar.gz: 36c1164fdefca9a1f9a53b9e845d4cdebfd66f1ae8084bfa0f3d4433a8bcf834003694d71453fb81d46a55ffe73b95c919c5799652b0d14de3d2bf01b40a1f58

data/app/kern/pagers/pg_sockio.js

@@ -2,10 +2,14 @@
 <% [0].each do |i| %>
   //Destination for events sent from the sockio driver
   function __pg_sockio<%= i %>_xevent_handler(ep, ename, einfo) {
+    //Events
+    //update - From the sockio interface, an update refers to a page update
+    //interval - Every 15 seconds, from the callout timer.
     if (ename === "update") {
       //If changes_id was given
       if (einfo.changes_id !== undefined) {
-        vm_mark_changes_synced(vm_cache[pg_sockio<%= i %>_ns][einfo.page._id], einfo.changes_id)
+        //This is a friendly function that will ignore mis-matches of changes_id
+        vm_mark_changes_synced(vm_cache[pg_sockio<%= i %>_ns][einfo.page._id], einfo.changes_id);
       }
 
       //If page exists, then we need to rebase the page, this will actually
@@ -15,10 +19,39 @@
         vm_rebase(vm_cache[pg_sockio<%= i %>_ns][einfo.page._id], einfo.page);
       }
 
+      //Mark page as synced if it contains no changes, we don't need to check base
+      //here because if it has base, it has __changes as well
+      if (einfo.page.__changes === undefined) {
+        vm_pg_unmark_needs_sync(pg_sockio<%= i %>_ns, einfo.page._id)
+      }
+
       //Write out page
       vm_transaction_begin();
         vm_cache_write(pg_sockio<%= i %>_ns, einfo.page);
       vm_transaction_end();
+    } else if (ename === "interval") {
+
+      //Create watch_list
+      var watch_list = [];
+      var watched_keys = Object.keys(vm_notify_map[pg_sockio<%= i %>_ns]);
+      for (var i = 0; i < watched_keys.length; ++i) {
+        var key = watched_keys[i];
+        watch_list.push(key);
+
+        //If entry exists, put in the hash value
+        if (vm_cache[pg_sockio<%= i %>_ns][key] === undefined) {
+          watch_list.push(null);
+        } else {
+          watch_list.push(vm_cache[pg_sockio<%= i %>_ns][key]._hash);
+        }
+      }
+
+      //Synchronize the watchlist with the server
+      var resync_info = {
+        watch_list: watch_list
+      };
+
+      SEND("net", "if_sockio_send", pg_sockio<%= i %>_bp, "resync", resync_info);
     } else {
       <% if @debug %>
         throw "pg_sockio<%= i %>_xevent_handler received an event called: " + ename + "that it does not know how to handle. This event should never have even been forwarded, but you may have missed adding the handler code if you did request a forward"
@@ -46,6 +79,9 @@
     //Signal that the socket.io driver should forward all events to the socket defined by pg_sockio{N}_bp
     //to the endpoint (with the same reference)
     SEND("net", "if_sockio_fwd", pg_sockio<%= i %>_bp, "update", pg_sockio<%= i %>_bp);
+
+    //Request a timer every 15 seconds
+    reg_interval(pg_sockio<%= i %>_bp, "interval", 15*4);
   }
 
   function pg_sockio<%= i %>_watch(id, page) {
@@ -56,6 +92,10 @@
   }
 
   function pg_sockio<%= i %>_unwatch(id) {
+    var info = {
+      page_id: id
+    }
+    SEND("net", "if_sockio_send", pg_sockio<%= i %>_bp, "unwatch", info);
   }
 
   function pg_sockio<%= i %>_write(page) {
@@ -70,6 +110,12 @@
       vm_cache_write(pg_sockio<%= i %>_ns, page); 
     vm_transaction_end();
 
+    //Mark pages as needing a synchronization
+    vm_pg_mark_needs_sync(pg_sockio<%= i %>_ns, page._id);
+  }
+
+  function pg_sockio<%= i %>_sync(page_id) {
+    var page = vm_cache[pg_sockio<%= i %>_ns][page_id];
     //Clone page and send a copy to the server
     var copied = vm_copy_page(page);
     var info = {page: copied, changes: page.__changes, changes_id: page.__changes_id};

data/app/kern/services/vm.rb

@@ -793,6 +793,9 @@ service :vm do
     }
 
     vm_notify_map[params.ns][params.id].splice(midx, 1);
+    
+    //Remove from notify map if no entries exist
+    if (vm_notify_map[params.ns][params.id].length === 0) { delete vm_notify_map[params.ns][params.id]; }
 
     delete vm_bp_to_nmap[bp][params.ns][params.id];
 

data/docs/kernel_api.md

@@ -39,6 +39,9 @@ instead.
       0 is the master root view. Returns base pointer. `event_gw` is a pointer to a `vc`. If it is null, then any events coming in will not be sent to
       somewhere else if they do not match any 'on' for the current action.
 
+##Timers
+  * See [./callout.md](Callout) for how to register a timer
+
 ##Things that are compiled into the kernel from user given data
 `MODS` - A list of modules that was specified in `./app/drivers/$PLATFORM/config.yml`
 `PLATFORM` - The platform that this kernel was compiled with

data/docs/mod/persist.md

@@ -11,7 +11,7 @@ Persistance management. Loosely based on redis.
 `if_per_set_f(ns, key, tp)` - Tell the driver to dereference the telepointer and to save it to disk.
 
 For race conditions, e.g, an asynchronous set is followed by a synchronous get, it is undefined as to what that behavior that will be.
-It is expected that the kernel should manage the write-back cache and that the driver should not attempt a write back cache unless
+I If the page does not exist, the hash value is null.t is expected that the kernel should manage the write-back cache and that the driver should not attempt a write back cache unless
 it is convenient to do so.
 
 ###Kernel interrupts

data/docs/services/vm.md

@@ -222,6 +222,15 @@ The pager synchronization daemon is embodied in the function called `vm_pg_sync_
         `__changes_id` of the page matches `changes_id`. If the page is based (implying the base page has changes and the page has changes as all base
         pages have changes), then if the `changes_id` matches the **base** `__changes_id` , the `__base` is removed from the page. If `changes_id`
         does not match in either of the cases, then nothing happends. This may happend if a synchronization errousouly comes in.
+    * **Why do we have both `vm_rebase` and `vm_mark_changes_synced`?**
+      * They are used under similar circumstances. You always `vm_mark_changes_synced` before calling `vm_rebase` on a page. The reasoning is that
+          `vm_rebase` will assume that the cached page does not contain changes if they are present in `older`. If you didn't do this, then the
+          cached page would be rebased and could contain changes even though it's already been rebased on an older page. E.g. `newer[changes, nobase]`
+          rebased would be `older[changes, nobase]` where `changes` are equal on the `newer` and `older` but clearly that's incorrect. Another way of
+          looking at it is that `vm_rebase` is saying that you are rebasing **on an authority** page and therefore needs no evidence that the page was
+          an authority (which is why the `changes_id` can be stripped).  Method 3 of looking at it is that `vm_rebase` on a `newer[changes,
+          based[changes, nobase]]` with `older` where `older` contains the changes of `newer.__base.__changes`, would result in `older` having
+          `newer.__base.__changes` fast-forwarded over it, which it would already contain those changes.
 ###Non functional (functional as is in lambda calculus, or lisp (no **global** state changes but may modify parameters)
 ####Pager specific
   * `vm_cache_write(ns,  page)` - Save a page to cache memory. This will not recalculate the page hash. The page will be stored in `vm_cache[ns][id]` by.
@@ -229,7 +238,8 @@ The pager synchronization daemon is embodied in the function called `vm_pg_sync_
     * The page_id is added to the `vm_unsynced` with the value of 0; see above in `Datatypes & Structures` for details. i.e.
         `vm_unsynced[$PAGER_NS][page_id] = 0`
     *  the pager's routine of `sync` is called immediately. The page must exist in cache at this point.
-  * `vm_pg_unmark_needs_sync(ns, page_id)` - Removes the page from the pending synchronization queue `delete vm_unsynced[$PAGER_NS][page_id]`)
+  * `vm_pg_unmark_needs_sync(ns, page_id)` - Removes the page from the pending synchronization queue `delete vm_unsynced[$PAGER_NS][page_id]`). If
+      it's not in the synchronization queue, then nothing will happend
 
 ### <a name='user_page_modification_helpers'></a>User page modification helpers (Controller Macros)
 You should never directly edit a page in user land; if you do; the pager has no way of knowing that you made modifications. Additionally, if you have multiple controllers watching a page, and it is modified in one controller, those other controllers

data/docs/services/vm/pagers.md

@@ -78,7 +78,10 @@ This pager connects to a socket.io server via the `sockio` module.
         that the endpoint shares the same value as the socket's base pointer.
   * **Functions**
     * `init` - Will begin trying to establish a connection to the server. When pages are written, 
-    * `watch` - Signals to the socket.io server that a page is being watched via `watch` event with parameters `{page_id:}`
+    * `watch` - Signals to the socket.io server that a page is being watched via `watch` event with parameters `{page_id:}`. Additionally, at a periodic interval (10 seconds) by default,
+      the server will be notified of the entire watch-list via the event `resync` which contains the key `watch_list` which will return an array of [page_id, page_hash, page_id, page_hash...] where page_id is the pages `_id` and
+      `page_hash` is the page's `__hash`. If the page does not exist, the hash value is null.
+      
     * `unwatch` - Signals to the socket.io server that a page is no longer being watched via `unwatch` event with parameters `{page_id:}` 
     * `write` - 
        * Sends the sockio server the `write` event with a hash containing `page`, and optionally (`changes`, and `changes_id`). The `page` contains only the basic

data/docs/todo.md

@@ -10,6 +10,8 @@
   2. The pager synchronization daemon and functions associated with marking paging as needing synchronization will attempt to sync at one time and not
   store the data if there is a crash or exit before synchronization completes. Furethermore, too many unsynced pages will wreck havok as they wil be
   dispatched at the same time via the synchronization daemon.
+  3. A `watch` request on a page does technically asynchronously dispatch a disk read request at the same time the pager's request goes through. If a pager was to write
+     a page before this disk read, directly via `vm_cache_write`, then the cached page would not be loaded in time.
 
 ###Ideas for improvement
   0. The `Raise` function should not actually signal the controller, it could just inline that

data/lib/flok/version.rb

@@ -1,3 +1,3 @@
 module Flok
-  VERSION = "0.0.82"
+  VERSION = "0.0.83"
 end

data/spec/env/kern.rb

@@ -31,8 +31,7 @@ shared_context "kern" do
 
     #Will return everything put into the 'dump' dictionary (pre-defined for your convenience)
     def evald str
-      self.eval "dump = {}"
-      self.eval str
+      self.eval "dump = {}; #{str}"
       _dump = self.dump("dump")
 
       return DumpHelper.new(_dump)
@@ -204,6 +203,10 @@ shared_context "kern" do
       end
     end
 
+    def ignore
+      @q.shift
+    end
+
     #Expect the queue to not contain a message matching
     def expect_not_to_contain msg_name, &block
       original_q = JSON.parse(@q.to_json)

data/spec/kern/assets/vm/controller12.rb

@@ -1,43 +1,38 @@
 controller :my_controller do
   services :vm
 
+  on_entry %{
+      read_res_params = [];
+  }
+
   action :my_action do
     on_entry %{
-      var entry = {
-        hello: "world"
-      }
-
-      page = NewPage("array", "test");
-      SetPageHead(page, "head");
-      SetPageNext(page, "next");
-      EntryInsert(page, 0, entry);
-
-      page2 = CopyPage(page)
-      EntryInsert(page, 0, entry);
-
-
-      var watch_info = {
-        ns: "spec",
-        id: "test"
-      }
-
-      var write_info = {
-        ns: "spec",
-        page: page
-      };
+    }
 
-      var write_info2 = {
-        ns: "spec",
-        page: page2
-      };
+    on "write_first", %{
+      var page = vm_create_page("test")
+      info_write = {ns: "spec", page: page}
+      Request("vm", "write", info_write);
+      context.page = page;
+    }
 
+    on "watch_first", %{
+      var info_watch = {ns: "spec", id: "test"}
+      Request("vm", "watch", info_watch);
+    }
 
-      read_res_params = [];
-      Request("vm", "write", write_info);
-      Request("vm", "watch", watch_info);
-      Request("vm", "write", write_info2);
+    on "modify_first", %{
+      var page = vm_copy_page(context.page);
+      page.entries.push({
+        _sig: "test",
+        _id: "test",
+        value: "test"
+      });
+      var info_write = {ns: "spec", page: page};
+      Request("vm", "write", info_write);
     }
 
+
     on "read_res", %{
       read_res_params.push(params);
     }

data/spec/kern/assets/vm/pg_sockio/unmark_changes.rb

@@ -0,0 +1,34 @@
+controller :my_controller do
+  services :vm
+
+  action :my_action do
+    on_entry %{
+    }
+
+    on "create_page", %{
+      var page = vm_create_page("test");
+      var info = {ns: "sockio", page: page};
+      Request("vm", "write", info);
+
+      var info2 = {ns: "sockio", id: "test"};
+      Request("vm", "watch", info2);
+    }
+
+    on "modify_page", %{
+      //Modify the page
+      var new_page = vm_copy_page(read_page);
+
+      new_page.entries.push({
+        _id: gen_id(),
+        _sig: gen_id(),
+        value: "foo"
+      });
+      var info = {ns: "sockio", page: new_page}
+      Request("vm", "write", info);
+    }
+
+    on "read_res", %{
+      read_page = params;
+    }
+  end
+end

data/spec/kern/assets/vm/pg_sockio/watch.rb

@@ -3,16 +3,17 @@ controller :my_controller do
 
   action :my_action do
     on_entry %{
-      var watch_info = {
-        ns: "sockio",
-        id: "test", 
-      };
-
+      var watch_info = { ns: "sockio", id: "test", };
       Request("vm", "watch", watch_info);
     }
 
     on "read_res", %{
       read_res_params = params;
     }
+
+    on "unwatch", %{
+      var unwatch_info = { ns: "sockio", id: "test"};
+      Request("vm", "unwatch", unwatch_info);
+    }
   end
 end

data/spec/kern/assets/vm/pg_sockio/watch3.rb

@@ -0,0 +1,31 @@
+controller :my_controller do
+  services :vm
+
+  action :my_action do
+    on_entry %{
+      //Setup watch
+      var watch_info0 = { ns: "sockio", id: "test0", };
+      var watch_info1 = { ns: "sockio", id: "test1", };
+
+      Request("vm", "watch", watch_info0);
+      Request("vm", "watch", watch_info1);
+    }
+
+    on "write_test1", %{
+      //Write test1
+      var test1 = vm_create_page("test1");
+      var write_info1 = { ns: "sockio", page: test1};
+      Request("vm", "write", write_info1);
+    }
+
+    on "read_res", %{
+      read_res_params = params;
+    }
+
+    #Unwatch test1 page
+    on "unwatch_test1", %{
+      var unwatch_info1 = { ns: "sockio", id: "test1", };
+      Request("vm", "unwatch", unwatch_info1);
+    }
+  end
+end

data/spec/kern/vm_service_spec.rb

@@ -363,32 +363,35 @@ RSpec.describe "kern:vm_service" do
     expect(read_res_params).to eq(vm_write_list)
   end
 
-  it "non-sync watch does send two watch callbacks to a controller if there is cached content" do
+  it "non-sync watch does send two watch callbacks to a controller if there is cached content followed by a write" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/controller12.rb'), File.read("./spec/kern/assets/vm/config4.rb") 
 
     ctx.eval %{
       base = _embed("my_controller", 1, {}, null);
     }
 
-    #Should not have read anything at this point in time
-    read_res_params = JSON.parse(ctx.eval("JSON.stringify(read_res_params)"))
-    expect(read_res_params.length).to eq(0)
+    #Step 1. Write a page into cache
+    ################################################################################
+    #Trigger controller 'write_first'
+    @driver.int "int_event", [ ctx.eval("base"), "write_first", {} ]
 
-    ctx.eval("int_dispatch([])")
+    #Write should have trigger a disk read (to ensure there is no page in cache) vhich we respond
+    #with nothing
+    @driver.ignore_up_to "if_per_get"
+    @driver.int "int_per_get_res", ["vm", "spec", "test", nil]
+    ################################################################################
 
-    #Now we read the first after it de-queued
-    read_res_params = JSON.parse(ctx.eval("JSON.stringify(read_res_params)"))
-    expect(read_res_params.length).to eq(1)
+    #Step 2. Watch that page
+    ################################################################################
+    #Trigger controller 'watch_first'
+    @driver.int "int_event", [ ctx.eval("base"), "watch_first", {} ]
 
-    ctx.eval("int_dispatch([])")
+    #Asynchronous dispatch
+    100.times { @ctx.eval("int_dispatch()")}
 
-    #And now the second
-    read_res_params = JSON.parse(ctx.eval("JSON.stringify(read_res_params)"))
-    expect(read_res_params.length).to eq(2)
-
-    #And they should have been read in order
-    vm_write_list = JSON.parse(ctx.eval("JSON.stringify(vm_write_list)"));
-    expect(read_res_params).to eq(vm_write_list)
+    #Should have triggered the read_res
+    expect(@ctx.dump("read_res_params").count).to eq(1)
+    ################################################################################
   end
 
   it "vm_cache_write does not tell controllers an update has occurred if the page requested to cache was already cached" do

data/spec/kern/vm_sockio_pager_spec.rb

@@ -60,6 +60,26 @@ RSpec.describe "kern:sockio_pager" do
     @driver.mexpect "if_sockio_fwd", [Integer, "update", dump["pg_sockio0_bp"]], 1
   end
 
+  it "Does send an unwatch request via socket.io when a page is unwatched" do
+    ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
+    ctx.eval %{
+      //Call embed on main root view
+      base = _embed("my_controller", 0, {}, null);
+
+      //Drain queue
+      int_dispatch([]);
+    }
+
+    @driver.int "int_event", [ @ctx.eval("base"), "unwatch", {} ]
+
+    #Expect an unwatch request
+    @driver.ignore_up_to "if_sockio_send", 1 do |e|
+      e[1] == "unwatch"
+    end
+    unwatch_msg = @driver.get "if_sockio_send", 1
+    expect(unwatch_msg[2]).to eq({"page_id" => "test"})
+  end
+
   it "Does send a watch request via socket.io when a page is watched" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
     ctx.eval %{
@@ -77,7 +97,90 @@ RSpec.describe "kern:sockio_pager" do
     }], 1
   end
 
-  it "Does write a page to vm_cache that **does** already exist as <unbased, nochanges> the page receives an 'update' response from the external socket.io without a changes id" do
+  it "Does send a watch request at periodic intervals of all pages that are currently watched and then does not send pages that have been unwatched" do
+    ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch3.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
+    ctx.eval %{
+      //Call embed on main root view
+      base = _embed("my_controller", 0, {}, null);
+
+      //Drain queue
+      int_dispatch([]);
+    }
+
+    #Get through the first two watches triggered by the original watch
+    @driver.ignore_up_to "if_sockio_send", 1
+    @driver.mexpect "if_sockio_send", [Integer, "watch", {
+      "page_id" => "test0"
+    }], 1
+    @driver.ignore_up_to "if_sockio_send", 1
+    @driver.mexpect "if_sockio_send", [Integer, "watch", {
+      "page_id" => "test1"
+    }], 1
+
+    #Now we wait 15 seconds
+    (15*4).times { @driver.int "int_timer" }
+
+    #Now we should have a message for the synhronization of watchlist
+    @driver.ignore_up_to("if_sockio_send", 1) { |e| next e[1] == "resync" }
+    resync_res = @driver.get "if_sockio_send", 1
+    resync_info = resync_res[2] #Hash on end contains the actual data from the message
+    expect(resync_info.keys).to include("watch_list"); #Should have a watch list
+
+    #Check the watchlist we got, first get the hash values for the pages
+    expected_watch_list = []
+    expected_watch_list += ["test0", nil]
+    expected_watch_list += ["test1", nil]
+    expect(resync_info["watch_list"]).to eq(expected_watch_list)
+
+    #Test 2 - Now we are changing a page, so test1 should have a hash value
+    #######################################################################################################
+    #Update test1, which will try to read from disk, respond with a blank page
+    @driver.int "int_event", [ @ctx.eval("base"), "write_test1", {} ]
+    @driver.ignore_up_to "if_per_get", 2 do |e|
+      next e[2] == "test1"
+    end
+    @driver.int "int_per_get_res", ["vm", "sockio", "test1", nil]
+
+    #Now we wait 15 seconds (again)
+    (15*4).times { @driver.int "int_timer" }
+
+    #Now we should have a message for the synhronization of watchlist
+    @driver.ignore #it's incomplete... so
+    @driver.ignore_up_to("if_sockio_send", 1) { |e| next e[1] == "resync" }
+    resync_res = @driver.get "if_sockio_send", 1
+    resync_info = resync_res[2] #Hash on end contains the actual data from the message
+    expect(resync_info.keys).to include("watch_list"); #Should have a watch list
+
+    #Check the watchlist we got, first get the hash values for the pages
+    expected_watch_list = []
+    expected_watch_list += ["test0", nil]
+    expected_watch_list += ["test1", @ctx.eval("vm_cache.sockio.test1._hash")]
+    expect(resync_info["watch_list"]).to eq(expected_watch_list)
+    #######################################################################################################
+
+    #Test 3 - Now we unwatch a page
+    #######################################################################################################
+    #Unwatch test1
+    @driver.int "int_event", [ @ctx.eval("base"), "unwatch_test1", {} ]
+
+    #Now we wait 15 seconds (again)
+    (15*4).times { @driver.int "int_timer" }
+
+    #Now we should have a message for the synhronization of watchlist
+    @driver.ignore #it's incomplete... so
+    @driver.ignore_up_to("if_sockio_send", 1) { |e| next e[1] == "resync" }
+    resync_res = @driver.get "if_sockio_send", 1
+    resync_info = resync_res[2] #Hash on end contains the actual data from the message
+    expect(resync_info.keys).to include("watch_list"); #Should have a watch list
+
+    #Check the watchlist we got, first get the hash values for the pages
+    expected_watch_list = []
+    expected_watch_list += ["test0", nil]
+    expect(resync_info["watch_list"]).to eq(expected_watch_list)
+    #######################################################################################################
+  end
+
+  it "Does write a page to vm_cache that **does** already exist as <unbased, nochanges> the page receives an 'update' response from the external socket.io without a changes id (server result should be written into cache as-is); should no longer exist in unsynced" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch2.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
     dump = ctx.evald %{
       //Call embed on main root view
@@ -88,6 +191,9 @@ RSpec.describe "kern:sockio_pager" do
 
       //pg_sockio0 socket address & the endpoint for the event callback
       dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
     }
 
     #sockio driver should have been signaled (which it should respond asynchronously, and presumabely, after the disk)
@@ -144,9 +250,14 @@ RSpec.describe "kern:sockio_pager" do
     expect(post_read_res_dump["read_res_params"][1]["entries"]).to eq([
       {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"}
     ])
+
+    #Should no longer be unsynced
+    vm_unsynced = @ctx.dump("vm_unsynced")
+      expect(vm_unsynced["sockio"]).to eq({
+    })
   end
 
-  it "Does write a page to vm_cache that **does** already exist as <unbased, changes> receives an 'update' response from the external socket.io without a changes_id" do
+  it "Does write a page to vm_cache that **does** already exist as <unbased, changes> receives an 'update' response from the external socket.io without a changes_id. Should still be in the vm_unsynced" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch2.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
     dump = ctx.evald %{
       //Call embed on main root view
@@ -157,6 +268,9 @@ RSpec.describe "kern:sockio_pager" do
 
       //pg_sockio0 socket address & the endpoint for the event callback
       dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
     }
 
     #sockio driver should have been signaled (which it should respond asynchronously, and presumabely, after the disk)
@@ -225,9 +339,195 @@ RSpec.describe "kern:sockio_pager" do
       {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"},
       {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
     ])
+
+    #Should still be unsynced as it contains changes (we only removed changes on __base which is double buffered)
+    vm_unsynced = @ctx.dump("vm_unsynced")
+    expect(vm_unsynced["sockio"]).to eq({
+      "test" => 0
+    })
+
+  end
+
+  it "Does write a page to vm_cache that **does** already exist as <unbased, changes> receives an 'update' response from the external socket.io with a mis-matching changes_id. Should still be in the vm_unsynced" do
+    ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch2.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
+    dump = ctx.evald %{
+      //Call embed on main root view
+      dump.base = _embed("my_controller", 0, {}, null);
+
+      //Drain queue
+      int_dispatch([]);
+
+      //pg_sockio0 socket address & the endpoint for the event callback
+      dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
+    }
+
+    #sockio driver should have been signaled (which it should respond asynchronously, and presumabely, after the disk)
+    @driver.ignore_up_to "if_sockio_send"
+    @driver.mexpect "if_sockio_send", [Integer, "watch", {
+      "page_id" => "test"
+    }], 1
+
+    #The disk should have been signaled
+    @driver.ignore_up_to "if_per_get"
+    @driver.mexpect "if_per_get", ["vm", "sockio", "test"], 2
+
+
+    #The disk should respond with a page that contains changes
+    @driver.int "int_per_get_res", ["vm", "sockio", "test", {
+      "_id" => "test",
+      "_next" => nil,
+      "_head" => nil,
+      "entries" => [
+        {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
+        {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}
+      ],
+      "__changes" => [
+        ["+", 0, {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"}],
+        ["+", 1, {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}],
+        ["-", "foo3"],
+      ],
+      "__changes_id" => "foo"
+    }]
+
+    #We (driver sockio) received a watch request for a page with the id 'test'
+    #Now we are imagining that the socket.io driver received back some
+    #data and is now signaling to the kernel that data is available (as it sends to an
+    #event endpoint equal to the socket bp)
+    @driver.int "int_event", [dump["pg_sockio0_bp"], "update", {page: {
+      _id: "test",
+      _next: nil,
+      _head: nil,
+      entries: [
+        {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"},
+        {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"}
+      ],
+    }, changes_id: "foo2"}]
+
+    post_read_res_dump = ctx.evald %{
+      for (var i = 0; i < 100; ++i) {
+        //Drain queue (vm_cache_write defers to controller)
+        int_dispatch([]);
+      }
+
+      dump.read_res_params = read_res_params;
+    }
+
+    #The controller should have received a notification that a page was updated twice, one 
+    #for the disk response and one for the pager response
+    expect(post_read_res_dump["read_res_params"].length).to eq(2)
+    expect(post_read_res_dump["read_res_params"][0]["entries"]).to eq([
+      {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
+      {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}
+    ])
+
+    #Next page should be rebased ontop of the incomming page, such that changes are played *over* it
+    #which includes deletion of foo3
+    expect(post_read_res_dump["read_res_params"][1]["entries"]).to eq([
+      {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
+      {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"},
+      {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
+    ])
+
+    #Should still be unsynced as it contains changes (we only removed changes on __base which is double buffered)
+    vm_unsynced = @ctx.dump("vm_unsynced")
+    expect(vm_unsynced["sockio"]).to eq({
+      "test" => 0
+    })
+  end
+
+  it "Does write a page to vm_cache that **does** already exist as <unbased, changes> receives an 'update' response from the external socket.io with matching changes_id. Should not still be in the vm_unsynced" do
+    ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch2.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
+    dump = ctx.evald %{
+      //Call embed on main root view
+      dump.base = _embed("my_controller", 0, {}, null);
+
+      //Drain queue
+      int_dispatch([]);
+
+      //pg_sockio0 socket address & the endpoint for the event callback
+      dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
+    }
+
+    #sockio driver should have been signaled (which it should respond asynchronously, and presumabely, after the disk)
+    @driver.ignore_up_to "if_sockio_send"
+    @driver.mexpect "if_sockio_send", [Integer, "watch", {
+      "page_id" => "test"
+    }], 1
+
+    #The disk should have been signaled
+    @driver.ignore_up_to "if_per_get"
+    @driver.mexpect "if_per_get", ["vm", "sockio", "test"], 2
+
+
+    #The disk should respond with a page that contains changes
+    @driver.int "int_per_get_res", ["vm", "sockio", "test", {
+      "_id" => "test",
+      "_next" => nil,
+      "_head" => nil,
+      "entries" => [
+        {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
+        {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}
+      ],
+      "__changes" => [
+        ["+", 0, {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"}],
+        ["+", 1, {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}],
+        ["-", "foo3"],
+      ],
+      "__changes_id" => "foo"
+    }]
+
+    #We (driver sockio) received a watch request for a page with the id 'test'
+    #Now we are imagining that the socket.io driver received back some
+    #data and is now signaling to the kernel that data is available (as it sends to an
+    #event endpoint equal to the socket bp)
+    @driver.int "int_event", [dump["pg_sockio0_bp"], "update", {page: {
+      _id: "test",
+      _next: nil,
+      _head: nil,
+      entries: [
+        {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"},
+        {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"}
+      ],
+    }, changes_id: "foo"}]
+
+    post_read_res_dump = ctx.evald %{
+      for (var i = 0; i < 100; ++i) {
+        //Drain queue (vm_cache_write defers to controller)
+        int_dispatch([]);
+      }
+
+      dump.read_res_params = read_res_params;
+    }
+
+    #The controller should have received a notification that a page was updated twice, one 
+    #for the disk response and one for the pager response
+    expect(post_read_res_dump["read_res_params"].length).to eq(2)
+    expect(post_read_res_dump["read_res_params"][0]["entries"]).to eq([
+      {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
+      {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}
+    ])
+
+    #Next page should be rebased ontop of the incomming page, such that changes are played *over* it
+    #which includes deletion of foo3
+    expect(post_read_res_dump["read_res_params"][1]["entries"]).to eq([
+      {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"},
+      {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
+    ])
+
+    #Should still be unsynced as it contains changes (we only removed changes on __base which is double buffered)
+    vm_unsynced = @ctx.dump("vm_unsynced")
+    expect(vm_unsynced["sockio"]).to eq({
+    })
   end
 
-  it "Does write a page to vm_cache that **does** already exist as <based<unbased, changes>, changes> receives an 'update' response from the external socket.io without a changes_id" do
+
+  it "Does write a page to vm_cache that **does** already exist as <based<unbased, changes>, changes> receives an 'update' response from the external socket.io without a changes_id. Should still exist in vm_unsynced" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch2.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
     dump = ctx.evald %{
       //Call embed on main root view
@@ -238,6 +538,9 @@ RSpec.describe "kern:sockio_pager" do
 
       //pg_sockio0 socket address & the endpoint for the event callback
       dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
     }
 
     #sockio driver should have been signaled (which it should respond asynchronously, and presumabely, after the disk)
@@ -323,9 +626,15 @@ RSpec.describe "kern:sockio_pager" do
         {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
         {"_id" => "foo5", "_sig" => "foo5", "value" => "bar5"},
     ])
+
+    #Should still be unsynced as it contains changes (we only removed changes on __base which is double buffered)
+    vm_unsynced = @ctx.dump("vm_unsynced")
+    expect(vm_unsynced["sockio"]).to eq({
+      "test" => 0
+    })
   end
 
-  it "Does write a page to vm_cache that **does** already exist as <unbased, changes> receives an 'update' response from the external socket.io **with** an existing changes_id" do
+  it "Does write a page to vm_cache that **does** already exist as [changes, based[changes, unbased]] receives an 'update' response from the external socket.io **with** an existing changes_id but keeps that page in vm_unsynced" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch2.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
     dump = ctx.evald %{
       //Call embed on main root view
@@ -336,6 +645,9 @@ RSpec.describe "kern:sockio_pager" do
 
       //pg_sockio0 socket address & the endpoint for the event callback
       dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
     }
 
     #sockio driver should have been signaled (which it should respond asynchronously, and presumabely, after the disk)
@@ -349,7 +661,7 @@ RSpec.describe "kern:sockio_pager" do
     @driver.mexpect "if_per_get", ["vm", "sockio", "test"], 2
 
 
-    #The disk should respond with a page that contains changes
+    #The disk should respond with a page that contains <based<nobase, changes>, changes>
     @driver.int "int_per_get_res", ["vm", "sockio", "test", {
       "_id" => "test",
       "_next" => nil,
@@ -363,25 +675,37 @@ RSpec.describe "kern:sockio_pager" do
         ["+", 1, {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}],
         ["-", "foo3"],
       ],
-      "__changes_id" => "foo"
+      "__changes_id" => "foo",
+      "__base" => {
+        "_id" => "test",
+        "_next" => nil,
+        "_head" => nil,
+        "entries" => [
+          {"_id" => "fooX", "_sig" => "fooX", "value" => "barX"},
+          {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"}
+        ],
+        "__changes_id" => "foo2",
+        "__changes" => [
+          ["-", "fooX"],
+          ["+", 1, {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"}]
+        ]
+      }
     }]
 
     #We (driver sockio) received a watch request for a page with the id 'test'
     #Now we are imagining that the socket.io driver received back some
     #data and is now signaling to the kernel that data is available (as it sends to an
     #event endpoint equal to the socket bp)
-    @driver.int "int_event", [dump["pg_sockio0_bp"], "update", {
-      page: {
-        _id: "test",
-        _next: nil,
-        _head: nil,
-        entries: [
-          {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"},
-          {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"}
-        ],
-      }, 
-      changes_id: "foo"
-    }]
+    @driver.int "int_event", [dump["pg_sockio0_bp"], "update", {page: {
+      _id: "test",
+      _next: nil,
+      _head: nil,
+      entries: [
+        {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
+        {"_id" => "foo5", "_sig" => "foo5", "value" => "bar5"},
+        {"_id" => "fooX", "_sig" => "fooX", "value" => "barX"},
+      ],
+    }, changes_id: "foo2"}]
 
     post_read_res_dump = ctx.evald %{
       for (var i = 0; i < 100; ++i) {
@@ -399,22 +723,26 @@ RSpec.describe "kern:sockio_pager" do
       {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
       {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"}
     ])
-    #Changes should be present in second copy
-    expect(post_read_res_dump["read_res_params"][0]["__changes_id"]).not_to eq(nil)
-
 
-    #Next page should be rebased ontop of the incomming page, however, since changes_id matches
-    #current changes on the page, the page is just replaced (as changes were aknowledged and there
-    #is nothing to repair)
+    #Next version is a double replay.  First, the server page is called the new 'base', then changes from the 
+    #old base are played ontop of the server page. Then the top-level changes are recalculated based on this new page,
+    #and then replayed on the server's page *again* (a linked copy where the first replayed sits at __base.
     expect(post_read_res_dump["read_res_params"][1]["entries"]).to eq([
-      {"_id" => "foo3", "_sig" => "foo3", "value" => "bar3"},
-      {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
+        {"_id" => "foo1", "_sig" => "foo1", "value" => "bar1"},
+        {"_id" => "foo2", "_sig" => "foo2", "value" => "bar2"},
+        {"_id" => "foo4", "_sig" => "foo4", "value" => "bar4"},
+        {"_id" => "foo5", "_sig" => "foo5", "value" => "bar5"},
+        {"_id" => "fooX", "_sig" => "fooX", "value" => "barX"},
     ])
-    #Changes should not be present in second copy
-    expect(post_read_res_dump["read_res_params"][1]["__changes_id"]).to eq(nil)
+
+    #Should still be unsynced as it contains changes (we only removed changes on __base which is double buffered)
+    vm_unsynced = @ctx.dump("vm_unsynced")
+    expect(vm_unsynced["sockio"]).to eq({
+      "test" => 0
+    })
   end
 
-  it "Does write a page to vm_cache that **does not** already exist when the page receives an 'update' response from the external socket.io" do
+  it "Does write a page to vm_cache that **does not** already exist when the page receives an 'update' response from the external socket.io. Should not exist in vm_unsynced anymore" do
     ctx = flok_new_user File.read('./spec/kern/assets/vm/pg_sockio/watch.rb'), File.read("./spec/kern/assets/vm/pg_sockio/config.rb") 
     dump = ctx.evald %{
       //Call embed on main root view
@@ -425,6 +753,9 @@ RSpec.describe "kern:sockio_pager" do
 
       //pg_sockio0 socket address & the endpoint for the event callback
       dump.pg_sockio0_bp = pg_sockio0_bp;
+
+      //Mark page as unsynced manually
+      vm_unsynced["sockio"]["test"] = 0;
     }
 
     #We received a watch request for a page with the id 'test'
@@ -450,6 +781,11 @@ RSpec.describe "kern:sockio_pager" do
     expect(post_read_res_dump["read_res_params"]["entries"]).to eq([
       {"_id" => "foo", "_sig" => "foo", "value" => "bar"}
     ])
+
+    #Should not be in the queue anymore
+    vm_unsynced = @ctx.dump("vm_unsynced")
+    expect(vm_unsynced["sockio"]).to eq({
+    })
   end
 
   it "Does accept writes of pages that don't currently exist in cache; they go into vm_cache as-is and are sent to the sockio driver" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flok
 version: !ruby/object:Gem::Version
-  version: 0.0.82
+  version: 0.0.83
 platform: ruby
 authors:
 - seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-06 00:00:00.000000000 Z
+date: 2015-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: execjs
@@ -1278,8 +1278,10 @@ files:
 - spec/kern/assets/vm/pg_sockio/config.rb
 - spec/kern/assets/vm/pg_sockio/config_no_url.rb
 - spec/kern/assets/vm/pg_sockio/nothing.rb
+- spec/kern/assets/vm/pg_sockio/unmark_changes.rb
 - spec/kern/assets/vm/pg_sockio/watch.rb
 - spec/kern/assets/vm/pg_sockio/watch2.rb
+- spec/kern/assets/vm/pg_sockio/watch3.rb
 - spec/kern/assets/vm/pg_sockio/write.rb
 - spec/kern/assets/vm/pg_sockio/write2.rb
 - spec/kern/assets/vm/pg_sockio/write3.rb
@@ -2230,8 +2232,10 @@ test_files:
 - spec/kern/assets/vm/pg_sockio/config.rb
 - spec/kern/assets/vm/pg_sockio/config_no_url.rb
 - spec/kern/assets/vm/pg_sockio/nothing.rb
+- spec/kern/assets/vm/pg_sockio/unmark_changes.rb
 - spec/kern/assets/vm/pg_sockio/watch.rb
 - spec/kern/assets/vm/pg_sockio/watch2.rb
+- spec/kern/assets/vm/pg_sockio/watch3.rb
 - spec/kern/assets/vm/pg_sockio/write.rb
 - spec/kern/assets/vm/pg_sockio/write2.rb
 - spec/kern/assets/vm/pg_sockio/write3.rb