flok 0.0.41 → 0.0.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d5f0a382decc21b5e55f99815a7b23ecf1f53009
-  data.tar.gz: b5f74e961712e188e97fc94da0b51fbac5e664b2
+  metadata.gz: bebf06696d30319a7889c7039753077ad38eda75
+  data.tar.gz: 4c6bc9ed05585a54a65a5a4e995e16a60c85351f
 SHA512:
-  metadata.gz: 56c35e867d9766f03239f2ce0343fac6ba34f4af931b6b30e1b69ce3bd5c13e2d3dd8b9a11e9e6d16dbf87247a1757f68d7eedd26fe20da14297a558cd95394c
-  data.tar.gz: 48bb0f9c0b18b43582b0960bcd661f233fbe3d471025b41402a32dc32a343a40a23e108741f5fcb9b076e74555c572979828f88cdc1bf3f4cc8d23c254a6d68c
+  metadata.gz: 2075905fba3ce9231fd8cdec4ed32e5ac7bed5b41be301cb75e9de651951b631239e4fa5d0f2166f2d1a4f89c3e6e04ab4319ee8e6d6a7c8bb03c54d4cdef322
+  data.tar.gz: 591a0e811654e08ad243c574cfa58a8681cf522df86bf610c3c8ebdff141700f6d4fec6c35c05066a7019d8f9940b852f1dd31fb893dcf7c23b83249dc935604

data/app/kern/pagers/pg_dummy.js

@@ -0,0 +1,18 @@
+//Configure pg_dummy0
+<% [0].each do |i| %>
+  function pg_dummy<%= i %>_init(ns, options) {
+    pg_dummy<%= i %>_init_params = {ns: ns, options: options};
+    pg_dummy<%= i %>_ns = ns;
+
+    pg_dummy<%= i %>_spec_did_init = true;
+  }
+
+  function pg_dummy<%= i %>_watch(id, page) {
+  }
+
+  function pg_dummy<%= i %>_unwatch(id) {
+  }
+
+  function pg_dummy<%= i %>_write(page) {
+  }
+<% end %>

data/app/kern/services/vm.rb

@@ -24,8 +24,25 @@ service :vm do
 
     //Cache
     function vm_cache_write(ns, page) {
+      <% if @debug %>
+        if (vm_transaction_in_progress === false) { throw "vm_cache_write called but a transaction was not in progress. Make sure to call vm_transaction_begin and vm_transaction_end" }
+        if (vm_transaction_ns !== null && vm_transaction_ns !== ns) { throw "vm_cache_write called, and is within a vm_transaction but the ns given: " + ns + " does not match the transaction ns of: " + vm_transaction_ns };
+      <% end %>
+
+      //Namespace is needed for vm_transaction_end
+      vm_transaction_ns = ns;
+
+      vm_rehash_page(page);
+
       var old = vm_cache[ns][page._id];
-      if (old && old._hash == page._hash) { return; }
+      if (old) {
+        //Same, don't do anything
+        if (old._hash === page._hash) { return; }
+
+        //Diff
+        vm_transaction_diffs.push(vm_diff(old, page));
+        vm_transaction_changed_ids.push(page._id);
+      }
 
       vm_dirty[ns][page._id] = page;
       vm_cache[ns][page._id] = page;
@@ -39,39 +56,6 @@ service :vm do
       }
     }
 
-    function vm_rehash_page(page) {
-      var z = 0;
-
-      //head and next are optional
-      if (page._head) { var z = crc32(0, page._head) }
-      if (page._next) { z = crc32(z, page._next) }
-
-      z = crc32(z, page._id)
-
-      //Hash differently based on type
-      if (page._type === "array") {
-        var e = page.entries;
-        for (var i = 0; i < e.length; ++i) {
-          z = crc32(z, e[i]._sig);
-        }
-      } else if (page._type === "hash") {
-        var keys = Object.keys(page.entries);
-        var e = page.entries;
-        var q = 0;
-        for (var i = 0; i < keys.length; ++i) {
-          var _sig = e[keys[i]]._sig;
-          var r = crc32(0, _sig);
-          q = q + r;
-        }
-        q = +q;
-        z = crc32(z, q.toString());
-      } <% if @debug %> else {
-        throw "vm_rehash_page got an unspported type: "+page._type;
-      }
-      <% end %>
-      page._hash = z.toString();
-    }
-
     function vm_pageout() {
       <% @options[:pagers].each do |p| %>
         //Get id_to_page mappings
@@ -111,6 +95,349 @@ service :vm do
     <% if @debug %>
       vm_write_list = [];
     <% end %>
+
+    //Generic Page Helpers
+    ///////////////////////////////////////////////////////////////////////////
+    function vm_create_page(id) {
+      if (id === undefined) {
+        id = gen_id();
+      }
+
+      var page = {
+        _id: id,
+        _head: null,
+        _next: null,
+        _hash: null,
+        entries: [],
+        __index: {},
+      };
+
+      return page;
+    }
+
+    function vm_copy_page(page) {
+      var page = {
+        _id: page._id,
+        _head: page._head,
+        _next: page._next,
+        _hash: page._hash,
+        entries: JSON.parse(JSON.stringify(page.entries)),
+      };
+
+      return page;
+    }
+
+    function vm_rehash_page(page) {
+      var z = 0;
+
+      //head and next are optional
+      if (page._head) { var z = crc32(0, page._head) }
+      if (page._next) { z = crc32(z, page._next) }
+
+      z = crc32(z, page._id)
+
+      //Hash differently based on type
+      var e = page.entries;
+      for (var i = 0; i < e.length; ++i) {
+        z = crc32(z, e[i]._sig);
+      }
+
+      page._hash = z.toString();
+    }
+
+    function vm_reindex_page(page) {
+      page.__index = {};
+      for (var i = 0; i < page.entries.length; ++i) {
+        page.__index[page.entries[i]._id] = i;
+      }
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+    //vm_diff helpers
+    ///////////////////////////////////////////////////////////////////////////
+    function vm_diff(old_page, new_page) {
+      var diff_log = [];
+      if (old_page._head !== new_page._head) {
+        diff_log.push(["HEAD_M", new_page._head])
+      }
+
+      if (old_page._next !== new_page._next) {
+        diff_log.push(["NEXT_M", new_page._next])
+      }
+
+      var from_entries = old_page.entries;
+      var to_entries = new_page.entries;
+
+      //Calculated lists
+      var ins = [];
+      var dels = [];
+      var moves = [];
+      var modify = [];
+
+      //a_prime is Union (ordered) of from
+      //b_prime is Union (ordered) of to
+      var a_prime = [];
+      var b_prime = [];
+
+      //Save all entry sigs
+      var from_entries_sig  = [];
+      for (var i = 0; i < from_entries.length; ++i) {
+        from_entries_sig[from_entries[i]._id] = from_entries[i]._sig;
+      }
+
+      //Need to re-index page for the modify code which needs to know the index
+      //of the id of the new entry
+      vm_reindex_page(new_page);
+
+      //Save all the entry sigs
+      var to_entries_sig  = [];
+      for (var i = 0; i < to_entries.length; ++i) {
+        to_entries_sig[to_entries[i]._id] = to_entries[i]._sig;
+      }
+
+      //I. Calculate all elements in to_entries that are not in from_entries
+      //for each one of those elements, mark it as insertion and remove them in reverse order.
+      for (var i = 0; i < to_entries.length; ++i) {
+        //Does the entry *not* exist in from_entries?
+        var to_entry_id = to_entries[i]._id;
+        if (from_entries_sig[to_entry_id] === undefined) {
+          ins.push(["+", i, to_entries[i]]);
+        } else {
+          //The entry *does* exist, therefore it must be part of the shared
+          b_prime.push(to_entries[i]._id);
+        }
+      }
+
+      for (var i = 0; i < from_entries.length; ++i) {
+        var from_entry_id = from_entries[i]._id;
+        if (to_entries_sig[from_entry_id] === undefined) {
+          dels.push(["-", from_entries[i]._id]);
+        } else {
+          a_prime.push(from_entries[i]._id);
+
+          if (from_entries[i]._sig != to_entries_sig[from_entry_id]) {
+            modify.push(["M", new_page.entries[new_page.__index[from_entry_id]]]);
+          }
+        }
+      }
+
+      //*==================================*
+      //| Wild UNOPTIMIZED ALGORITHM       |
+      //|                                  |
+      //| appeared!                        |
+      //|                                v |
+      //*==================================*
+      while(1) {
+        var wdiff = 0;
+        var wb_index;
+        var wa_index;
+
+        for (var i = 0; i < a_prime.length; ++i) {
+          var b_index = b_prime.indexOf(a_prime[i]);
+          var diff = b_index - i;
+
+          if (Math.abs(diff) > Math.abs(wdiff)) {
+            wdiff = diff;
+            wa_index = i;
+            wb_index = b_index;
+          }
+        }
+
+        if (Math.abs(wdiff) > 0) {
+          var r = a_prime.splice(wa_index, 1);
+          a_prime.splice(wb_index, 0, r[0]);
+
+          moves.push([">", wb_index, r[0]]);
+        } else {
+          break
+        }
+      }
+
+      var res = diff_log.concat(dels).concat(modify).concat(moves).concat(ins);
+      return res;
+    }
+
+    function vm_diff_replay(page, diff) {
+      for (var i = 0; i < diff.length; ++i) {
+        vm_reindex_page(page);
+        var e = diff[i];
+
+        //vm_diff type
+        var type = e[0];
+        if (type === "+") {
+          var eindex = e[1];
+          var entry = e[2];
+
+          //Ignore insertion if an element already exists with the given id
+          if (page["__index"][entry["_id"]] === undefined) {
+            //Insertion
+            page.entries.splice(eindex, 0, entry);
+          }
+        } else if (type === ">") {
+          var eindex = e[1];
+          var entry_id = e[2];
+
+          var current_index = page["__index"][entry_id];
+          if (current_index !== undefined) {
+            var entry = page.entries.splice(current_index, 1)[0];
+            page.entries.splice(eindex, 0, entry);
+          }
+        } else if (type === "M") {
+          var entry = e[1];
+
+          //Take out old, put in new
+          if (page["__index"][entry["_id"]] !== undefined) {
+            page.entries.splice(page["__index"][entry["_id"]], 1, entry);
+          }
+        } else if (type === "-") {
+          var eid = e[1];
+
+          var index = page.__index[eid];
+
+          //Take out
+          if (page["__index"][eid] !== undefined) {
+            page.entries.splice(index, 1);
+          }
+        } else if (type === "HEAD_M") {
+          page._head = e[1];
+        } else if (type === "NEXT_M") {
+          page._next = e[1];
+        }
+      }
+    } 
+    ///////////////////////////////////////////////////////////////////////////
+
+    //Commit helpers
+    ///////////////////////////////////////////////////////////////////////////
+    function vm_commit(older, newer) {
+      newer.__changes_id = gen_id();
+
+      if (older.__changes && !older.__base) {
+        newer.__base = older;
+      } else if (older.__changes) {
+        newer.__base = older.__base;
+      }
+
+      if (older.__base) {
+        newer.__changes = vm_diff(older.__base, newer);
+      } else {
+        newer.__changes = vm_diff(older, newer);
+      }
+    }
+
+    function vm_rebase(newer, older) {
+      if (newer.__changes && !newer.__base) {
+        <% if @debug %>
+          if (newer.__changes_id === undefined) {
+            throw "__changes_id did not exist on newer: " + JSON.stringify(newer) + " but it did have __changes";
+          }
+        <% end %>
+        older.__changes = newer.__changes;
+        older.__changes_id = newer.__changes_id;
+
+        vm_diff_replay(older, older.__changes);
+      } else if (newer.__changes && newer.__base) {
+        <% if @debug %>
+          if (newer.__changes_id === undefined) {
+            throw "__changes_id did not exist on newer: " + JSON.stringify(newer) + " but it did have __changes";
+          }
+        <% end %>
+
+        //Reconstruct the __base by playing newer.__base.__changes ontop of older (which is the base we are rebasing on)
+        //Imagine that you texted a teacher changes, but are unsure whether that teacher has received those changes, meanwhile,
+        //the teacher texts you a new fresh copy of the page. You must now keep track of the changes you texted her (newer.__base.__changes)
+        //while still being able to create a new list of changes for any future changes that you make (as we diff pages to create the changes)
+        //So we reconstruct the newer.__base page  by taking what the teacher gave us, trash the newer.__base page, but replay the changes
+        //that newer.__base.__changes had onto the copy the teacher gave us. E.g. we cross out "Sally" on our list, text teacher that we crossed
+        //out sally. Teacher gave us a new list that has "Bill" Crossed out. We Then take the new list and cross out "Sally" and call that our new
+        //base page.
+        vm_diff_replay(older, newer.__base.__changes);
+
+        //Copy the page, we need to use the copy as a '__base' page because we want the non-copied older page to be the non-base version. (And we
+        //will make it the 'non' base version by again, replaying changes from the 'newer.__changes') after setting the __base to the copy.
+        var older_copy = vm_copy_page(older);
+        older_copy.__changes = newer.__base.__changes;
+        older_copy.__changes_id = newer.__base.__changes_id;
+        vm_reindex_page(older_copy);
+        older.__base = older_copy;
+
+        //Now update the older page w/ the `newer.__changes`
+        vm_diff_replay(older, newer.__changes);
+
+        //Calculate diff for older
+        older.__changes = vm_diff(older.__base, older);
+        older.__changes_id = gen_id();
+      }
+    }
+
+    function vm_mark_changes_synced(page, changes_id) {
+      if (page.__base === undefined && changes_id === page.__changes_id) {
+        delete page.__changes;
+        delete page.__changes_id;
+      } else if (page.__base !== undefined && changes_id === page.__base.__changes_id) {
+        delete page.__base;
+      }
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+    //vm transaction helpers
+    ///////////////////////////////////////////////////////////////////////////
+    vm_transaction_in_progress = false;
+    function vm_transaction_begin() {
+      <% if @debug %>
+        if (vm_transaction_in_progress === true) { throw "vm_transaction_begin called but a transaction was already in progress" }
+      <% end %>
+      vm_transaction_in_progress = true;
+      vm_transaction_diffs = [];
+      vm_transaction_changed_ids = [];
+      vm_transaction_ns = null;
+    }
+
+    function vm_transaction_end() {
+      <% if @debug %>
+        if (vm_transaction_in_progress === false) { throw "vm_transaction_end called but vm_transaction_begin was never called" }
+      <% end %>
+      vm_transaction_in_progress = false;
+
+      for (var i = 0; i < vm_transaction_changed_ids.length; ++i) {
+        var page_id = vm_transaction_changed_ids[i];
+        var bps = vm_notify_map[vm_transaction_ns][page_id];
+        if (bps !== undefined) {
+          pieces = [];
+          for (var x = 0; x < vm_transaction_diffs[i].length; ++x) {
+            //Get diff entry
+            var diff_entry = vm_transaction_diffs[i][x];
+            pieces.push(diff_entry);
+
+            //For all listening controllers
+            for (var y = 0; y < bps.length; ++y) {
+              var bp = bps[y];
+
+              if (diff_entry[0] === "M") {
+                int_event_defer(bp, "entry_modify", {page_id: page_id, entry: diff_entry[1]});
+              } else if (diff_entry[0] === "-") {
+                int_event_defer(bp, "entry_del", {page_id: page_id, entry_id: diff_entry[1]});
+              } else if (diff_entry[0] === ">") {
+                var eindex = diff_entry[1];
+                var eid = diff_entry[2];
+                int_event_defer(bp, "entry_move", {entry_id: eid, from_page_id: page_id, to_page_id: page_id, to_page_index: eindex});
+              } else if (diff_entry[0] === "+") {
+                var eindex = diff_entry[1];
+                var entry = diff_entry[2];
+                int_event_defer(bp, "entry_ins", {page_id: page_id, index: eindex, entry: entry});
+              } else if (diff_entry[0] === "NEXT_M") {
+                int_event_defer(bp, "next_changed", {page_id: page_id, value: diff_entry[1]});
+              } else if (diff_entry[0] === "HEAD_M") {
+                int_event_defer(bp, "head_changed", {page_id: page_id, value: diff_entry[1]});
+              } 
+            }
+          }
+
+          //throw JSON.stringify(pieces);
+        }
+      }
+    }
+    ///////////////////////////////////////////////////////////////////////////
   }
 
   on_wakeup %{

data/docs/services/vm.md

@@ -8,35 +8,20 @@ Each pager belongs to a *namespace*; page faults hit a namespace and then the pa
 Fun aside; Because of the hashing schemantics; this paging system solves the age old problem of ... how do you show that data has changed *now* when to be assured that you have perferctly synchronized data with the server?;... you need to do a 3-way handshake with the updates.  You could have a network server pager that supports writes but dosen't forward those to the network. That way, you can locally modify the page and then if the modifications were guessed correctly, the server would not even send back a page modification update! (Locally, the page would have been propogated as well).  In the meantime, after modifying the local page, you would send a real network request to the server which would in turn update it's own paging system but at that point, the server would check in with you about your pages, but miraculously, because you gussed the updated page correctly, no modifications will need to be made. You could even purposefully put a 'not_synced' key in and actually show the user when the page was correctly synchronized.
 
 ##Pages
-Each page is either of a `array` type or `hash` type.
-
-###Array type
+###Example
 ```ruby
 page_example = {
   _head: <<uuid STR or NULL>>,
-  _type: "array",
   _next: <<uuid STR or NULL>,
   _id: <<uuid STR>,
   entries: [
     {_id: <<uuid STR>>, _sig: <<random_signature for inserts and modifies STR>>},
     ...
   ],
-  _hash: <<CRC32 >
-}
-```
-
-###Hash type
-```ruby
-page_example = {
-  _head: <<uuid STR or NULL>>,
-  _type: "hash",
-  _next: <<uuid STR or NULL>,
-  _id: <<uuid STR>
-  entries: {
-    "my_id0" => {_sig: <<random_signature for inserts and modifies STR>>},
-    ...
-  },
-  _hash: <<CRC32 >
+  _hash: <<CRC32>>,
+  __index: {
+    entry_id: entry_index,
+  }
 }
 ```
 
@@ -44,10 +29,8 @@ page_example = {
   * `_next (string or null)` - The next element on this list. If `_next` is non-existant, then this page is the endpoint of the list.
   * `_id (string)` - The name of this page. Even if every key changed, the `_id` will not change. This is supposed to indicate, semantically, that this page still *means* the same thing.  For example, imagine a page.  If all entries were to be **removed** from this page and new entries were **inserted** on this page, then it would be semantically sound to say that the entries were **changed**.
   * `entries`
-    * `_type == 'array'`
-      * An array of dictionaries. Each element contains a `_id` that is analogous to the page `_id`. (These are not the same, but carry the same semantics).  Entries also have a `_sig` which should be a generated hash value that changes when the entry changes.
-    * `_type == 'hash'`
-      * A dictionary of dictionaries. Entries have a `_sig` which should be a generated hash value that changes when the entry changes.
+    * An array of dictionaries. Each element contains a `_id` that is analogous to the page `_id`. (These are not the same, but carry the same semantics).  Entries also have a `_sig` which should be a generated hash value that changes when the entry changes.
+  * `__index` - A dictionary mapping entry `_id` into an index of the `entries` array.
   * `_hash (string)` - All entry `_id's`, `_next`, the page `_id`, and `head` are hashed togeather. Any changes to this page will cause this `_hash` to change which makes it a useful way to check if a page is modified and needs to be updated. The hash function is an ordered CRC32 function run in the following order.  See [Calculating Page Hash](#calculating_page_hash).
 
 ------
@@ -71,6 +54,21 @@ Assuming a crc function of `crc32(seed, string)`
 
 ------
 
+##Schemas & Data-Types
+
+####`vm_diff_entry`
+See [VM Diff](./vm/diff.md) for specific information.
+
+###`Based page`
+A based page contains the additional keys of `__base` and `__changes`, and these keys are not `null`. Optionally, it may contain the keys
+`__base_sync` (which is also not null).
+  * `__base` - A copy of the fully synchronized page (fully embedded)
+  * `__changes` - An `vm_diff` array of changes from either `__base`, or if not `null` and not `undefined`, the `__base_sync` page.
+  * `__base_sync` - An optional key, serves the same purpose as `__base`, but when synchronizing, the `__base_sync` is used for `__changes` as
+      `__base_sync` holds a full copy of the currently in sync page.
+
+Pages that are being synhronized are known as a `based in-sync page`.
+
 ##Configuration
 The paging service may be configured in your `./config/services.rb`. You must set an array of pagers where each pager is responsible for a particular
 namespace. See [VM Pagers](./vm/pagers.md) for more info.
@@ -130,7 +128,8 @@ if (page is not resident in memory && not_synchronous) {
   * Parameters
     * `ns` - The namespace of the page, e.g. 'user'
     * `id` - Watching the page that contains this in the `_id` field
-    * `sync (optional)` - If set to `true` then the disk read and cache read will be performed synchronously. Additionally, all future cache reads / updates will be performed synchronously.
+    * `sync (optional)` - If set to `true` then the cache read will be performed synchronously; however, the disk read will still be performed
+        asynchronously. Additionally, all future cache reads / updates will be performed synchronously.
   * Event Responses
     * `read_res` - Whenever a change occurs to a page or the first read.
     * Returns an immutable page in params
@@ -167,12 +166,44 @@ Pageout is embodied in the function named `vm_pageout()`. This will asynchronous
     must support `unwatch` removal which we only receive the `bp`, `ns`, and `key`.
 
 ##Helper Methods
-###Pager specific
-  * `vm_cache_write(ns,  page)` - Save a page to cache memory. This will not recalculate the page hash. The page will be stored in `vm_cache[ns][id]` by.
 
-###Page modification
-  * `vm_rehash_page(page)` - Calculates the hash for a page and modifies that page with the new `_hash` field. If the `_hash` field does not exist, it
+###Functional
+####Page modification (assuming inputs are modifiable)
+  * **Generic Page**
+    * `vm_create_page(id)` - **this does not write anything to memory. It has no side effects except returning a hash**.
+    * `vm_create_page()` -  Same as vm_create_page, but generates an id fore you.
+    * `vm_copy_page(page)` - Creates a copy of the page. Only copies the `_head`, `_next`, `_id`, `entries`, `_hash`
+    * `vm_rehash_page(page)` - Calculates the hash for a page and modifies that page with the new `_hash` field. If the `_hash` field does not exist, it
       will create it
+    * `vm_reindex_page(page)` - Recalculates the `__index` field of the page. If `__index` does not exist, it is added.
+  * **Diff helpers**
+    * See [VM Diff](./vm/diff.md) section on *Functional Kernel
+  * **Commit helpers**
+    * `vm_commit(older, newer)` - Modifications will be done to `newer`. It is assumed that `newer` is neither based nor changed. This is typical of a
+        new page creation. It is assumed that `older` is either `[unbased, nochanges]`, `[unbased, changes]` or `[based[unbased, changes], changes]`.
+       You would use this when a page is being written over a page that already exists. This will mark page as having changes.
+          1. `older: [unbased, nochanges]` - `newer.__changes` will equal `vm_diff(older, newer)` and `newer.__changes_id` will be generated.
+          2. `older: [unbased, changes]` - `newer.__base` will point to `older`. `newer.__changes` will equal `vm_diff(older, newer)` and
+          `newer__changes_id` will be generated.
+          3. `older: [based[unbased, changes], changes]]` - `newer.__base` will point to `older.__base`. Then `newer.__changes` will equal
+          `vm_diff(older.__base, newer)` and `newer.__changes_id` will be generated.
+    * `vm_rebase(newer, older)` - Modifications are done to `older`. It is assumed that `older` is not based nor changed. This is typical of a
+        synchronized page from a server. It is assumed that `newer` is either `[unbased, nochanges]`, `[unbased, changes]` or `[based[unbased,
+        changes], changes]`.
+          1. `newer: [unbased, nochanges]` - No changes as `newer` does not contain any changes, therefore, `older` is the *truth*.
+          2. `newer: [unbased, changes]` - `older` takes `newer.__changes` and `newer.__changes_id`. `older` then replays `older.__changes` on itself.
+          3. `newer: [based[unbased, changes], changes]]`
+            1. `older` takes `newer.__base.__changes` and `newer.__base.__changes_id`. `older` then replays `older.__changes` onto itself.
+            2. `older` clones itself, let that clone be called `oldest`. `older.__base` is set to `oldest`.
+            3. `older` replays `newer.__changes` onto itself.
+            4. `older` then calculates `__changes` based off `oldest`.
+    * `vm_mark_changes_synced(page, changes_id)` - Will reverse the steps of `vm_commit`. If the page has changes but is not based, then the changes are removed if the
+        `__changes_id` of the page matches `changes_id`. If the page is based (implying the base page has changes and the page has changes as all base
+        pages have changes), then if the `changes_id` matches the **base** `__changes_id` , the `__base` is removed from the page. If `changes_id`
+        does not match in either of the cases, then nothing happends. This may happend if a synchronization errousouly comes in.
+###Non functional
+####Pager specific
+  * `vm_cache_write(ns,  page)` - Save a page to cache memory. This will not recalculate the page hash. The page will be stored in `vm_cache[ns][id]` by.
 
 ### <a name='user_page_modification_helpers'></a>User page modification helpers (Controller Macros)
 You should never directly edit a page in user land; if you do; the pager has no way of knowing that you made modifications. Additionally, if you have multiple controllers watching a page, and it is modified in one controller, those other controllers
@@ -183,19 +214,17 @@ Aside, modifying a page goes against the semantics of the vm system; you're thin
 If you're creating a new page, please use these macros as well; just switch out `CopyPage` for `NewPage`.
 
 ####Per entry
-  * `NewPage(type, id)` - Returns a new blank page; internally creates a page that has a null `_next`, `_head`, and `entries` array with 0 elements. type can either be `array` or `hash`. `_id` is generated if it is not passed.
+  * `NewPage(type, id)` - Returns a new blank page; internally creates a page that has a null `_next`, `_head`, and `entries` array with 0 elements.  `_id` is generated if it is not passed.
   * `CopyPage(page)` - Copies a page and returns the new page. Internally this copies the entire page with the exception of the
       `_hash` field.
-      
-  * For both `array` and `hashes`, the following functions work (albeit different semantics). For array types, the `eindex` is an integer in the array, For hash types, the `eindex` is a key inside the dictionary.
-    * `EntryDel(page, eindex)` - Remove a single entry from a page. (Internally this deletes the array entry).
-    * `EntryInsert(page, eindex, entry)` - Insert an entry, entry should be a dictionary value. 
-      * For arrays, this generates the `_sig` and `_id` for you.
-      * For hashes, this generates the `_sig` for you.
-    * `EntryMutable(page, eindex)` - Set a mutable entry at a specific index which you can then modify. The signature is changed for you. You can not
-        use this with dot syntax like `EntryMutable(page, eindex).id = 'foo'`, you may only get a variable.
-    * `SetPageNext(page, id)` - Sets the `_next` id for the page
-    * `SetPageHead(page, id)` - Sets the `_head` id for the page
+  * `EntryDel(page, eid)` - Remove a single entry from a page. (Internally this deletes the array entry).
+  * `EntryInsertAtIndex(page, eindex, entry)` - Insert an entry at a specific index. This generates the `_sig` and `_id` for you.
+  * `EntryInsertAtId(page, eid, entry)` - Insert an entry with a particular `_id`. This generates `_sig` for you. It will be put at the end of the
+      array
+  * `EntryMutable(page, eid)` - Set a mutable entry at a specific index which you can then modify. The signature is changed for you. You can not
+      use this with dot syntax like `EntryMutable(page, eindex).id = 'foo'`, you may only get a variable.
+  * `SetPageNext(page, id)` - Sets the `_next` id for the page
+  * `SetPageHead(page, id)` - Sets the `_head` id for the page
 
 Here is an example of a page being modified inside a controller after a `read_res`
 ```js