@happy0/flumeview-level-custom 0.0.0-stage → 2.0.0

package/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Gordon Martin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+# flumeview-level-custom

package/package.json

@@ -1,6 +1,35 @@
 {
   "name": "@happy0/flumeview-level-custom",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "2.0.0",
+  "description": "",
+  "license": "MIT",
+  "author": "Gordon Martin",
+  "type": "commonjs",
+  "main": "src/index.js",
+  "scripts": {
+    "test": "vitest run .",
+    "prettier": "prettier . --write"
+  },
+  "devDependencies": {
+    "flumecodec": "^0.0.1",
+    "flumedb": "^2.1.8",
+    "flumelog-memory": "^1.0.2",
+    "flumelog-offset": "^3.4.4",
+    "prettier": "^3.9.10",
+    "pull-abortable": "^4.1.1",
+    "vitest": "^5.0.5"
+  },
+  "dependencies": {
+    "charwise": "^3.0.1",
+    "level": "^10.0.0",
+    "level-read-stream": "^2.0.0",
+    "ltgt": "^2.2.1",
+    "obv": "^0.0.1",
+    "pull-level": "^2.0.4",
+    "pull-live": "^1.0.1",
+    "pull-pushable": "^2.2.0",
+    "pull-stream": "^3.7.0",
+    "pull-write": "^1.1.4",
+    "stream-to-pull-stream": "^1.7.3"
+  }
+}

package/src/flumeView.js

@@ -0,0 +1,104 @@
+const Obv = require("obv");
+const path = require("path");
+const charwise = require("charwise");
+const { Level } = require("level");
+const pull = require("pull-stream");
+const { createPullStream } = require("./stream/read");
+const { METADATA_KEY } = require("./metadata");
+
+module.exports = function (version, createWriter, filter) {
+  return function (log, name) {
+    const dir = path.dirname(log.filename);
+
+    const since = Obv();
+    const db = createLevelDb(dir, log, name);
+    let closed = false;
+
+    startConsumingLog(db, since, version);
+
+    let writer;
+
+    function destroy(cb) {
+      cb = cb || (() => {});
+      since.set(-1);
+      db.clear().then(cb);
+    }
+
+    async function startConsumingLog(db, since) {
+      const value = await db.get(METADATA_KEY, { keyEncoding: "utf8" });
+
+      if (!value) {
+        // No existing entry, we're consuming the log from the start
+        since.set(-1);
+      } else if (value.version === version) {
+        // The version has not changed, resume consuming the log from where we left off
+        // last time
+        since.set(value.since);
+      } else {
+        // version has changed, wipe db and start over.
+        destroy(() => {});
+      }
+    }
+
+    return {
+      // flumeview interface
+      methods: { get: "async", read: "source" },
+      since: since,
+      createSink: function (cb) {
+        writer = createWriter(db, since, version, cb);
+
+        if (!filter) {
+          return writer;
+        } else {
+          return pull(
+            pull.filter((item) => filter(item.value)),
+            writer,
+          );
+        }
+      },
+      destroy: destroy,
+      close: function (cb) {
+        cb = cb || (() => {});
+
+        closed = true;
+
+        if (writer) {
+          writer.abort(() => db.close().then(cb));
+        } else {
+          db.close().then(cb);
+        }
+      },
+      get: function (key, cb) {
+        db.get(key).then((result) => {
+          if (result === undefined) {
+            cb(result);
+          } else {
+            cb(null, result);
+          }
+        });
+      },
+      /**
+       *
+       * @param {*} opts The level DB stream key range options ( see 'level-read-stream' library ) as well as an optional 'live' parameter to indicate whether the
+       * stream should stay live an emit live updates. IF live, a {sync:true} object will be emitted when the stream is about to emit
+       * live values and each item will have a 'type' field (put, del) indicating whether the database operation type as well as the key
+       * and value
+       * @returns a pull-stream source of database entries satisfying the input range
+       */
+      read: function (opts) {
+        return createPullStream(db, opts);
+      },
+    };
+  };
+};
+
+function createLevelDb(dir, log, name) {
+  if (!log.filename) {
+    throw new Error("flumeview-level-custom can only be used with a file log");
+  }
+
+  return new Level(path.join(dir, name), {
+    keyEncoding: charwise,
+    valueEncoding: "json",
+  });
+}

package/src/index.js

@@ -0,0 +1,7 @@
+const flumeView = require("./flumeView");
+const mapperBuilder = require("./mapperBuilder");
+const metadata = require("./metadata");
+
+module.exports.FlumeViewCustom = flumeView;
+module.exports.mapperBuilder = mapperBuilder;
+module.exports.metadata = metadata;

package/src/mapperBuilder.js

@@ -0,0 +1,37 @@
+const Write = require("pull-write");
+const { createMetadataUpdateItem } = require("./metadata");
+
+module.exports.sequentialMapper = function (asyncMapper) {
+  const MAX_ITEM_BUFFER_SIZE = 0;
+
+  return function (db, since, version, streamEndCb) {
+    return Write(
+      function write(batch, cb) {
+        if (batch.length !== 1) {
+          cb(
+            new Error(
+              "Batch should never have more or less than one item for sequential mapper.",
+            ),
+          );
+        } else {
+          const item = batch[0];
+          const metadata = createMetadataUpdateItem(version, item.seq);
+          const update = asyncMapper(db, item.value);
+
+          Promise.resolve(update)
+            .then((items) => {
+              return db.batch([metadata, ...items]);
+            })
+            .then((_) => {
+              since.set(item.seq);
+              cb();
+            })
+            .catch((e) => console.error(e));
+        }
+      },
+      null,
+      MAX_ITEM_BUFFER_SIZE,
+      streamEndCb,
+    );
+  };
+};

package/src/metadata.js

@@ -0,0 +1,16 @@
+const METADATA_KEY = "\x00";
+
+function createMetadataUpdateBatchItem(version, seq) {
+  return {
+    key: METADATA_KEY,
+    value: { version: version, since: seq },
+    valueEncoding: "json",
+    keyEncoding: "utf8",
+    type: "put",
+  };
+}
+
+module.exports = {
+  METADATA_KEY,
+  createMetadataUpdateItem: createMetadataUpdateBatchItem,
+};

package/src/stream/post.js

@@ -0,0 +1,36 @@
+var ltgt = require("ltgt");
+
+function eq(a, b) {
+  if (Buffer.isBuffer(a) && Buffer.isBuffer(b)) {
+    return beq(a, b);
+  } else return a === b;
+}
+
+module.exports = function post(db, opts, each) {
+  if (!each) ((each = opts), (opts = {}));
+
+  var encode =
+    (opts && opts.keyEncoding && opts.keyEncoding.encode) ||
+    (db.options && db.options.keyEncoding && db.options.keyEncoding.encode) ||
+    function (x) {
+      return x;
+    };
+
+  var _opts = ltgt.toLtgt(opts, {}, encode);
+
+  function cmp(key) {
+    return ltgt.contains(_opts, encode(key));
+  }
+
+  function onWrite(ary) {
+    ary.forEach(function (op) {
+      if (cmp(op.key)) each(op);
+    });
+  }
+
+  db.on("write", onWrite);
+
+  return function () {
+    db.removeListener("write", onWrite);
+  };
+};

package/src/stream/read.js

@@ -0,0 +1,48 @@
+const { EntryStream, KeyStream, ValueStream } = require("level-read-stream");
+const toPull = require("stream-to-pull-stream");
+const post = require("./post");
+const pushable = require("pull-pushable");
+const Live = require("pull-live");
+
+function old(db, opts) {
+  const stream = createStream(db, opts);
+  return toPull.read1(stream);
+}
+
+function live(db, opts) {
+  const p = pushable(function (err) {
+    if (opts.onAbort) opts.onAbort(err);
+    cleanup();
+  });
+
+  const cleanup = post(db, opts, function (ch) {
+    if (opts.keys === false) p.push(ch.value);
+    else if (opts.values === false) p.push(ch.key);
+    else {
+      p.push({ type: ch.type, key: ch.key, value: ch.value });
+    }
+  });
+
+  return p;
+}
+
+function createStream(db, opts) {
+  if (opts.keys && !opts.values) {
+    return new KeyStream(db, opts);
+  } else if (!opts.keys && opts.values) {
+    return new ValueStream(db, opts);
+  } else {
+    return new EntryStream(db, opts);
+  }
+}
+
+module.exports.createPullStream = function (db, opts) {
+  return Live(
+    function (opts) {
+      return old(db, opts);
+    },
+    function (opts) {
+      return live(db, opts);
+    },
+  )(opts);
+};

package/test/e2e.test.js

@@ -0,0 +1,236 @@
+const { FlumeViewCustom, mapperBuilder } = require("../src/index");
+const Flume = require("flumedb");
+const util = require("util");
+const pull = require("pull-stream");
+const Log = require("flumelog-offset");
+const codec = require("flumecodec");
+const { rmSync } = require("fs");
+
+function pullCollectPromise(source) {
+  return new Promise((resolve, reject) => {
+    return pull(
+      source,
+      pull.collect((err, result) => {
+        if (result) {
+          resolve(result);
+        } else {
+          reject(err);
+        }
+      }),
+    );
+  });
+}
+
+function chessView(flumeDb) {
+  async function chessMapper(db, item) {
+    // Simplified - in a really scuttlebutt environment we'd have to deal with invite acceptances / game ends arriving
+    // out of order from different user's feeds
+
+    if (item.value.content.type === "chess_invite") {
+      const gameId = item.key;
+
+      const entry = {
+        id: gameId,
+        inviter: item.value.author,
+        invitee: item.value.content.inviting,
+        white:
+          item.value.content.myColor === "white"
+            ? item.value.author
+            : item.value.content.inviting,
+        black:
+          item.value.content.myColor === "black"
+            ? item.value.author
+            : item.value.content.inviting,
+        state: "invite",
+      };
+
+      return [
+        {
+          type: "put",
+          key: `/games/${gameId}`,
+          value: entry,
+        },
+        {
+          type: "put",
+          key: `/player/inviter/${entry.inviter}/id/${gameId}`,
+          value: entry,
+        },
+        {
+          type: "put",
+          key: `/player/invited/${entry.invitee}/id/${gameId}`,
+          value: entry,
+        },
+      ];
+    } else if (item.value.content.type === "chess_invite_accept") {
+      const gameId = item.value.content.root;
+      const existingEntry = await db.get(`/games/${gameId}`);
+
+      existingEntry.state = "accepted";
+
+      const deletions = [
+        {
+          type: "del",
+          key: `/player/inviter/${existingEntry.inviter}/id/${gameId}`,
+        },
+        {
+          type: "del",
+          key: `/player/invited/${existingEntry.invitee}/id/${gameId}`,
+        },
+      ];
+
+      const insertions = [
+        { type: "put", key: `/games/${gameId}`, value: existingEntry },
+        {
+          type: "put",
+          key: `/player/live_game/${existingEntry.inviter}/game/${gameId}`,
+          value: existingEntry,
+        },
+        {
+          type: "put",
+          key: `/player/live_game/${existingEntry.invitee}/game/${gameId}`,
+          value: existingEntry,
+        },
+      ];
+
+      return [...deletions, ...insertions];
+    }
+  }
+
+  const chessFlumeView = FlumeViewCustom(
+    1,
+    mapperBuilder.sequentialMapper(chessMapper),
+  );
+
+  const db = flumeDb.use("flumeviewCustom", chessFlumeView);
+
+  const chessView = db["flumeviewCustom"];
+  const append = util.promisify(db.append);
+
+  return {
+    append: (item) => {
+      return append(item);
+    },
+    getOutboundInvites: function (inviter) {
+      return chessView.read({
+        gt: `/player/inviter/${inviter}`,
+        lt: `/player/j`,
+      });
+    },
+    getInboundInvites: function (invitee) {
+      return chessView.read({
+        gt: `/player/invited/${invitee}/`,
+        lt: `/player/invited/${invitee}/\uffff`,
+      });
+    },
+    getGamesInProgress: function (playerId) {
+      return chessView.read({
+        gt: `/player/live_game/${playerId}/`,
+        lt: `/player/live_game/${playerId}/\uffff`,
+      });
+    },
+  };
+}
+
+describe("End to end scenario", () => {
+  let db;
+
+  afterEach(async () => {
+    if (db) {
+      db.close(() => {});
+      db = null;
+      rmSync("/tmp/flumeview-level-custom", { recursive: true, force: true });
+    }
+  });
+
+  test("Chess game index scenario invites (non-live)", async () => {
+    db = Flume(
+      Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+    );
+
+    const chess = chessView(db);
+
+    const items = [
+      {
+        key: "%Zf7y1aoZ/51P5IzF6GEvXNRkBqs+X7r+hG5djhQfwpM=.sha256",
+        value: {
+          previous: "%EA7jGV3XotJ3qQRMwpDbswj2xfHkEFCbZtSiXfr6d6E=.sha256",
+          sequence: 19942,
+          author: "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+          timestamp: 1791495457292,
+          hash: "sha256",
+          content: {
+            type: "chess_invite",
+            inviting: "@STgkEO48Kucd57i7x0vBqfkz/euvt2XdglwF6gZLtdk=.ed25519",
+            myColor: "black",
+            branch: [
+              "%AxUibLnrkZrzA6bk15LpGGDepPZOR2nEn24Kql0YYho=.sha256",
+              "%t/8R6p+0Hkq7vDacYwiMOoKNzRf9DD2Ukhiyhxmsroc=.sha256",
+              "%01ksKwoAAkrY+bjYdP1WCgtZHHGGcZ8bXmEoYnUlyn4=.sha256",
+              "%9oK3ltfnvH9/7h5sxD6Cu+IYoSXcOddTMPJ5L6dLsUM=.sha256",
+              "%ldUBk14tkOXNoLECOP9BYDyCyBh4p9KvfHAIzpwcK9k=.sha256",
+            ],
+          },
+          signature:
+            "LSq6Ul3o71i7QlPANr3xEhuazhHhPyByLoCGtxv+Nhu37VeI35UOIjCXGBfUNl+9c4f54y6XRNZXS4/ctQvhAg==.sig.ed25519",
+        },
+      },
+      {
+        key: "%GpeF1QW5NEiyMDSnotDYoU/2YDFAVVVNYJDleKpnhtw=.sha256",
+        value: {
+          previous: "%sat8SZr1QshItA3IZK4e6221k8i+3NOf0JLH6PlgIzw=.sha256",
+          sequence: 5561,
+          author: "@STgkEO48Kucd57i7x0vBqfkz/euvt2XdglwF6gZLtdk=.ed25519",
+          timestamp: 1793634892038,
+          hash: "sha256",
+          content: {
+            type: "chess_invite_accept",
+            root: "%Zf7y1aoZ/51P5IzF6GEvXNRkBqs+X7r+hG5djhQfwpM=.sha256",
+            branch: "%Zf7y1aoZ/51P5IzF6GEvXNRkBqs+X7r+hG5djhQfwpM=.sha256",
+          },
+          signature:
+            "S/ZvvE9VNtkcI6vMJmijlwU8kDAjKLlWhAMenA/tZQRs1MOsDnl3AySfNMAkNEIJw4pUP0//EeTUOFPQgECwCA==.sig.ed25519",
+        },
+        timestamp: 1793639323422,
+        rts: 1793634892038,
+      },
+    ];
+
+    await chess.append(items[0]);
+
+    const source = chess.getInboundInvites(
+      "@STgkEO48Kucd57i7x0vBqfkz/euvt2XdglwF6gZLtdk=.ed25519",
+    );
+
+    const result = await pullCollectPromise(source);
+
+    expect(result.length).toStrictEqual(1);
+
+    const invitedSource = chess.getOutboundInvites(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+    const invitedResult = await pullCollectPromise(invitedSource);
+    expect(invitedResult.length).toStrictEqual(1);
+
+    const inProgressSource = chess.getGamesInProgress(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+    const inProgressResult = await pullCollectPromise(inProgressSource);
+    expect(inProgressResult.length).toStrictEqual(0);
+
+    // After invite acceptance the invites should disappear and there should be an 'in progress' result
+    await chess.append(items[1]);
+
+    const invitedSource2 = chess.getInboundInvites(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+    const invitedResult2 = await pullCollectPromise(invitedSource2);
+    expect(invitedResult2.length).toStrictEqual(0);
+
+    const inProgressSource2 = chess.getGamesInProgress(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+
+    const inProgressResult2 = await pullCollectPromise(inProgressSource2);
+    expect(inProgressResult2.length).toStrictEqual(1);
+  });
+});

package/test/index.test.js

@@ -0,0 +1,352 @@
+const Log = require("flumelog-offset");
+const { mapperBuilder, FlumeViewCustom } = require("../src");
+const Flume = require("flumedb");
+const codec = require("flumecodec");
+const util = require("util");
+const { rmSync } = require("fs");
+const pull = require("pull-stream");
+const Aborter = require("pull-abortable");
+
+describe("flumeview-level-custom flumeview builder tests", () => {
+  function defaultWriter() {
+    return mapperBuilder.sequentialMapper(defaultMapper);
+  }
+
+  function createWriter(mapper) {
+    return mapperBuilder.sequentialMapper(mapper);
+  }
+
+  function defaultMapper(db, item) {
+    return Promise.resolve([{ type: "put", key: item.id, value: item.value }]);
+  }
+
+  let db = null;
+
+  afterEach(async () => {
+    if (db) {
+      db.close(() => {});
+      db = null;
+      rmSync("/tmp/flumeview-level-custom", { recursive: true, force: true });
+    }
+  });
+
+  describe("get", () => {
+    test("Test without optional filter function parameter set", async () => {
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
+
+      const get = util.promisify(db["flumeviewCustom"].get);
+      const append = util.promisify(db.append);
+
+      await append({ id: "testaroonie", value: "testarino" });
+
+      const result = await get("testaroonie");
+
+      expect(result).toStrictEqual("testarino");
+    });
+
+    test("Test with optional filter function parameter set", async () => {
+      const filter = (item) => item.type !== "post";
+
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter(), filter));
+
+      const append = util.promisify(db.append);
+      const get = util.promisify(db["flumeviewCustom"].get);
+
+      await append({ id: "testaroonie", value: "value" });
+      await append({ id: "testaroonie2", value: "value2", type: "post" });
+      await append({ id: "testaroonie3", value: "value3" });
+
+      const result1 = await get("testaroonie");
+      const result2 = await get("testaroonie2");
+      const result3 = await get("testaroonie3");
+
+      expect(result1).toStrictEqual("value");
+      expect(result2).toStrictEqual(undefined);
+      expect(result3).toStrictEqual("value3");
+    });
+  });
+
+  describe("read", () => {
+    test("non-live read", async () => {
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
+
+      const append = util.promisify(db.append);
+      const read = db["flumeviewCustom"].read;
+
+      await append({ id: "testaroonie", value: "value" });
+      await append({ id: "testaroonie2", value: "value2", type: "post" });
+      await append({ id: "testaroonie3", value: "value3" });
+
+      const source = read({
+        live: false,
+        gte: "testaroonie",
+        lte: "testaroonie2",
+      });
+
+      const result = await new Promise((resolve, reject) => {
+        const result = [];
+        pull(
+          source,
+          pull.drain(
+            (item) => result.push(item),
+            (err) => {
+              err ? reject(err) : resolve(result);
+            },
+          ),
+        );
+      });
+
+      expect(result.length).toStrictEqual(2);
+
+      expect(result[0]).toStrictEqual({ key: "testaroonie", value: "value" });
+      expect(result[1]).toStrictEqual({ key: "testaroonie2", value: "value2" });
+    });
+
+    test("Read live stream", async () => {
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
+
+      const append = util.promisify(db.append);
+      const read = db["flumeviewCustom"].read;
+
+      await append({ id: "testaroonie", value: "value" });
+      await append({ id: "testaroonie2", value: "value2", type: "post" });
+
+      const source = read({
+        live: true,
+        gte: "testaroonie",
+      });
+
+      const aborter = Aborter();
+
+      const resultPromise = new Promise((resolve, reject) => {
+        const result = [];
+        pull(
+          source,
+          aborter,
+          pull.drain(
+            (item) => {
+              if (item.value === "value") {
+                // Wait until the stream gets going so that we get this as a 'live' stream event
+                append({ id: "testaroonie3", value: "value3" });
+              }
+
+              if (item.value === "value3") {
+                aborter.abort();
+              }
+
+              result.push(item);
+            },
+            (err) => {
+              err ? reject(err) : resolve(result);
+            },
+          ),
+        );
+      });
+
+      const result = await resultPromise;
+
+      // One item is a 'sync' message indicating the stream is now live
+      expect(result.length).toStrictEqual(4);
+
+      expect(result[0]).toStrictEqual({ key: "testaroonie", value: "value" });
+      expect(result[1]).toStrictEqual({ key: "testaroonie2", value: "value2" });
+      expect(result[3]).toStrictEqual({
+        key: "testaroonie3",
+        value: "value3",
+        type: "put",
+      });
+    });
+
+    test("Read live stream (deletion)", async () => {
+      function mapper(db, item) {
+        const type = item.type || "put";
+        return [{ type: type, key: item.id, value: item.value, type: type }];
+      }
+
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, createWriter(mapper)));
+
+      const append = util.promisify(db.append);
+      const read = db["flumeviewCustom"].read;
+
+      await append({ id: "testaroonie", value: "value" });
+      await append({ id: "testaroonie2", value: { test: "value2" } });
+
+      const source = read({
+        live: true,
+        gte: "testaroonie",
+      });
+
+      const aborter = Aborter();
+
+      const resultPromise = new Promise((resolve, reject) => {
+        const result = [];
+        pull(
+          source,
+          aborter,
+          pull.drain(
+            (item) => {
+              if (item.key === "testaroonie") {
+                // Append once we're reading from the stream so that it appears once we go live
+                append({ id: "testaroonie", type: "del" });
+              }
+
+              if (item.type === "del") {
+                aborter.abort();
+              }
+
+              result.push(item);
+            },
+            (err) => {
+              err ? reject(err) : resolve(result);
+            },
+          ),
+        );
+      });
+
+      const result = await resultPromise;
+
+      expect(result.length).toStrictEqual(4);
+
+      expect(result[0]).toStrictEqual({ key: "testaroonie", value: "value" });
+      expect(result[1]).toStrictEqual({
+        key: "testaroonie2",
+        value: { test: "value2" },
+      });
+      // Sync message expected inbetween
+      expect(result[3]).toStrictEqual({
+        key: "testaroonie",
+        type: "del",
+        value: undefined,
+      });
+    });
+
+    test("Read live stream (update key)", () => {});
+
+    test("Read stream (keys only)", () => {});
+
+    test("Read stream (values only)", () => {});
+
+    test("Read stream (live, keys only)", () => {});
+
+    test("Read stream (live, values only)", () => {});
+
+    test("Read stream live, custom key encoding", () => {});
+  });
+
+  describe("Offset tracking", () => {
+    test("Starts from last offset reached on startup", async () => {
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
+
+      const get = util.promisify(db["flumeviewCustom"].get);
+      const append = util.promisify(db.append);
+
+      await append({ id: "testaroonie", value: "testarino" });
+
+      const result = await get("testaroonie");
+
+      expect(result).toStrictEqual("testarino");
+
+      // Append a couple of new items before mounting the view
+
+      db.close(() => {});
+
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+      );
+
+      const append2 = util.promisify(db.append);
+      await append2({ id: "testaroonie2", value: "testarino2" });
+      await append2({ id: "testaroonie3", value: "testarino3" });
+
+      db.close(() => {});
+
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
+
+      const read = db["flumeviewCustom"].read;
+
+      const source = read({
+        live: false,
+        gte: "testaroonie",
+        values: true,
+        keys: false,
+      });
+
+      // The stream will pause until the indexing has caught up with our new items (since has reached the log seq),
+      // then emit all the new items
+      const result2 = await new Promise((resolve, reject) => {
+        const result = [];
+        pull(
+          source,
+          pull.drain(
+            (item) => {
+              result.push(item);
+            },
+            (err) => {
+              err ? reject(err) : resolve(result);
+            },
+          ),
+        );
+      });
+
+      expect(result2.length).toStrictEqual(3);
+
+      expect(result2[0]).toStrictEqual("testarino");
+      expect(result2[1]).toStrictEqual("testarino2");
+      expect(result2[2]).toStrictEqual("testarino3");
+    });
+
+    test("Rebuilds the index if the new version is greater than the current index", async () => {
+      function newSchemaMapper(db, item) {
+        return [
+          {
+            type: "put",
+            key: item.id,
+            value: { item: item.value, version: 2 },
+          },
+        ];
+      }
+
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
+
+      const get = util.promisify(db["flumeviewCustom"].get);
+      const append = util.promisify(db.append);
+
+      await append({ id: "testaroonie", value: "testarino" });
+
+      const result = await get("testaroonie");
+
+      expect(result).toStrictEqual("testarino");
+
+      db.close(() => {});
+
+      db = Flume(
+        Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+      ).use(
+        "flumeviewCustom",
+        FlumeViewCustom(2, createWriter(newSchemaMapper)),
+      );
+
+      const get2 = util.promisify(db["flumeviewCustom"].get);
+
+      const updatedResult = await get2("testaroonie");
+
+      expect(updatedResult).toStrictEqual({ item: "testarino", version: 2 });
+    });
+  });
+});

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    globals: true,
+    maxConcurrency: 1,
+  },
+});