@happy0/flumeview-level-custom 1.0.0 → 2.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@happy0/flumeview-level-custom",
-  "version": "1.0.0",
+  "version": "2.0.1",
   "description": "",
   "license": "MIT",
   "author": "Gordon Martin",

package/src/flumeView.js

@@ -0,0 +1,104 @@
+const Obv = require("obv");
+const path = require("path");
+const charwise = require("charwise");
+const { Level } = require("level");
+const pull = require("pull-stream");
+const { createPullStream } = require("./stream/read");
+const { METADATA_KEY } = require("./metadata");
+
+module.exports = function (version, createWriter, filter) {
+  return function (log, name) {
+    const dir = path.dirname(log.filename);
+
+    const since = Obv();
+    const db = createLevelDb(dir, log, name);
+    let closed = false;
+
+    startConsumingLog(db, since, version);
+
+    let writer;
+
+    function destroy(cb) {
+      cb = cb || (() => {});
+      since.set(-1);
+      db.clear().then(cb);
+    }
+
+    async function startConsumingLog(db, since) {
+      const value = await db.get(METADATA_KEY, { keyEncoding: "utf8" });
+
+      if (!value) {
+        // No existing entry, we're consuming the log from the start
+        since.set(-1);
+      } else if (value.version === version) {
+        // The version has not changed, resume consuming the log from where we left off
+        // last time
+        since.set(value.since);
+      } else {
+        // version has changed, wipe db and start over.
+        destroy(() => {});
+      }
+    }
+
+    return {
+      // flumeview interface
+      methods: { get: "async", read: "source" },
+      since: since,
+      createSink: function (cb) {
+        writer = createWriter(db, since, version, cb);
+
+        if (!filter) {
+          return writer;
+        } else {
+          return pull(
+            pull.filter((item) => filter(item.value)),
+            writer,
+          );
+        }
+      },
+      destroy: destroy,
+      close: function (cb) {
+        cb = cb || (() => {});
+
+        closed = true;
+
+        if (writer) {
+          writer.abort(() => db.close().then(cb));
+        } else {
+          db.close().then(cb);
+        }
+      },
+      get: function (key, cb) {
+        db.get(key).then((result) => {
+          if (result === undefined) {
+            cb(result);
+          } else {
+            cb(null, result);
+          }
+        });
+      },
+      /**
+       *
+       * @param {*} opts The level DB stream key range options ( see 'level-read-stream' library ) as well as an optional 'live' parameter to indicate whether the
+       * stream should stay live an emit live updates. IF live, a {sync:true} object will be emitted when the stream is about to emit
+       * live values and each item will have a 'type' field (put, del) indicating whether the database operation type as well as the key
+       * and value
+       * @returns a pull-stream source of database entries satisfying the input range
+       */
+      read: function (opts) {
+        return createPullStream(db, opts);
+      },
+    };
+  };
+};
+
+function createLevelDb(dir, log, name) {
+  if (!log.filename) {
+    throw new Error("flumeview-level-custom can only be used with a file log");
+  }
+
+  return new Level(path.join(dir, name), {
+    keyEncoding: charwise,
+    valueEncoding: "json",
+  });
+}

package/src/index.js

@@ -1,164 +1,7 @@
-const Obv = require("obv");
-const path = require("path");
-const Write = require("pull-write");
-const charwise = require("charwise");
-const { Level } = require("level");
-const pull = require("pull-stream");
-const { createPullStream } = require("./stream/read");
+const flumeViewLevelCustom = require("./flumeView");
+const mapperBuilder = require("./mapperBuilder");
+const metadata = require("./metadata");
 
-const METADATA_KEY = "\x00";
-
-const MAX_ITEM_BUFFER_SIZE = 0;
-
-/**
- *
- * @param {*} version the version of the flumeview we're building. If this value differs to the value stored in the view's metadata, we
- * will rebuild the index from scratch.
- * @param {*} asyncMapper A function that takes a level database and the current flumelog item value as parameters. It should return an array of
- * batch operation objects to be written to the database. It's important that these are returned rather than inserted via the db parameter so that the
- * flumeview metadata can be updated at the same time as the new entries with the current log position atomically.
- * @param {*} filter Optional function to filter out items in the logstream by prior to mapping them to database entries
- * @returns
- */
-module.exports = function (version, asyncMapper, filter) {
-  return function (log, name) {
-    const dir = path.dirname(log.filename);
-
-    const since = Obv();
-    const db = createLevelDb(dir, log, name);
-    let closed = false;
-
-    startConsumingLog(db, since, version);
-
-    let writer;
-
-    async function mapEntry(item) {
-      const items = await asyncMapper(db, item);
-      return items;
-    }
-
-    function createWriter(db, streamEndCb) {
-      return Write(
-        function write(batch, cb) {
-          if (closed) {
-            return cb(new Error("database closed while index was building"));
-          } else if (batch.length !== 1) {
-            // For v1 we're only exposing an interface that allows one item to be processed at a time.
-            // For future versions we may consider batching optimisations
-            throw new Error(
-              "Batch should never have more or less than one item.",
-            );
-          } else {
-            const item = batch[0];
-
-            const metadata = {
-              key: METADATA_KEY,
-              value: { version: version, since: item.seq },
-              valueEncoding: "json",
-              keyEncoding: "utf8",
-              type: "put",
-            };
-
-            const update = mapEntry(item.value);
-
-            update
-              .then((items) => {
-                return db.batch([metadata, ...items]);
-              })
-              .then((_) => {
-                since.set(item.seq);
-                cb();
-              })
-              .catch((e) => console.error(e));
-          }
-        },
-        null,
-        MAX_ITEM_BUFFER_SIZE,
-        streamEndCb,
-      );
-    }
-
-    function destroy(cb) {
-      cb = cb || (() => {});
-      since.set(-1);
-      db.clear().then(cb);
-    }
-
-    async function startConsumingLog(db, since) {
-      const value = await db.get(METADATA_KEY, { keyEncoding: "utf8" });
-
-      if (!value) {
-        // No existing entry, we're consuming the log from the start
-        since.set(-1);
-      } else if (value.version === version) {
-        // The version has not changed, resume consuming the log from where we left off
-        // last time
-        since.set(value.since);
-      } else {
-        // version has changed, wipe db and start over.
-        destroy(() => {});
-      }
-    }
-
-    return {
-      // flumeview interface
-      methods: { get: "async", read: "source" },
-      since: since,
-      createSink: function (cb) {
-        writer = createWriter(db, cb);
-
-        if (!filter) {
-          return writer;
-        } else {
-          return pull(
-            pull.filter((item) => filter(item.value)),
-            writer,
-          );
-        }
-      },
-      destroy: destroy,
-      close: function (cb) {
-        cb = cb || (() => {});
-
-        closed = true;
-
-        if (writer) {
-          writer.abort(() => db.close().then(cb));
-        } else {
-          db.close().then(cb);
-        }
-      },
-      get: function (key, cb) {
-        db.get(key).then((result) => {
-          if (result === undefined) {
-            cb(result);
-          } else {
-            cb(null, result);
-          }
-        });
-      },
-      /**
-       *
-       * @param {*} opts The level DB stream key range options ( see 'level-read-stream' library ) as well as an optional 'live' parameter to indicate whether the
-       * stream should stay live an emit live updates. IF live, a {sync:true} object will be emitted when the stream is about to emit
-       * live values and each item will have a 'type' field (put, del) indicating whether the database operation type as well as the key
-       * and value
-       * @returns a pull-stream source of database entries satisfying the input range
-       */
-      read: function (opts) {
-        return createPullStream(db, opts);
-      },
-    };
-  };
-};
-
-function createLevelDb(dir, log, name) {
-  if (!log.filename) {
-    throw new Error("flumeview-level-custom can only be used with a file log");
-  }
-
-  return new Level(path.join(dir, name), {
-    keyEncoding: charwise,
-    valueEncoding: "json",
-  });
-}
+module.exports.FlumeViewLevelCustom = flumeViewLevelCustom;
+module.exports.mapperBuilder = mapperBuilder;
+module.exports.metadata = metadata;

package/src/mapperBuilder.js

@@ -0,0 +1,37 @@
+const Write = require("pull-write");
+const { createMetadataUpdateItem } = require("./metadata");
+
+module.exports.sequentialMapper = function (asyncMapper) {
+  const MAX_ITEM_BUFFER_SIZE = 0;
+
+  return function (db, since, version, streamEndCb) {
+    return Write(
+      function write(batch, cb) {
+        if (batch.length !== 1) {
+          cb(
+            new Error(
+              "Batch should never have more or less than one item for sequential mapper.",
+            ),
+          );
+        } else {
+          const item = batch[0];
+          const metadata = createMetadataUpdateItem(version, item.seq);
+          const update = asyncMapper(db, item.value);
+
+          Promise.resolve(update)
+            .then((items) => {
+              return db.batch([metadata, ...items]);
+            })
+            .then((_) => {
+              since.set(item.seq);
+              cb();
+            })
+            .catch((e) => console.error(e));
+        }
+      },
+      null,
+      MAX_ITEM_BUFFER_SIZE,
+      streamEndCb,
+    );
+  };
+};

package/src/metadata.js

@@ -0,0 +1,16 @@
+const METADATA_KEY = "\x00";
+
+function createMetadataUpdateBatchItem(version, seq) {
+  return {
+    key: METADATA_KEY,
+    value: { version: version, since: seq },
+    valueEncoding: "json",
+    keyEncoding: "utf8",
+    type: "put",
+  };
+}
+
+module.exports = {
+  METADATA_KEY,
+  createMetadataUpdateItem: createMetadataUpdateBatchItem,
+};

package/test/e2e.test.js

@@ -0,0 +1,236 @@
+const { FlumeViewCustom, mapperBuilder } = require("../src/index");
+const Flume = require("flumedb");
+const util = require("util");
+const pull = require("pull-stream");
+const Log = require("flumelog-offset");
+const codec = require("flumecodec");
+const { rmSync } = require("fs");
+
+function pullCollectPromise(source) {
+  return new Promise((resolve, reject) => {
+    return pull(
+      source,
+      pull.collect((err, result) => {
+        if (result) {
+          resolve(result);
+        } else {
+          reject(err);
+        }
+      }),
+    );
+  });
+}
+
+function chessView(flumeDb) {
+  async function chessMapper(db, item) {
+    // Simplified - in a really scuttlebutt environment we'd have to deal with invite acceptances / game ends arriving
+    // out of order from different user's feeds
+
+    if (item.value.content.type === "chess_invite") {
+      const gameId = item.key;
+
+      const entry = {
+        id: gameId,
+        inviter: item.value.author,
+        invitee: item.value.content.inviting,
+        white:
+          item.value.content.myColor === "white"
+            ? item.value.author
+            : item.value.content.inviting,
+        black:
+          item.value.content.myColor === "black"
+            ? item.value.author
+            : item.value.content.inviting,
+        state: "invite",
+      };
+
+      return [
+        {
+          type: "put",
+          key: `/games/${gameId}`,
+          value: entry,
+        },
+        {
+          type: "put",
+          key: `/player/inviter/${entry.inviter}/id/${gameId}`,
+          value: entry,
+        },
+        {
+          type: "put",
+          key: `/player/invited/${entry.invitee}/id/${gameId}`,
+          value: entry,
+        },
+      ];
+    } else if (item.value.content.type === "chess_invite_accept") {
+      const gameId = item.value.content.root;
+      const existingEntry = await db.get(`/games/${gameId}`);
+
+      existingEntry.state = "accepted";
+
+      const deletions = [
+        {
+          type: "del",
+          key: `/player/inviter/${existingEntry.inviter}/id/${gameId}`,
+        },
+        {
+          type: "del",
+          key: `/player/invited/${existingEntry.invitee}/id/${gameId}`,
+        },
+      ];
+
+      const insertions = [
+        { type: "put", key: `/games/${gameId}`, value: existingEntry },
+        {
+          type: "put",
+          key: `/player/live_game/${existingEntry.inviter}/game/${gameId}`,
+          value: existingEntry,
+        },
+        {
+          type: "put",
+          key: `/player/live_game/${existingEntry.invitee}/game/${gameId}`,
+          value: existingEntry,
+        },
+      ];
+
+      return [...deletions, ...insertions];
+    }
+  }
+
+  const chessFlumeView = FlumeViewCustom(
+    1,
+    mapperBuilder.sequentialMapper(chessMapper),
+  );
+
+  const db = flumeDb.use("flumeviewCustom", chessFlumeView);
+
+  const chessView = db["flumeviewCustom"];
+  const append = util.promisify(db.append);
+
+  return {
+    append: (item) => {
+      return append(item);
+    },
+    getOutboundInvites: function (inviter) {
+      return chessView.read({
+        gt: `/player/inviter/${inviter}/`,
+        lt: `/player/inviter/${inviter}/\uffff`,
+      });
+    },
+    getInboundInvites: function (invitee) {
+      return chessView.read({
+        gt: `/player/invited/${invitee}/`,
+        lt: `/player/invited/${invitee}/\uffff`,
+      });
+    },
+    getGamesInProgress: function (playerId) {
+      return chessView.read({
+        gt: `/player/live_game/${playerId}/`,
+        lt: `/player/live_game/${playerId}/\uffff`,
+      });
+    },
+  };
+}
+
+describe("End to end scenario", () => {
+  let db;
+
+  afterEach(async () => {
+    if (db) {
+      db.close(() => {});
+      db = null;
+      rmSync("/tmp/flumeview-level-custom", { recursive: true, force: true });
+    }
+  });
+
+  test("Chess game index scenario invites (non-live)", async () => {
+    db = Flume(
+      Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
+    );
+
+    const chess = chessView(db);
+
+    const items = [
+      {
+        key: "%Zf7y1aoZ/51P5IzF6GEvXNRkBqs+X7r+hG5djhQfwpM=.sha256",
+        value: {
+          previous: "%EA7jGV3XotJ3qQRMwpDbswj2xfHkEFCbZtSiXfr6d6E=.sha256",
+          sequence: 19942,
+          author: "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+          timestamp: 1791495457292,
+          hash: "sha256",
+          content: {
+            type: "chess_invite",
+            inviting: "@STgkEO48Kucd57i7x0vBqfkz/euvt2XdglwF6gZLtdk=.ed25519",
+            myColor: "black",
+            branch: [
+              "%AxUibLnrkZrzA6bk15LpGGDepPZOR2nEn24Kql0YYho=.sha256",
+              "%t/8R6p+0Hkq7vDacYwiMOoKNzRf9DD2Ukhiyhxmsroc=.sha256",
+              "%01ksKwoAAkrY+bjYdP1WCgtZHHGGcZ8bXmEoYnUlyn4=.sha256",
+              "%9oK3ltfnvH9/7h5sxD6Cu+IYoSXcOddTMPJ5L6dLsUM=.sha256",
+              "%ldUBk14tkOXNoLECOP9BYDyCyBh4p9KvfHAIzpwcK9k=.sha256",
+            ],
+          },
+          signature:
+            "LSq6Ul3o71i7QlPANr3xEhuazhHhPyByLoCGtxv+Nhu37VeI35UOIjCXGBfUNl+9c4f54y6XRNZXS4/ctQvhAg==.sig.ed25519",
+        },
+      },
+      {
+        key: "%GpeF1QW5NEiyMDSnotDYoU/2YDFAVVVNYJDleKpnhtw=.sha256",
+        value: {
+          previous: "%sat8SZr1QshItA3IZK4e6221k8i+3NOf0JLH6PlgIzw=.sha256",
+          sequence: 5561,
+          author: "@STgkEO48Kucd57i7x0vBqfkz/euvt2XdglwF6gZLtdk=.ed25519",
+          timestamp: 1793634892038,
+          hash: "sha256",
+          content: {
+            type: "chess_invite_accept",
+            root: "%Zf7y1aoZ/51P5IzF6GEvXNRkBqs+X7r+hG5djhQfwpM=.sha256",
+            branch: "%Zf7y1aoZ/51P5IzF6GEvXNRkBqs+X7r+hG5djhQfwpM=.sha256",
+          },
+          signature:
+            "S/ZvvE9VNtkcI6vMJmijlwU8kDAjKLlWhAMenA/tZQRs1MOsDnl3AySfNMAkNEIJw4pUP0//EeTUOFPQgECwCA==.sig.ed25519",
+        },
+        timestamp: 1793639323422,
+        rts: 1793634892038,
+      },
+    ];
+
+    await chess.append(items[0]);
+
+    const source = chess.getInboundInvites(
+      "@STgkEO48Kucd57i7x0vBqfkz/euvt2XdglwF6gZLtdk=.ed25519",
+    );
+
+    const result = await pullCollectPromise(source);
+
+    expect(result.length).toStrictEqual(1);
+
+    const invitedSource = chess.getOutboundInvites(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+    const invitedResult = await pullCollectPromise(invitedSource);
+    expect(invitedResult.length).toStrictEqual(1);
+
+    const inProgressSource = chess.getGamesInProgress(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+    const inProgressResult = await pullCollectPromise(inProgressSource);
+    expect(inProgressResult.length).toStrictEqual(0);
+
+    // After invite acceptance the invites should disappear and there should be an 'in progress' result
+    await chess.append(items[1]);
+
+    const invitedSource2 = chess.getInboundInvites(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+    const invitedResult2 = await pullCollectPromise(invitedSource2);
+    expect(invitedResult2.length).toStrictEqual(0);
+
+    const inProgressSource2 = chess.getGamesInProgress(
+      "@RJ09Kfs3neEZPrbpbWVDxkN92x9moe3aPusOMOc4S2I=.ed25519",
+    );
+
+    const inProgressResult2 = await pullCollectPromise(inProgressSource2);
+    expect(inProgressResult2.length).toStrictEqual(1);
+  });
+});

package/test/index.test.js

@@ -1,5 +1,5 @@
-var Log = require("flumelog-offset");
-const FlumeviewCustom = require("../src");
+const Log = require("flumelog-offset");
+const { mapperBuilder, FlumeViewCustom } = require("../src");
 const Flume = require("flumedb");
 const codec = require("flumecodec");
 const util = require("util");
@@ -8,25 +8,33 @@ const pull = require("pull-stream");
 const Aborter = require("pull-abortable");
 
 describe("flumeview-level-custom flumeview builder tests", () => {
-  function mapper(db, item) {
-    return [{ type: "put", key: item.id, value: item.value }];
+  function defaultWriter() {
+    return mapperBuilder.sequentialMapper(defaultMapper);
+  }
+
+  function createWriter(mapper) {
+    return mapperBuilder.sequentialMapper(mapper);
+  }
+
+  function defaultMapper(db, item) {
+    return Promise.resolve([{ type: "put", key: item.id, value: item.value }]);
   }
 
   let db = null;
 
-  describe("get", () => {
-    afterEach(async () => {
-      if (db) {
-        db.close(() => {});
-        db = null;
-        rmSync("/tmp/flumeview-level-custom", { recursive: true, force: true });
-      }
-    });
+  afterEach(async () => {
+    if (db) {
+      db.close(() => {});
+      db = null;
+      rmSync("/tmp/flumeview-level-custom", { recursive: true, force: true });
+    }
+  });
 
+  describe("get", () => {
     test("Test without optional filter function parameter set", async () => {
       db = Flume(
         Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
 
       const get = util.promisify(db["flumeviewCustom"].get);
       const append = util.promisify(db.append);
@@ -43,7 +51,7 @@ describe("flumeview-level-custom flumeview builder tests", () => {
 
       db = Flume(
         Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper, filter));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter(), filter));
 
       const append = util.promisify(db.append);
       const get = util.promisify(db["flumeviewCustom"].get);
@@ -63,18 +71,10 @@ describe("flumeview-level-custom flumeview builder tests", () => {
   });
 
   describe("read", () => {
-    afterEach(async () => {
-      if (db) {
-        db.close(() => {});
-        db = null;
-        rmSync("/tmp/flumeview-level-custom", { recursive: true, force: true });
-      }
-    });
-
     test("non-live read", async () => {
       db = Flume(
         Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
 
       const append = util.promisify(db.append);
       const read = db["flumeviewCustom"].read;
@@ -111,7 +111,7 @@ describe("flumeview-level-custom flumeview builder tests", () => {
     test("Read live stream", async () => {
       db = Flume(
         Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
 
       const append = util.promisify(db.append);
       const read = db["flumeviewCustom"].read;
@@ -173,7 +173,7 @@ describe("flumeview-level-custom flumeview builder tests", () => {
 
       db = Flume(
         Log("/tmp/flumeview-level-custom/log.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, createWriter(mapper)));
 
       const append = util.promisify(db.append);
       const read = db["flumeviewCustom"].read;
@@ -247,7 +247,7 @@ describe("flumeview-level-custom flumeview builder tests", () => {
     test("Starts from last offset reached on startup", async () => {
       db = Flume(
         Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
 
       const get = util.promisify(db["flumeviewCustom"].get);
       const append = util.promisify(db.append);
@@ -274,7 +274,7 @@ describe("flumeview-level-custom flumeview builder tests", () => {
 
       db = Flume(
         Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
 
       const read = db["flumeviewCustom"].read;
 
@@ -322,7 +322,7 @@ describe("flumeview-level-custom flumeview builder tests", () => {
 
       db = Flume(
         Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(1, mapper));
+      ).use("flumeviewCustom", FlumeViewCustom(1, defaultWriter()));
 
       const get = util.promisify(db["flumeviewCustom"].get);
       const append = util.promisify(db.append);
@@ -337,7 +337,10 @@ describe("flumeview-level-custom flumeview builder tests", () => {
 
       db = Flume(
         Log("/tmp/flumeview-level-custom/log2.offset", 1024, codec.json),
-      ).use("flumeviewCustom", FlumeviewCustom(2, newSchemaMapper));
+      ).use(
+        "flumeviewCustom",
+        FlumeViewCustom(2, createWriter(newSchemaMapper)),
+      );
 
       const get2 = util.promisify(db["flumeviewCustom"].get);
 

package/vitest.config.ts

@@ -3,5 +3,6 @@ import { defineConfig } from "vitest/config";
 export default defineConfig({
   test: {
     globals: true,
+    maxConcurrency: 1,
   },
 });