@ethersphere/bee-js 13.0.1-upcoming.gd3bc2f5 → 13.1.0

package/README.md

@@ -127,6 +127,8 @@ The `toString` method uses `toHex`.
 | SOCReader  | SingleOwnerChunk reader | `bee.soc.makeReader`  |
 | FeedWriter | Feed writer             | `bee.feed.makeWriter` |
 | FeedReader | Feed reader             | `bee.feed.makeReader` |
+| RollingFeedWriter | Rolling feed writer | `bee.rollingFeed.makeWriter` |
+| RollingFeedReader | Rolling feed reader | `bee.rollingFeed.makeReader` |
 
 ### Bee API
 
@@ -332,6 +334,62 @@ const bee = new Bee('http://localhost:1633')
 const uploadResult = await bee.collection.uploadFromDirectory(batchId, './path/to/gallery/')
 ```
 
+### Rolling feed (periodically-restarting sequential feed)
+
+A rolling feed avoids the unbounded growth of a plain sequential feed by restarting it every
+`periodLength` seconds, so old postage-batch eviction never breaks the latest update. See
+[ROLLING_FEED.md](./ROLLING_FEED.md) for the full design.
+
+A rolling feed only stays readable while the writer keeps publishing. Keeping it alive is the
+application's job, not the SDK's, so a writer belongs on a timer — republish on every tick, even
+when the data has not changed.
+
+#### Writer
+
+```js
+import { Bee, PrivateKey, Topic } from '@ethersphere/bee-js'
+
+const bee = new Bee('http://localhost:1633')
+const topic = Topic.fromString('my-feed')
+const signer = new PrivateKey('...')
+const periodLength = 600 // 10 minutes
+
+const writer = bee.rollingFeed.makeWriter(topic, signer, periodLength)
+
+let latest = 'Hello, World!'
+await writer.uploadPayload(batchId, latest)
+
+// tick twice per period so a slow or delayed write still lands inside its own period
+const handle = setInterval(async () => {
+  try {
+    await writer.uploadPayload(batchId, latest)
+  } catch (error) {
+    console.error('rolling feed heartbeat failed', error)
+  }
+}, (periodLength / 2) * 1000)
+
+process.on('SIGTERM', () => clearInterval(handle))
+```
+
+If the writer was down long enough to leave gaps behind, call `writer.catchUp(batchId)` before
+resuming the timer to backfill the missed periods. It throws when no populated period is found
+within `maxBackfill`, so it is for restarts, not for the very first run.
+
+#### Reader
+
+```js
+import { Bee, EthAddress, Topic } from '@ethersphere/bee-js'
+
+const bee = new Bee('http://localhost:1633')
+const topic = Topic.fromString('my-feed')
+const owner = new EthAddress('...')
+const periodLength = 600 // must match the writer
+
+const reader = bee.rollingFeed.makeReader(topic, owner, periodLength)
+const result = await reader.downloadPayload()
+console.log(result.payload.toUtf8()) // prints 'Hello, World!'
+```
+
 ### Customize http/https agent and headers
 
 ```js

package/dist/cjs/bee.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Bee = void 0;
-const cafe_utility_1 = require("cafe-utility");
 const core_sdk_1 = require("@ethersphere/core-sdk");
+const cafe_utility_1 = require("cafe-utility");
 const envelope_1 = require("./api/envelope");
 const rchash_1 = require("./api/rchash");
 const soc_1 = require("./chunk/soc");
@@ -18,6 +18,7 @@ const file_1 = require("./modules/file");
 const grantee_1 = require("./modules/grantee");
 const messaging_1 = require("./modules/messaging");
 const pin_1 = require("./modules/pin");
+const rolling_feed_1 = require("./modules/rolling-feed");
 const settlement_1 = require("./modules/settlement");
 const soc_2 = require("./modules/soc");
 const stake_1 = require("./modules/stake");
@@ -90,6 +91,7 @@ class Bee {
         this.chunk = new chunk_1.Chunk(context);
         this.file = new file_1.File(context);
         this.collection = new collection_1.Collection(context);
+        this.rollingFeed = new rolling_feed_1.RollingFeed(context);
     }
     /**
      * Creates a Content Addressed Chunk.

package/dist/cjs/index.js

@@ -36,7 +36,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Bee = exports.Size = exports.Utils = exports.Duration = exports.SUPPORTED_BEE_VERSION_EXACT = exports.SUPPORTED_BEE_VERSION = exports.MantarayNode = exports.TransactionId = exports.Topic = exports.Stamper = exports.Span = exports.Signature = exports.Reference = exports.PublicKey = exports.PrivateKey = exports.PeerAddress = exports.Identifier = exports.FeedIndex = exports.EthAddress = exports.ChunkSplitter = exports.ChunkBuilder = exports.Bytes = exports.BatchId = void 0;
+exports.Bee = exports.SUPPORTED_BEE_VERSION_EXACT = exports.SUPPORTED_BEE_VERSION = exports.Size = exports.Utils = exports.Duration = exports.RollingFeedWriter = exports.RollingFeedReader = exports.MantarayNode = exports.TransactionId = exports.Topic = exports.Stamper = exports.Span = exports.Signature = exports.Reference = exports.PublicKey = exports.PrivateKey = exports.PeerAddress = exports.Identifier = exports.FeedIndex = exports.EthAddress = exports.ChunkSplitter = exports.ChunkBuilder = exports.Bytes = exports.BatchId = void 0;
 const bee_1 = require("./bee");
 Object.defineProperty(exports, "Bee", { enumerable: true, get: function () { return bee_1.Bee; } });
 var core_sdk_1 = require("@ethersphere/core-sdk");
@@ -58,9 +58,9 @@ Object.defineProperty(exports, "Topic", { enumerable: true, get: function () { r
 Object.defineProperty(exports, "TransactionId", { enumerable: true, get: function () { return core_sdk_1.TransactionId; } });
 var manifest_1 = require("./manifest/manifest");
 Object.defineProperty(exports, "MantarayNode", { enumerable: true, get: function () { return manifest_1.MantarayNode; } });
-var version_1 = require("./version");
-Object.defineProperty(exports, "SUPPORTED_BEE_VERSION", { enumerable: true, get: function () { return version_1.SUPPORTED_BEE_VERSION; } });
-Object.defineProperty(exports, "SUPPORTED_BEE_VERSION_EXACT", { enumerable: true, get: function () { return version_1.SUPPORTED_BEE_VERSION_EXACT; } });
+var rolling_feed_1 = require("./modules/rolling-feed");
+Object.defineProperty(exports, "RollingFeedReader", { enumerable: true, get: function () { return rolling_feed_1.RollingFeedReader; } });
+Object.defineProperty(exports, "RollingFeedWriter", { enumerable: true, get: function () { return rolling_feed_1.RollingFeedWriter; } });
 __exportStar(require("./types"), exports);
 __exportStar(require("./utils/constants"), exports);
 var duration_1 = require("./utils/duration");
@@ -70,3 +70,6 @@ exports.Utils = __importStar(require("./utils/expose"));
 var size_1 = require("./utils/size");
 Object.defineProperty(exports, "Size", { enumerable: true, get: function () { return size_1.Size; } });
 __exportStar(require("./utils/tokens"), exports);
+var version_1 = require("./version");
+Object.defineProperty(exports, "SUPPORTED_BEE_VERSION", { enumerable: true, get: function () { return version_1.SUPPORTED_BEE_VERSION; } });
+Object.defineProperty(exports, "SUPPORTED_BEE_VERSION_EXACT", { enumerable: true, get: function () { return version_1.SUPPORTED_BEE_VERSION_EXACT; } });

package/dist/cjs/modules/rolling-feed.js

@@ -0,0 +1,188 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RollingFeedReader = exports.RollingFeedWriter = exports.RollingFeed = void 0;
+const core_sdk_1 = require("@ethersphere/core-sdk");
+const __1 = require("..");
+const feed_1 = require("../api/feed");
+const soc_1 = require("../chunk/soc");
+const feed_2 = require("../feed");
+const identifier_1 = require("../feed/identifier");
+const error_1 = require("../utils/error");
+// RollingFeedReader.downloadPayload/downloadReference only ever fall back one period, so
+// backfilling deeper than that serves no reader by default - see catchUp's maxBackfill param.
+const DEFAULT_MAX_BACKFILL = 1;
+function periodIndex(t, periodLength) {
+    if (periodLength <= 0) {
+        throw new __1.BeeError('Period length must be greater than zero!');
+    }
+    return Math.floor(t / periodLength);
+}
+function topicFor(baseTopic, periodIdx) {
+    const baseTopicBytes = baseTopic.toUint8Array();
+    return new core_sdk_1.Topic((0, core_sdk_1.keccak256)(__1.Bytes.concat(baseTopicBytes, (0, core_sdk_1.numberToUint64)(BigInt(periodIdx), 'BE'))));
+}
+async function isPeriodPopulated(requestOptions, owner, topic) {
+    try {
+        await (0, feed_1.probeFeed)(requestOptions, owner, topic);
+        return true;
+    }
+    catch (e) {
+        if (e instanceof error_1.BeeResponseError) {
+            return false;
+        }
+        throw e;
+    }
+}
+async function fetchLatestReference(requestOptions, owner, topic) {
+    const { feedIndex } = await (0, feed_1.probeFeed)(requestOptions, owner, topic);
+    const update = await (0, feed_2.downloadFeedUpdate)(requestOptions, owner, topic, feedIndex, true);
+    return {
+        reference: new core_sdk_1.Reference(update.payload.toUint8Array()),
+        feedIndex,
+        feedIndexNext: feedIndex.next(),
+    };
+}
+class RollingFeed {
+    constructor(context) {
+        this.context = context;
+    }
+    makeWriter(baseTopic, signer, periodLength) {
+        return new RollingFeedWriter(this.context, baseTopic, signer, periodLength);
+    }
+    makeReader(baseTopic, owner, periodLength) {
+        return new RollingFeedReader(this.context, baseTopic, owner, periodLength);
+    }
+}
+exports.RollingFeed = RollingFeed;
+class RollingFeedWriter {
+    constructor(context, baseTopic, signer, periodLength) {
+        this.context = context;
+        this.baseTopic = baseTopic;
+        this.signer = signer;
+        this.periodLength = periodLength;
+    }
+    async uploadPayload(postageBatchId, payload, options) {
+        const requestOptions = this.context.getRequestOptionsForCall();
+        const stamp = new core_sdk_1.BatchId(postageBatchId);
+        const { currentTopic, nextTopic, mirrorOptions } = this.currentAndNextTopics(options);
+        const [result] = await Promise.all([
+            (0, feed_2.updateFeedWithPayload)(requestOptions, this.signer, currentTopic, payload, stamp, options),
+            (0, feed_2.updateFeedWithPayload)(requestOptions, this.signer, nextTopic, payload, stamp, mirrorOptions),
+        ]);
+        return result;
+    }
+    async uploadReference(postageBatchId, reference, options) {
+        const requestOptions = this.context.getRequestOptionsForCall();
+        const stamp = new core_sdk_1.BatchId(postageBatchId);
+        const { currentTopic, nextTopic, mirrorOptions } = this.currentAndNextTopics(options);
+        const [result] = await Promise.all([
+            (0, feed_2.updateFeedWithReference)(requestOptions, this.signer, currentTopic, reference, stamp, options),
+            (0, feed_2.updateFeedWithReference)(requestOptions, this.signer, nextTopic, reference, stamp, mirrorOptions),
+        ]);
+        return result;
+    }
+    /**
+     * True unless the period right before `periodIdx` (default: current) never got mirrored
+     * forward into it, i.e. the writer went silent for at least one whole period.
+     */
+    async isCaughtUp(periodIdx) {
+        const requestOptions = this.context.getRequestOptionsForCall();
+        const targetPeriod = periodIdx ?? periodIndex(Date.now() / 1000, this.periodLength);
+        const owner = this.signer.publicKey().address();
+        return isPeriodPopulated(requestOptions, owner, topicFor(this.baseTopic, targetPeriod));
+    }
+    /**
+     * Backfills up to `maxBackfill` periods strictly between the last populated one and
+     * `periodIdx` (default: current) with that period's last known payload/reference. Never
+     * writes `periodIdx` itself - keeping it populated during silence is the caller's job (a
+     * periodic `uploadPayload`/`uploadReference` heartbeat), not catchUp's. A conditional
+     * "write it if empty" would race that heartbeat: there's no compare-and-swap on a SOC
+     * address, so a stale write can still land after a concurrent fresh one and shadow it.
+     *
+     * `maxBackfill` bounds both the backward scan and the backfill itself, since scanning
+     * further back than you're willing to backfill only wastes round trips. Defaults to 1,
+     * matching the reader's one-period fallback - nothing deeper is ever read anyway. Throws
+     * if no populated period is found within that bound.
+     */
+    async catchUp(postageBatchId, periodIdx, maxBackfill = DEFAULT_MAX_BACKFILL) {
+        const requestOptions = this.context.getRequestOptionsForCall();
+        const stamp = new core_sdk_1.BatchId(postageBatchId);
+        const owner = this.signer.publicKey().address();
+        const targetPeriod = periodIdx ?? periodIndex(Date.now() / 1000, this.periodLength);
+        // periods before 0 can't exist (period index is derived from Unix time), so the scan
+        // must not probe them even when the lookback window would otherwise reach that far
+        const scanFloor = Math.max(0, targetPeriod - 1 - maxBackfill);
+        let lastGoodPeriod = targetPeriod - 1;
+        while (lastGoodPeriod >= scanFloor) {
+            if (await isPeriodPopulated(requestOptions, owner, topicFor(this.baseTopic, lastGoodPeriod))) {
+                break;
+            }
+            lastGoodPeriod--;
+        }
+        if (lastGoodPeriod < scanFloor) {
+            throw new __1.BeeError(`No populated period found within ${maxBackfill} periods to catch up from!`);
+        }
+        if (lastGoodPeriod === targetPeriod - 1) {
+            return; // no gap - nothing to backfill
+        }
+        const sourceTopic = topicFor(this.baseTopic, lastGoodPeriod);
+        const { feedIndex: sourceIndex } = await (0, feed_1.probeFeed)(requestOptions, owner, sourceTopic);
+        const sourceChunk = await (0, feed_2.downloadFeedUpdateAsCAC)(requestOptions, owner, sourceTopic, sourceIndex);
+        const periodsToBackfill = Array.from({ length: targetPeriod - 1 - lastGoodPeriod }, (_, i) => lastGoodPeriod + 1 + i);
+        await Promise.all(periodsToBackfill.map(async (period) => {
+            const identifier = (0, identifier_1.makeFeedIdentifier)(topicFor(this.baseTopic, period), 0);
+            return (0, soc_1.uploadSingleOwnerChunkWithWrappedChunk)(requestOptions, this.signer, stamp, identifier, sourceChunk);
+        }));
+    }
+    currentAndNextTopics(options) {
+        const currentPeriod = periodIndex(Date.now() / 1000, this.periodLength);
+        return {
+            currentTopic: topicFor(this.baseTopic, currentPeriod),
+            nextTopic: topicFor(this.baseTopic, currentPeriod + 1),
+            mirrorOptions: { ...options, index: undefined },
+        };
+    }
+}
+exports.RollingFeedWriter = RollingFeedWriter;
+class RollingFeedReader {
+    constructor(context, baseTopic, owner, periodLength) {
+        this.context = context;
+        this.baseTopic = baseTopic;
+        this.owner = owner;
+        this.periodLength = periodLength;
+    }
+    /**
+     * Reads the current period's feed; falls back to the previous period once if empty,
+     * to tolerate clock skew between writer and reader.
+     */
+    async downloadPayload(options) {
+        const requestOptions = this.context.getRequestOptionsForCall();
+        const currentPeriod = periodIndex(Date.now() / 1000, this.periodLength);
+        try {
+            return await (0, feed_1.fetchLatestFeedUpdate)(requestOptions, this.owner, topicFor(this.baseTopic, currentPeriod), options);
+        }
+        catch (e) {
+            if (!(e instanceof error_1.BeeResponseError)) {
+                throw e;
+            }
+            return (0, feed_1.fetchLatestFeedUpdate)(requestOptions, this.owner, topicFor(this.baseTopic, currentPeriod - 1), options);
+        }
+    }
+    /**
+     * Same as `downloadPayload`, but for a reference to data uploaded elsewhere.
+     */
+    async downloadReference() {
+        const requestOptions = this.context.getRequestOptionsForCall();
+        const currentPeriod = periodIndex(Date.now() / 1000, this.periodLength);
+        try {
+            return await fetchLatestReference(requestOptions, this.owner, topicFor(this.baseTopic, currentPeriod));
+        }
+        catch (e) {
+            if (!(e instanceof error_1.BeeResponseError)) {
+                throw e;
+            }
+            return fetchLatestReference(requestOptions, this.owner, topicFor(this.baseTopic, currentPeriod - 1));
+        }
+    }
+}
+exports.RollingFeedReader = RollingFeedReader;