@graffiti-garden/implementation-decentralized 0.0.3 → 0.0.5

package/dist/esm/3-protocol/4-graffiti.js

@@ -105,6 +105,7 @@ const MESSAGE_LABEL_UNLABELED = 0;
 const MESSAGE_LABEL_VALID = 1;
 const MESSAGE_LABEL_TRASH = 2;
 const MESSAGE_LABEL_INVALID = 3;
+const CONCURRENCY = 16;
 class GraffitiDecentralized {
   dids = new DecentralizedIdentifiers();
   authorization = new Authorization();
@@ -547,82 +548,90 @@ class GraffitiDecentralized {
     );
     let indexedIteratorNexts = iterators.map(async (it, index) => indexedSingleEndpointQueryNext(it, index));
     let active = indexedIteratorNexts.length;
-    while (active > 0) {
-      const next = await Promise.race(indexedIteratorNexts);
-      if (next.error !== void 0) {
-        indexedIteratorNexts[next.index] = new Promise(() => {
-        });
-        active--;
-        yield {
-          error: next.error,
-          origin: allInboxes[next.index].serviceEndpoint
-        };
-      } else if (next.result.done) {
-        const inbox = allInboxes[next.index];
-        cursors[inbox.serviceEndpoint] = next.result.value;
-        indexedIteratorNexts[next.index] = new Promise(() => {
-        });
-        active--;
-      } else {
-        indexedIteratorNexts[next.index] = indexedSingleEndpointQueryNext(
-          iterators[next.index],
-          next.index
-        );
-        const { object, tombstone, tags: receivedTags } = next.result.value;
-        if (tombstone) {
-          if (tombstones.get(object.url) === true) continue;
-          tombstones.set(object.url, true);
+    try {
+      while (active > 0) {
+        const next = await Promise.race(indexedIteratorNexts);
+        if (next.error !== void 0) {
+          indexedIteratorNexts[next.index] = new Promise(() => {
+          });
+          active--;
           yield {
-            tombstone,
-            object: { url: object.url }
+            error: next.error,
+            origin: allInboxes[next.index].serviceEndpoint
           };
+        } else if (next.result.done) {
+          const inbox = allInboxes[next.index];
+          cursors[inbox.serviceEndpoint] = next.result.value;
+          indexedIteratorNexts[next.index] = new Promise(() => {
+          });
+          active--;
         } else {
-          if (tombstones.get(object.url) === false) continue;
-          const matchedTagIndices = tags.reduce(
-            (acc, tag, tagIndex) => {
-              for (const receivedTag of receivedTags) {
-                if (tag.length === receivedTag.length && tag.every((b, i) => receivedTag[i] === b)) {
-                  acc.push(tagIndex);
-                  break;
-                }
-              }
-              return acc;
-            },
-            []
+          indexedIteratorNexts[next.index] = indexedSingleEndpointQueryNext(
+            iterators[next.index],
+            next.index
           );
-          const matchedChannels = matchedTagIndices.map(
-            (index) => channels[index]
-          );
-          if (matchedChannels.length === 0) {
+          const { object, tombstone, tags: receivedTags } = next.result.value;
+          if (tombstone) {
+            if (tombstones.get(object.url) === true) continue;
+            tombstones.set(object.url, true);
             yield {
-              error: new Error(
-                "Inbox returned object without matching channels"
-              ),
-              origin: allInboxes[next.index].serviceEndpoint
+              tombstone,
+              object: { url: object.url }
             };
-          }
-          tombstones.set(object.url, false);
-          yield {
-            object: {
-              ...object,
-              channels: matchedChannels
+          } else {
+            if (tombstones.get(object.url) === false) continue;
+            const matchedTagIndices = tags.reduce(
+              (acc, tag, tagIndex) => {
+                for (const receivedTag of receivedTags) {
+                  if (tag.length === receivedTag.length && tag.every((b, i) => receivedTag[i] === b)) {
+                    acc.push(tagIndex);
+                    break;
+                  }
+                }
+                return acc;
+              },
+              []
+            );
+            const matchedChannels = matchedTagIndices.map(
+              (index) => channels[index]
+            );
+            if (matchedChannels.length === 0) {
+              yield {
+                error: new Error(
+                  "Inbox returned object without matching channels"
+                ),
+                origin: allInboxes[next.index].serviceEndpoint
+              };
             }
-          };
+            tombstones.set(object.url, false);
+            yield {
+              object: {
+                ...object,
+                channels: matchedChannels
+              }
+            };
+          }
         }
       }
+    } finally {
+      await Promise.all(
+        iterators.map(async (it) => {
+          await it.return("");
+        })
+      );
     }
     return {
       cursor: JSON.stringify({
         channels,
         cursors
-      }),
-      continue: (session2) => this.discoverMeta(channels, schema, cursors, session2)
+      })
     };
   }
   discover = (...args) => {
     const [channels, schema, session] = args;
     return this.discoverMeta(channels, schema, {}, session);
   };
+  // @ts-ignore
   continueDiscover = (...args) => {
     const [cursor, session] = args;
     let channels;
@@ -769,62 +778,164 @@ class GraffitiDecentralized {
       queryArguments.cursor,
       inboxToken
     );
-    while (true) {
-      const itResult = await iterator.next();
-      if (itResult.done) return itResult.value;
-      const result = itResult.value;
-      const label = result.l;
-      if (label !== MESSAGE_LABEL_VALID && label !== MESSAGE_LABEL_UNLABELED && label !== MESSAGE_LABEL_TRASH)
-        continue;
-      const messageId = result.id;
-      const { o: object, m: metadataBytes, t: receivedTags } = result.m;
-      let metadata;
-      try {
-        const metadataRaw = dagCborDecode(metadataBytes);
-        metadata = MessageMetadataSchema.parse(metadataRaw);
-      } catch (e) {
+    const inFlight = [];
+    let doneValue = null;
+    try {
+      while (true) {
+        while (doneValue === null && inFlight.length < CONCURRENCY) {
+          const itResult = await iterator.next();
+          if (itResult.done) {
+            doneValue = itResult.value;
+            break;
+          }
+          const processPromise = this.processOneLabeledMessage(
+            inboxEndpoint,
+            itResult.value,
+            inboxToken,
+            recipient
+          ).catch((e) => {
+            throw e;
+          });
+          inFlight.push(processPromise);
+        }
+        const nextProcessedPromise = inFlight.shift();
+        if (!nextProcessedPromise) {
+          if (doneValue !== null) return doneValue;
+          throw new Error("Process queue empty but no return value");
+        }
+        const processed = await nextProcessedPromise;
+        if (processed) yield processed;
+      }
+    } finally {
+      await iterator.return("");
+    }
+  }
+  async processOneLabeledMessage(inboxEndpoint, result, inboxToken, recipient) {
+    const label = result.l;
+    if (label !== MESSAGE_LABEL_VALID && label !== MESSAGE_LABEL_UNLABELED && label !== MESSAGE_LABEL_TRASH)
+      return;
+    const messageId = result.id;
+    const { o: object, m: metadataBytes, t: receivedTags } = result.m;
+    let metadata;
+    try {
+      const metadataRaw = dagCborDecode(metadataBytes);
+      metadata = MessageMetadataSchema.parse(metadataRaw);
+    } catch (e) {
+      this.inboxes.label(
+        inboxEndpoint,
+        messageId,
+        MESSAGE_LABEL_INVALID,
+        inboxToken
+      );
+      return;
+    }
+    const {
+      [MESSAGE_DATA_STORAGE_BUCKET_KEY]: storageBucketKey,
+      [MESSAGE_DATA_TOMBSTONED_MESSAGE_ID_KEY]: tombstonedMessageId
+    } = metadata;
+    const allowedTickets = MESSAGE_DATA_ALLOWED_TICKETS_KEY in metadata ? metadata[MESSAGE_DATA_ALLOWED_TICKETS_KEY] : void 0;
+    const announcements = MESSAGE_DATA_ANNOUNCEMENTS_KEY in metadata ? metadata[MESSAGE_DATA_ANNOUNCEMENTS_KEY] : void 0;
+    if (label === MESSAGE_LABEL_VALID) {
+      return {
+        messageId,
+        object,
+        storageBucketKey,
+        allowedTickets,
+        tags: receivedTags,
+        announcements
+      };
+    } else if (label === MESSAGE_LABEL_TRASH) {
+      if (!tombstonedMessageId) return;
+      const past = await this.inboxes.get(
+        inboxEndpoint,
+        tombstonedMessageId,
+        inboxToken
+      );
+      if (!past || past[LABELED_MESSAGE_MESSAGE_KEY][MESSAGE_OBJECT_KEY].url !== object.url)
+        return;
+      if (past[LABELED_MESSAGE_LABEL_KEY] !== MESSAGE_LABEL_TRASH) {
         this.inboxes.label(
           inboxEndpoint,
-          messageId,
-          MESSAGE_LABEL_INVALID,
+          tombstonedMessageId,
+          MESSAGE_LABEL_TRASH,
           inboxToken
         );
-        continue;
       }
-      const {
-        [MESSAGE_DATA_STORAGE_BUCKET_KEY]: storageBucketKey,
-        [MESSAGE_DATA_TOMBSTONED_MESSAGE_ID_KEY]: tombstonedMessageId
-      } = metadata;
-      const allowedTickets = MESSAGE_DATA_ALLOWED_TICKETS_KEY in metadata ? metadata[MESSAGE_DATA_ALLOWED_TICKETS_KEY] : void 0;
-      const announcements = MESSAGE_DATA_ANNOUNCEMENTS_KEY in metadata ? metadata[MESSAGE_DATA_ANNOUNCEMENTS_KEY] : void 0;
-      if (label === MESSAGE_LABEL_VALID) {
-        yield {
-          messageId,
-          object,
-          storageBucketKey,
-          allowedTickets,
-          tags: receivedTags,
-          announcements
-        };
-        continue;
-      } else if (label === MESSAGE_LABEL_TRASH) {
-        if (!tombstonedMessageId) continue;
-        const past = await this.inboxes.get(
-          inboxEndpoint,
-          tombstonedMessageId,
-          inboxToken
+      return {
+        messageId,
+        tombstone: true,
+        object,
+        storageBucketKey,
+        allowedTickets,
+        tags: receivedTags,
+        announcements
+      };
+    }
+    let validationError = void 0;
+    try {
+      const actor = object.actor;
+      const actorDocument = await this.dids.resolve(actor);
+      const storageBucketService = actorDocument?.service?.find(
+        (service) => service.id === DID_SERVICE_ID_GRAFFITI_STORAGE_BUCKET && service.type === DID_SERVICE_TYPE_GRAFFITI_STORAGE_BUCKET
+      );
+      if (!storageBucketService) {
+        throw new GraffitiErrorNotFound(
+          `Actor ${actor} has no storage bucket service`
         );
-        if (!past || past[LABELED_MESSAGE_MESSAGE_KEY][MESSAGE_OBJECT_KEY].url !== object.url)
-          continue;
-        if (past[LABELED_MESSAGE_LABEL_KEY] !== MESSAGE_LABEL_TRASH) {
+      }
+      if (typeof storageBucketService.serviceEndpoint !== "string") {
+        throw new GraffitiErrorNotFound(
+          `Actor ${actor} does not have a valid storage bucket endpoint`
+        );
+      }
+      const storageBucketEndpoint = storageBucketService.serviceEndpoint;
+      const objectBytes = await this.storageBuckets.get(
+        storageBucketEndpoint,
+        storageBucketKey,
+        MAX_OBJECT_SIZE_BYTES
+      );
+      if (MESSAGE_DATA_ALLOWED_TICKET_KEY in metadata && !recipient) {
+        throw new GraffitiErrorForbidden(
+          `Recipient is required when allowed ticket is present`
+        );
+      }
+      const privateObjectInfo = allowedTickets ? { allowedTickets } : MESSAGE_DATA_ALLOWED_TICKET_KEY in metadata ? {
+        recipient: recipient ?? "null",
+        allowedTicket: metadata[MESSAGE_DATA_ALLOWED_TICKET_KEY],
+        allowedIndex: metadata[MESSAGE_DATA_ALLOWED_TICKET_INDEX_KEY]
+      } : void 0;
+      await this.objectEncoding.validate(
+        object,
+        receivedTags,
+        objectBytes,
+        privateObjectInfo
+      );
+    } catch (e) {
+      validationError = e;
+    }
+    if (tombstonedMessageId) {
+      if (validationError instanceof GraffitiErrorNotFound) {
+        this.inboxes.get(inboxEndpoint, tombstonedMessageId, inboxToken).then((result2) => {
+          if (
+            // Make sure that it actually references the object being deleted
+            result2 && result2[LABELED_MESSAGE_MESSAGE_KEY][MESSAGE_OBJECT_KEY].url === object.url && // And that the object is not already marked as trash
+            result2[LABELED_MESSAGE_LABEL_KEY] !== MESSAGE_LABEL_TRASH
+          ) {
+            this.inboxes.label(
+              inboxEndpoint,
+              tombstonedMessageId,
+              MESSAGE_LABEL_TRASH,
+              inboxToken
+            );
+          }
           this.inboxes.label(
             inboxEndpoint,
-            tombstonedMessageId,
+            messageId,
             MESSAGE_LABEL_TRASH,
             inboxToken
           );
-        }
-        yield {
+        });
+        return {
           messageId,
           tombstone: true,
           object,
@@ -833,117 +944,48 @@ class GraffitiDecentralized {
           tags: receivedTags,
           announcements
         };
-        continue;
-      }
-      let validationError = void 0;
-      try {
-        const actor = object.actor;
-        const actorDocument = await this.dids.resolve(actor);
-        const storageBucketService = actorDocument?.service?.find(
-          (service) => service.id === DID_SERVICE_ID_GRAFFITI_STORAGE_BUCKET && service.type === DID_SERVICE_TYPE_GRAFFITI_STORAGE_BUCKET
+      } else {
+        console.error("Recieved an incorrect tombstone object");
+        console.error(validationError);
+        this.inboxes.label(
+          inboxEndpoint,
+          messageId,
+          MESSAGE_LABEL_INVALID,
+          inboxToken
         );
-        if (!storageBucketService) {
-          throw new GraffitiErrorNotFound(
-            `Actor ${actor} has no storage bucket service`
-          );
-        }
-        if (typeof storageBucketService.serviceEndpoint !== "string") {
-          throw new GraffitiErrorNotFound(
-            `Actor ${actor} does not have a valid storage bucket endpoint`
-          );
-        }
-        const storageBucketEndpoint = storageBucketService.serviceEndpoint;
-        const objectBytes = await this.storageBuckets.get(
-          storageBucketEndpoint,
-          storageBucketKey,
-          MAX_OBJECT_SIZE_BYTES
+      }
+    } else {
+      if (validationError === void 0) {
+        this.inboxes.label(
+          inboxEndpoint,
+          messageId,
+          MESSAGE_LABEL_VALID,
+          inboxToken
         );
-        if (MESSAGE_DATA_ALLOWED_TICKET_KEY in metadata && !recipient) {
-          throw new GraffitiErrorForbidden(
-            `Recipient is required when allowed ticket is present`
-          );
-        }
-        const privateObjectInfo = allowedTickets ? { allowedTickets } : MESSAGE_DATA_ALLOWED_TICKET_KEY in metadata ? {
-          recipient: recipient ?? "null",
-          allowedTicket: metadata[MESSAGE_DATA_ALLOWED_TICKET_KEY],
-          allowedIndex: metadata[MESSAGE_DATA_ALLOWED_TICKET_INDEX_KEY]
-        } : void 0;
-        await this.objectEncoding.validate(
+        return {
+          messageId,
           object,
-          receivedTags,
-          objectBytes,
-          privateObjectInfo
+          storageBucketKey,
+          tags: receivedTags,
+          allowedTickets,
+          announcements
+        };
+      } else if (validationError instanceof GraffitiErrorNotFound) {
+        this.inboxes.label(
+          inboxEndpoint,
+          messageId,
+          MESSAGE_LABEL_TRASH,
+          inboxToken
         );
-      } catch (e) {
-        validationError = e;
-      }
-      if (tombstonedMessageId) {
-        if (validationError instanceof GraffitiErrorNotFound) {
-          this.inboxes.get(inboxEndpoint, tombstonedMessageId, inboxToken).then((result2) => {
-            if (
-              // Make sure that it actually references the object being deleted
-              result2 && result2[LABELED_MESSAGE_MESSAGE_KEY][MESSAGE_OBJECT_KEY].url === object.url && // And that the object is not already marked as trash
-              result2[LABELED_MESSAGE_LABEL_KEY] !== MESSAGE_LABEL_TRASH
-            ) {
-              this.inboxes.label(
-                inboxEndpoint,
-                tombstonedMessageId,
-                MESSAGE_LABEL_TRASH,
-                inboxToken
-              );
-            }
-            this.inboxes.label(
-              inboxEndpoint,
-              messageId,
-              MESSAGE_LABEL_TRASH,
-              inboxToken
-            );
-          });
-          yield {
-            messageId,
-            tombstone: true,
-            object,
-            storageBucketKey,
-            allowedTickets,
-            tags: receivedTags,
-            announcements
-          };
-        } else {
-          console.error("Recieved an incorrect object");
-          console.error(validationError);
-          this.inboxes.label(
-            inboxEndpoint,
-            messageId,
-            MESSAGE_LABEL_INVALID,
-            inboxToken
-          );
-        }
       } else {
-        if (validationError === void 0) {
-          this.inboxes.label(
-            inboxEndpoint,
-            messageId,
-            MESSAGE_LABEL_VALID,
-            inboxToken
-          );
-          yield {
-            messageId,
-            object,
-            storageBucketKey,
-            tags: receivedTags,
-            allowedTickets,
-            announcements
-          };
-        } else {
-          console.error("Recieved an incorrect object");
-          console.error(validationError);
-          this.inboxes.label(
-            inboxEndpoint,
-            messageId,
-            MESSAGE_LABEL_INVALID,
-            inboxToken
-          );
-        }
+        console.error("Recieved an incorrect object");
+        console.error(validationError);
+        this.inboxes.label(
+          inboxEndpoint,
+          messageId,
+          MESSAGE_LABEL_INVALID,
+          inboxToken
+        );
       }
     }
   }