@graffiti-garden/implementation-local 0.6.2 → 0.6.4

package/src/database.ts

@@ -63,9 +63,17 @@ export interface GraffitiLocalOptions {
    * If not provided, an internal instance will be created.
    */
   ajv?: Ajv;
+  /**
+   * Wait at least this long (in milliseconds) before continuing a stream.
+   * A basic form of rate limiting. Defaults to 1 seconds.
+   */
+  continueBuffer?: number;
 }
 
 const DEFAULT_ORIGIN = "graffiti:local:";
+
+// During stream continuations, return objects
+// that have possibly already been seen over this window
 const LAST_MODIFIED_BUFFER = 60000;
 
 type GraffitiObjectWithTombstone = GraffitiObjectBase & { tombstone: boolean };
@@ -240,6 +248,9 @@ export class GraffitiLocalDatabase
   get: Graffiti["get"] = async (...args) => {
     const [urlObject, schema, session] = args;
 
+    // TODO: Rate limit getting the same object
+    // over and over
+
     const docsAll = await this.allDocsAtLocation(urlObject);
 
     // Filter out ones not allowed
@@ -596,7 +607,10 @@ export class GraffitiLocalDatabase
     channels?: string[],
     processedIds?: Set<string>,
   ): AsyncGenerator<GraffitiObjectStreamContinueEntry<Schema>> {
-    const showTombstones = ifModifiedSince !== undefined;
+    if (ifModifiedSince !== undefined) {
+      // Subtract a minute to make sure we don't miss any objects
+      ifModifiedSince -= LAST_MODIFIED_BUFFER;
+    }
 
     const result = await (
       await this.db
@@ -613,7 +627,8 @@ export class GraffitiLocalDatabase
       if (processedIds?.has(doc._id)) continue;
       processedIds?.add(doc._id);
 
-      if (!showTombstones && doc.tombstone) continue;
+      // If this is not a continuation, skip tombstones
+      if (ifModifiedSince === undefined && doc.tombstone) continue;
 
       const object = this.extractGraffitiObject(doc);
 
@@ -636,6 +651,19 @@ export class GraffitiLocalDatabase
     }
   }
 
+  protected async waitToContinue(ifModifiedSince: number | undefined) {
+    if (ifModifiedSince === undefined) return;
+    const continueBuffer = this.options.continueBuffer ?? 1000;
+    const timeElapsedSinceContinue = Date.now() - ifModifiedSince;
+    if (timeElapsedSinceContinue < continueBuffer) {
+      // Continue was called too soon,
+      // wait a bit before continuing
+      await new Promise((resolve) =>
+        setTimeout(resolve, continueBuffer - timeElapsedSinceContinue),
+      );
+    }
+  }
+
   protected async *discoverMeta<Schema extends JSONSchema>(
     args: Parameters<typeof Graffiti.prototype.discover<Schema>>,
     ifModifiedSince?: number,
@@ -643,6 +671,8 @@ export class GraffitiLocalDatabase
     GraffitiObjectStreamContinueEntry<Schema>,
     number | undefined
   > {
+    await this.waitToContinue(ifModifiedSince);
+
     const [channels, schema, session] = args;
     const validate = compileGraffitiObjectSchema(await this.ajv, schema);
     const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
@@ -673,8 +703,7 @@ export class GraffitiLocalDatabase
       for await (const result of iterator) yield result;
     }
 
-    // Subtract a minute to make sure we don't miss any objects
-    return startTime - LAST_MODIFIED_BUFFER;
+    return startTime;
   }
 
   protected async *recoverOrphansMeta<Schema extends JSONSchema>(
@@ -684,6 +713,8 @@ export class GraffitiLocalDatabase
     GraffitiObjectStreamContinueEntry<Schema>,
     number | undefined
   > {
+    await this.waitToContinue(ifModifiedSince);
+
     const [schema, session] = args;
     const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
       schema,
@@ -708,7 +739,7 @@ export class GraffitiLocalDatabase
 
     for await (const result of iterator) yield result;
 
-    return startTime - LAST_MODIFIED_BUFFER;
+    return startTime;
   }
 
   protected discoverCursor(