@graffiti-garden/implementation-local 0.6.0 → 0.6.1

package/src/database.ts

@@ -4,9 +4,8 @@ import type {
   GraffitiObjectUrl,
   JSONSchema,
   GraffitiSession,
-  GraffitiObject,
-  GraffitiStream,
-  ChannelStats,
+  GraffitiObjectStreamContinue,
+  GraffitiObjectStreamContinueEntry,
 } from "@graffiti-garden/api";
 import {
   GraffitiErrorNotFound,
@@ -22,7 +21,6 @@ import {
   compileGraffitiObjectSchema,
   unpackObjectUrl,
 } from "./utilities.js";
-import { Repeater } from "@repeaterjs/repeater";
 import type Ajv from "ajv";
 import type { applyPatch } from "fast-json-patch";
 
@@ -68,6 +66,7 @@ export interface GraffitiLocalOptions {
 }
 
 const DEFAULT_ORIGIN = "graffiti:local:";
+const LAST_MODIFIED_BUFFER = 60000;
 
 type GraffitiObjectWithTombstone = GraffitiObjectBase & { tombstone: boolean };
 
@@ -587,103 +586,105 @@ export class GraffitiLocalDatabase
     };
   }
 
-  protected discoverMeta<Schema extends JSONSchema>(
-    channels: string[],
-    schema: Schema,
-    session?: GraffitiSession | null,
-    ifModifiedSince?: number,
-  ): GraffitiStream<GraffitiObject<Schema>> {
-    const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
-      schema,
-      ifModifiedSince,
-    );
-
-    // Don't return tombstones on the first pass
+  protected async *streamObjects<Schema extends JSONSchema>(
+    index: string,
+    startkey: string,
+    endkey: string,
+    validate: ReturnType<typeof compileGraffitiObjectSchema<Schema>>,
+    session: GraffitiSession | undefined | null,
+    ifModifiedSince: number | undefined,
+    channels?: string[],
+    processedIds?: Set<string>,
+  ): AsyncGenerator<GraffitiObjectStreamContinueEntry<Schema>> {
     const showTombstones = ifModifiedSince !== undefined;
 
-    const repeater: GraffitiStream<GraffitiObject<typeof schema>> =
-      // @ts-ignore
-      new Repeater(async (push, stop) => {
-        const validate = compileGraffitiObjectSchema(await this.ajv, schema);
+    const result = await (
+      await this.db
+    ).query<GraffitiObjectWithTombstone>(index, {
+      startkey,
+      endkey,
+      include_docs: true,
+    });
 
-        const processedIds = new Set<string>();
+    for (const row of result.rows) {
+      const doc = row.doc;
+      if (!doc) continue;
 
-        for (const channel of channels) {
-          const keyPrefix = encodeURIComponent(channel) + "/";
-          const startkey = keyPrefix + startKeySuffix;
-          const endkey = keyPrefix + endKeySuffix;
+      if (processedIds?.has(doc._id)) continue;
+      processedIds?.add(doc._id);
 
-          const result = await (
-            await this.db
-          ).query<GraffitiObjectWithTombstone>(
-            "indexes/objectsPerChannelAndLastModified",
-            { startkey, endkey, include_docs: true },
-          );
+      if (!showTombstones && doc.tombstone) continue;
 
-          for (const row of result.rows) {
-            const doc = row.doc;
-            if (!doc) continue;
+      const object = this.extractGraffitiObject(doc);
 
-            if (!showTombstones && doc.tombstone) continue;
+      if (channels) {
+        if (!isActorAllowedGraffitiObject(object, session)) continue;
+        maskGraffitiObject(object, channels, session);
+      }
 
-            const object = this.extractGraffitiObject(doc);
+      if (!validate(object)) continue;
 
-            if (!ifModifiedSince || object.lastModified > ifModifiedSince) {
-              ifModifiedSince = object.lastModified;
-            }
+      yield doc.tombstone
+        ? {
+            tombstone: true,
+            object: {
+              url: object.url,
+              lastModified: object.lastModified,
+            },
+          }
+        : { object };
+    }
+  }
 
-            // Don't double return the same object
-            // (which can happen if it's in multiple channels)
-            if (processedIds.has(doc._id)) continue;
-            processedIds.add(doc._id);
+  protected async *discoverMeta<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.discover<Schema>>,
+    ifModifiedSince?: number,
+  ): AsyncGenerator<
+    GraffitiObjectStreamContinueEntry<Schema>,
+    number | undefined
+  > {
+    const [channels, schema, session] = args;
+    const validate = compileGraffitiObjectSchema(await this.ajv, schema);
+    const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
+      schema,
+      ifModifiedSince,
+    );
 
-            // Make sure the user is allowed to see it
-            if (!isActorAllowedGraffitiObject(doc, session)) continue;
+    const processedIds = new Set<string>();
 
-            // Mask out the allowed list and channels
-            // if the user is not the owner
-            maskGraffitiObject(object, channels, session);
+    const startTime = new Date().getTime();
 
-            // Check that it matches the schema
-            if (validate(object)) {
-              await push({
-                value: object,
-                ...(doc.tombstone ? { tombstone: true } : {}),
-              });
-            }
-          }
-        }
-        stop();
-
-        const cursor: string =
-          "discover:" +
-          JSON.stringify({
-            channels,
-            schema,
-            ifModifiedSince,
-            actor: session?.actor,
-          });
-        return {
-          cursor,
-          continue: () =>
-            this.continueStream(cursor, session) as GraffitiStream<
-              GraffitiObject<Schema>
-            >,
-        };
-      });
+    for (const channel of channels) {
+      const keyPrefix = encodeURIComponent(channel) + "/";
+      const startkey = keyPrefix + startKeySuffix;
+      const endkey = keyPrefix + endKeySuffix;
 
-    return repeater;
-  }
+      const iterator = this.streamObjects<Schema>(
+        "indexes/objectsPerChannelAndLastModified",
+        startkey,
+        endkey,
+        validate,
+        session,
+        ifModifiedSince,
+        channels,
+        processedIds,
+      );
 
-  discover: Graffiti["discover"] = (...args) => {
-    return this.discoverMeta<(typeof args)[1]>(...args);
-  };
+      for await (const result of iterator) yield result;
+    }
+
+    // Subtract a minute to make sure we don't miss any objects
+    return startTime - LAST_MODIFIED_BUFFER;
+  }
 
-  protected recoverOrphansMeta<Schema extends JSONSchema>(
-    schema: Schema,
-    session: GraffitiSession,
+  protected async *recoverOrphansMeta<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.recoverOrphans<Schema>>,
     ifModifiedSince?: number,
-  ): GraffitiStream<GraffitiObject<Schema>> {
+  ): AsyncGenerator<
+    GraffitiObjectStreamContinueEntry<Schema>,
+    number | undefined
+  > {
+    const [schema, session] = args;
     const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
       schema,
       ifModifiedSince,
@@ -692,141 +693,138 @@ export class GraffitiLocalDatabase
     const startkey = keyPrefix + startKeySuffix;
     const endkey = keyPrefix + endKeySuffix;
 
-    // Don't return tombstones on the first pass
-    const showTombstones = ifModifiedSince !== undefined;
+    const validate = compileGraffitiObjectSchema(await this.ajv, schema);
 
-    const repeater: GraffitiStream<GraffitiObject<Schema>> =
-      // @ts-ignore
-      new Repeater(async (push, stop) => {
-        const validate = compileGraffitiObjectSchema(await this.ajv, schema);
-
-        const result = await (
-          await this.db
-        ).query<GraffitiObjectWithTombstone>(
-          "indexes/orphansPerActorAndLastModified",
-          {
-            startkey,
-            endkey,
-            include_docs: true,
-          },
-        );
+    const startTime = new Date().getTime();
 
-        for (const row of result.rows) {
-          const doc = row.doc;
-          if (!doc) continue;
+    const iterator = this.streamObjects<Schema>(
+      "indexes/orphansPerActorAndLastModified",
+      startkey,
+      endkey,
+      validate,
+      session,
+      ifModifiedSince,
+    );
 
-          if (!showTombstones && doc.tombstone) continue;
+    for await (const result of iterator) yield result;
 
-          if (!ifModifiedSince || doc.lastModified > ifModifiedSince) {
-            ifModifiedSince = doc.lastModified;
-          }
+    return startTime - LAST_MODIFIED_BUFFER;
+  }
+
+  protected async *discoverContinue<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.discover<Schema>>,
+    ifModifiedSince?: number,
+  ): GraffitiObjectStreamContinue<Schema> {
+    const iterator = this.discoverMeta(args, ifModifiedSince);
 
-          // No masking/access necessary because
-          // the objects are all owned by the querier
+    while (true) {
+      const result = await iterator.next();
+      if (result.done) {
+        const ifModifiedSince = result.value;
+        return {
+          continue: () => this.discoverContinue<Schema>(args, ifModifiedSince),
+          cursor: "",
+        };
+      }
+      yield result.value;
+    }
+  }
 
-          const object = this.extractGraffitiObject(doc);
-          if (validate(object)) {
-            await push({
-              value: object,
-              ...(doc.tombstone ? { tombstone: true } : {}),
-            });
-          }
+  discover: Graffiti["discover"] = (...args) => {
+    const iterator = this.discoverMeta(args);
+
+    const this_ = this;
+    return (async function* () {
+      while (true) {
+        const result = await iterator.next();
+        if (result.done) {
+          return {
+            continue: () =>
+              this_.discoverContinue<(typeof args)[1]>(args, result.value),
+            cursor: "",
+          };
         }
-        stop();
-        const cursor: string =
-          "recover-orphans:" +
-          JSON.stringify({
-            schema,
-            actor: session.actor,
-            ifModifiedSince,
-          });
+        // Make sure to filter out tombstones
+        if (result.value.tombstone) continue;
+        yield result.value;
+      }
+    })();
+  };
+
+  protected async *recoverContinue<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.recoverOrphans<Schema>>,
+    ifModifiedSince?: number,
+  ): GraffitiObjectStreamContinue<Schema> {
+    const iterator = this.recoverOrphansMeta(args, ifModifiedSince);
+
+    while (true) {
+      const result = await iterator.next();
+      if (result.done) {
+        const ifModifiedSince = result.value;
         return {
-          cursor,
-          continue: () =>
-            this.continueStream(cursor, session) as GraffitiStream<
-              GraffitiObject<Schema>
-            >,
+          continue: () => this.recoverContinue<Schema>(args, ifModifiedSince),
+          cursor: "",
         };
-      });
-
-    return repeater;
+      }
+      yield result.value;
+    }
   }
 
   recoverOrphans: Graffiti["recoverOrphans"] = (...args) => {
-    return this.recoverOrphansMeta<(typeof args)[0]>(...args);
+    const iterator = this.recoverOrphansMeta(args);
+
+    const this_ = this;
+    return (async function* () {
+      while (true) {
+        const result = await iterator.next();
+        if (result.done) {
+          return {
+            continue: () =>
+              this_.recoverContinue<(typeof args)[0]>(args, result.value),
+            cursor: "",
+          };
+        }
+        // Make sure to filter out tombstones
+        if (result.value.tombstone) continue;
+        yield result.value;
+      }
+    })();
   };
 
   channelStats: Graffiti["channelStats"] = (session) => {
-    const repeater: ReturnType<typeof Graffiti.prototype.channelStats> =
-      new Repeater(async (push, stop) => {
-        const keyPrefix = encodeURIComponent(session.actor) + "/";
-        const result = await (
-          await this.db
-        ).query("indexes/channelStatsPerActor", {
-          startkey: keyPrefix,
-          endkey: keyPrefix + "\uffff",
-          reduce: true,
-          group: true,
-        });
-        for (const row of result.rows) {
-          const channelEncoded = row.key.split("/")[1];
-          if (typeof channelEncoded !== "string") continue;
-          const { count, max: lastModified } = row.value;
-          if (typeof count !== "number" || typeof lastModified !== "number")
-            continue;
-          await push({
-            value: {
-              channel: decodeURIComponent(channelEncoded),
-              count,
-              lastModified,
-            },
-          });
-        }
-        stop();
-        const cursor = "channel-stats";
-        return {
-          cursor,
-          continue: () =>
-            this.continueStream(
-              cursor,
-              session,
-            ) as GraffitiStream<ChannelStats>,
-        };
+    const this_ = this;
+    return (async function* () {
+      const keyPrefix = encodeURIComponent(session.actor) + "/";
+      const result = await (
+        await this_.db
+      ).query("indexes/channelStatsPerActor", {
+        startkey: keyPrefix,
+        endkey: keyPrefix + "\uffff",
+        reduce: true,
+        group: true,
       });
-
-    return repeater;
+      for (const row of result.rows) {
+        const channelEncoded = row.key.split("/")[1];
+        if (typeof channelEncoded !== "string") continue;
+        const { count, max: lastModified } = row.value;
+        if (typeof count !== "number" || typeof lastModified !== "number")
+          continue;
+        yield {
+          value: {
+            channel: decodeURIComponent(channelEncoded),
+            count,
+            lastModified,
+          },
+        };
+      }
+    })();
   };
 
-  continueStream: Graffiti["continueStream"] = (cursor, session) => {
-    if (cursor === "channel-stats") {
-      if (!session) {
-        throw new GraffitiErrorForbidden(
-          "You must be logged in to continue the stream",
-        );
-      }
-      return this.channelStats(session);
-    } else if (cursor.startsWith("recover-orphans:")) {
-      const { schema, actor, ifModifiedSince } = JSON.parse(
-        cursor.slice("recover-orphans:".length),
-      );
-      if (!session || session.actor !== actor) {
-        throw new GraffitiErrorForbidden(
-          "You must be logged in as the actor same actor who started the stream",
-        );
-      }
-      return this.recoverOrphansMeta(schema, session, ifModifiedSince);
-    } else if (cursor.startsWith("discover:")) {
-      const { channels, schema, actor, ifModifiedSince } = JSON.parse(
-        cursor.slice("discover:".length),
-      );
-      if (session?.actor !== actor) {
-        throw new GraffitiErrorForbidden(
-          "You must be logged in as the actor same actor who started the stream",
-        );
-      }
-      return this.discoverMeta(channels, schema, session, ifModifiedSince);
-    } else {
-      throw new GraffitiErrorNotFound("Cursor not found");
-    }
+  continueObjectStream: Graffiti["continueObjectStream"] = (
+    cursor,
+    session,
+  ) => {
+    // TODO: Implement this
+    throw new GraffitiErrorNotFound("Cursor not found");
   };
 }

package/src/index.ts

@@ -27,7 +27,7 @@ export class GraffitiLocal extends Graffiti {
   discover: Graffiti["discover"];
   recoverOrphans: Graffiti["recoverOrphans"];
   channelStats: Graffiti["channelStats"];
-  continueStream: Graffiti["continueStream"];
+  continueObjectStream: Graffiti["continueObjectStream"];
 
   constructor(options?: GraffitiLocalOptions) {
     super();
@@ -43,7 +43,20 @@ export class GraffitiLocal extends Graffiti {
       graffitiPouchDbBase.recoverOrphans.bind(graffitiPouchDbBase);
     this.channelStats =
       graffitiPouchDbBase.channelStats.bind(graffitiPouchDbBase);
-    this.continueStream =
-      graffitiPouchDbBase.continueStream.bind(graffitiPouchDbBase);
+    this.continueObjectStream =
+      graffitiPouchDbBase.continueObjectStream.bind(graffitiPouchDbBase);
   }
 }
+
+function myFunction<T extends boolean>(
+  flag: T,
+): T extends true ? string : number {
+  if (!flag) {
+    return "Hello" as T extends true ? string : number;
+  } else {
+    return 42 as T extends true ? string : number;
+  }
+}
+// Usage
+const a = myFunction(false); // Type is number
+const b = myFunction(true); // Type is string