@graffiti-garden/implementation-local 0.5.1 → 0.6.1

package/src/database.ts

@@ -4,6 +4,8 @@ import type {
   GraffitiObjectUrl,
   JSONSchema,
   GraffitiSession,
+  GraffitiObjectStreamContinue,
+  GraffitiObjectStreamContinueEntry,
 } from "@graffiti-garden/api";
 import {
   GraffitiErrorNotFound,
@@ -16,11 +18,9 @@ import {
   applyGraffitiPatch,
   maskGraffitiObject,
   isActorAllowedGraffitiObject,
-  isObjectNewer,
   compileGraffitiObjectSchema,
-  unpackLocationOrUri,
+  unpackObjectUrl,
 } from "./utilities.js";
-import { Repeater } from "@repeaterjs/repeater";
 import type Ajv from "ajv";
 import type { applyPatch } from "fast-json-patch";
 
@@ -38,18 +38,18 @@ export interface GraffitiLocalOptions {
   pouchDBOptions?: PouchDB.Configuration.DatabaseConfiguration;
   /**
    * Includes the scheme and other information (possibly domain name)
-   * to prefix prefixes all URIs put in the system. Defaults to `graffiti:local`.
+   * to prefix prefixes all URLs put in the system. Defaults to `graffiti:local`.
    */
   origin?: string;
   /**
-   * Whether to allow putting objects at arbtirary URIs, i.e.
-   * URIs that are *not* prefixed with the origin or not generated
+   * Whether to allow putting objects at arbtirary URLs, i.e.
+   * URLs that are *not* prefixed with the origin or not generated
    * by the system. Defaults to `false`.
    *
    * Allows this implementation to be used as a client-side cache
    * for remote sources.
    */
-  allowSettingArbitraryUris?: boolean;
+  allowSettingArbitraryUrls?: boolean;
   /**
    * Whether to allow the user to set the lastModified field
    * when putting objects. Defaults to `false`.
@@ -58,12 +58,6 @@ export interface GraffitiLocalOptions {
    * for remote sources.
    */
   allowSettinngLastModified?: boolean;
-  /**
-   * The time in milliseconds to keep tombstones before deleting them.
-   * See the {@link https://api.graffiti.garden/classes/Graffiti.html#discover | `discover` }
-   * documentation for more information.
-   */
-  tombstoneRetention?: number;
   /**
    * An optional Ajv instance to use for schema validation.
    * If not provided, an internal instance will be created.
@@ -71,27 +65,21 @@ export interface GraffitiLocalOptions {
   ajv?: Ajv;
 }
 
-const DEFAULT_TOMBSTONE_RETENTION = 86400000; // 1 day in milliseconds
 const DEFAULT_ORIGIN = "graffiti:local:";
+const LAST_MODIFIED_BUFFER = 60000;
+
+type GraffitiObjectWithTombstone = GraffitiObjectBase & { tombstone: boolean };
 
 /**
  * An implementation of only the database operations of the
  * GraffitiAPI without synchronization or session management.
  */
 export class GraffitiLocalDatabase
-  implements
-    Pick<
-      Graffiti,
-      | "get"
-      | "put"
-      | "patch"
-      | "delete"
-      | "discover"
-      | "recoverOrphans"
-      | "channelStats"
-    >
+  implements Omit<Graffiti, "login" | "logout" | "sessionEvents">
 {
-  protected db_: Promise<PouchDB.Database<GraffitiObjectBase>> | undefined;
+  protected db_:
+    | Promise<PouchDB.Database<GraffitiObjectWithTombstone>>
+    | undefined;
   protected applyPatch_: Promise<typeof applyPatch> | undefined;
   protected ajv_: Promise<Ajv> | undefined;
   protected readonly options: GraffitiLocalOptions;
@@ -105,7 +93,7 @@ export class GraffitiLocalDatabase
           name: "graffitiDb",
           ...this.options.pouchDBOptions,
         };
-        const db = new PouchDB<GraffitiObjectBase>(
+        const db = new PouchDB<GraffitiObjectWithTombstone>(
           pouchDbOptions.name,
           pouchDbOptions,
         );
@@ -115,7 +103,7 @@ export class GraffitiLocalDatabase
             _id: "_design/indexes",
             views: {
               objectsPerChannelAndLastModified: {
-                map: function (object: GraffitiObjectBase) {
+                map: function (object: GraffitiObjectWithTombstone) {
                   const paddedLastModified = object.lastModified
                     .toString()
                     .padStart(15, "0");
@@ -128,7 +116,7 @@ export class GraffitiLocalDatabase
                 }.toString(),
               },
               orphansPerActorAndLastModified: {
-                map: function (object: GraffitiObjectBase) {
+                map: function (object: GraffitiObjectWithTombstone) {
                   if (object.channels.length === 0) {
                     const paddedLastModified = object.lastModified
                       .toString()
@@ -143,7 +131,7 @@ export class GraffitiLocalDatabase
                 }.toString(),
               },
               channelStatsPerActor: {
-                map: function (object: GraffitiObjectBase) {
+                map: function (object: GraffitiObjectWithTombstone) {
                   if (object.tombstone) return;
                   object.channels.forEach(function (channel) {
                     const id =
@@ -178,7 +166,7 @@ export class GraffitiLocalDatabase
     return this.db_;
   }
 
-  get applyPatch() {
+  protected get applyPatch() {
     if (!this.applyPatch_) {
       this.applyPatch_ = (async () => {
         const { applyPatch } = await import("fast-json-patch");
@@ -188,7 +176,7 @@ export class GraffitiLocalDatabase
     return this.applyPatch_;
   }
 
-  get ajv() {
+  protected get ajv() {
     if (!this.ajv_) {
       this.ajv_ = this.options.ajv
         ? Promise.resolve(this.options.ajv)
@@ -200,6 +188,20 @@ export class GraffitiLocalDatabase
     return this.ajv_;
   }
 
+  protected extractGraffitiObject(
+    object: GraffitiObjectWithTombstone,
+  ): GraffitiObjectBase {
+    const { value, channels, allowed, url, actor, lastModified } = object;
+    return {
+      value,
+      channels,
+      allowed,
+      url,
+      actor,
+      lastModified,
+    };
+  }
+
   constructor(options?: GraffitiLocalOptions) {
     this.options = options ?? {};
     this.origin = this.options.origin ?? DEFAULT_ORIGIN;
@@ -208,13 +210,13 @@ export class GraffitiLocalDatabase
     }
   }
 
-  protected async allDocsAtLocation(locationOrUri: GraffitiObjectUrl | string) {
-    const uri = unpackLocationOrUri(locationOrUri) + "/";
+  protected async allDocsAtLocation(objectUrl: string | GraffitiObjectUrl) {
+    const url = unpackObjectUrl(objectUrl) + "/";
     const results = await (
       await this.db
     ).allDocs({
-      startkey: uri,
-      endkey: uri + "\uffff", // \uffff is the last unicode character
+      startkey: url,
+      endkey: url + "\uffff", // \uffff is the last unicode character
       include_docs: true,
     });
     const docs = results.rows
@@ -222,7 +224,7 @@ export class GraffitiLocalDatabase
       // Remove undefined docs
       .reduce<
         PouchDB.Core.ExistingDocument<
-          GraffitiObjectBase & PouchDB.Core.AllDocsMeta
+          GraffitiObjectWithTombstone & PouchDB.Core.AllDocsMeta
         >[]
       >((acc, doc) => {
         if (doc) acc.push(doc);
@@ -231,14 +233,14 @@ export class GraffitiLocalDatabase
     return docs;
   }
 
-  protected docId(location: GraffitiObjectUrl) {
-    return location.url + "/" + randomBase64();
+  protected docId(objectUrl: GraffitiObjectUrl) {
+    return objectUrl.url + "/" + randomBase64();
   }
 
   get: Graffiti["get"] = async (...args) => {
-    const [locationOrUri, schema, session] = args;
+    const [urlObject, schema, session] = args;
 
-    const docsAll = await this.allDocsAtLocation(locationOrUri);
+    const docsAll = await this.allDocsAtLocation(urlObject);
 
     // Filter out ones not allowed
     const docs = docsAll.filter((doc) =>
@@ -250,10 +252,20 @@ export class GraffitiLocalDatabase
       );
 
     // Get the most recent document
-    const doc = docs.reduce((a, b) => (isObjectNewer(a, b) ? a : b));
+    const doc = docs.reduce((a, b) =>
+      a.lastModified > b.lastModified ||
+      (a.lastModified === b.lastModified && !a.tombstone && b.tombstone)
+        ? a
+        : b,
+    );
+
+    if (doc.tombstone) {
+      throw new GraffitiErrorNotFound(
+        "The object you are trying to get either does not exist or you are not allowed to see it",
+      );
+    }
 
-    // Strip out the _id and _rev
-    const { _id, _rev, _conflicts, _attachments, ...object } = doc;
+    const object = this.extractGraffitiObject(doc);
 
     // Mask out the allowed list and channels
     // if the user is not the owner
@@ -275,7 +287,7 @@ export class GraffitiLocalDatabase
    * spared.
    */
   protected async deleteAtLocation(
-    locationOrUri: GraffitiObjectUrl | string,
+    url: GraffitiObjectUrl | string,
     options: {
       keepLatest?: boolean;
       session?: GraffitiSession;
@@ -283,7 +295,7 @@ export class GraffitiLocalDatabase
       keepLatest: false,
     },
   ) {
-    const docsAtLocationAll = await this.allDocsAtLocation(locationOrUri);
+    const docsAtLocationAll = await this.allDocsAtLocation(url);
     const docsAtLocationAllowed = options.session
       ? docsAtLocationAll.filter((doc) =>
           isActorAllowedGraffitiObject(doc, options.session),
@@ -353,10 +365,8 @@ export class GraffitiLocalDatabase
         const { id } = resultOrError;
         const deletedDoc = docsToDelete.find((doc) => doc._id === id);
         if (deletedDoc) {
-          const { _id, _rev, _conflicts, _attachments, ...object } = deletedDoc;
           deletedObject = {
-            ...object,
-            tombstone: true,
+            ...this.extractGraffitiObject(deletedDoc),
             lastModified,
           };
           break;
@@ -368,8 +378,8 @@ export class GraffitiLocalDatabase
   }
 
   delete: Graffiti["delete"] = async (...args) => {
-    const [locationOrUri, session] = args;
-    const deletedObject = await this.deleteAtLocation(locationOrUri, {
+    const [url, session] = args;
+    const deletedObject = await this.deleteAtLocation(url, {
       session,
     });
     if (!deletedObject) {
@@ -392,7 +402,7 @@ export class GraffitiLocalDatabase
         oldObject = await this.get(objectPartial.url, {}, session);
       } catch (e) {
         if (e instanceof GraffitiErrorNotFound) {
-          if (!this.options.allowSettingArbitraryUris) {
+          if (!this.options.allowSettingArbitraryUrls) {
             throw new GraffitiErrorNotFound(
               "The object you are trying to replace does not exist or you are not allowed to see it",
             );
@@ -413,7 +423,7 @@ export class GraffitiLocalDatabase
         objectPartial.lastModified) ||
       new Date().getTime();
 
-    const object: GraffitiObjectBase = {
+    const object: GraffitiObjectWithTombstone = {
       value: objectPartial.value,
       channels: objectPartial.channels,
       allowed: objectPartial.allowed,
@@ -448,10 +458,10 @@ export class GraffitiLocalDatabase
   };
 
   patch: Graffiti["patch"] = async (...args) => {
-    const [patch, locationOrUri, session] = args;
+    const [patch, url, session] = args;
     let originalObject: GraffitiObjectBase;
     try {
-      originalObject = await this.get(locationOrUri, {}, session);
+      originalObject = await this.get(url, {}, session);
     } catch (e) {
       if (e instanceof GraffitiErrorNotFound) {
         throw new GraffitiErrorNotFound(
@@ -465,10 +475,6 @@ export class GraffitiLocalDatabase
       throw new GraffitiErrorForbidden(
         "The object you are trying to patch is owned by another actor",
       );
-    } else if (originalObject.tombstone) {
-      throw new GraffitiErrorNotFound(
-        "The object you are trying to patch has been deleted",
-      );
     }
 
     // Patch it outside of the database
@@ -512,6 +518,7 @@ export class GraffitiLocalDatabase
       await this.db
     ).put({
       ...patchObject,
+      tombstone: false,
       _id: this.docId(patchObject),
     });
 
@@ -522,12 +529,14 @@ export class GraffitiLocalDatabase
 
     return {
       ...originalObject,
-      tombstone: true,
       lastModified: patchObject.lastModified,
     };
   };
 
-  protected queryLastModifiedSuffixes(schema: JSONSchema) {
+  protected queryLastModifiedSuffixes(
+    schema: JSONSchema,
+    lastModified?: number,
+  ) {
     // Use the index for queries over ranges of lastModified
     let startKeySuffix = "";
     let endKeySuffix = "\uffff";
@@ -538,7 +547,10 @@ export class GraffitiLocalDatabase
     ) {
       const lastModifiedSchema = schema.properties.lastModified;
 
-      const minimum = lastModifiedSchema.minimum;
+      const minimum =
+        lastModified && lastModifiedSchema.minimum
+          ? Math.max(lastModified, lastModifiedSchema.minimum)
+          : (lastModified ?? lastModifiedSchema.minimum);
       const exclusiveMinimum = lastModifiedSchema.exclusiveMinimum;
 
       let intMinimum: number | undefined;
@@ -574,136 +586,245 @@ export class GraffitiLocalDatabase
     };
   }
 
-  discover: Graffiti["discover"] = (...args) => {
-    const [channels, schema, session] = args;
+  protected async *streamObjects<Schema extends JSONSchema>(
+    index: string,
+    startkey: string,
+    endkey: string,
+    validate: ReturnType<typeof compileGraffitiObjectSchema<Schema>>,
+    session: GraffitiSession | undefined | null,
+    ifModifiedSince: number | undefined,
+    channels?: string[],
+    processedIds?: Set<string>,
+  ): AsyncGenerator<GraffitiObjectStreamContinueEntry<Schema>> {
+    const showTombstones = ifModifiedSince !== undefined;
+
+    const result = await (
+      await this.db
+    ).query<GraffitiObjectWithTombstone>(index, {
+      startkey,
+      endkey,
+      include_docs: true,
+    });
 
-    const { startKeySuffix, endKeySuffix } =
-      this.queryLastModifiedSuffixes(schema);
+    for (const row of result.rows) {
+      const doc = row.doc;
+      if (!doc) continue;
 
-    const repeater: ReturnType<
-      typeof Graffiti.prototype.discover<typeof schema>
-    > = new Repeater(async (push, stop) => {
-      const validate = compileGraffitiObjectSchema(await this.ajv, schema);
+      if (processedIds?.has(doc._id)) continue;
+      processedIds?.add(doc._id);
 
-      const processedIds = new Set<string>();
+      if (!showTombstones && doc.tombstone) continue;
 
-      for (const channel of channels) {
-        const keyPrefix = encodeURIComponent(channel) + "/";
-        const startkey = keyPrefix + startKeySuffix;
-        const endkey = keyPrefix + endKeySuffix;
+      const object = this.extractGraffitiObject(doc);
 
-        const result = await (
-          await this.db
-        ).query<GraffitiObjectBase>(
-          "indexes/objectsPerChannelAndLastModified",
-          { startkey, endkey, include_docs: true },
-        );
+      if (channels) {
+        if (!isActorAllowedGraffitiObject(object, session)) continue;
+        maskGraffitiObject(object, channels, session);
+      }
 
-        for (const row of result.rows) {
-          const doc = row.doc;
-          if (!doc) continue;
+      if (!validate(object)) continue;
 
-          const { _id, _rev, ...object } = doc;
+      yield doc.tombstone
+        ? {
+            tombstone: true,
+            object: {
+              url: object.url,
+              lastModified: object.lastModified,
+            },
+          }
+        : { object };
+    }
+  }
 
-          // Don't double return the same object
-          // (which can happen if it's in multiple channels)
-          if (processedIds.has(_id)) continue;
-          processedIds.add(_id);
+  protected async *discoverMeta<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.discover<Schema>>,
+    ifModifiedSince?: number,
+  ): AsyncGenerator<
+    GraffitiObjectStreamContinueEntry<Schema>,
+    number | undefined
+  > {
+    const [channels, schema, session] = args;
+    const validate = compileGraffitiObjectSchema(await this.ajv, schema);
+    const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
+      schema,
+      ifModifiedSince,
+    );
 
-          // Make sure the user is allowed to see it
-          if (!isActorAllowedGraffitiObject(doc, session)) continue;
+    const processedIds = new Set<string>();
 
-          // Mask out the allowed list and channels
-          // if the user is not the owner
-          maskGraffitiObject(object, channels, session);
+    const startTime = new Date().getTime();
 
-          // Check that it matches the schema
-          if (validate(object)) {
-            await push({ value: object });
-          }
-        }
-      }
-      stop();
-      return {
-        tombstoneRetention:
-          this.options.tombstoneRetention ?? DEFAULT_TOMBSTONE_RETENTION,
-      };
-    });
+    for (const channel of channels) {
+      const keyPrefix = encodeURIComponent(channel) + "/";
+      const startkey = keyPrefix + startKeySuffix;
+      const endkey = keyPrefix + endKeySuffix;
 
-    return repeater;
-  };
+      const iterator = this.streamObjects<Schema>(
+        "indexes/objectsPerChannelAndLastModified",
+        startkey,
+        endkey,
+        validate,
+        session,
+        ifModifiedSince,
+        channels,
+        processedIds,
+      );
 
-  recoverOrphans: Graffiti["recoverOrphans"] = (schema, session) => {
-    const { startKeySuffix, endKeySuffix } =
-      this.queryLastModifiedSuffixes(schema);
+      for await (const result of iterator) yield result;
+    }
+
+    // Subtract a minute to make sure we don't miss any objects
+    return startTime - LAST_MODIFIED_BUFFER;
+  }
+
+  protected async *recoverOrphansMeta<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.recoverOrphans<Schema>>,
+    ifModifiedSince?: number,
+  ): AsyncGenerator<
+    GraffitiObjectStreamContinueEntry<Schema>,
+    number | undefined
+  > {
+    const [schema, session] = args;
+    const { startKeySuffix, endKeySuffix } = this.queryLastModifiedSuffixes(
+      schema,
+      ifModifiedSince,
+    );
     const keyPrefix = encodeURIComponent(session.actor) + "/";
     const startkey = keyPrefix + startKeySuffix;
     const endkey = keyPrefix + endKeySuffix;
 
-    const repeater: ReturnType<
-      typeof Graffiti.prototype.recoverOrphans<typeof schema>
-    > = new Repeater(async (push, stop) => {
-      const validate = compileGraffitiObjectSchema(await this.ajv, schema);
+    const validate = compileGraffitiObjectSchema(await this.ajv, schema);
 
-      const result = await (
-        await this.db
-      ).query<GraffitiObjectBase>("indexes/orphansPerActorAndLastModified", {
-        startkey,
-        endkey,
-        include_docs: true,
-      });
+    const startTime = new Date().getTime();
 
-      for (const row of result.rows) {
-        const doc = row.doc;
-        if (!doc) continue;
+    const iterator = this.streamObjects<Schema>(
+      "indexes/orphansPerActorAndLastModified",
+      startkey,
+      endkey,
+      validate,
+      session,
+      ifModifiedSince,
+    );
+
+    for await (const result of iterator) yield result;
 
-        // No masking/access necessary because
-        // the objects are all owned by the querier
+    return startTime - LAST_MODIFIED_BUFFER;
+  }
 
-        const { _id, _rev, ...object } = doc;
-        if (validate(object)) {
-          await push({ value: object });
+  protected async *discoverContinue<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.discover<Schema>>,
+    ifModifiedSince?: number,
+  ): GraffitiObjectStreamContinue<Schema> {
+    const iterator = this.discoverMeta(args, ifModifiedSince);
+
+    while (true) {
+      const result = await iterator.next();
+      if (result.done) {
+        const ifModifiedSince = result.value;
+        return {
+          continue: () => this.discoverContinue<Schema>(args, ifModifiedSince),
+          cursor: "",
+        };
+      }
+      yield result.value;
+    }
+  }
+
+  discover: Graffiti["discover"] = (...args) => {
+    const iterator = this.discoverMeta(args);
+
+    const this_ = this;
+    return (async function* () {
+      while (true) {
+        const result = await iterator.next();
+        if (result.done) {
+          return {
+            continue: () =>
+              this_.discoverContinue<(typeof args)[1]>(args, result.value),
+            cursor: "",
+          };
         }
+        // Make sure to filter out tombstones
+        if (result.value.tombstone) continue;
+        yield result.value;
       }
-      stop();
-      return {
-        tombstoneRetention:
-          this.options.tombstoneRetention ?? DEFAULT_TOMBSTONE_RETENTION,
-      };
-    });
+    })();
+  };
 
-    return repeater;
+  protected async *recoverContinue<Schema extends JSONSchema>(
+    args: Parameters<typeof Graffiti.prototype.recoverOrphans<Schema>>,
+    ifModifiedSince?: number,
+  ): GraffitiObjectStreamContinue<Schema> {
+    const iterator = this.recoverOrphansMeta(args, ifModifiedSince);
+
+    while (true) {
+      const result = await iterator.next();
+      if (result.done) {
+        const ifModifiedSince = result.value;
+        return {
+          continue: () => this.recoverContinue<Schema>(args, ifModifiedSince),
+          cursor: "",
+        };
+      }
+      yield result.value;
+    }
+  }
+
+  recoverOrphans: Graffiti["recoverOrphans"] = (...args) => {
+    const iterator = this.recoverOrphansMeta(args);
+
+    const this_ = this;
+    return (async function* () {
+      while (true) {
+        const result = await iterator.next();
+        if (result.done) {
+          return {
+            continue: () =>
+              this_.recoverContinue<(typeof args)[0]>(args, result.value),
+            cursor: "",
+          };
+        }
+        // Make sure to filter out tombstones
+        if (result.value.tombstone) continue;
+        yield result.value;
+      }
+    })();
   };
 
   channelStats: Graffiti["channelStats"] = (session) => {
-    const repeater: ReturnType<typeof Graffiti.prototype.channelStats> =
-      new Repeater(async (push, stop) => {
-        const keyPrefix = encodeURIComponent(session.actor) + "/";
-        const result = await (
-          await this.db
-        ).query("indexes/channelStatsPerActor", {
-          startkey: keyPrefix,
-          endkey: keyPrefix + "\uffff",
-          reduce: true,
-          group: true,
-        });
-        for (const row of result.rows) {
-          const channelEncoded = row.key.split("/")[1];
-          if (typeof channelEncoded !== "string") continue;
-          const { count, max: lastModified } = row.value;
-          if (typeof count !== "number" || typeof lastModified !== "number")
-            continue;
-          await push({
-            value: {
-              channel: decodeURIComponent(channelEncoded),
-              count,
-              lastModified,
-            },
-          });
-        }
-        stop();
+    const this_ = this;
+    return (async function* () {
+      const keyPrefix = encodeURIComponent(session.actor) + "/";
+      const result = await (
+        await this_.db
+      ).query("indexes/channelStatsPerActor", {
+        startkey: keyPrefix,
+        endkey: keyPrefix + "\uffff",
+        reduce: true,
+        group: true,
       });
+      for (const row of result.rows) {
+        const channelEncoded = row.key.split("/")[1];
+        if (typeof channelEncoded !== "string") continue;
+        const { count, max: lastModified } = row.value;
+        if (typeof count !== "number" || typeof lastModified !== "number")
+          continue;
+        yield {
+          value: {
+            channel: decodeURIComponent(channelEncoded),
+            count,
+            lastModified,
+          },
+        };
+      }
+    })();
+  };
 
-    return repeater;
+  continueObjectStream: Graffiti["continueObjectStream"] = (
+    cursor,
+    session,
+  ) => {
+    // TODO: Implement this
+    throw new GraffitiErrorNotFound("Cursor not found");
   };
 }