@warp-drive-mirror/experiments 0.2.6-beta.0 → 0.2.6-beta.2

package/dist/unpkg/prod/data-worker.js

@@ -0,0 +1,366 @@
+import { DocumentStorage } from './document-storage.js';
+import { assertPrivateStore } from '@warp-drive-mirror/core/store/-private';
+import { SkipCache } from '@warp-drive-mirror/core/types/request';
+
+const WorkerScope = globalThis.SharedWorkerGlobalScope;
+class DataWorker {
+  constructor(UserStore, options) {
+    // disable if running on main thread
+    if (typeof window !== 'undefined') {
+      return;
+    }
+    this.store = new UserStore();
+    this.threads = new Map();
+    this.pending = new Map();
+    this.options = Object.assign({
+      persisted: false,
+      scope: ''
+    }, options);
+    this.isSharedWorker = WorkerScope && globalThis instanceof WorkerScope;
+    this.initialize();
+  }
+  initialize() {
+    // enable the CacheHandler to access the worker
+    this.store._worker = this;
+    if (this.options.persisted) {
+      // will be accessed by the worker's CacheHandler off of store
+      this.storage = new DocumentStorage({
+        scope: this.options.scope
+      });
+    }
+    if (this.isSharedWorker) {
+      globalThis.onconnect = e => {
+        const port = e.ports[0];
+        port.onmessage = event => {
+          const {
+            type
+          } = event.data;
+          switch (type) {
+            case 'connect':
+              this.setupThread(event.data.thread, port);
+              break;
+          }
+        };
+        port.start();
+      };
+    } else {
+      globalThis.onmessage = event => {
+        const {
+          type
+        } = event.data;
+        switch (type) {
+          case 'connect':
+            this.setupThread(event.data.thread, event.ports[0]);
+            break;
+        }
+      };
+    }
+  }
+  setupThread(thread, port) {
+    this.threads.set(thread, port);
+    this.pending.set(thread, new Map());
+    port.onmessage = event => {
+      if (event.type === 'close') {
+        this.threads.delete(thread);
+        return;
+      }
+      const {
+        type
+      } = event.data;
+      switch (type) {
+        case 'abort':
+          this.abortRequest(event.data);
+          break;
+        case 'request':
+          void this.request(prepareRequest(event.data));
+          break;
+      }
+    };
+  }
+  abortRequest(event) {
+    const {
+      thread,
+      id
+    } = event;
+    const future = this.pending.get(thread).get(id);
+    if (future) {
+      future.abort();
+      this.pending.get(thread).delete(id);
+    }
+  }
+  async request(event) {
+    const {
+      thread,
+      id,
+      data
+    } = event;
+    try {
+      const future = this.store.request(data);
+      this.pending.get(thread).set(id, future);
+      const result = await future;
+      this.threads.get(thread)?.postMessage({
+        type: 'success-response',
+        id,
+        thread,
+        data: prepareResponse(result)
+      });
+    } catch (error) {
+      if (isAbortError(error)) return;
+      this.threads.get(thread)?.postMessage({
+        type: 'error-response',
+        id,
+        thread,
+        data: error
+      });
+    } finally {
+      this.pending.get(thread).delete(id);
+    }
+  }
+}
+function softCloneResponse(response) {
+  if (!response) return null;
+  const clone = {};
+  if (response.headers) {
+    clone.headers = Array.from(response.headers);
+  }
+  clone.ok = response.ok;
+  clone.redirected = response.redirected;
+  clone.status = response.status;
+  clone.statusText = response.statusText;
+  clone.type = response.type;
+  clone.url = response.url;
+  return clone;
+}
+function isAbortError(error) {
+  return error instanceof Error && error.name === 'AbortError';
+}
+function prepareResponse(result) {
+  const newResponse = {
+    response: softCloneResponse(result.response),
+    content: result.content
+  };
+  return newResponse;
+}
+function prepareRequest(event) {
+  if (event.data.headers) {
+    event.data.headers = new Headers(event.data.headers);
+  }
+  return event;
+}
+
+const MUTATION_OPS = new Set(['createRecord', 'updateRecord', 'deleteRecord']);
+
+/**
+ * In a Worker, any time we are asked to make a request, data needs to be returned.
+ * background requests are ergo no different than foreground requests.
+ * @internal
+ */
+function calcShouldFetch(store, request, hasCachedValue, identifier) {
+  const {
+    cacheOptions
+  } = request;
+  return request.op && MUTATION_OPS.has(request.op) || cacheOptions?.reload || cacheOptions?.backgroundReload || !hasCachedValue || (store.lifetimes && identifier ? store.lifetimes.isHardExpired(identifier, store) || store.lifetimes.isSoftExpired(identifier, store) : false);
+}
+function isMutation(request) {
+  return Boolean(request.op && MUTATION_OPS.has(request.op));
+}
+function isCacheAffecting(document) {
+  if (!isMutation(document.request)) {
+    return true;
+  }
+  // a mutation combined with a 204 has no cache impact when no known records were involved
+  // a createRecord with a 201 with an empty response and no known records should similarly
+  // have no cache impact
+
+  if (document.request.op === 'createRecord' && document.response?.status === 201) {
+    return document.content ? Object.keys(document.content).length > 0 : false;
+  }
+  return document.response?.status !== 204;
+}
+function isAggregateError(error) {
+  return error instanceof AggregateError || error.name === 'AggregateError' && Array.isArray(error.errors);
+}
+// TODO @runspired, consider if we should deep freeze errors (potentially only in debug) vs cloning them
+function cloneError(error) {
+  const isAggregate = isAggregateError(error);
+  const cloned = isAggregate ? new AggregateError(structuredClone(error.errors), error.message) : new Error(error.message);
+  cloned.stack = error.stack;
+  cloned.error = error.error;
+
+  // copy over enumerable properties
+  Object.assign(cloned, error);
+  return cloned;
+}
+
+/**
+ * A simplified CacheHandler that hydrates ResourceDataDocuments from the cache
+ * with their referenced resources.
+ *
+ */
+const CacheHandler = {
+  request(context, next) {
+    // if we have no cache or no cache-key skip cache handling
+    if (!context.request.store || context.request.cacheOptions?.[SkipCache]) {
+      return next(context.request);
+    }
+    const {
+      store
+    } = context.request;
+    const identifier = store.cacheKeyManager.getOrCreateDocumentIdentifier(context.request);
+    const peeked = identifier ? store.cache.peekRequest(identifier) : null;
+    if (identifier && !peeked) {
+      // if we are using persisted cache, we should attempt to populate the in-memory cache now
+      const worker = store._worker;
+      if (worker?.storage) {
+        return worker.storage.getDocument(identifier).then(document => {
+          if (document) {
+            store.cache.put(document);
+          }
+          return completeRequest(identifier, store, context, next);
+        }).catch(e => {
+          return completeRequest(identifier, store, context, next);
+        });
+      }
+    }
+    return completeRequest(identifier, store, context, next);
+  }
+};
+function completeRequest(identifier, store, context, next) {
+  const peeked = identifier ? store.cache.peekRequest(identifier) : null;
+  // In a Worker, any time we are asked to make a request, data needs to be returned.
+  // background requests are ergo no different than foreground requests.
+  if (calcShouldFetch(store, context.request, !!peeked, identifier)) {
+    return fetchContentAndHydrate(next, context, identifier);
+  }
+  context.setResponse(peeked.response);
+  if ('error' in peeked) {
+    throw peeked;
+  }
+  return maybeUpdateObjects(store, peeked.content);
+}
+function maybeUpdateObjects(store, document) {
+  if (!document) {
+    return document;
+  }
+  if (Array.isArray(document.data)) {
+    const data = document.data.map(identifier => {
+      return store.cache.peek(identifier);
+    });
+    return Object.assign({}, document, {
+      data
+    });
+  } else {
+    // @ts-expect-error FIXME investigate why document.data won't accept the signature
+    const data = document.data ? store.cache.peek(document.data) : null;
+    return Object.assign({}, document, {
+      data
+    });
+  }
+}
+function maybeUpdatePersistedCache(store, document, resourceDocument) {
+  const worker = store._worker;
+  if (!worker?.storage) {
+    return;
+  }
+  if (!document && resourceDocument) {
+    // we have resources to update but not a full request to cache
+    void worker.storage.putResources(resourceDocument, resourceIdentifier => {
+      return store.cache.peek(resourceIdentifier);
+    });
+  } else if (document) {
+    void worker.storage.putDocument(document, resourceIdentifier => {
+      return store.cache.peek(resourceIdentifier);
+    });
+  }
+}
+function updateCacheForSuccess(store, request, document) {
+  let response = null;
+  if (isMutation(request)) {
+    const record = request.data?.record || request.records?.[0];
+    if (record) {
+      // @ts-expect-error while this is valid, we should update the CacheHandler for transactional saves
+      response = store.cache.didCommit(record, document);
+
+      // a mutation combined with a 204 has no cache impact when no known records were involved
+      // a createRecord with a 201 with an empty response and no known records should similarly
+      // have no cache impact
+    } else if (isCacheAffecting(document)) {
+      response = store.cache.put(document);
+      maybeUpdatePersistedCache(store, null, response);
+    }
+  } else {
+    response = store.cache.put(document);
+    if (response.lid) {
+      const identifier = store.cacheKeyManager.getOrCreateDocumentIdentifier(request);
+      const full = store.cache.peekRequest(identifier);
+      maybeUpdatePersistedCache(store, full);
+    }
+  }
+  return maybeUpdateObjects(store, response);
+}
+function handleFetchSuccess(store, request, identifier, document) {
+  let response;
+  assertPrivateStore(store);
+  store._join(() => {
+    response = updateCacheForSuccess(store, request, document);
+  });
+  if (store.lifetimes?.didRequest) {
+    store.lifetimes.didRequest(request, document.response, identifier, store);
+  }
+  return response;
+}
+function updateCacheForError(store, request, error) {
+  if (isMutation(request)) {
+    // TODO similar to didCommit we should spec this to be similar to cache.put for handling full response
+    // currently we let the response remain undefiend.
+    const errors = error && error.content && typeof error.content === 'object' && 'errors' in error.content && Array.isArray(error.content.errors) ? error.content.errors : undefined;
+    const record = request.data?.record || request.records?.[0];
+    store.cache.commitWasRejected(record, errors);
+  } else {
+    const identifier = store.cacheKeyManager.getOrCreateDocumentIdentifier(request);
+    if (identifier) {
+      maybeUpdatePersistedCache(store, error);
+    }
+    return store.cache.put(error);
+  }
+}
+function handleFetchError(store, request, identifier, error) {
+  if (request.signal?.aborted) {
+    throw error;
+  }
+  assertPrivateStore(store);
+  let response;
+  store._join(() => {
+    response = updateCacheForError(store, request, error);
+  });
+  if (identifier && store.lifetimes?.didRequest) {
+    store.lifetimes.didRequest(request, error.response, identifier, store);
+  }
+  if (isMutation(request)) {
+    throw error;
+  }
+  const newError = cloneError(error);
+  newError.content = response;
+  throw newError;
+}
+function fetchContentAndHydrate(next, context, identifier) {
+  const {
+    request
+  } = context;
+  const {
+    store
+  } = context.request;
+  if (isMutation(request)) {
+    // TODO should we handle multiple records in request.records by iteratively calling willCommit for each
+    const record = request.data?.record || request.records?.[0];
+    if (record) {
+      store.cache.willCommit(record, context);
+    }
+  }
+  if (store.lifetimes?.willRequest) {
+    store.lifetimes.willRequest(request, identifier, store);
+  }
+  return next(request).then(document => handleFetchSuccess(store, request, identifier, document), error => handleFetchError(store, request, identifier, error));
+}
+
+export { CacheHandler, DataWorker };

package/dist/unpkg/prod/document-storage.js

@@ -0,0 +1,339 @@
+const WARP_DRIVE_STORAGE_FILE_NAME = 'warp-drive_document-storage';
+const WARP_DRIVE_STORAGE_VERSION = 1;
+
+/**
+ * DocumentStorage is specifically designed around WarpDrive Cache and Request concepts.
+ *
+ * CacheFileDocument is a StructuredDocument (request response) whose `content` is
+ * the ResourceDocument returned by inserting the request into a Store's Cache.
+ */
+
+/**
+ * A CacheDocument is a reconstructed request response that rehydrates ResourceDocument
+ * with the associated resources based on their identifiers.
+ */
+
+class InternalDocumentStorage {
+  constructor(options) {
+    this.options = options;
+    this._lastModified = 0;
+    this._invalidated = true;
+    this._fileHandle = this._open(options.scope);
+    this._channel = Object.assign(new BroadcastChannel(options.scope), {
+      onmessage: this._onMessage.bind(this)
+    });
+  }
+  _onMessage(_event) {
+    this._invalidated = true;
+  }
+  async _open(scope) {
+    const directoryHandle = await navigator.storage.getDirectory();
+    const fileHandle = await directoryHandle.getFileHandle(scope, {
+      create: true
+    });
+    return fileHandle;
+  }
+  async _read() {
+    if (this._filePromise) {
+      return this._filePromise;
+    }
+    if (this._invalidated) {
+      const updateFile = async () => {
+        const fileHandle = await this._fileHandle;
+        const file = await fileHandle.getFile();
+        const lastModified = file.lastModified;
+        if (lastModified === this._lastModified && this._cache) {
+          return this._cache;
+        }
+        const contents = await file.text();
+        const cache = contents ? JSON.parse(contents) : {
+          documents: [],
+          resources: []
+        };
+        const documents = new Map(cache.documents);
+        const resources = new Map(cache.resources);
+        const cacheMap = {
+          documents,
+          resources
+        };
+        this._cache = cacheMap;
+        this._invalidated = false;
+        this._lastModified = lastModified;
+        return cacheMap;
+      };
+      this._filePromise = updateFile();
+      await this._filePromise;
+      this._filePromise = null;
+    }
+    return this._cache;
+  }
+  async _patch(documentKey, document, updatedResources) {
+    const fileHandle = await this._fileHandle;
+    // secure a lock before getting latest state
+    const writable = await fileHandle.createWritable();
+    const cache = await this._read();
+    cache.documents.set(documentKey, document);
+    updatedResources.forEach((resource, key) => {
+      cache.resources.set(key, resource);
+    });
+    const documents = [...cache.documents.entries()];
+    const resources = [...cache.resources.entries()];
+    const cacheFile = {
+      documents,
+      resources
+    };
+    await writable.write(JSON.stringify(cacheFile));
+    await writable.close();
+    this._channel.postMessage({
+      type: 'patch',
+      key: documentKey,
+      resources: [...updatedResources.keys()]
+    });
+  }
+  async getDocument(key) {
+    const cache = await this._read();
+    // clone the document to avoid leaking the internal cache
+    const document = safeDocumentHydrate(cache.documents.get(key.lid));
+    if (!document) {
+      return null;
+    }
+
+    // expand the document with the resources
+    if (document.content) {
+      if (docHasData(document.content)) {
+        let data = null;
+        if (Array.isArray(document.content.data)) {
+          data = document.content.data.map(resourceIdentifier => {
+            const resource = cache.resources.get(resourceIdentifier.lid);
+            if (!resource) {
+              throw new Error(`Resource not found for ${resourceIdentifier.lid}`);
+            }
+
+            // clone the resource to avoid leaking the internal cache
+            return structuredClone(resource);
+          });
+        } else if (document.content.data) {
+          const resource = cache.resources.get(document.content.data.lid);
+          if (!resource) {
+            throw new Error(`Resource not found for ${document.content.data.lid}`);
+          }
+
+          // clone the resource to avoid leaking the internal cache
+          data = structuredClone(resource);
+        }
+        if (document.content.included) {
+          const included = document.content.included.map(resourceIdentifier => {
+            const resource = cache.resources.get(resourceIdentifier.lid);
+            if (!resource) {
+              throw new Error(`Resource not found for ${resourceIdentifier.lid}`);
+            }
+
+            // clone the resource to avoid leaking the internal cache
+            return structuredClone(resource);
+          });
+          document.content.included = included;
+        }
+        document.content.data = data;
+      }
+    }
+    return document;
+  }
+  async putDocument(document, resourceCollector) {
+    const resources = new Map();
+    if (!document.content) {
+      throw new Error(`Document content is missing, only finalized documents can be stored`);
+    }
+    if (!document.content.lid) {
+      throw new Error(`Document content is missing a lid, only documents with a cache-key can be stored`);
+    }
+    if (docHasData(document.content)) {
+      this._getResources(document.content, resourceCollector, resources);
+    }
+    await this._patch(document.content.lid, safeDocumentSerialize(document), resources);
+  }
+  _getResources(document, resourceCollector, resources = new Map()) {
+    if (Array.isArray(document.data)) {
+      document.data.forEach(resourceIdentifier => {
+        const resource = resourceCollector(resourceIdentifier);
+        resources.set(resourceIdentifier.lid, structuredClone(resource));
+      });
+    } else if (document.data) {
+      const resource = resourceCollector(document.data);
+      resources.set(document.data.lid, structuredClone(resource));
+    }
+    if (document.included) {
+      document.included.forEach(resourceIdentifier => {
+        const resource = resourceCollector(resourceIdentifier);
+        resources.set(resourceIdentifier.lid, structuredClone(resource));
+      });
+    }
+    return resources;
+  }
+  async putResources(document, resourceCollector) {
+    const fileHandle = await this._fileHandle;
+    // secure a lock before getting latest state
+    const writable = await fileHandle.createWritable();
+    const cache = await this._read();
+    const updatedResources = this._getResources(document, resourceCollector);
+    updatedResources.forEach((resource, key) => {
+      cache.resources.set(key, resource);
+    });
+    const documents = [...cache.documents.entries()];
+    const resources = [...cache.resources.entries()];
+    const cacheFile = {
+      documents,
+      resources
+    };
+    await writable.write(JSON.stringify(cacheFile));
+    await writable.close();
+    this._channel.postMessage({
+      type: 'patch',
+      key: null,
+      resources: [...updatedResources.keys()]
+    });
+  }
+  async clear(reset) {
+    const fileHandle = await this._fileHandle;
+    const writable = await fileHandle.createWritable();
+    await writable.write('');
+    await writable.close();
+    this._invalidated = true;
+    this._lastModified = 0;
+    this._cache = null;
+    this._filePromise = null;
+    this._channel.postMessage({
+      type: 'clear'
+    });
+    if (!reset) {
+      this._channel.close();
+      this._channel = null;
+      if (!this.options.isolated) {
+        Storages.delete(this.options.scope);
+      }
+    }
+  }
+}
+function safeDocumentSerialize(document) {
+  const doc = document;
+  const newDoc = {};
+  if ('request' in doc) {
+    newDoc.request = prepareRequest(doc.request);
+  }
+  if ('response' in doc) {
+    newDoc.response = prepareResponse(doc.response);
+  }
+  if ('content' in doc) {
+    newDoc.content = structuredClone(doc.content);
+  }
+  return newDoc;
+}
+function prepareRequest(request) {
+  const {
+    signal,
+    headers
+  } = request;
+  const requestCopy = Object.assign({}, request);
+  delete requestCopy.store;
+  if (signal instanceof AbortSignal) {
+    delete requestCopy.signal;
+  }
+  if (headers instanceof Headers) {
+    requestCopy.headers = Array.from(headers);
+  }
+  return requestCopy;
+}
+function prepareResponse(response) {
+  if (!response) return null;
+  const clone = {};
+  if (response.headers) {
+    clone.headers = Array.from(response.headers);
+  }
+  clone.ok = response.ok;
+  clone.redirected = response.redirected;
+  clone.status = response.status;
+  clone.statusText = response.statusText;
+  clone.type = response.type;
+  clone.url = response.url;
+  return clone;
+}
+function safeDocumentHydrate(document) {
+  const doc = document;
+  const newDoc = {};
+  if ('request' in doc) {
+    const headers = new Headers(doc.request.headers);
+    const req = Object.assign({}, doc.request, {
+      headers
+    });
+    newDoc.request = new Request(doc.request.url ?? '', req);
+  }
+  if ('response' in doc) {
+    const headers = new Headers(doc.response.headers);
+    const resp = Object.assign({}, doc.response, {
+      headers
+    });
+    newDoc.response = new Response(null, resp);
+  }
+  if ('content' in doc) {
+    newDoc.content = structuredClone(doc.content);
+  }
+  return newDoc;
+}
+function docHasData(doc) {
+  return 'data' in doc;
+}
+const Storages = new Map();
+
+/**
+ * DocumentStorage is a wrapper around the StorageManager API that provides
+ * a simple interface for reading and updating documents and requests.
+ *
+ * Some goals for this experiment:
+ *
+ * - optimize for storing requests/documents
+ * - optimize for storing resources
+ * - optimize for looking up resources associated to a document
+ * - optimize for notifying cross-tab when data is updated
+ *
+ * optional features:
+ *
+ * - support for offline mode
+ * - ?? support for relationship based cache traversal
+ * - a way to index records by type + another field (e.g updatedAt/createAt/name)
+ *   such that simple queries can be done without having to scan all records
+ */
+class DocumentStorage {
+  constructor(options = {}) {
+    options.isolated = options.isolated ?? false;
+    options.scope = options.scope ?? 'default';
+    const fileName = `${WARP_DRIVE_STORAGE_FILE_NAME}@version_${WARP_DRIVE_STORAGE_VERSION}:${options.scope}`;
+    if (!options.isolated && Storages.has(fileName)) {
+      const storage = Storages.get(fileName);
+      if (storage) {
+        this._storage = storage.deref();
+        return;
+      }
+    }
+    const storage = new InternalDocumentStorage({
+      scope: fileName,
+      isolated: options.isolated
+    });
+    this._storage = storage;
+    if (!options.isolated) {
+      Storages.set(fileName, new WeakRef(storage));
+    }
+  }
+  getDocument(key) {
+    return this._storage.getDocument(key);
+  }
+  putDocument(document, resourceCollector) {
+    return this._storage.putDocument(document, resourceCollector);
+  }
+  putResources(document, resourceCollector) {
+    return this._storage.putResources(document, resourceCollector);
+  }
+  clear(reset) {
+    return this._storage.clear(reset);
+  }
+}
+
+export { DocumentStorage };