@adobe/helix-google-support 1.0.0 → 1.2.0

package/CHANGELOG.md

@@ -1,3 +1,24 @@
+# [1.2.0](https://github.com/adobe/helix-google-support/compare/v1.1.0...v1.2.0) (2022-04-28)
+
+
+### Features
+
+* add get file|doc from path ([#9](https://github.com/adobe/helix-google-support/issues/9)) ([4a2e927](https://github.com/adobe/helix-google-support/commit/4a2e9272f4328e936e65b7144d04e516ac602625))
+
+# [1.1.0](https://github.com/adobe/helix-google-support/compare/v1.0.1...v1.1.0) (2022-04-28)
+
+
+### Features
+
+* add cache invalidate for drive items ([#8](https://github.com/adobe/helix-google-support/issues/8)) ([7aa2c80](https://github.com/adobe/helix-google-support/commit/7aa2c8061facb2d3c45929356c23370d83686f55))
+
+## [1.0.1](https://github.com/adobe/helix-google-support/compare/v1.0.0...v1.0.1) (2022-04-20)
+
+
+### Bug Fixes
+
+* minor update ([#6](https://github.com/adobe/helix-google-support/issues/6)) ([8d7cd50](https://github.com/adobe/helix-google-support/commit/8d7cd50c8162da12c4d4430833416fac0f2eda93))
+
 # 1.0.0 (2022-04-13)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-google-support",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "Helix Google Support",
   "main": "src/index.js",
   "type": "module",
@@ -40,16 +40,17 @@
     "@semantic-release/changelog": "6.0.1",
     "@semantic-release/git": "10.0.1",
     "@semantic-release/npm": "9.0.1",
-    "c8": "7.11.0",
+    "c8": "7.11.2",
     "codecov": "3.8.3",
-    "eslint": "8.13.0",
+    "eslint": "8.14.0",
     "eslint-plugin-header": "3.1.1",
     "eslint-plugin-import": "2.26.0",
     "husky": "7.0.4",
     "junit-report-builder": "3.0.0",
-    "lint-staged": "12.3.7",
+    "lint-staged": "12.4.0",
     "mocha": "9.2.2",
     "mocha-multi-reporters": "1.5.1",
+    "nock": "13.2.4",
     "semantic-release": "19.0.2"
   },
   "lint-staged": {

package/src/GoogleClient.d.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+declare interface ICachePlugin {}
+
+declare interface DriveItemInfo {}
+
+declare interface GoogleClientOptions {
+  log:Console;
+  clientId:string;
+  clientSecret:string;
+  redirectUri:string;
+  cachePlugin?:ICachePlugin;
+}
+
+/**
+ * Google client
+ */
+declare class GoogleClient {
+  /**
+   * Sets the global item cache options. It internally creates a new LRU
+   * with the given options
+   *
+   * @param opts
+   */
+  static setItemCacheOptions(opts:object);
+
+  /**
+   * Returns a url for a google drive id.
+   * @param {string} id
+   * @returns {string}
+   */
+  static id2Url(id:string):string;
+
+  constructor(opts:GoogleClientOptions);
+
+  init():Promise<GoogleClient>;
+
+  generateAuthUrl(...args):Promise<string>;
+
+  /**
+   * Sets the credentials
+   * @param tokens
+   * @returns {Promise<void>}
+   */
+  setCredentials(tokens:object):Promise<void>;
+
+  /**
+   * Returns the token for the given code
+   * @param {string} code
+   * @returns {Promise<*>}
+   */
+  getToken(code:string):Promise<object>;
+
+  /**
+   * @param {string} parentId
+   * @param {string[]} pathSegments
+   * @param {string} parentPath
+   * @returns {Promise<DriveItemInfo[]>|null}
+   */
+  getUncachedItemsFromSegments(parentId:string, pathSegments:string[], parentPath:string):Promise<DriveItemInfo[]>;
+
+  /**
+   * @param {string} parentId
+   * @param {string} pathSegments
+   * @param {string} parentPath
+   * @returns {Promise<DriveItemInfo[]>|null}
+   */
+  getDriveItemsFromSegments(parentId:string, pathSegments:string[], parentPath:string):Promise<DriveItemInfo[]>;
+
+  /**
+   * returns the items hierarchy for the given path and root id, starting with the given path.
+   * @param {string} parentId
+   * @param {string} path
+   * @return {DriveItemInfo[]}
+   */
+  getDriveItemsFromSegments(parentId:string, path:string):Promise<DriveItemInfo[]>;
+
+  /**
+   * returns the items hierarchy for the given item, starting with the given id
+   * @param {string} fileId
+   * @param {object} roots
+   * @returns {Promise<DriveItemInfo[]>}
+   */
+  getItemsFromId(fileId:string, roots:object):Promise<DriveItemInfo[]>;
+
+  /**
+   * Returns the (cached) item for the given path or {@code null} if the item cannot be found.
+   * The item will contains a `invalidate()` method which can be used to remove it from the cache.
+   *
+   * @param {string} parentId
+   * @param {string} path
+   * @returns {Promise<DriveItemInfo>}
+   */
+  getItemFromPath(parentId:string, path:string):Promise<DriveItemInfo>;
+
+  /**
+   * Returns an (uncached) file directly via the google api
+   * @param {string} fileId
+   * @returns {string} file data
+   */
+  getFile(fileId:string):Promise<string>;
+
+  /**
+   * Fetches the file data from the give path. If the file with the internal id could not be
+   * fetched, the item cache is invalidated and the operation is retried. this is to support
+   * moved items.
+   * @param {string} parentId
+   * @param {string} path
+   * @param {boolean} noRetry {@code true} to avoid retry
+   * @returns {Promise<string>|null} The data of the file or {@code null} if the file does not exist
+   */
+  getFileFromPath(parentId:string, path:string, noRetry:boolean):Promise<string>;
+
+  /**
+   * Returns an (uncached) document directly via the google api
+   * @param {string} documentId
+   * @returns {object} document
+   */
+  getDocument(documentId:string):Promise<object>;
+
+  /**
+   * Fetches the document from the give path. If the document with the internal id could not be
+   * fetched, the item cache is invalidated and the operation is retried. this is to support
+   * moved items.
+   * @param {string} parentId
+   * @param {string} path
+   * @param {boolean} noRetry {@code true} to avoid retry
+   * @returns {Promise<object>|null} The document or {@code null} if the document does not exist
+   */
+  getDocumentFromPath(parentId:string, path:string, noRetry:boolean):Promise<object>;
+}

package/src/GoogleClient.js

@@ -9,15 +9,65 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import LRU from 'lru-cache';
 import { google } from 'googleapis';
-import { editDistance, sanitizeName, splitByExtension } from '@adobe/helix-onedrive-support/utils';
+import {
+  editDistance, sanitizeName, splitByExtension,
+} from '@adobe/helix-onedrive-support/utils';
+import { StatusCodeError } from '@adobe/helix-onedrive-support';
 import { GoogleTokenCache } from './GoogleTokenCache.js';
-import cache from './cache.js';
+
+let lru = new LRU({ max: 1000, ttl: 60000 });
+
+/**
+ * @typedef DriveItemInfo {
+ * @property {string} id
+ * @property {string} name
+ * @property {string} url
+ * @property {string} path
+ */
+
+/**
+ * Adds the last modified property if defined in item
+ * @param {DriveItemInfo} itemInfo
+ * @param item
+ * @returns {DriveItemInfo}
+ */
+function addLastModified(itemInfo, item) {
+  if (item.modifiedTime) {
+    // eslint-disable-next-line no-param-reassign
+    itemInfo.lastModified = Date.parse(item.modifiedTime);
+  }
+  return itemInfo;
+}
+
+function createPathSegments(path) {
+  const pathSegments = path.split('/');
+  if (!pathSegments[0]) {
+    // if path starts with '/' the first segment is empty
+    pathSegments.shift();
+  }
+  return pathSegments;
+}
+
+function getCacheKey(parentId, pathSegments) {
+  return `${parentId}:${pathSegments.join('/')}`;
+}
 
 /**
  * Google auth client
  */
 export class GoogleClient {
+  /**
+   * Sets the global item cache options. It internally creates a new LRU
+   * with the given options
+   *
+   * @param opts
+   */
+  static setItemCacheOptions(opts) {
+    lru = new LRU(opts);
+  }
+
   /**
    * Returns a url for a google drive id.
    * @param {string} id
@@ -35,39 +85,38 @@ export class GoogleClient {
    * @param {ICachePlugin} plugin
    */
   constructor(opts) {
-    this.log = opts.log;
-    this.auth = new google.auth.OAuth2(
-      opts.clientId,
-      opts.clientSecret,
-      opts.redirectUri,
-    );
-    this.cachePlugin = opts.cachePlugin;
-    this.cache = new GoogleTokenCache(opts.cachePlugin).withLog(opts.log);
-
-    /// hack to capture tokens, since the emit handler is not awaited in the google client
-    const originalRefreshTokenNoCache = this.auth.refreshTokenNoCache.bind(this.auth);
-    this.auth.refreshTokenNoCache = async (...args) => {
-      const ret = await originalRefreshTokenNoCache(...args);
-      await this.cache.store(ret.tokens);
-      return ret;
-    };
+    Object.assign(this, {
+      log: opts.log,
+      auth: new google.auth.OAuth2(
+        opts.clientId,
+        opts.clientSecret,
+        opts.redirectUri,
+      ),
+    });
+
+    if (opts.cachePlugin) {
+      this.cachePlugin = opts.cachePlugin;
+      this.cache = new GoogleTokenCache(opts.cachePlugin).withLog(opts.log);
+      /// hack to capture tokens, since the emit handler is not awaited in the google client
+      const originalRefreshTokenNoCache = this.auth.refreshTokenNoCache.bind(this.auth);
+      this.auth.refreshTokenNoCache = async (...args) => {
+        const ret = await originalRefreshTokenNoCache(...args);
+        await this.cache.store(ret.tokens);
+        return ret;
+      };
+    }
 
     this.drive = google.drive({
       version: 'v3',
       auth: this.auth,
     });
-
-    /**
-     * Cached version of `getUncachedItemsFromPath`
-     */
-    this.getDriveItemsFromPath = cache(this.getUncachedItemsFromPath.bind(this), {
-      hash: (fn, path, parentId) => `${parentId}:${path}`,
-    });
   }
 
   async init() {
-    await this.cache.load();
-    this.auth.setCredentials(this.cache.tokens);
+    if (this.cache) {
+      await this.cache.load();
+      this.auth.setCredentials(this.cache.tokens);
+    }
     return this;
   }
 
@@ -75,28 +124,40 @@ export class GoogleClient {
     return this.auth.generateAuthUrl(...args);
   }
 
+  /**
+   * Sets the credentials
+   * @param tokens
+   * @returns {Promise<void>}
+   */
   async setCredentials(tokens) {
-    await this.cache.store(tokens);
+    if (this.cache) {
+      await this.cache.store(tokens);
+    }
     this.auth.setCredentials(tokens);
   }
 
+  /**
+   * Returns the token for the given code
+   * @param {string} code
+   * @returns {Promise<*>}
+   */
   async getToken(code) {
-    const resp = await this.auth.getToken(code);
-    await this.cache.store(resp.tokens);
-    return resp;
+    const { tokens } = await this.auth.getToken(code);
+    if (this.cache) {
+      await this.cache.store(tokens);
+    }
+    return tokens;
   }
 
   /**
-   * @param {Drive} drive
-   * @param {AdminContext} context
-   * @param {string} path
    * @param {string} parentId
+   * @param {string[]} pathSegments
    * @param {string} parentPath
-   * @returns {Promise<EditFolderInfo[]>}
+   * @returns {Promise<DriveItemInfo[]>|null}
    */
-  async getUncachedItemsFromPath(path, parentId, parentPath) {
+  async getUncachedItemsFromSegments(parentId, pathSegments, parentPath) {
     const { log, drive } = this;
-    const [name, ...rest] = path.split('/');
+    const name = pathSegments.shift();
     const [baseName, ext] = splitByExtension(name);
     const sanitizedName = sanitizeName(baseName);
 
@@ -106,7 +167,7 @@ export class GoogleClient {
         `'${parentId}' in parents`,
         'and trashed=false',
         // folder if path continues, sheet otherwise
-        `and mimeType ${rest.length ? '=' : '!='} 'application/vnd.google-apps.folder'`,
+        `and mimeType ${pathSegments.length ? '=' : '!='} 'application/vnd.google-apps.folder'`,
       ].join(' '),
       fields: 'nextPageToken, files(id, name, modifiedTime)',
       includeItemsFromAllDrives: true,
@@ -163,50 +224,83 @@ export class GoogleClient {
     }
 
     const itemPath = `${parentPath}/${sanitizedName}`;
-    const children = rest.length
+    const children = pathSegments.length
       // eslint-disable-next-line no-use-before-define
-      ? await this.getDriveItemsFromPath(rest.join('/'), item.id, itemPath)
+      ? await this.getDriveItemsFromSegments(item.id, pathSegments, itemPath)
       : [];
 
     if (!children) {
       return null;
     }
-
-    const pathItem = {
+    const pathItem = addLastModified({
       name,
       path: itemPath,
       id: item.id,
-      lastModified: Date.parse(item.modifiedTime),
-    };
+    }, item);
+
+    if (children.length) {
+      // add parent references not-enumerable to avoid deep structures during serialization
+      Object.defineProperty(children[children.length - 1], 'parent', {
+        enumerable: false,
+        value: pathItem,
+      });
+    }
 
     return [...children, pathItem];
   }
 
+  /**
+   * @param {string} parentId
+   * @param {string} pathSegments
+   * @param {string} parentPath
+   * @returns {Promise<DriveItemInfo[]>|null}
+   */
+  async getDriveItemsFromSegments(parentId, pathSegments, parentPath) {
+    const key = getCacheKey(parentId, pathSegments);
+    let items = lru.get(key);
+    if (items) {
+      return items;
+    }
+    items = await this.getUncachedItemsFromSegments(parentId, pathSegments, parentPath);
+    if (items) {
+      lru.set(key, items);
+      // add invalidation function as not-enumerable to keep compatible objects
+      Object.defineProperty(items[items.length - 1], 'invalidate', {
+        enumerable: false,
+        value() {
+          lru.delete(key);
+          this.parent?.invalidate();
+        },
+      });
+    }
+    return items;
+  }
+
   /**
    * returns the items hierarchy for the given path and root id, starting with the given path.
-   * @param context
-   * @param path
-   * @param rootId
+   * @param {string} parentId
+   * @param {string} path
+   * @return {DriveItemInfo[]}
    */
-  async getItemsFromPath(path, rootId) {
-    const result = await this.getDriveItemsFromPath(path, rootId, '');
+  async getItemsFromPath(parentId, path) {
+    const segs = createPathSegments(path);
+    const result = await this.getDriveItemsFromSegments(parentId, segs, '');
     if (!result) {
-      return null;
+      return [];
     }
     return [...result, {
       name: '',
       path: '/',
-      id: rootId,
+      id: parentId,
     }];
   }
 
   /**
    * returns the items hierarchy for the given item, starting with the given id
-   * @param {AdminContext} context
-   * @param {Drive} drive
    * @param {string} fileId
    * @param {object} roots
-   * @returns {Promise<EditFolderInfo[]>}
+   * @private
+   * @returns {Promise<DriveItemInfo[]>}
    */
   async getItems(fileId, roots) {
     const { log } = this;
@@ -223,62 +317,170 @@ export class GoogleClient {
     const root = roots[fileId];
     if (root) {
       // stop at mount root
-      return [{
+      return [addLastModified({
         id: fileId,
         name: '',
         path: root,
-        lastModified: Date.parse(data.modifiedTime),
-      }];
+      }, data)];
     }
 
     const parentId = data.parents ? data.parents[0] : '';
     if (!parentId) {
       // outside mountpoint
-      return [{
+      return [addLastModified({
         id: fileId,
         name: data.name,
         path: `/root:/${data.name}`,
-        lastModified: Date.parse(data.modifiedTime),
-      }];
+      }, data)];
     }
 
     const ancestors = await this.getItems(data.parents[0], roots);
     const parentPath = ancestors[0].path.replace(/\/+$/, '');
-    ancestors.unshift({
+    ancestors.unshift(addLastModified({
       id: fileId,
       name: data.name,
       path: `${parentPath}/${data.name}`,
-      lastModified: Date.parse(data.modifiedTime),
-    });
+    }, data));
     return ancestors;
   }
 
   /**
-   * @param {AdminContext} context
    * @param {string} fileId
    * @param {object} roots
-   * @returns {Promise<EditFolderInfo[]>}
+   * @returns {Promise<DriveItemInfo[]>}
    */
   async getItemsFromId(fileId, roots) {
     const { log } = this;
     try {
       return await this.getItems(fileId, roots);
     } catch (e) {
+      if (e.response && e.response.status === 404) {
+        log.warn(`unable to get items for ${fileId}. Not found`);
+        return [];
+      }
       log.warn(`unable to get items for ${fileId}. ${e}`);
-      return [];
+      throw e;
     }
   }
 
   /**
-   * @param {AdminContext} context
+   * Returns the (cached) item for the given path or {@code null} if the item cannot be found.
+   * The item will contains a `invalidate()` method which can be used to remove it from the cache.
+   *
+   * @param {string} parentId
+   * @param {string} path
+   * @returns {Promise<DriveItemInfo>}
+   */
+  async getItemFromPath(parentId, path) {
+    const segs = createPathSegments(path);
+    const items = await this.getDriveItemsFromSegments(parentId, segs, '');
+    const item = items?.[0];
+    if (!item) {
+      return null;
+    }
+    return item;
+  }
+
+  /**
+   * Returns an (uncached) file directly via the google api
    * @param {string} fileId
    * @returns {string} file data
    */
   async getFile(fileId) {
-    const res = await this.drive.files.get({
-      fileId,
-      alt: 'media',
+    try {
+      const res = await this.drive.files.get({
+        fileId,
+        alt: 'media',
+      });
+      return res.data;
+    } catch (e) {
+      if (e.response && e.response.status === 404) {
+        throw new StatusCodeError(`Not Found: ${fileId}`, 404);
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * Fetches the file data from the give path. If the file with the internal id could not be
+   * fetched, the item cache is invalidated and the operation is retried. this is to support
+   * moved items.
+   * @param {string} parentId
+   * @param {string} path
+   * @param {boolean} noRetry {@code true} to avoid retry
+   * @returns {Promise<string>|null} The data of the file or {@code null} if the file does not exist
+   */
+  async getFileFromPath(parentId, path, noRetry) {
+    const item = await this.getItemFromPath(parentId, path);
+    if (!item) {
+      return null;
+    }
+    try {
+      // noinspection ES6RedundantAwait (need to catch exception)
+      return await this.getFile(item.id);
+    } catch (e) {
+      if (e.statusCode === 404) {
+        if (noRetry) {
+          return null;
+        }
+        this.log.info(`file ${item.id} does not exist - invalidating cache and retry`);
+        item.invalidate();
+        return this.getFileFromPath(parentId, path, true);
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  /**
+   * Returns an (uncached) document directly via the google api
+   * @param {string} documentId
+   * @returns {object} document
+   */
+  async getDocument(documentId) {
+    const docs = google.docs({
+      version: 'v1',
+      auth: this.auth,
     });
-    return res.data;
+    try {
+      const res = await docs.documents.get({ documentId });
+      return res.data;
+    } catch (e) {
+      if (e.response && e.response.status === 404) {
+        throw new StatusCodeError(`Not Found: ${documentId}`, 404);
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * Fetches the document from the give path. If the document with the internal id could not be
+   * fetched, the item cache is invalidated and the operation is retried. this is to support
+   * moved items.
+   * @param {string} parentId
+   * @param {string} path
+   * @param {boolean} noRetry {@code true} to avoid retry
+   * @returns {Promise<object>|null} The document or {@code null} if the document does not exist
+   */
+  async getDocumentFromPath(parentId, path, noRetry) {
+    const item = await this.getItemFromPath(parentId, path);
+    if (!item) {
+      return null;
+    }
+    try {
+      // noinspection ES6RedundantAwait (need to catch exception)
+      return await this.getDocument(item.id);
+    } catch (e) {
+      if (e.statusCode === 404) {
+        if (noRetry) {
+          return null;
+        }
+        this.log.info(`document ${item.id} does not exist - invalidating cache and retry`);
+        item.invalidate();
+        return this.getDocumentFromPath(parentId, path, true);
+      } else {
+        throw e;
+      }
+    }
   }
 }

package/src/cache.js

@@ -1,78 +0,0 @@
-/*
- * Copyright 2020 Adobe. All rights reserved.
- * This file is licensed to you under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under
- * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
- * OF ANY KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-import LRU from 'lru-cache';
-
-let lru = new LRU({ max: 1000, ttl: 60000 });
-
-/**
- * Returns a memoized version of the function `fn`.
- * @param {function} fn a function to memoize
- * @param {object} opts caching options
- * @param {function} opts.hash a hash function to build a cache key.
- * The hash function will be called with the `fn` and a list of
- * it's arguments. A good hashing function will discard irrelevant
- * arguments and join the resulting arguments
- * @param {function} opts.cacheresult a predicate function that will be called
- * with the result of the function execution. Only when this function returns
- * `true` will the result be cached. The default is that all results are
- * cached.
- * @param {function} opts.cacheerror a predicate function that will be called
- * with the error that the function execution throws. Only when this
- * function returns `true` will the error be cached. The default is that
- * errors are never cached, so that each new invocation will be uncached.
- */
-export default function cache(fn, opts = {}) {
-  const {
-    hash = (...args) => args.join(),
-    cacheresult = () => true,
-    cacheerror = () => false,
-  } = opts;
-  return async function cached(...args) {
-    const key = hash(fn, ...args);
-    const entry = lru.get(key);
-    if (entry) {
-      if (entry.err) {
-        throw entry.err;
-      }
-      return entry.ok;
-    }
-    try {
-      // invoke the function
-      const result = await fn(...args);
-      if (cacheresult(result)) {
-        // store the result under ok if permitted
-        lru.set(key, { ok: result });
-      }
-      // and return the result
-      return result;
-    } catch (err) {
-      if (cacheerror(err)) {
-        // store the error under err if permitted
-        lru.set(key, { err });
-      }
-      // and throw the error
-      throw err;
-    }
-  };
-}
-
-/**
- * Resets the QuickLRU cache with new options. Existing cache entries
- * will be cleared.
- * @param {object} opts options
- * @param {integer} opts.maxSize maximum size of the cache
- */
-cache.options = (opts) => {
-  lru = new LRU(opts);
-
-  return cache;
-};