@pipedream/google_drive 0.3.3 → 0.4.0

package/sources/new-shared-drive/new-shared-drive.mjs

@@ -0,0 +1,68 @@
+import googleDrive from "../../google_drive.app.mjs";
+
+export default {
+  key: "google_drive-new-shared-drive",
+  name: "New Shared Drive",
+  description: "Emits a new event any time a shared drive is created.",
+  version: "0.0.9",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    googleDrive,
+    db: "$.service.db",
+    timer: {
+      label: "Polling interval",
+      description: "Interval to poll the Google Drive API for new shared drives",
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 60 * 15, // 30 minutes
+      },
+    },
+  },
+  hooks: {
+    async deploy() {
+      const { drives: initDrives } = await this.googleDrive.listDrivesInPage();
+      for (const drive of initDrives) {
+        const newDrive = await this.googleDrive.getDrive(drive.id);
+        const meta = this.generateMeta(newDrive);
+        this.$emit(newDrive, meta);
+      }
+
+      this._setKnownDrives(initDrives.map((drive) => drive.id));
+    },
+  },
+  methods: {
+    _getKnownDrives() {
+      return this.db.get("driveIds");
+    },
+    _setKnownDrives(driveIds) {
+      this.db.set("driveIds", Array.from(driveIds));
+    },
+    generateMeta(drive) {
+      const ts = new Date(drive.createdTime).getTime();
+      return {
+        id: drive.id,
+        summary: drive.name,
+        ts,
+      };
+    },
+  },
+  async run() {
+    const knownDrives = new Set(this._getKnownDrives());
+    const drivesStream = this.googleDrive.listDrives();
+    for await (const drive of drivesStream) {
+      if (knownDrives.has(drive.id)) {
+        // We've already seen this drive, so we skip it
+        continue;
+      }
+
+      knownDrives.add(drive.id);
+
+      const newDrive = await this.googleDrive.getDrive(drive.id);
+      const meta = this.generateMeta(newDrive);
+      this.$emit(newDrive, meta);
+    }
+
+    this._setKnownDrives(knownDrives);
+  },
+};

package/utils.mjs

@@ -0,0 +1,247 @@
+import fs from "fs";
+import { axios } from "@pipedream/platform";
+import {
+  MY_DRIVE_VALUE,
+  LEGACY_MY_DRIVE_VALUE,
+  MAX_FILE_OPTION_PATH_SEGMENTS,
+} from "./constants.mjs";
+
+/**
+ * Returns whether the specified drive ID corresponds to the authenticated
+ * user's My Drive or not
+ *
+ * @param {String} drive the ID value of a Google Drive
+ * @returns `true` only when the specified drive is the user's 'My Drive'
+ */
+function isMyDrive(drive) {
+  return drive === MY_DRIVE_VALUE || drive === LEGACY_MY_DRIVE_VALUE;
+}
+
+/**
+ * Returns a valid Google Drive ID to be used in Google Drive API calls
+ *
+ * @param {String} drive the ID value of a Google Drive, as provided by the
+ * `drive` prop definition of this app
+ * @returns the proper Google Drive ID to be used in Google Drive API calls
+ */
+function getDriveId(drive) {
+  return isMyDrive(drive)
+    ? null
+    : drive;
+}
+
+/**
+ * Gets an options object to be used in functions that use the
+ * [the `drive.drives.list` API](https://bit.ly/3AiWE1x).
+ *
+ * @param {String} drive the ID value of a Google Drive, as provided by the
+ * `drive` prop definition of this app
+ * @param {object} [baseOpts = {}] - an object containing extra/optional
+ * parameters to be fed to the GDrive API call, as defined in [the API
+ * docs](https://bit.ly/3AnQDR1)
+ *
+ * @returns an object containing the options
+ */
+function getListFilesOpts(drive, baseOpts = {}) {
+  // Use default options (e.g., `corpora=drive`) for `files.list` if `drive` is
+  // empty or is "My Drive". Otherwise, use the "drive" corpus and include
+  // `supportsAllDrives` param.
+  const opts = (!drive || isMyDrive(drive))
+    ? baseOpts
+    : {
+      ...baseOpts,
+      corpora: "drive",
+      driveId: getDriveId(drive),
+      includeItemsFromAllDrives: true,
+      supportsAllDrives: true,
+    };
+  return opts;
+}
+
+/**
+ * Returns a file stream from a file URL or file path to be used in Google Drive
+ * API calls
+ *
+ * @param {Object} opts - an object containing options for getting a file stream
+ * @param {String} opts.fileUrl - the url of a file to download to a Readable
+ * stream
+ * @param {String} opts.filePath - the path to a file from which to create a
+ * Readable stream
+ * @returns {stream.Readable} a Readable stream from the file URL or file path
+ */
+async function getFileStream({
+  $, fileUrl, filePath,
+}) {
+  return fileUrl
+    ? (await axios($ ?? this, {
+      url: fileUrl,
+      method: "GET",
+      responseType: "stream",
+    }))
+    : fs.createReadStream(filePath);
+}
+
+/**
+ * Truncate an array of path segments from its base
+ *
+ * @param {String[]} pathArr - the array of path segments
+ * @returns the truncated array whose first element is "..." if truncated
+ */
+function truncatePath(pathArr) {
+  if (pathArr.length <= MAX_FILE_OPTION_PATH_SEGMENTS) {
+    return pathArr;
+  }
+  return [
+    "...",
+    ...pathArr.slice(-1 * (MAX_FILE_OPTION_PATH_SEGMENTS - 1)),
+  ];
+}
+
+/**
+ * Builds an object mapping file IDs to arrays of file/folder ID path segments from the drive's root
+ * folder to each file, using the `file.parents` property and a list of folders in the drive
+ *
+ * @see
+ * {@link https://developers.google.com/drive/api/v3/reference/files Google Drive File Resource}
+ *
+ * @param {object[]} files - the array of files for which to build paths
+ * @param {object[]} folders - the array of folders in the drive
+ * @returns {Object.<string, string[]>} the object mapping file IDs to arrays of path segments
+ */
+function buildFilePaths(files = [], folders = []) {
+  const folderIdToFolder = folders.reduce((acc, cur) => {
+    acc[cur.id] = cur;
+    return acc;
+  }, {});
+  const paths = {};
+  // Recursive function that returns an array of file `id`s representing the path to a file if
+  // requisite parent folders are available (in `file.parents`) to the requesting user, or an array
+  // containing the file ID otherwise
+  const pathToFile = (file) => {
+    if (!file) {
+      // unretrieved folder or root folder
+      return [];
+    }
+    if (paths[file.id] !== undefined) {
+      return paths[file.id];
+    }
+    if (!file.parents) {
+      // file belongs to a different drive and user does not have access to the parent
+      return [
+        file.id,
+      ];
+    }
+    let parentPath;
+    for (const parent of file.parents) {
+      parentPath = pathToFile(folderIdToFolder[parent]);
+      paths[parent] = parentPath;
+      if (parentPath?.[0]) {
+        break;
+      }
+    }
+    return [
+      ...parentPath,
+      file.id,
+    ];
+  };
+  files.forEach((file) => {
+    paths[file.id] = pathToFile(file);
+  });
+  return paths;
+}
+
+/**
+ * Builds an object mapping file IDs to arrays of file/folder name path segments from the drive's
+ * root folder to each file, using the `file.parents` property and a list of folders in the drive
+ *
+ * @param {object[]} files - the array of files for which to build paths
+ * @param {object[]} folders - the array of folders in the drive
+ * @returns {Object.<string, string[]>} the object mapping file IDs to arrays of path segments
+ */
+function buildFileNamePaths(files = [], folders = []) {
+  const fileIdToFile = files.concat(folders).reduce((acc, cur) => {
+    acc[cur.id] = cur;
+    return acc;
+  }, {});
+  const fileIdToPath = buildFilePaths(files, folders);
+  return Object.fromEntries(Object.entries(fileIdToPath).map(([
+    id,
+    path,
+  ]) => ([
+    id,
+    path.filter((id) => fileIdToFile[id]?.name)
+      .map((id) => fileIdToFile[id]?.name),
+  ])));
+}
+
+/**
+ * Gets an object mapping file IDs to string paths from the drive's root folder to each file, if the
+ * file's `parents` are available to the requesting user
+ *
+ * @param {object[]} files - the array of files for which to get file paths
+ * @param {object[]} folders - the array of folders in the drive
+ * @returns {Object.<string, string>} the object mapping file IDs to file paths
+ */
+function getFilePaths(files = [], folders = []) {
+  const fileIdToNamePath = buildFileNamePaths(files, folders);
+  return Object.fromEntries(Object.entries(fileIdToNamePath).map(([
+    id,
+    path,
+  ]) => ([
+    id,
+    truncatePath(path).join(" > "),
+  ])));
+}
+
+/**
+ * Return an object compose of non-empty string valued properties of `obj`
+ *
+ * @param {Object} obj - the source object
+ * @param {String[]} [fromKeys] - keys of properties in `obj` to omit if corresponding value is
+ * empty string, or all keys by default
+ * @returns the new object
+ */
+function omitEmptyStringValues(obj, fromKeys) {
+  return Object.fromEntries(
+    // eslint-disable-next-line multiline-ternary,array-element-newline,array-bracket-newline
+    Object.entries(obj).filter(([ k, v ]) => {
+      return (fromKeys && !fromKeys.includes(k)) || v !== "";
+    }),
+  );
+}
+
+/**
+ * A utility function that accepts a string as an argument and reformats it in
+ * order to remove newline characters and consecutive spaces. Useful when
+ * dealing with very long templated strings that are split into multiple lines.
+ *
+ * @example
+ * // returns "This is a much cleaner string"
+ * toSingleLineString(`
+ *   This is a much
+ *   cleaner string
+ * `);
+ *
+ * @param {string}  multiLineString the input string to reformat
+ * @returns a formatted string based on the content of the input argument,
+ * without newlines and multiple spaces
+ * Source: {@linkcode ../aws/sources/common/utils.mjs utils.mjs}.
+ */
+function toSingleLineString(multiLineString) {
+  return multiLineString
+    .trim()
+    .replace(/\n/g, " ")
+    .replace(/\s{2,}/g, " ");
+}
+
+export {
+  MY_DRIVE_VALUE,
+  isMyDrive,
+  getDriveId,
+  getListFilesOpts,
+  getFileStream,
+  omitEmptyStringValues,
+  toSingleLineString,
+  buildFilePaths,
+  getFilePaths,
+};

package/LICENSE

@@ -1,7 +0,0 @@
-Copyright 2020 Pipedream, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/google_drive.app.js

@@ -1,212 +0,0 @@
-const axios = require("axios");
-const { google } = require("googleapis");
-
-const GOOGLE_DRIVE_UPDATE_TYPES = [
-  "add",
-  "sync",
-  "remove",
-  "update",
-  "trash",
-  "untrash",
-  "change",
-];
-
-module.exports = {
-  type: "app",
-  app: "google_drive",
-  propDefinitions: {
-    watchedDrive: {
-      type: "string",
-      label: "Drive",
-      description: "The drive you want to watch for changes",
-      async options({ page, prevContext }) {
-        const { nextPageToken } = prevContext;
-        return await this.listDrives(nextPageToken);
-      },
-    },
-    updateTypes: {
-      type: "string[]",
-      label: "Types of updates",
-      description:
-        "The types of updates you want to watch for on these files. [See Google's docs](https://developers.google.com/drive/api/v3/push#understanding-drive-api-notification-events).",
-      // https://developers.google.com/drive/api/v3/push#understanding-drive-api-notification-events
-      default: GOOGLE_DRIVE_UPDATE_TYPES,
-      options: GOOGLE_DRIVE_UPDATE_TYPES,
-    },
-    watchForPropertiesChanges: {
-      type: "boolean",
-      label: "Watch for changes to file properties",
-      description:
-        "Watch for changes to [file properties](https://developers.google.com/drive/api/v3/properties) in addition to changes to content. **Defaults to `false`, watching for only changes to content**.",
-      optional: true,
-      default: false,
-    },
-  },
-  methods: {
-    // Returns a drive object authenticated with the user's access token
-    drive() {
-      const auth = new google.auth.OAuth2();
-      auth.setCredentials({ access_token: this.$auth.oauth_access_token });
-      return google.drive({ version: "v3", auth });
-    },
-    // Google's push notifications provide a URL to the resource that changed,
-    // which we can use to fetch the file's metadata. So we use axios here
-    // (vs. the Node client) to get that.
-    async getFileMetadata(url) {
-      return (
-        await axios({
-          method: "GET",
-          headers: {
-            Authorization: `Bearer ${this.$auth.oauth_access_token}`,
-          },
-          url,
-        })
-      ).data;
-    },
-    async getChanges(pageToken, driveId) {
-      const drive = this.drive();
-      // As with many of the methods for Google Drive, we must
-      // pass a request of a different shape when we're requesting
-      // changes for My Drive (null driveId) vs. a shared drive
-      let changeRequest;
-      if (driveId) {
-        changeRequest = {
-          driveId,
-          pageToken,
-          includeItemsFromAllDrives: true,
-          supportsAllDrives: true,
-        };
-      } else {
-        changeRequest = {
-          pageToken,
-        };
-      }
-      const { changes, newStartPageToken } = (
-        await drive.changes.list(changeRequest)
-      ).data;
-
-      // Some changes do not include an associated file object. Return only those that do
-      const changedFiles = changes
-        .map((change) => change.file)
-        .filter((f) => typeof f === "object");
-
-      return {
-        changedFiles,
-        newStartPageToken,
-      };
-    },
-    async getPageToken() {
-      const drive = this.drive();
-      return (await drive.changes.getStartPageToken({})).data.startPageToken;
-    },
-    async listDrives(pageToken) {
-      const drive = this.drive();
-      const resp = await drive.drives.list({ pageToken });
-      const { drives, nextPageToken } = resp.data;
-      // "My Drive" isn't returned from the list of drives,
-      // so we add it to the list and assign it a static
-      // ID that we can refer to when we need.
-      const options = [{ label: "My Drive", value: "myDrive" }];
-      for (const d of drives) {
-        options.push({ label: d.name, value: d.id });
-      }
-      return {
-        options,
-        context: { nextPageToken },
-      };
-    },
-    async listFiles(opts) {
-      const drive = this.drive();
-      // Listing files in My Drive and a shared drive requires
-      // mutually-exclusive options, so we accept an object of
-      // opts from the caller and pass them to the list method.
-      const resp = await drive.files.list(opts);
-      const { files, nextPageToken } = resp.data;
-      const options = files.map((file) => {
-        return { label: file.name, value: file.id };
-      });
-      return {
-        options,
-        context: { nextPageToken },
-      };
-    },
-    _makeWatchRequestBody(id, address) {
-      return {
-        id, // the component-specific channel ID, a UUID
-        type: "web_hook",
-        address, // the component-specific HTTP endpoint
-      };
-    },
-    async watchDrive(id, address, pageToken, driveId) {
-      const drive = this.drive();
-      const requestBody = this._makeWatchRequestBody(id, address);
-
-      // Google expects an entirely different object to be passed
-      // when you make a watch request for My Drive vs. a shared drive
-      // "My Drive" doesn't have a driveId, so if this method is called
-      // without a driveId, we make a watch request for My Drive
-      let watchRequest;
-      if (driveId) {
-        watchRequest = {
-          driveId,
-          pageToken,
-          requestBody,
-          includeItemsFromAllDrives: true,
-          supportsAllDrives: true,
-        };
-      } else {
-        watchRequest = {
-          pageToken,
-          requestBody,
-        };
-      }
-      // When watching for changes to an entire account, we must pass a pageToken,
-      // which points to the moment in time we want to start watching for changes:
-      // https://developers.google.com/drive/api/v3/manage-changes
-      const { expiration, resourceId } = (
-        await drive.changes.watch(watchRequest)
-      ).data;
-      console.log(`Watch request for drive successful, expiry: ${expiration}`);
-      return {
-        expiration: parseInt(expiration),
-        resourceId,
-      };
-    },
-    async watchFile(id, address, fileId) {
-      const drive = this.drive();
-      const requestBody = this._makeWatchRequestBody(id, address);
-      const { expiration, resourceId } = (
-        await drive.files.watch({
-          fileId,
-          requestBody,
-          supportsAllDrives: true,
-        })
-      ).data;
-      console.log(
-        `Watch request for file ${fileId} successful, expiry: ${expiration}`
-      );
-      return {
-        expiration: parseInt(expiration),
-        resourceId,
-      };
-    },
-    async stopNotifications(id, resourceId) {
-      // id = channelID
-      // See https://github.com/googleapis/google-api-nodejs-client/issues/627
-      const drive = this.drive();
-
-      // If for some reason the channel doesn't exist, this throws an error
-      // It's OK for this to fail in those cases, since we'll renew the channel
-      // immediately after trying to stop it if we still want notifications,
-      // so we squash the error, log it, and move on.
-      try {
-        await drive.channels.stop({ resource: { id, resourceId } });
-        console.log(`Stopped push notifications on channel ${id}`);
-      } catch (err) {
-        console.error(
-          `Failed to stop channel ${id} for resource ${resourceId}: ${err}`
-        );
-      }
-    },
-  },
-};