@pipedream/google_drive 0.3.2 → 0.4.1

package/sources/new-or-modified-comments/new-or-modified-comments.mjs

@@ -0,0 +1,104 @@
+// This source processes changes to any files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push and here:
+// https://developers.google.com/drive/api/v3/manage-changes
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+import { GOOGLE_DRIVE_NOTIFICATION_CHANGE } from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-or-modified-comments",
+  name: "New or Modified Comments",
+  description:
+    "Emits a new event any time a file comment is added, modified, or deleted in your linked Google Drive",
+  version: "0.0.9",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  hooks: {
+    ...common.hooks,
+    async activate() {
+      await common.hooks.activate.bind(this)();
+      this._setInitTime(Date.now());
+    },
+    async deactivate() {
+      await common.hooks.deactivate.bind(this)();
+      this._setInitTime(null);
+    },
+  },
+  methods: {
+    ...common.methods,
+    _getInitTime() {
+      return this.db.get("initTime");
+    },
+    _setInitTime(initTime) {
+      this.db.set("initTime", initTime);
+    },
+    _getLastCommentTimeForFile(fileId) {
+      return this.db.get(fileId) || this._getInitTime();
+    },
+    _updateLastCommentTimeForFile(fileId, commentTime) {
+      this.db.set(fileId, commentTime);
+    },
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: commentId,
+        content: summary,
+        modifiedTime: tsString,
+      } = data;
+      const { "x-goog-message-number": eventId } = headers;
+      return {
+        id: `${commentId}-${eventId}`,
+        summary,
+        ts: Date.parse(tsString),
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      for (const file of changedFiles) {
+        const lastCommentTimeForFile = this._getLastCommentTimeForFile(file.id);
+        let maxModifiedTime = lastCommentTimeForFile;
+        const commentsStream = this.googleDrive.listComments(
+          file.id,
+          lastCommentTimeForFile,
+        );
+
+        for await (const comment of commentsStream) {
+          const commentTime = Date.parse(comment.modifiedTime);
+          if (commentTime <= lastCommentTimeForFile) {
+            continue;
+          }
+
+          const eventToEmit = {
+            comment,
+            file,
+            change: {
+              state: headers["x-goog-resource-state"],
+              resourceURI: headers["x-goog-resource-uri"],
+
+              // Additional details about the changes. Possible values: content,
+              // parents, children, permissions.
+              changed: headers["x-goog-changed"],
+            },
+          };
+          const meta = this.generateMeta(comment, headers);
+          this.$emit(eventToEmit, meta);
+
+          maxModifiedTime = Math.max(maxModifiedTime, commentTime);
+          this._updateLastCommentTimeForFile(file.id, maxModifiedTime);
+        }
+      }
+    },
+  },
+};

package/sources/new-or-modified-files/new-or-modified-files.mjs

@@ -0,0 +1,66 @@
+// This source processes changes to any files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push and here:
+// https://developers.google.com/drive/api/v3/manage-changes
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+import {
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-or-modified-files",
+  name: "New or Modified Files",
+  description:
+    "Emits a new event any time any file in your linked Google Drive is added, modified, or deleted",
+  version: "0.0.20",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: fileId,
+        name: summary,
+        modifiedTime: tsString,
+      } = data;
+      const { "x-goog-message-number": eventId } = headers;
+      return {
+        id: `${fileId}-${eventId}`,
+        summary,
+        ts: Date.parse(tsString),
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      for (const file of changedFiles) {
+        const eventToEmit = {
+          file,
+          change: {
+            state: headers["x-goog-resource-state"],
+            resourceURI: headers["x-goog-resource-uri"],
+            changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+          },
+        };
+        const meta = this.generateMeta(file, headers);
+        this.$emit(eventToEmit, meta);
+      }
+    },
+  },
+};

package/sources/new-or-modified-folders/new-or-modified-folders.mjs

@@ -0,0 +1,86 @@
+// This source processes changes to any files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push and here:
+// https://developers.google.com/drive/api/v3/manage-changes
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+import {
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-or-modified-folders",
+  name: "New or Modified Folders",
+  description:
+    "Emits a new event any time any folder in your linked Google Drive is added, modified, or deleted",
+  version: "0.0.9",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    _getLastModifiedTimeForFile(fileId) {
+      return this.db.get(fileId);
+    },
+    _setModifiedTimeForFile(fileId, modifiedTime) {
+      this.db.set(fileId, modifiedTime);
+    },
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, ts) {
+      const {
+        id: fileId,
+        name: summary,
+      } = data;
+      return {
+        id: `${fileId}-${ts}`,
+        summary,
+        ts,
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      const files = changedFiles.filter(
+        // API docs that define Google Drive folders:
+        // https://developers.google.com/drive/api/v3/folder
+        (file) => file.mimeType === "application/vnd.google-apps.folder",
+      );
+
+      for (const file of files) {
+        // The changelog is updated each time a folder is opened. Check the
+        // folder's `modifiedTime` to see if the folder has been modified.
+        const fileInfo = await this.googleDrive.getFile(file.id);
+
+        const lastModifiedTimeForFile = this._getLastModifiedTimeForFile(file.id);
+        const modifiedTime = Date.parse(fileInfo.modifiedTime);
+        if (lastModifiedTimeForFile == modifiedTime) continue;
+
+        const eventToEmit = {
+          file,
+          change: {
+            state: headers["x-goog-resource-state"],
+            resourceURI: headers["x-goog-resource-uri"],
+            changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+          },
+        };
+        const meta = this.generateMeta(file, modifiedTime);
+        this.$emit(eventToEmit, meta);
+
+        this._setModifiedTimeForFile(file.id, modifiedTime);
+      }
+    },
+  },
+};

package/sources/new-shared-drive/new-shared-drive.mjs

@@ -0,0 +1,68 @@
+import googleDrive from "../../google_drive.app.mjs";
+
+export default {
+  key: "google_drive-new-shared-drive",
+  name: "New Shared Drive",
+  description: "Emits a new event any time a shared drive is created.",
+  version: "0.0.9",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    googleDrive,
+    db: "$.service.db",
+    timer: {
+      label: "Polling interval",
+      description: "Interval to poll the Google Drive API for new shared drives",
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 60 * 15, // 30 minutes
+      },
+    },
+  },
+  hooks: {
+    async deploy() {
+      const { drives: initDrives } = await this.googleDrive.listDrivesInPage();
+      for (const drive of initDrives) {
+        const newDrive = await this.googleDrive.getDrive(drive.id);
+        const meta = this.generateMeta(newDrive);
+        this.$emit(newDrive, meta);
+      }
+
+      this._setKnownDrives(initDrives.map((drive) => drive.id));
+    },
+  },
+  methods: {
+    _getKnownDrives() {
+      return this.db.get("driveIds");
+    },
+    _setKnownDrives(driveIds) {
+      this.db.set("driveIds", Array.from(driveIds));
+    },
+    generateMeta(drive) {
+      const ts = new Date(drive.createdTime).getTime();
+      return {
+        id: drive.id,
+        summary: drive.name,
+        ts,
+      };
+    },
+  },
+  async run() {
+    const knownDrives = new Set(this._getKnownDrives());
+    const drivesStream = this.googleDrive.listDrives();
+    for await (const drive of drivesStream) {
+      if (knownDrives.has(drive.id)) {
+        // We've already seen this drive, so we skip it
+        continue;
+      }
+
+      knownDrives.add(drive.id);
+
+      const newDrive = await this.googleDrive.getDrive(drive.id);
+      const meta = this.generateMeta(newDrive);
+      this.$emit(newDrive, meta);
+    }
+
+    this._setKnownDrives(knownDrives);
+  },
+};

package/utils.mjs

@@ -0,0 +1,267 @@
+import fs from "fs";
+import { axios } from "@pipedream/platform";
+import {
+  MY_DRIVE_VALUE,
+  LEGACY_MY_DRIVE_VALUE,
+  MAX_FILE_OPTION_PATH_SEGMENTS,
+} from "./constants.mjs";
+
+/**
+ * Returns whether the specified drive ID corresponds to the authenticated
+ * user's My Drive or not
+ *
+ * @param {String} drive the ID value of a Google Drive
+ * @returns `true` only when the specified drive is the user's 'My Drive'
+ */
+function isMyDrive(drive) {
+  return drive === MY_DRIVE_VALUE || drive === LEGACY_MY_DRIVE_VALUE;
+}
+
+/**
+ * Returns a valid Google Drive ID to be used in Google Drive API calls
+ *
+ * @param {String} drive the ID value of a Google Drive, as provided by the
+ * `drive` prop definition of this app
+ * @returns the proper Google Drive ID to be used in Google Drive API calls
+ */
+function getDriveId(drive) {
+  return isMyDrive(drive)
+    ? null
+    : drive;
+}
+
+/**
+ * Gets an options object to be used in functions that use the
+ * [the `drive.drives.list` API](https://bit.ly/3AiWE1x).
+ *
+ * @param {String} drive the ID value of a Google Drive, as provided by the
+ * `drive` prop definition of this app
+ * @param {object} [baseOpts = {}] - an object containing extra/optional
+ * parameters to be fed to the GDrive API call, as defined in [the API
+ * docs](https://bit.ly/3AnQDR1)
+ *
+ * @returns an object containing the options
+ */
+function getListFilesOpts(drive, baseOpts = {}) {
+  // Use default options (e.g., `corpora=drive`) for `files.list` if `drive` is
+  // empty or is "My Drive". Otherwise, use the "drive" corpus and include
+  // `supportsAllDrives` param.
+  const opts = (!drive || isMyDrive(drive))
+    ? baseOpts
+    : {
+      ...baseOpts,
+      corpora: "drive",
+      driveId: getDriveId(drive),
+      includeItemsFromAllDrives: true,
+      supportsAllDrives: true,
+    };
+  return opts;
+}
+
+/**
+ * Returns a file stream from a file URL or file path to be used in Google Drive
+ * API calls
+ *
+ * @param {Object} opts - an object containing options for getting a file stream
+ * @param {String} opts.fileUrl - the url of a file to download to a Readable
+ * stream
+ * @param {String} opts.filePath - the path to a file from which to create a
+ * Readable stream
+ * @returns {stream.Readable} a Readable stream from the file URL or file path
+ */
+async function getFileStream({
+  $, fileUrl, filePath,
+}) {
+  return fileUrl
+    ? (await axios($ ?? this, {
+      url: fileUrl,
+      method: "GET",
+      responseType: "stream",
+    }))
+    : fs.createReadStream(filePath);
+}
+
+function streamToBuffer(stream) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    stream.on("data", (chunk) => {
+      chunks.push(chunk);
+    }).on("end", () => {
+      resolve(Buffer.concat(chunks));
+    // eslint-disable-next-line newline-per-chained-call
+    }).on("error", (err) => {
+      reject(err);
+    });
+  });
+}
+
+function byteToMB(byte) {
+  return byte / 1024 / 1024;
+}
+
+/**
+ * Truncate an array of path segments from its base
+ *
+ * @param {String[]} pathArr - the array of path segments
+ * @returns the truncated array whose first element is "..." if truncated
+ */
+function truncatePath(pathArr) {
+  if (pathArr.length <= MAX_FILE_OPTION_PATH_SEGMENTS) {
+    return pathArr;
+  }
+  return [
+    "...",
+    ...pathArr.slice(-1 * (MAX_FILE_OPTION_PATH_SEGMENTS - 1)),
+  ];
+}
+
+/**
+ * Builds an object mapping file IDs to arrays of file/folder ID path segments from the drive's root
+ * folder to each file, using the `file.parents` property and a list of folders in the drive
+ *
+ * @see
+ * {@link https://developers.google.com/drive/api/v3/reference/files Google Drive File Resource}
+ *
+ * @param {object[]} files - the array of files for which to build paths
+ * @param {object[]} folders - the array of folders in the drive
+ * @returns {Object.<string, string[]>} the object mapping file IDs to arrays of path segments
+ */
+function buildFilePaths(files = [], folders = []) {
+  const folderIdToFolder = folders.reduce((acc, cur) => {
+    acc[cur.id] = cur;
+    return acc;
+  }, {});
+  const paths = {};
+  // Recursive function that returns an array of file `id`s representing the path to a file if
+  // requisite parent folders are available (in `file.parents`) to the requesting user, or an array
+  // containing the file ID otherwise
+  const pathToFile = (file) => {
+    if (!file) {
+      // unretrieved folder or root folder
+      return [];
+    }
+    if (paths[file.id] !== undefined) {
+      return paths[file.id];
+    }
+    if (!file.parents) {
+      // file belongs to a different drive and user does not have access to the parent
+      return [
+        file.id,
+      ];
+    }
+    let parentPath;
+    for (const parent of file.parents) {
+      parentPath = pathToFile(folderIdToFolder[parent]);
+      paths[parent] = parentPath;
+      if (parentPath?.[0]) {
+        break;
+      }
+    }
+    return [
+      ...parentPath,
+      file.id,
+    ];
+  };
+  files.forEach((file) => {
+    paths[file.id] = pathToFile(file);
+  });
+  return paths;
+}
+
+/**
+ * Builds an object mapping file IDs to arrays of file/folder name path segments from the drive's
+ * root folder to each file, using the `file.parents` property and a list of folders in the drive
+ *
+ * @param {object[]} files - the array of files for which to build paths
+ * @param {object[]} folders - the array of folders in the drive
+ * @returns {Object.<string, string[]>} the object mapping file IDs to arrays of path segments
+ */
+function buildFileNamePaths(files = [], folders = []) {
+  const fileIdToFile = files.concat(folders).reduce((acc, cur) => {
+    acc[cur.id] = cur;
+    return acc;
+  }, {});
+  const fileIdToPath = buildFilePaths(files, folders);
+  return Object.fromEntries(Object.entries(fileIdToPath).map(([
+    id,
+    path,
+  ]) => ([
+    id,
+    path.filter((id) => fileIdToFile[id]?.name)
+      .map((id) => fileIdToFile[id]?.name),
+  ])));
+}
+
+/**
+ * Gets an object mapping file IDs to string paths from the drive's root folder to each file, if the
+ * file's `parents` are available to the requesting user
+ *
+ * @param {object[]} files - the array of files for which to get file paths
+ * @param {object[]} folders - the array of folders in the drive
+ * @returns {Object.<string, string>} the object mapping file IDs to file paths
+ */
+function getFilePaths(files = [], folders = []) {
+  const fileIdToNamePath = buildFileNamePaths(files, folders);
+  return Object.fromEntries(Object.entries(fileIdToNamePath).map(([
+    id,
+    path,
+  ]) => ([
+    id,
+    truncatePath(path).join(" > "),
+  ])));
+}
+
+/**
+ * Return an object compose of non-empty string valued properties of `obj`
+ *
+ * @param {Object} obj - the source object
+ * @param {String[]} [fromKeys] - keys of properties in `obj` to omit if corresponding value is
+ * empty string, or all keys by default
+ * @returns the new object
+ */
+function omitEmptyStringValues(obj, fromKeys) {
+  return Object.fromEntries(
+    // eslint-disable-next-line multiline-ternary,array-element-newline,array-bracket-newline
+    Object.entries(obj).filter(([ k, v ]) => {
+      return (fromKeys && !fromKeys.includes(k)) || v !== "";
+    }),
+  );
+}
+
+/**
+ * A utility function that accepts a string as an argument and reformats it in
+ * order to remove newline characters and consecutive spaces. Useful when
+ * dealing with very long templated strings that are split into multiple lines.
+ *
+ * @example
+ * // returns "This is a much cleaner string"
+ * toSingleLineString(`
+ *   This is a much
+ *   cleaner string
+ * `);
+ *
+ * @param {string}  multiLineString the input string to reformat
+ * @returns a formatted string based on the content of the input argument,
+ * without newlines and multiple spaces
+ * Source: {@linkcode ../aws/sources/common/utils.mjs utils.mjs}.
+ */
+function toSingleLineString(multiLineString) {
+  return multiLineString
+    .trim()
+    .replace(/\n/g, " ")
+    .replace(/\s{2,}/g, " ");
+}
+
+export {
+  MY_DRIVE_VALUE,
+  isMyDrive,
+  getDriveId,
+  getListFilesOpts,
+  getFileStream,
+  omitEmptyStringValues,
+  toSingleLineString,
+  buildFilePaths,
+  getFilePaths,
+  streamToBuffer,
+  byteToMB,
+};