@pipedream/sharepoint 0.7.2 → 0.8.1

package/sources/updated-file-instant/updated-file-instant.mjs

@@ -0,0 +1,407 @@
+import { randomUUID } from "crypto";
+import sharepoint from "../../sharepoint.app.mjs";
+import { WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS } from "../../common/constants.mjs";
+
+export default {
+  key: "sharepoint-updated-file-instant",
+  name: "File Updated or Deleted (Instant)",
+  description: "Emit a new event when specific files are updated or deleted in a SharePoint document library",
+  version: "0.0.2",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    sharepoint,
+    db: "$.service.db",
+    http: {
+      type: "$.interface.http",
+      customResponse: true,
+    },
+    timer: {
+      label: "Subscription renewal schedule",
+      description:
+        "Microsoft Graph subscriptions expire after 30 days. " +
+        "This timer automatically renews the subscription. " +
+        "**You should not need to modify this schedule.**",
+      type: "$.interface.timer",
+      static: {
+        intervalSeconds: WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS,
+      },
+      hidden: true,
+    },
+    siteId: {
+      propDefinition: [
+        sharepoint,
+        "siteId",
+      ],
+    },
+    driveId: {
+      propDefinition: [
+        sharepoint,
+        "driveId",
+        (c) => ({
+          siteId: c.siteId,
+        }),
+      ],
+    },
+    folderId: {
+      propDefinition: [
+        sharepoint,
+        "folderId",
+        (c) => ({
+          siteId: c.siteId,
+          driveId: c.driveId,
+        }),
+      ],
+      description: "Optional: Select a folder to browse files from. Leave empty to browse from the drive root. " +
+        "This helps you navigate to the files you want to monitor.",
+    },
+    fileIds: {
+      propDefinition: [
+        sharepoint,
+        "fileIds",
+        (c) => ({
+          siteId: c.siteId,
+          driveId: c.driveId,
+          folderId: c.folderId,
+        }),
+      ],
+      label: "Files to Monitor",
+      description:
+        "Select one or more files to monitor for changes. " +
+        "You'll receive a real-time event whenever any of these files are modified or deleted.\n\n" +
+        "**Important:** Only the selected files will trigger events. Changes to other files in the drive will be ignored. " +
+        "This ensures you only receive notifications for the documents you care about.",
+    },
+  },
+  hooks: {
+    async activate() {
+      // Generate a unique client state for validating incoming webhooks
+      const clientState = randomUUID();
+
+      // Resolve wrapped prop values
+      const driveId = this.sharepoint.resolveWrappedValue(this.driveId);
+
+      // Create subscription on the drive root
+      // We'll filter to specific files when notifications arrive
+      const subscription = await this.sharepoint.createSubscription({
+        resource: `drives/${driveId}/root`,
+        notificationUrl: this.http.endpoint,
+        changeType: "updated",
+        clientState,
+      });
+
+      console.log(
+        `Created subscription ${subscription.id}, expires: ${subscription.expirationDateTime}`,
+      );
+
+      // Store subscription metadata
+      this._setSubscription({
+        id: subscription.id,
+        expirationDateTime: subscription.expirationDateTime,
+        clientState,
+      });
+
+      // Store the file IDs we're monitoring (unwrap labeled values and parse JSON)
+      const wrappedFileIds = this.sharepoint.resolveWrappedArrayValues(this.fileIds);
+      // Parse JSON strings to extract just the IDs, handle objects, and trim strings
+      const fileIds = wrappedFileIds.map((fileId) => {
+        // Handle object values directly
+        if (typeof fileId === "object" && fileId !== null) {
+          if (!fileId.id) {
+            console.log(`Warning: Object fileId missing 'id' field: ${JSON.stringify(fileId)}`);
+            return String(fileId);
+          }
+          return fileId.id;
+        }
+
+        // Handle string values - trim whitespace first
+        if (typeof fileId === "string") {
+          const trimmedFileId = fileId.trim();
+          if (trimmedFileId.startsWith("{")) {
+            try {
+              const parsed = JSON.parse(trimmedFileId);
+              if (!parsed || !parsed.id) {
+                throw new Error("Parsed object missing 'id' field");
+              }
+              return parsed.id;
+            } catch (e) {
+              console.log(`Warning: Failed to parse fileId: ${trimmedFileId}, error: ${e.message}`);
+              return trimmedFileId;
+            }
+          }
+          return trimmedFileId;
+        }
+
+        return fileId;
+      });
+      this._setMonitoredFileIds(fileIds);
+
+      // Initialize delta tracking - get current state so we only see future changes
+      const deltaResponse = await this.sharepoint.getDriveDelta({
+        driveId,
+      });
+      // Follow pagination to get the final deltaLink
+      let nextLink = deltaResponse["@odata.nextLink"];
+      let deltaLink = deltaResponse["@odata.deltaLink"];
+      while (nextLink && !deltaLink) {
+        const nextResponse = await this.sharepoint.getDriveDelta({
+          driveId,
+          deltaLink: nextLink,
+        });
+        nextLink = nextResponse["@odata.nextLink"];
+        deltaLink = nextResponse["@odata.deltaLink"];
+      }
+      this._setDeltaLink(deltaLink);
+      console.log("Initialized delta tracking");
+    },
+    async deactivate() {
+      const subscription = this._getSubscription();
+      if (subscription?.id) {
+        try {
+          await this.sharepoint.deleteSubscription({
+            subscriptionId: subscription.id,
+          });
+          console.log(`Deleted subscription ${subscription.id}`);
+        } catch (err) {
+          console.log(`Error deleting subscription: ${err.message}`);
+        }
+      }
+
+      // Clear stored state
+      this._setSubscription(null);
+      this._setMonitoredFileIds(null);
+      this._setDeltaLink(null);
+    },
+  },
+  methods: {
+    _getSubscription() {
+      return this.db.get("subscription");
+    },
+    _setSubscription(subscription) {
+      this.db.set("subscription", subscription);
+    },
+    _getMonitoredFileIds() {
+      return this.db.get("monitoredFileIds") || [];
+    },
+    _setMonitoredFileIds(fileIds) {
+      this.db.set("monitoredFileIds", fileIds);
+    },
+    _getDeltaLink() {
+      return this.db.get("deltaLink");
+    },
+    _setDeltaLink(deltaLink) {
+      this.db.set("deltaLink", deltaLink);
+    },
+    async renewSubscription() {
+      const subscription = this._getSubscription();
+      if (!subscription?.id) {
+        console.log("No subscription to renew");
+        return {
+          success: true,
+          skipped: true,
+        };
+      }
+
+      try {
+        const updated = await this.sharepoint.updateSubscription({
+          subscriptionId: subscription.id,
+        });
+
+        this._setSubscription({
+          ...subscription,
+          expirationDateTime: updated.expirationDateTime,
+        });
+
+        console.log(
+          `Renewed subscription ${subscription.id}, expires: ${updated.expirationDateTime}`,
+        );
+        return {
+          success: true,
+        };
+      } catch (error) {
+        const status = error.response?.status;
+
+        // Subscription not found - needs recreation
+        if (status === 404) {
+          console.log("Subscription not found, will recreate...");
+          return {
+            success: false,
+            shouldRecreate: true,
+          };
+        }
+
+        // Auth errors - don't retry
+        if ([
+          401,
+          403,
+        ].includes(status)) {
+          console.error(`Auth error renewing subscription: ${error.message}`);
+          return {
+            success: false,
+            shouldRecreate: false,
+          };
+        }
+
+        // Other errors - log but don't recreate (will retry on next timer tick)
+        console.error(`Error renewing subscription: ${error.message}`);
+        return {
+          success: false,
+          shouldRecreate: false,
+        };
+      }
+    },
+    generateMeta(file) {
+      // Use lastModifiedDateTime for updated files, deletedDateTime for deleted files
+      // Fall back to current time only if neither is available
+      let ts;
+      if (file.lastModifiedDateTime) {
+        ts = Date.parse(file.lastModifiedDateTime);
+      } else if (file.deletedDateTime) {
+        ts = Date.parse(file.deletedDateTime);
+      } else {
+        ts = Date.now();
+      }
+
+      const action = file.deleted
+        ? "deleted"
+        : "updated";
+      return {
+        id: `${file.id}-${ts}`,
+        summary: `File ${action}: ${file.name}`,
+        ts,
+      };
+    },
+  },
+  async run(event) {
+    // Handle subscription validation request from Microsoft
+    // https://learn.microsoft.com/en-us/graph/change-notifications-delivery-webhooks#notificationurl-validation
+    if (event.query?.validationToken) {
+      console.log("Responding to validation request");
+      this.http.respond({
+        status: 200,
+        headers: {
+          "Content-Type": "text/plain",
+        },
+        body: event.query.validationToken,
+      });
+      return;
+    }
+
+    // Handle timer event - renew subscription
+    if (event.timestamp) {
+      const result = await this.renewSubscription();
+
+      if (!result.success && result.shouldRecreate) {
+        console.log("Recreating subscription...");
+        await this.hooks.activate.call(this);
+      }
+      return;
+    }
+
+    // Handle webhook notification
+    const { body } = event;
+
+    if (!body?.value?.length) {
+      console.log("No notifications in webhook payload");
+      this.http.respond({
+        status: 202,
+        body: "",
+      });
+      return;
+    }
+
+    // Filter to only valid notifications for this subscription
+    const subscription = this._getSubscription();
+    const clientState = subscription?.clientState;
+
+    const validNotifications = body.value.filter((notification) => {
+      if (notification.clientState !== clientState) {
+        console.log(
+          `Warning: Ignoring notification with unexpected clientState: ${notification.clientState}`,
+        );
+        return false;
+      }
+      return true;
+    });
+
+    if (validNotifications.length === 0) {
+      console.log("No valid notifications after clientState filtering");
+      this.http.respond({
+        status: 202,
+        body: "",
+      });
+      return;
+    }
+
+    // Acknowledge receipt after validation
+    this.http.respond({
+      status: 202,
+      body: "",
+    });
+
+    // Use delta API to find what actually changed
+    const driveId = this.sharepoint.resolveWrappedValue(this.driveId);
+    const monitoredFileIds = this._getMonitoredFileIds();
+    let deltaLink = this._getDeltaLink();
+
+    console.log("Monitored file IDs:", JSON.stringify(monitoredFileIds));
+    console.log("Fetching delta changes...");
+
+    const changedFiles = [];
+    let hasMore = true;
+
+    while (hasMore) {
+      const deltaResponse = await this.sharepoint.getDriveDelta({
+        driveId,
+        deltaLink,
+      });
+
+      // Find files that changed and are in our monitored list
+      // Include both updated files (item.file) and deleted files (item.deleted)
+      for (const item of deltaResponse.value || []) {
+        if ((item.file || item.deleted) && monitoredFileIds.includes(item.id)) {
+          changedFiles.push(item);
+        }
+      }
+
+      // Update for next iteration or final storage
+      if (deltaResponse["@odata.nextLink"]) {
+        deltaLink = deltaResponse["@odata.nextLink"];
+      } else {
+        deltaLink = deltaResponse["@odata.deltaLink"];
+        hasMore = false;
+      }
+    }
+
+    // Store the new deltaLink for next time
+    this._setDeltaLink(deltaLink);
+
+    console.log(`Found ${changedFiles.length} changed monitored files`);
+
+    // Emit events for each changed file
+    for (const file of changedFiles) {
+      // Delta response may not include downloadUrl - fetch fresh if needed
+      // Skip fetching download URL for deleted files (they no longer exist)
+      let downloadUrl = file["@microsoft.graph.downloadUrl"];
+      if (!downloadUrl && !file.deleted) {
+        try {
+          const freshFile = await this.sharepoint.getDriveItem({
+            driveId,
+            fileId: file.id,
+          });
+          downloadUrl = freshFile["@microsoft.graph.downloadUrl"];
+        } catch (err) {
+          console.log(`Could not fetch download URL for ${file.name}: ${err.message}`);
+        }
+      }
+
+      this.$emit(
+        {
+          file,
+          downloadUrl,
+        },
+        this.generateMeta(file),
+      );
+    }
+  },
+};

package/sources/updated-list-item/updated-list-item.mjs

@@ -5,7 +5,7 @@ export default {
   key: "sharepoint-updated-list-item",
   name: "Updated List Item",
   description: "Emit new event when a list item is updated in Microsoft Sharepoint.",
-  version: "0.0.9",
+  version: "0.0.10",
   type: "source",
   dedupe: "unique",
   props: {

package/actions/select-files/select-files.mjs

@@ -1,198 +0,0 @@
-import sharepoint from "../../sharepoint.app.mjs";
-
-export default {
-  key: "sharepoint-select-files",
-  name: "Select Files",
-  description: "A file picker action that allows browsing and selecting one or more files from SharePoint. Returns the selected files' metadata including pre-authenticated download URLs. [See the documentation](https://learn.microsoft.com/en-us/graph/api/driveitem-get)",
-  version: "0.0.3",
-  type: "action",
-  annotations: {
-    destructiveHint: false,
-    openWorldHint: true,
-    readOnlyHint: true,
-  },
-  props: {
-    sharepoint,
-    siteId: {
-      propDefinition: [
-        sharepoint,
-        "siteId",
-      ],
-      withLabel: true,
-    },
-    driveId: {
-      propDefinition: [
-        sharepoint,
-        "driveId",
-        (c) => ({
-          siteId: c.siteId?.value || c.siteId,
-        }),
-      ],
-      withLabel: true,
-    },
-    folderId: {
-      propDefinition: [
-        sharepoint,
-        "folderId",
-        (c) => ({
-          siteId: c.siteId?.value || c.siteId,
-          driveId: c.driveId?.value || c.driveId,
-        }),
-      ],
-      label: "Folder",
-      description: "The folder to browse. Leave empty to browse the root of the drive.",
-      optional: true,
-      withLabel: true,
-    },
-    fileOrFolderIds: {
-      propDefinition: [
-        sharepoint,
-        "fileOrFolderId",
-        (c) => ({
-          siteId: c.siteId?.value || c.siteId,
-          driveId: c.driveId?.value || c.driveId,
-          folderId: c.folderId?.value || c.folderId,
-        }),
-      ],
-      type: "string[]",
-      label: "Files or Folders",
-      description: "Select one or more files, or select a folder and click 'Refresh Fields' to browse into it",
-      withLabel: true,
-    },
-  },
-  methods: {
-    resolveValue(prop) {
-      if (!prop) return null;
-      if (typeof prop === "object" && prop?.value) {
-        return prop.value;
-      }
-      return prop;
-    },
-    parseFileOrFolder(value) {
-      if (!value) return null;
-      const resolved = this.resolveValue(value);
-      try {
-        return JSON.parse(resolved);
-      } catch {
-        return {
-          id: resolved,
-          isFolder: false,
-        };
-      }
-    },
-    parseFileOrFolderList(values) {
-      if (!values) return [];
-      const list = Array.isArray(values)
-        ? values
-        : [
-          values,
-        ];
-      return list.map((v) => this.parseFileOrFolder(v)).filter(Boolean);
-    },
-  },
-  async run({ $ }) {
-    const selections = this.parseFileOrFolderList(this.fileOrFolderIds);
-
-    if (selections.length === 0) {
-      throw new Error("Please select at least one file or folder");
-    }
-
-    const siteId = this.resolveValue(this.siteId);
-    const driveId = this.resolveValue(this.driveId);
-
-    // Separate files and folders
-    const folders = selections.filter((s) => s.isFolder);
-    const files = selections.filter((s) => !s.isFolder);
-
-    // If only folders selected, return folder info
-    if (files.length === 0 && folders.length > 0) {
-      const folderNames = folders.map((f) => f.name).join(", ");
-      $.export("$summary", `Selected ${folders.length} folder(s): ${folderNames}. Set one as the Folder ID and refresh to browse its contents.`);
-      return {
-        type: "folders",
-        folders: folders.map((f) => ({
-          id: f.id,
-          name: f.name,
-        })),
-        message: "To browse a folder, set it as the folderId and reload props",
-      };
-    }
-
-    // Fetch metadata for all selected files in parallel, handling individual failures
-    const settledResults = await Promise.allSettled(
-      files.map(async (selected) => {
-        const file = await this.sharepoint.getDriveItem({
-          $,
-          siteId,
-          driveId,
-          fileId: selected.id,
-        });
-
-        const downloadUrl = file["@microsoft.graph.downloadUrl"];
-
-        return {
-          ...file,
-          downloadUrl,
-          _meta: {
-            siteId,
-            driveId,
-            fileId: selected.id,
-          },
-        };
-      }),
-    );
-
-    // Separate successful and failed results
-    const fileResults = [];
-    const errors = [];
-
-    settledResults.forEach((result, index) => {
-      if (result.status === "fulfilled") {
-        fileResults.push(result.value);
-      } else {
-        const selected = files[index];
-        const errorMessage = result.reason?.message || String(result.reason);
-        console.error(`Failed to fetch file ${selected.id} (${selected.name}): ${errorMessage}`);
-        errors.push({
-          fileId: selected.id,
-          fileName: selected.name,
-          error: errorMessage,
-        });
-      }
-    });
-
-    // If all files failed, throw an error
-    if (fileResults.length === 0 && errors.length > 0) {
-      throw new Error(`Failed to fetch all selected files: ${errors.map((e) => e.fileName).join(", ")}`);
-    }
-
-    // If single file, return it directly for backwards compatibility
-    if (fileResults.length === 1 && folders.length === 0 && errors.length === 0) {
-      $.export("$summary", `Selected file: ${fileResults[0].name}`);
-      return fileResults[0];
-    }
-
-    // Multiple files: return as array
-    const fileNames = fileResults.map((f) => f.name).join(", ");
-    const summaryParts = [
-      `Selected ${fileResults.length} file(s): ${fileNames}`,
-    ];
-    if (errors.length > 0) {
-      summaryParts.push(`Failed to fetch ${errors.length} file(s): ${errors.map((e) => e.fileName).join(", ")}`);
-    }
-    $.export("$summary", summaryParts.join(". "));
-
-    return {
-      files: fileResults,
-      ...(errors.length > 0 && {
-        errors,
-      }),
-      ...(folders.length > 0 && {
-        folders: folders.map((f) => ({
-          id: f.id,
-          name: f.name,
-        })),
-      }),
-    };
-  },
-};