@pipedream/google_drive 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/actions/add-comment/add-comment.mjs +1 -1
- package/actions/add-file-sharing-preference/add-file-sharing-preference.mjs +1 -1
- package/actions/copy-file/copy-file.mjs +1 -1
- package/actions/create-file-from-template/create-file-from-template.mjs +1 -1
- package/actions/create-file-from-text/create-file-from-text.mjs +1 -1
- package/actions/create-folder/create-folder.mjs +1 -1
- package/actions/create-shared-drive/create-shared-drive.mjs +1 -1
- package/actions/create-text-file/create-text-file.mjs +1 -1
- package/actions/delete-comment/delete-comment.mjs +1 -1
- package/actions/delete-file/delete-file.mjs +1 -1
- package/actions/delete-reply/delete-reply.mjs +1 -1
- package/actions/delete-shared-drive/delete-shared-drive.mjs +1 -1
- package/actions/download-file/download-file.mjs +26 -7
- package/actions/find-file/find-file.mjs +1 -1
- package/actions/find-folder/find-folder.mjs +1 -1
- package/actions/find-forms/find-forms.mjs +1 -1
- package/actions/find-spreadsheets/find-spreadsheets.mjs +1 -1
- package/actions/get-comment/get-comment.mjs +1 -1
- package/actions/get-current-user/get-current-user.mjs +1 -1
- package/actions/get-file/get-file.mjs +1 -1
- package/actions/get-file-by-id/get-file-by-id.mjs +1 -1
- package/actions/get-folder-id-for-path/get-folder-id-for-path.mjs +1 -1
- package/actions/get-reply/get-reply.mjs +1 -1
- package/actions/get-shared-drive/get-shared-drive.mjs +1 -1
- package/actions/get-user-details/get-user-details.mjs +1 -1
- package/actions/is-folder-ancestor/is-folder-ancestor.mjs +1 -1
- package/actions/list-access-proposals/list-access-proposals.mjs +1 -1
- package/actions/list-comments/list-comments.mjs +1 -1
- package/actions/list-files/list-files.mjs +1 -1
- package/actions/list-mime-type-options/list-mime-type-options.mjs +1 -1
- package/actions/list-permissions/list-permissions.mjs +1 -1
- package/actions/list-replies/list-replies.mjs +1 -1
- package/actions/list-shared-drives/list-shared-drives.mjs +1 -1
- package/actions/list-theme-id-options/list-theme-id-options.mjs +1 -1
- package/actions/move-file/move-file.mjs +1 -1
- package/actions/move-file-to-trash/move-file-to-trash.mjs +1 -1
- package/actions/remove-file-sharing-permission/remove-file-sharing-permission.mjs +1 -1
- package/actions/remove-sharing/remove-sharing.mjs +1 -1
- package/actions/reply-to-comment/reply-to-comment.mjs +1 -1
- package/actions/resolve-access-proposal/resolve-access-proposal.mjs +1 -1
- package/actions/resolve-comment/resolve-comment.mjs +1 -1
- package/actions/search-files/search-files.mjs +1 -1
- package/actions/search-shared-drives/search-shared-drives.mjs +1 -1
- package/actions/share-file/share-file.mjs +1 -1
- package/actions/trash-file/trash-file.mjs +1 -1
- package/actions/update-comment/update-comment.mjs +1 -1
- package/actions/update-file/update-file.mjs +1 -1
- package/actions/update-reply/update-reply.mjs +1 -1
- package/actions/update-shared-drive/update-shared-drive.mjs +1 -1
- package/actions/upload-file/upload-file.mjs +1 -1
- package/common/constants.mjs +22 -12
- package/common/utils.mjs +32 -0
- package/google_drive.app.mjs +90 -11
- package/package.json +1 -1
- package/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs +9 -6
- package/sources/changes-to-specific-files/changes-to-specific-files.mjs +1 -1
- package/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs +9 -6
- package/sources/common-dedupe-changes.mjs +22 -11
- package/sources/common-webhook.mjs +104 -40
- package/sources/new-access-proposal/new-access-proposal.mjs +1 -1
- package/sources/new-files-instant/new-files-instant.mjs +4 -1
- package/sources/new-files-instant-polling/new-files-instant-polling.mjs +38 -47
- package/sources/new-files-shared-drive/new-files-shared-drive.mjs +2 -16
- package/sources/new-or-modified-comments/new-or-modified-comments.mjs +10 -7
- package/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs +1 -1
- package/sources/new-or-modified-files/new-or-modified-files.mjs +20 -1
- package/sources/new-or-modified-files-polling/new-or-modified-files-polling.mjs +37 -36
- package/sources/new-or-modified-folders/new-or-modified-folders.mjs +52 -50
- package/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs +55 -63
- package/sources/new-shared-drive/new-shared-drive.mjs +1 -1
- package/sources/new-spreadsheet/new-spreadsheet.mjs +1 -1
- package/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs +38 -47
package/common/constants.mjs
CHANGED
|
@@ -122,17 +122,10 @@ const LEGACY_MY_DRIVE_VALUE = "myDrive";
|
|
|
122
122
|
*/
|
|
123
123
|
const WEBHOOK_SUBSCRIPTION_EXPIRATION_TIME_MILLISECONDS = 24 * 60 * 60 * 1000;
|
|
124
124
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
* due to an expired subscription not being renewed on time.
|
|
130
|
-
*
|
|
131
|
-
* More information can be found in the API docs:
|
|
132
|
-
* https://developers.google.com/drive/api/v3/push#optional-properties
|
|
133
|
-
*/
|
|
134
|
-
const WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS =
|
|
135
|
-
(WEBHOOK_SUBSCRIPTION_EXPIRATION_TIME_MILLISECONDS * 0.95) / 1000;
|
|
125
|
+
// The timer also resumes capped backlogs, not just renewals
|
|
126
|
+
const WEBHOOK_TIMER_INTERVAL_SECONDS = 10 * 60;
|
|
127
|
+
|
|
128
|
+
const WEBHOOK_RENEWAL_WINDOW_MILLISECONDS = 60 * 60 * 1000;
|
|
136
129
|
|
|
137
130
|
/**
|
|
138
131
|
* The maximum number of path segments to include in an option label for a prop whose value is a
|
|
@@ -273,6 +266,16 @@ const RATE_LIMIT_ERROR_REASONS = [
|
|
|
273
266
|
|
|
274
267
|
const CHANGED_FILE_FIELDS = "kind,id,name,mimeType,parents,createdTime,modifiedTime,trashed,version,size,md5Checksum,webViewLink,lastModifyingUser";
|
|
275
268
|
|
|
269
|
+
const CHANGE_FILTER_FILE_FIELDS = "id,name,mimeType,parents,createdTime,modifiedTime";
|
|
270
|
+
|
|
271
|
+
const MAX_CHANGES_PAGES_PER_RUN = 10;
|
|
272
|
+
const DEFAULT_MAX_EMITS_PER_RUN = 1000;
|
|
273
|
+
const MAX_EMITS_PER_RUN = 5000;
|
|
274
|
+
|
|
275
|
+
const MAX_FILE_INTERVAL_ENTRIES = 2000;
|
|
276
|
+
|
|
277
|
+
const MASS_CHANGE_MAX_AGE_MILLISECONDS = 60 * 60 * 1000;
|
|
278
|
+
|
|
276
279
|
/** Google Workspace types that `stashFile` can export to PDF. */
|
|
277
280
|
const PDF_EXPORTABLE_MIME_TYPES = [
|
|
278
281
|
"application/vnd.google-apps.document",
|
|
@@ -294,7 +297,8 @@ export {
|
|
|
294
297
|
MY_DRIVE_VALUE,
|
|
295
298
|
LEGACY_MY_DRIVE_VALUE,
|
|
296
299
|
WEBHOOK_SUBSCRIPTION_EXPIRATION_TIME_MILLISECONDS,
|
|
297
|
-
|
|
300
|
+
WEBHOOK_TIMER_INTERVAL_SECONDS,
|
|
301
|
+
WEBHOOK_RENEWAL_WINDOW_MILLISECONDS,
|
|
298
302
|
MAX_FILE_OPTION_PATH_SEGMENTS,
|
|
299
303
|
PAGINATION_TOKEN_FIELD,
|
|
300
304
|
GOOGLE_DRIVE_MIME_TYPE_PREFIX,
|
|
@@ -328,5 +332,11 @@ export {
|
|
|
328
332
|
RETRYABLE_STATUS_CODES,
|
|
329
333
|
RATE_LIMIT_ERROR_REASONS,
|
|
330
334
|
CHANGED_FILE_FIELDS,
|
|
335
|
+
CHANGE_FILTER_FILE_FIELDS,
|
|
336
|
+
MAX_CHANGES_PAGES_PER_RUN,
|
|
337
|
+
DEFAULT_MAX_EMITS_PER_RUN,
|
|
338
|
+
MAX_EMITS_PER_RUN,
|
|
339
|
+
MAX_FILE_INTERVAL_ENTRIES,
|
|
340
|
+
MASS_CHANGE_MAX_AGE_MILLISECONDS,
|
|
331
341
|
PDF_EXPORTABLE_MIME_TYPES,
|
|
332
342
|
};
|
package/common/utils.mjs
CHANGED
|
@@ -349,6 +349,37 @@ function sanitizeFileName(name) {
|
|
|
349
349
|
return safe;
|
|
350
350
|
}
|
|
351
351
|
|
|
352
|
+
/**
|
|
353
|
+
* Minimal inline worker pool. Runs `worker` over `items` with at most `limit`
|
|
354
|
+
* calls in flight at any moment and returns the results in input order.
|
|
355
|
+
*
|
|
356
|
+
* @param {Array} items the items to process
|
|
357
|
+
* @param {Number} limit the max number of concurrent `worker` calls
|
|
358
|
+
* @param {Function} worker async function invoked as `worker(item, index)`
|
|
359
|
+
* @returns {Promise<Array>} the results, in the same order as `items`
|
|
360
|
+
*/
|
|
361
|
+
async function mapWithConcurrency(items, limit, worker) {
|
|
362
|
+
const results = new Array(items.length);
|
|
363
|
+
let cursor = 0;
|
|
364
|
+
|
|
365
|
+
const runNext = async () => {
|
|
366
|
+
while (cursor < items.length) {
|
|
367
|
+
const index = cursor;
|
|
368
|
+
cursor += 1;
|
|
369
|
+
results[index] = await worker(items[index], index);
|
|
370
|
+
}
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
const runners = [];
|
|
374
|
+
const poolSize = Math.min(limit, items.length);
|
|
375
|
+
for (let i = 0; i < poolSize; i++) {
|
|
376
|
+
runners.push(runNext());
|
|
377
|
+
}
|
|
378
|
+
await Promise.allSettled(runners);
|
|
379
|
+
|
|
380
|
+
return results;
|
|
381
|
+
}
|
|
382
|
+
|
|
352
383
|
async function stashFile(item, googleDrive, dir) {
|
|
353
384
|
const fileMetadata = await googleDrive.getFile(item.id, {
|
|
354
385
|
fields: "name,mimeType",
|
|
@@ -444,6 +475,7 @@ export {
|
|
|
444
475
|
getFilePaths,
|
|
445
476
|
getListFilesOpts,
|
|
446
477
|
isMyDrive,
|
|
478
|
+
mapWithConcurrency,
|
|
447
479
|
MY_DRIVE_VALUE,
|
|
448
480
|
omitEmptyStringValues,
|
|
449
481
|
parseObjectEntries,
|
package/google_drive.app.mjs
CHANGED
|
@@ -7,11 +7,14 @@ const mimeTypes = Object.keys(mimeDb);
|
|
|
7
7
|
|
|
8
8
|
import googleMimeTypes from "./actions/google-mime-types.mjs";
|
|
9
9
|
import {
|
|
10
|
+
DEFAULT_MAX_EMITS_PER_RUN,
|
|
10
11
|
GOOGLE_DRIVE_FOLDER_MIME_TYPE,
|
|
11
12
|
GOOGLE_DRIVE_GRANTEE_TYPES,
|
|
12
13
|
GOOGLE_DRIVE_UPDATE_TYPE_OPTIONS,
|
|
13
14
|
GOOGLE_DRIVE_UPDATE_TYPES,
|
|
14
15
|
GOOGLE_DRIVE_UPLOAD_TYPE_OPTIONS,
|
|
16
|
+
MAX_CHANGES_PAGES_PER_RUN,
|
|
17
|
+
MAX_EMITS_PER_RUN,
|
|
15
18
|
MY_DRIVE_VALUE,
|
|
16
19
|
RATE_LIMIT_ERROR_REASONS,
|
|
17
20
|
RETRYABLE_STATUS_CODES,
|
|
@@ -161,10 +164,19 @@ export default {
|
|
|
161
164
|
changesPageSize: {
|
|
162
165
|
type: "integer",
|
|
163
166
|
label: "Changes Page Size",
|
|
164
|
-
description: "Maximum number of changes to fetch per API call (
|
|
167
|
+
description: "Maximum number of changes to fetch per API call (max 1000). Lower values mean more API calls per run. Example: `500`",
|
|
165
168
|
min: 1,
|
|
166
169
|
max: 1000,
|
|
167
|
-
default:
|
|
170
|
+
default: 1000,
|
|
171
|
+
optional: true,
|
|
172
|
+
},
|
|
173
|
+
maxEmitsPerRun: {
|
|
174
|
+
type: "integer",
|
|
175
|
+
label: "Max Events Per Run",
|
|
176
|
+
description: `Stop after about this many events and resume on the next run (max ${MAX_EMITS_PER_RUN}). A single page of changes larger than this is still processed in full. Example: \`1000\``,
|
|
177
|
+
min: 1,
|
|
178
|
+
max: MAX_EMITS_PER_RUN,
|
|
179
|
+
default: DEFAULT_MAX_EMITS_PER_RUN,
|
|
168
180
|
optional: true,
|
|
169
181
|
},
|
|
170
182
|
filePath: {
|
|
@@ -449,17 +461,22 @@ export default {
|
|
|
449
461
|
* per page
|
|
450
462
|
* @param {string} [fileFields] - the file fields to return for each change,
|
|
451
463
|
* e.g. `id,name,parents`. Defaults to the API's minimal file fields
|
|
464
|
+
* @param {object} [options] - `stopOnRateLimit` ends the stream instead of
|
|
465
|
+
* throwing; `changeFilter` drops changes before they are yielded
|
|
452
466
|
* @yields
|
|
453
467
|
* @type {ChangesPage}
|
|
454
468
|
*/
|
|
455
|
-
async *listChanges(pageToken, driveId, pageSize = 1000, fileFields) {
|
|
469
|
+
async *listChanges(pageToken, driveId, pageSize = 1000, fileFields, options = {}) {
|
|
470
|
+
const {
|
|
471
|
+
stopOnRateLimit = false, changeFilter,
|
|
472
|
+
} = options;
|
|
456
473
|
const drive = this.drive();
|
|
457
474
|
let changeRequest = {
|
|
458
475
|
pageToken,
|
|
459
476
|
pageSize,
|
|
460
477
|
};
|
|
461
478
|
if (fileFields) {
|
|
462
|
-
changeRequest.fields = `nextPageToken,newStartPageToken,changes(file(${fileFields}))`;
|
|
479
|
+
changeRequest.fields = `nextPageToken,newStartPageToken,changes(time,file(${fileFields}))`;
|
|
463
480
|
}
|
|
464
481
|
|
|
465
482
|
// As with many of the methods for Google Drive, we must
|
|
@@ -475,9 +492,18 @@ export default {
|
|
|
475
492
|
}
|
|
476
493
|
|
|
477
494
|
while (true) {
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
495
|
+
let data;
|
|
496
|
+
try {
|
|
497
|
+
({ data } = await this.retryWithExponentialBackoff(
|
|
498
|
+
() => drive.changes.list(changeRequest),
|
|
499
|
+
));
|
|
500
|
+
} catch (error) {
|
|
501
|
+
if (!stopOnRateLimit || !this.isRateLimitError(error)) {
|
|
502
|
+
throw error;
|
|
503
|
+
}
|
|
504
|
+
console.log(`Rate limited by Google Drive (${error.message}). Stopping; remaining changes resume on the next run.`);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
481
507
|
const {
|
|
482
508
|
changes = [],
|
|
483
509
|
newStartPageToken,
|
|
@@ -486,13 +512,22 @@ export default {
|
|
|
486
512
|
|
|
487
513
|
// Some changes do not include an associated file object. Return only
|
|
488
514
|
// those that do
|
|
489
|
-
const
|
|
490
|
-
|
|
491
|
-
|
|
515
|
+
const withFiles = changes.filter((change) => typeof change.file === "object");
|
|
516
|
+
const kept = [];
|
|
517
|
+
for (const change of withFiles) {
|
|
518
|
+
if (!changeFilter || await changeFilter(change)) {
|
|
519
|
+
kept.push(change);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
if (kept.length < withFiles.length) {
|
|
523
|
+
console.log(`Skipped ${withFiles.length - kept.length} changes filtered by the source`);
|
|
524
|
+
}
|
|
525
|
+
const changedFiles = kept.map((change) => change.file);
|
|
492
526
|
|
|
493
527
|
yield {
|
|
494
528
|
changedFiles,
|
|
495
529
|
nextPageToken: nextPageToken || newStartPageToken,
|
|
530
|
+
isLastPage: Boolean(newStartPageToken),
|
|
496
531
|
};
|
|
497
532
|
|
|
498
533
|
if (newStartPageToken) {
|
|
@@ -504,6 +539,46 @@ export default {
|
|
|
504
539
|
changeRequest.pageToken = nextPageToken;
|
|
505
540
|
}
|
|
506
541
|
},
|
|
542
|
+
// Returns true once the feed is drained; stops early at the emit or page cap
|
|
543
|
+
async processChangesPages({
|
|
544
|
+
pageToken,
|
|
545
|
+
driveId,
|
|
546
|
+
pageSize,
|
|
547
|
+
fileFields,
|
|
548
|
+
maxEmits,
|
|
549
|
+
processPage,
|
|
550
|
+
savePageToken,
|
|
551
|
+
changeFilter,
|
|
552
|
+
}) {
|
|
553
|
+
const changedFilesStream = this.listChanges(pageToken, driveId, pageSize, fileFields, {
|
|
554
|
+
stopOnRateLimit: true,
|
|
555
|
+
changeFilter,
|
|
556
|
+
});
|
|
557
|
+
const cap = maxEmits ?? DEFAULT_MAX_EMITS_PER_RUN;
|
|
558
|
+
let pages = 0;
|
|
559
|
+
let emitted = 0;
|
|
560
|
+
for await (const {
|
|
561
|
+
changedFiles, nextPageToken, isLastPage,
|
|
562
|
+
} of changedFilesStream) {
|
|
563
|
+
if (emitted && emitted + changedFiles.length > cap) {
|
|
564
|
+
console.log(`Emitted ${emitted} events; remaining changes resume on the next run.`);
|
|
565
|
+
return false;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
emitted += (await processPage(changedFiles)) ?? changedFiles.length;
|
|
569
|
+
savePageToken(nextPageToken);
|
|
570
|
+
|
|
571
|
+
if (isLastPage) {
|
|
572
|
+
return true;
|
|
573
|
+
}
|
|
574
|
+
if (emitted >= cap || ++pages >= MAX_CHANGES_PAGES_PER_RUN) {
|
|
575
|
+
console.log(`Emitted ${emitted} events; remaining changes resume on the next run.`);
|
|
576
|
+
return false;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
// Rate limited
|
|
580
|
+
return false;
|
|
581
|
+
},
|
|
507
582
|
async getPageToken(driveId) {
|
|
508
583
|
const drive = this.drive();
|
|
509
584
|
const request = driveId
|
|
@@ -1703,7 +1778,11 @@ export default {
|
|
|
1703
1778
|
return (await drive.accessproposals.resolve(opts)).data;
|
|
1704
1779
|
},
|
|
1705
1780
|
isRetryableError(error, statusCode) {
|
|
1706
|
-
|
|
1781
|
+
return RETRYABLE_STATUS_CODES.includes(statusCode)
|
|
1782
|
+
|| this.isRateLimitError(error, statusCode);
|
|
1783
|
+
},
|
|
1784
|
+
isRateLimitError(error, statusCode = error.status || error.response?.status) {
|
|
1785
|
+
if (statusCode === 429) {
|
|
1707
1786
|
return true;
|
|
1708
1787
|
}
|
|
1709
1788
|
const errors = error.errors ?? error.response?.data?.error?.errors ?? [];
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@ export default {
|
|
|
14
14
|
key: "google_drive-changes-to-files-in-drive",
|
|
15
15
|
name: "Changes to Files in Drive",
|
|
16
16
|
description: "Emit new event when a change is made to one of the specified files. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/watch)",
|
|
17
|
-
version: "0.0
|
|
17
|
+
version: "0.1.0",
|
|
18
18
|
type: "source",
|
|
19
19
|
dedupe: "unique",
|
|
20
20
|
props: {
|
|
@@ -69,6 +69,7 @@ export default {
|
|
|
69
69
|
const { files } = await this.googleDrive.listFilesInPage(null, args);
|
|
70
70
|
|
|
71
71
|
await this.processChanges(files);
|
|
72
|
+
this.flushFileIntervals();
|
|
72
73
|
},
|
|
73
74
|
...common.hooks,
|
|
74
75
|
},
|
|
@@ -103,6 +104,9 @@ export default {
|
|
|
103
104
|
isFileRelevant(file) {
|
|
104
105
|
return this.files.includes(file.id);
|
|
105
106
|
},
|
|
107
|
+
isRelevantChange({ file }) {
|
|
108
|
+
return this.isFileRelevant(file);
|
|
109
|
+
},
|
|
106
110
|
getChanges(headers) {
|
|
107
111
|
if (!headers) {
|
|
108
112
|
return {
|
|
@@ -133,14 +137,13 @@ export default {
|
|
|
133
137
|
async processChanges(changedFiles, headers) {
|
|
134
138
|
console.log(`Processing ${changedFiles.length} changed files`);
|
|
135
139
|
|
|
136
|
-
|
|
140
|
+
// Filter first so unwatched files aren't recorded
|
|
141
|
+
const relevantFiles = changedFiles.filter((file) => this.isFileRelevant(file));
|
|
142
|
+
const filteredFiles = this.checkMinimumInterval(relevantFiles);
|
|
137
143
|
for (const file of filteredFiles) {
|
|
138
|
-
if (!this.isFileRelevant(file)) {
|
|
139
|
-
console.log(`Skipping event for irrelevant file ${file.id}`);
|
|
140
|
-
continue;
|
|
141
|
-
}
|
|
142
144
|
await this.processChange(file, headers);
|
|
143
145
|
}
|
|
146
|
+
return filteredFiles.length;
|
|
144
147
|
},
|
|
145
148
|
},
|
|
146
149
|
};
|
|
@@ -16,7 +16,7 @@ export default {
|
|
|
16
16
|
key: "google_drive-changes-to-specific-files",
|
|
17
17
|
name: "Changes to Specific Files",
|
|
18
18
|
description: "Watches for changes to specific files, emitting an event when a change is made to one of those files. To watch for changes to [shared drive](https://support.google.com/a/users/answer/9310351) files, use the **Changes to Specific Files (Shared Drive)** source instead.",
|
|
19
|
-
version: "0.3.
|
|
19
|
+
version: "0.3.16",
|
|
20
20
|
type: "source",
|
|
21
21
|
// Dedupe events based on the "x-goog-message-number" header for the target channel:
|
|
22
22
|
// https://developers.google.com/drive/api/v3/push#making-watch-requests
|
package/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs
CHANGED
|
@@ -29,7 +29,7 @@ export default {
|
|
|
29
29
|
key: "google_drive-changes-to-specific-files-shared-drive",
|
|
30
30
|
name: "Changes to Specific Files (Shared Drive)",
|
|
31
31
|
description: "Watches for changes to specific files in a shared drive, emitting an event when a change is made to one of those files",
|
|
32
|
-
version: "0.
|
|
32
|
+
version: "0.4.0",
|
|
33
33
|
type: "source",
|
|
34
34
|
// Dedupe events based on the "x-goog-message-number" header for the target channel:
|
|
35
35
|
// https://developers.google.com/drive/api/v3/push#making-watch-requests
|
|
@@ -74,6 +74,7 @@ export default {
|
|
|
74
74
|
const { files } = await this.googleDrive.listFilesInPage(null, args);
|
|
75
75
|
|
|
76
76
|
await this.processChanges(files);
|
|
77
|
+
this.flushFileIntervals();
|
|
77
78
|
},
|
|
78
79
|
...common.hooks,
|
|
79
80
|
},
|
|
@@ -108,6 +109,9 @@ export default {
|
|
|
108
109
|
isFileRelevant(file) {
|
|
109
110
|
return this.files.includes(file.id);
|
|
110
111
|
},
|
|
112
|
+
isRelevantChange({ file }) {
|
|
113
|
+
return this.isFileRelevant(file);
|
|
114
|
+
},
|
|
111
115
|
getChanges(headers) {
|
|
112
116
|
if (!headers) {
|
|
113
117
|
return {
|
|
@@ -138,14 +142,13 @@ export default {
|
|
|
138
142
|
async processChanges(changedFiles, headers) {
|
|
139
143
|
console.log(`Processing ${changedFiles.length} changed files`);
|
|
140
144
|
|
|
141
|
-
|
|
145
|
+
// Filter first so unwatched files aren't recorded
|
|
146
|
+
const relevantFiles = changedFiles.filter((file) => this.isFileRelevant(file));
|
|
147
|
+
const filteredFiles = this.checkMinimumInterval(relevantFiles);
|
|
142
148
|
for (const file of filteredFiles) {
|
|
143
|
-
if (!this.isFileRelevant(file)) {
|
|
144
|
-
console.log(`Skipping event for irrelevant file ${file.id}`);
|
|
145
|
-
continue;
|
|
146
|
-
}
|
|
147
149
|
await this.processChange(file, headers);
|
|
148
150
|
}
|
|
151
|
+
return filteredFiles.length;
|
|
149
152
|
},
|
|
150
153
|
},
|
|
151
154
|
sampleEmit,
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { MAX_FILE_INTERVAL_ENTRIES } from "../common/constants.mjs";
|
|
2
|
+
|
|
1
3
|
export default {
|
|
2
4
|
props: {
|
|
3
5
|
intervalAlert: {
|
|
@@ -19,7 +21,10 @@ You can change or disable this minimum interval using the prop \`Minimum Interva
|
|
|
19
21
|
},
|
|
20
22
|
methods: {
|
|
21
23
|
_getFileIntervals() {
|
|
22
|
-
|
|
24
|
+
if (!this._fileIntervals) {
|
|
25
|
+
this._fileIntervals = this.db.get("fileIntervals") ?? {};
|
|
26
|
+
}
|
|
27
|
+
return this._fileIntervals;
|
|
23
28
|
},
|
|
24
29
|
_setFileIntervals(value) {
|
|
25
30
|
this.db.set("fileIntervals", value);
|
|
@@ -40,20 +45,26 @@ You can change or disable this minimum interval using the prop \`Minimum Interva
|
|
|
40
45
|
}
|
|
41
46
|
|
|
42
47
|
const now = Date.now();
|
|
43
|
-
const minTimestamp = now - (this.perFileInterval * 1000 * 60);
|
|
44
48
|
const savedData = this._getFileIntervals();
|
|
45
|
-
Object.entries(savedData).forEach(([
|
|
46
|
-
key,
|
|
47
|
-
value,
|
|
48
|
-
]) => {
|
|
49
|
-
if (value < minTimestamp) {
|
|
50
|
-
delete savedData[key];
|
|
51
|
-
}
|
|
52
|
-
});
|
|
53
49
|
fileIds.forEach((id) => {
|
|
54
50
|
savedData[id] = now;
|
|
55
51
|
});
|
|
56
|
-
this.
|
|
52
|
+
this._fileIntervalsChanged = true;
|
|
53
|
+
},
|
|
54
|
+
flushFileIntervals() {
|
|
55
|
+
if (this._fileIntervalsChanged) {
|
|
56
|
+
const minTimestamp = Date.now() - (this.perFileInterval * 1000 * 60);
|
|
57
|
+
const entries = Object.entries(this._getFileIntervals())
|
|
58
|
+
.filter((entry) => entry[1] >= minTimestamp)
|
|
59
|
+
.sort((a, b) => b[1] - a[1])
|
|
60
|
+
.slice(0, MAX_FILE_INTERVAL_ENTRIES);
|
|
61
|
+
this._setFileIntervals(Object.fromEntries(entries));
|
|
62
|
+
}
|
|
63
|
+
this.resetFileIntervals();
|
|
64
|
+
},
|
|
65
|
+
resetFileIntervals() {
|
|
66
|
+
this._fileIntervals = null;
|
|
67
|
+
this._fileIntervalsChanged = false;
|
|
57
68
|
},
|
|
58
69
|
checkMinimumInterval(files) {
|
|
59
70
|
const filteredFiles = this.filterByMinimumInterval(files);
|
|
@@ -2,7 +2,11 @@ import includes from "lodash/includes.js";
|
|
|
2
2
|
import { v4 as uuid } from "uuid";
|
|
3
3
|
|
|
4
4
|
import googleDrive from "../google_drive.app.mjs";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
MY_DRIVE_VALUE,
|
|
7
|
+
WEBHOOK_RENEWAL_WINDOW_MILLISECONDS,
|
|
8
|
+
WEBHOOK_TIMER_INTERVAL_SECONDS,
|
|
9
|
+
} from "../common/constants.mjs";
|
|
6
10
|
import { getListFilesOpts } from "../common/utils.mjs";
|
|
7
11
|
import commonDedupeChanges from "./common-dedupe-changes.mjs";
|
|
8
12
|
|
|
@@ -19,8 +23,9 @@ export default {
|
|
|
19
23
|
googleDrive,
|
|
20
24
|
"watchedDrive",
|
|
21
25
|
],
|
|
22
|
-
description: "
|
|
26
|
+
description: "The drive to watch. Defaults to **My Drive**. To watch a [Shared Drive](https://support.google.com/a/users/answer/9310351), select it from this list.",
|
|
23
27
|
optional: false,
|
|
28
|
+
default: MY_DRIVE_VALUE,
|
|
24
29
|
},
|
|
25
30
|
timer: {
|
|
26
31
|
label: "Push notification renewal schedule",
|
|
@@ -28,16 +33,30 @@ export default {
|
|
|
28
33
|
"The Google Drive API requires occasional renewal of push notification subscriptions. **This runs in the background, so you should not need to modify this schedule**.",
|
|
29
34
|
type: "$.interface.timer",
|
|
30
35
|
static: {
|
|
31
|
-
intervalSeconds:
|
|
36
|
+
intervalSeconds: WEBHOOK_TIMER_INTERVAL_SECONDS,
|
|
32
37
|
},
|
|
33
38
|
hidden: true,
|
|
34
39
|
},
|
|
40
|
+
// Unused; kept so older deployments can still save
|
|
41
|
+
updateTypes: {
|
|
42
|
+
type: "string[]",
|
|
43
|
+
label: "Types of updates",
|
|
44
|
+
description: "Deprecated and ignored.",
|
|
45
|
+
optional: true,
|
|
46
|
+
hidden: true,
|
|
47
|
+
},
|
|
35
48
|
changesPageSize: {
|
|
36
49
|
propDefinition: [
|
|
37
50
|
googleDrive,
|
|
38
51
|
"changesPageSize",
|
|
39
52
|
],
|
|
40
53
|
},
|
|
54
|
+
maxEmitsPerRun: {
|
|
55
|
+
propDefinition: [
|
|
56
|
+
googleDrive,
|
|
57
|
+
"maxEmitsPerRun",
|
|
58
|
+
],
|
|
59
|
+
},
|
|
41
60
|
},
|
|
42
61
|
hooks: {
|
|
43
62
|
async activate() {
|
|
@@ -97,6 +116,38 @@ export default {
|
|
|
97
116
|
_setPageToken(pageToken) {
|
|
98
117
|
this.db.set("pageToken", pageToken);
|
|
99
118
|
},
|
|
119
|
+
_getBacklogHeaders() {
|
|
120
|
+
return this.db.get("backlogHeaders");
|
|
121
|
+
},
|
|
122
|
+
_setBacklogHeaders(headers) {
|
|
123
|
+
this.db.set("backlogHeaders", headers);
|
|
124
|
+
},
|
|
125
|
+
isSubscriptionExpiring(subscription) {
|
|
126
|
+
const expiration = Number(subscription?.expiration);
|
|
127
|
+
return !expiration || expiration - Date.now() < WEBHOOK_RENEWAL_WINDOW_MILLISECONDS;
|
|
128
|
+
},
|
|
129
|
+
async renewSubscription(subscription, channelID, pageToken) {
|
|
130
|
+
const {
|
|
131
|
+
newChannelID,
|
|
132
|
+
newPageToken,
|
|
133
|
+
expiration,
|
|
134
|
+
resourceId,
|
|
135
|
+
} = await this.googleDrive.renewSubscription(
|
|
136
|
+
this.drive,
|
|
137
|
+
subscription,
|
|
138
|
+
this.http.endpoint,
|
|
139
|
+
channelID,
|
|
140
|
+
pageToken,
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
this._setSubscription({
|
|
144
|
+
expiration,
|
|
145
|
+
resourceId,
|
|
146
|
+
});
|
|
147
|
+
this._setChannelID(newChannelID);
|
|
148
|
+
this._setPageToken(newPageToken);
|
|
149
|
+
return newPageToken;
|
|
150
|
+
},
|
|
100
151
|
isMyDrive(drive = this.drive) {
|
|
101
152
|
return googleDrive.methods.isMyDrive(drive);
|
|
102
153
|
},
|
|
@@ -128,6 +179,16 @@ export default {
|
|
|
128
179
|
getChangesFileFields() {
|
|
129
180
|
return undefined;
|
|
130
181
|
},
|
|
182
|
+
isRelevantChange() {
|
|
183
|
+
return true;
|
|
184
|
+
},
|
|
185
|
+
async getChangeFilter() {
|
|
186
|
+
return (change) => this.isRelevantChange(change);
|
|
187
|
+
},
|
|
188
|
+
// false: processChanges() ignores changedFiles, so run it once, not per page
|
|
189
|
+
consumesChangedFiles() {
|
|
190
|
+
return true;
|
|
191
|
+
},
|
|
131
192
|
/**
|
|
132
193
|
* This method is responsible for processing a list of changed files
|
|
133
194
|
* according to the event source's purpose. As an abstract method, it must
|
|
@@ -137,10 +198,36 @@ export default {
|
|
|
137
198
|
* by the API](https://bit.ly/3h7WeUa)
|
|
138
199
|
* @param {object} [headers] - an object containing the request headers of
|
|
139
200
|
* the webhook call made by Google Drive
|
|
201
|
+
* @returns {number} the number of events emitted
|
|
140
202
|
*/
|
|
141
203
|
processChanges() {
|
|
142
204
|
throw new Error("processChanges is not implemented");
|
|
143
205
|
},
|
|
206
|
+
async processChangesFrom(pageToken, headers) {
|
|
207
|
+
this.resetFileIntervals();
|
|
208
|
+
// Thrown runs keep db writes, so flush intervals with the token
|
|
209
|
+
try {
|
|
210
|
+
const caughtUp = await this.googleDrive.processChangesPages({
|
|
211
|
+
pageToken,
|
|
212
|
+
driveId: this.getDriveId(),
|
|
213
|
+
pageSize: this.changesPageSize,
|
|
214
|
+
fileFields: this.getChangesFileFields(),
|
|
215
|
+
maxEmits: this.maxEmitsPerRun,
|
|
216
|
+
changeFilter: await this.getChangeFilter(),
|
|
217
|
+
processPage: (changedFiles) => this.processChanges(changedFiles, headers),
|
|
218
|
+
savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
|
|
219
|
+
});
|
|
220
|
+
this._setBacklogHeaders(caughtUp
|
|
221
|
+
? null
|
|
222
|
+
: {
|
|
223
|
+
"x-goog-resource-state": headers["x-goog-resource-state"],
|
|
224
|
+
"x-goog-resource-uri": headers["x-goog-resource-uri"],
|
|
225
|
+
"x-goog-changed": headers["x-goog-changed"],
|
|
226
|
+
});
|
|
227
|
+
} finally {
|
|
228
|
+
this.flushFileIntervals();
|
|
229
|
+
}
|
|
230
|
+
},
|
|
144
231
|
},
|
|
145
232
|
async run(event) {
|
|
146
233
|
// This function is polymorphic: it can be triggered as a cron job, to make
|
|
@@ -152,25 +239,15 @@ export default {
|
|
|
152
239
|
|
|
153
240
|
// Component was invoked by timer
|
|
154
241
|
if (event.timestamp) {
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
expiration,
|
|
159
|
-
resourceId,
|
|
160
|
-
} = await this.googleDrive.renewSubscription(
|
|
161
|
-
this.drive,
|
|
162
|
-
subscription,
|
|
163
|
-
this.http.endpoint,
|
|
164
|
-
channelID,
|
|
165
|
-
pageToken,
|
|
166
|
-
);
|
|
242
|
+
const currentPageToken = this.isSubscriptionExpiring(subscription)
|
|
243
|
+
? await this.renewSubscription(subscription, channelID, pageToken)
|
|
244
|
+
: pageToken;
|
|
167
245
|
|
|
168
|
-
this.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
this._setPageToken(newPageToken);
|
|
246
|
+
const backlogHeaders = this._getBacklogHeaders();
|
|
247
|
+
if (backlogHeaders && this.consumesChangedFiles()) {
|
|
248
|
+
console.log("Resuming changes left by a capped run");
|
|
249
|
+
await this.processChangesFrom(currentPageToken, backlogHeaders);
|
|
250
|
+
}
|
|
174
251
|
return;
|
|
175
252
|
} else {
|
|
176
253
|
this.http.respond({
|
|
@@ -206,25 +283,12 @@ export default {
|
|
|
206
283
|
}
|
|
207
284
|
|
|
208
285
|
const driveId = this.getDriveId();
|
|
209
|
-
|
|
210
|
-
this.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
this.changesPageSize,
|
|
214
|
-
this.getChangesFileFields(),
|
|
215
|
-
);
|
|
216
|
-
for await (const changedFilesPage of changedFilesStream) {
|
|
217
|
-
const {
|
|
218
|
-
changedFiles,
|
|
219
|
-
nextPageToken,
|
|
220
|
-
} = changedFilesPage;
|
|
221
|
-
|
|
222
|
-
// Process all the changed files retrieved from the current page
|
|
223
|
-
await this.processChanges(changedFiles, headers);
|
|
224
|
-
|
|
225
|
-
// After successfully processing the changed files, we store the page
|
|
226
|
-
// token of the next page
|
|
227
|
-
this._setPageToken(nextPageToken);
|
|
286
|
+
if (!this.consumesChangedFiles()) {
|
|
287
|
+
await this.processChanges([], headers);
|
|
288
|
+
this._setPageToken(await this.googleDrive.getPageToken(driveId));
|
|
289
|
+
return;
|
|
228
290
|
}
|
|
291
|
+
|
|
292
|
+
await this.processChangesFrom(pageToken, headers);
|
|
229
293
|
},
|
|
230
294
|
};
|
|
@@ -6,7 +6,7 @@ export default {
|
|
|
6
6
|
key: "google_drive-new-access-proposal",
|
|
7
7
|
name: "New Access Proposal",
|
|
8
8
|
description: "Emit new event when a new access proposal is requested in Google Drive",
|
|
9
|
-
version: "0.0.
|
|
9
|
+
version: "0.0.21",
|
|
10
10
|
type: "source",
|
|
11
11
|
dedupe: "unique",
|
|
12
12
|
props: {
|
|
@@ -11,7 +11,7 @@ export default {
|
|
|
11
11
|
key: "google_drive-new-files-instant",
|
|
12
12
|
name: "New Files (Instant)",
|
|
13
13
|
description: "Emit new event when a new file is added in your linked Google Drive",
|
|
14
|
-
version: "0.
|
|
14
|
+
version: "0.3.0",
|
|
15
15
|
type: "source",
|
|
16
16
|
dedupe: "unique",
|
|
17
17
|
props: {
|
|
@@ -131,6 +131,9 @@ export default {
|
|
|
131
131
|
GOOGLE_DRIVE_NOTIFICATION_CHANGE,
|
|
132
132
|
];
|
|
133
133
|
},
|
|
134
|
+
consumesChangedFiles() {
|
|
135
|
+
return false;
|
|
136
|
+
},
|
|
134
137
|
async emitFiles(files) {
|
|
135
138
|
for (const file of files) {
|
|
136
139
|
if (!(await this.shouldProcess(file))) {
|