@credal/actions 0.2.142 → 0.2.144

package/dist/actions/providers/google-oauth/getDriveFileContentById.js

@@ -14,19 +14,20 @@ import { extractTextFromPdf } from "../../../utils/pdf.js";
 import { getGoogleDocContent, getGoogleSheetContent, getGoogleSlidesContent } from "../../../utils/google.js";
 import { read, utils } from "xlsx";
 import officeParser from "officeparser";
+const BASE_WEB_URL = "https://drive.google.com/file/d/";
+const BASE_API_URL = "https://www.googleapis.com/drive/v3/files/";
 const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
     var _b, _c, _d, _e;
     if (!authParams.authToken) {
         return { success: false, error: MISSING_AUTH_TOKEN };
     }
-    const BASE_URL = "https://www.googleapis.com/drive/v3/files/";
     const headers = { Authorization: `Bearer ${authParams.authToken}` };
     const { limit: charLimit, fileId } = params;
     const timeoutLimit = params.timeoutLimit !== undefined && params.timeoutLimit > 0 ? params.timeoutLimit * 1000 : 15000;
     const axiosClient = createAxiosClientWithTimeout(timeoutLimit);
     // helper to fetch drive metadata with fields we need (incl. shortcut details)
     const fetchMeta = (fid) => __awaiter(void 0, void 0, void 0, function* () {
-        const metaUrl = `${BASE_URL}${encodeURIComponent(fid)}` +
+        const metaUrl = `${BASE_API_URL}${encodeURIComponent(fid)}` +
             `?fields=name,mimeType,size,driveId,parents,` +
             `shortcutDetails(targetId,targetMimeType)` +
             `&supportsAllDrives=true`;
@@ -63,7 +64,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
             content = yield getGoogleSlidesContent(params.fileId, authParams.authToken, axiosClient, sharedDriveParam);
         }
         else if (mimeType === "application/pdf") {
-            const downloadUrl = `${BASE_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
+            const downloadUrl = `${BASE_API_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
             const downloadRes = yield axiosClient.get(downloadUrl, {
                 headers,
                 responseType: "arraybuffer",
@@ -80,7 +81,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
         }
         else if (mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
             mimeType === "application/msword") {
-            const downloadUrl = `${BASE_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
+            const downloadUrl = `${BASE_API_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
             const downloadRes = yield axiosClient.get(downloadUrl, {
                 headers,
                 responseType: "arraybuffer",
@@ -97,7 +98,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
             }
         }
         else if (mimeType === "text/plain" || mimeType === "text/html" || mimeType === "application/rtf") {
-            const downloadUrl = `${BASE_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
+            const downloadUrl = `${BASE_API_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
             const downloadRes = yield axiosClient.get(downloadUrl, {
                 headers,
                 responseType: "text",
@@ -109,7 +110,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
             mimeType === "text/tab-separated-values" ||
             mimeType === "application/rtf" ||
             mimeType === "application/json") {
-            const downloadUrl = `${BASE_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
+            const downloadUrl = `${BASE_API_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
             const downloadRes = yield axiosClient.get(downloadUrl, {
                 headers,
                 responseType: "arraybuffer",
@@ -120,7 +121,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
         }
         else if (mimeType === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ||
             mimeType === "application/vnd.ms-excel") {
-            const downloadUrl = `${BASE_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
+            const downloadUrl = `${BASE_API_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
             const downloadRes = yield axiosClient.get(downloadUrl, {
                 headers,
                 responseType: "arraybuffer",
@@ -139,7 +140,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
         }
         else if (mimeType === "application/vnd.openxmlformats-officedocument.presentationml.presentation") {
             // Handle modern PowerPoint files (.pptx only)
-            const downloadUrl = `${BASE_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
+            const downloadUrl = `${BASE_API_URL}${encodeURIComponent(params.fileId)}?alt=media${sharedDriveParam}`;
             const downloadRes = yield axiosClient.get(downloadUrl, {
                 headers,
                 responseType: "arraybuffer",
@@ -174,7 +175,7 @@ const getDriveFileContentById = (_a) => __awaiter(void 0, [_a], void 0, function
             results: [
                 {
                     name: fileName,
-                    url: `${BASE_URL}${encodeURIComponent(params.fileId)}`,
+                    url: `${BASE_WEB_URL}${params.fileId}`,
                     contents: { content, fileName, fileLength: originalLength },
                 },
             ],

package/dist/actions/providers/google-oauth/searchDriveByKeywordsAndGetFileContent.js

@@ -10,6 +10,20 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
 import searchDriveByQuery from "./searchDriveByQuery.js";
 import getDriveFileContentById from "./getDriveFileContentById.js";
+// Helper function to process files in batches with concurrency control
+const processBatch = (items_1, processor_1, ...args_1) => __awaiter(void 0, [items_1, processor_1, ...args_1], void 0, function* (items, processor, batchSize = 3) {
+    const results = [];
+    for (let i = 0; i < items.length; i += batchSize) {
+        const batch = items.slice(i, i + batchSize);
+        const batchResults = yield Promise.allSettled(batch.map(processor));
+        for (const result of batchResults) {
+            if (result.status === "fulfilled") {
+                results.push(result.value);
+            }
+        }
+    }
+    return results;
+});
 const searchDriveByKeywordsAndGetFileContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
     var _b;
     if (!authParams.authToken) {
@@ -29,13 +43,33 @@ const searchDriveByKeywordsAndGetFileContent = (_a) => __awaiter(void 0, [_a], v
     if (!searchResult.success) {
         return { success: false, error: searchResult.error };
     }
-    // For each file, fetch its content in parallel
     const files = (_b = searchResult.files) !== null && _b !== void 0 ? _b : [];
-    const contentPromises = files.map((file) => __awaiter(void 0, void 0, void 0, function* () {
+    // File types that are likely to fail or have no useful text content
+    const problematicMimeTypes = new Set([
+        "application/vnd.google-apps.form",
+        "application/vnd.google-apps.site",
+        "application/vnd.google-apps.map",
+        "application/vnd.google-apps.drawing",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation", // PowerPoint
+        "application/vnd.ms-powerpoint",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", // Excel (not supported yet)
+        "application/vnd.ms-excel",
+    ]);
+    // Filter out problematic files BEFORE processing to avoid wasting resources
+    const validFiles = files
+        .slice(0, limit)
+        .filter(file => file.id && file.name && !problematicMimeTypes.has(file.mimeType));
+    // Process only valid files in smaller batches to avoid overwhelming the API
+    const filesWithContent = yield processBatch(validFiles, (file) => __awaiter(void 0, void 0, void 0, function* () {
         var _a, _b, _c;
         try {
+            // Add timeout for individual file content requests with shorter timeout
             const contentResult = yield getDriveFileContentById({
-                params: { fileId: file.id, limit: maxChars },
+                params: {
+                    fileId: file.id,
+                    limit: maxChars,
+                    timeoutLimit: 2,
+                },
                 authParams,
             });
             return {
@@ -46,17 +80,16 @@ const searchDriveByKeywordsAndGetFileContent = (_a) => __awaiter(void 0, [_a], v
                 content: contentResult.success ? (_c = (_b = (_a = contentResult.results) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.contents) === null || _c === void 0 ? void 0 : _c.content : undefined,
             };
         }
-        catch (error) {
-            console.error(`Error fetching content for file ${file.id}:`, error);
+        catch (_d) {
             return {
                 id: file.id,
                 name: file.name,
                 mimeType: file.mimeType,
                 url: file.url,
+                content: undefined, // Gracefully handle errors
             };
         }
-    }));
-    const filesWithContent = yield Promise.all(contentPromises);
+    }), 5);
     // Return combined results
     return {
         success: true,

package/dist/actions/providers/google-oauth/searchDriveByQuery.js

@@ -37,14 +37,14 @@ const searchDriveByQuery = (_a) => __awaiter(void 0, [_a], void 0, function* ({
 });
 // Original search method - search all drives at once
 const searchAllDrivesAtOnce = (query, authToken, limit, orderByQuery) => __awaiter(void 0, void 0, void 0, function* () {
-    const allDrivesUrl = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&supportsAllDrives=true&includeItemsFromAllDrives=true&corpora=allDrives&pageSize=1000${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}`;
+    const allDrivesUrl = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&supportsAllDrives=true&includeItemsFromAllDrives=true&corpora=allDrives&pageSize=100${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}`;
     const allDrivesRes = axiosClient.get(allDrivesUrl, {
         headers: {
             Authorization: `Bearer ${authToken}`,
         },
     });
     // need to search domain wide separately because the allDrives search doesn't include domain wide files
-    const orgWideUrl = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&corpora=domain&pageSize=1000${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}`;
+    const orgWideUrl = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&corpora=domain&pageSize=100${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}`;
     const orgWideRes = axiosClient.get(orgWideUrl, {
         headers: {
             Authorization: `Bearer ${authToken}`,
@@ -76,7 +76,7 @@ const searchAllDrivesIndividually = (query, authToken, limit, orderByQuery) => _
         `q=${encodeURIComponent(query)}&` +
         `fields=files(id,name,mimeType,webViewLink),nextPageToken&` +
         `corpora=domain&` +
-        `pageSize=1000${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}`;
+        `pageSize=100${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}`;
     const domainDriveFunction = () => __awaiter(void 0, void 0, void 0, function* () {
         var _a, _b;
         const domainRes = yield axiosClient.get(domainUrl, {
@@ -148,11 +148,11 @@ const searchSingleDrive = (query, driveId, authToken, orderByQuery) => __awaiter
         let url;
         if (driveId === "root") {
             // Search in user's personal drive
-            url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink),nextPageToken&pageSize=1000${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}${nextPageToken ? `&pageToken=${nextPageToken}` : ""}`;
+            url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink),nextPageToken&pageSize=100${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}${nextPageToken ? `&pageToken=${nextPageToken}` : ""}`;
         }
         else {
             // Search in specific shared drive
-            url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(`${query} and parents in '${driveId}'`)}&fields=files(id,name,mimeType,webViewLink),nextPageToken&supportsAllDrives=true&includeItemsFromAllDrives=true&corpora=drive&driveId=${driveId}&pageSize=1000${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}${nextPageToken ? `&pageToken=${nextPageToken}` : ""}`;
+            url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(`${query} and parents in '${driveId}'`)}&fields=files(id,name,mimeType,webViewLink),nextPageToken&supportsAllDrives=true&includeItemsFromAllDrives=true&corpora=drive&driveId=${driveId}&pageSize=100${orderByQuery ? `&orderBy=${encodeURIComponent(orderByQuery)}` : ""}${nextPageToken ? `&pageToken=${nextPageToken}` : ""}`;
         }
         const res = yield axiosClient.get(url, {
             headers: {

package/dist/utils/google.js

@@ -263,45 +263,70 @@ export function getGoogleDocContent(fileId, authToken, axiosClient, sharedDriveP
             }
             else {
                 console.log("Error using Google Docs API", docsError);
-                // Fallback to Drive API export if Docs API fails
-                const exportUrl = `${GDRIVE_BASE_URL}${encodeURIComponent(fileId)}/export?mimeType=text/plain${sharedDriveParams}`;
-                const exportRes = yield axiosClient.get(exportUrl, {
-                    headers: { Authorization: `Bearer ${authToken}` },
-                    responseType: "text",
-                });
-                return exportRes.data;
+                // Check if it's a 404 or permission error - don't retry these
+                if (docsError && typeof docsError === "object" && "status" in docsError) {
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const status = docsError.status;
+                    if (status === 404 || status === 403) {
+                        throw new Error(`File not accessible (${status}): ${fileId}`);
+                    }
+                }
+                try {
+                    // Fallback to Drive API export if Docs API fails
+                    const exportUrl = `${GDRIVE_BASE_URL}${encodeURIComponent(fileId)}/export?mimeType=text/plain${sharedDriveParams}`;
+                    const exportRes = yield axiosClient.get(exportUrl, {
+                        headers: { Authorization: `Bearer ${authToken}` },
+                        responseType: "text",
+                    });
+                    return exportRes.data;
+                }
+                catch (_a) {
+                    throw new Error(`Unable to access document content: ${fileId}`);
+                }
             }
         }
     });
 }
 export function getGoogleSheetContent(fileId, authToken, axiosClient, sharedDriveParams) {
     return __awaiter(this, void 0, void 0, function* () {
+        // Use CSV export as primary method - it's much faster and more reliable for large sheets
+        // The Sheets API with includeGridData can timeout on large spreadsheets
         try {
-            const sheetsUrl = `https://sheets.googleapis.com/v4/spreadsheets/${fileId}?includeGridData=true`;
-            const sheetsRes = yield axiosClient.get(sheetsUrl, {
-                headers: {
-                    Authorization: `Bearer ${authToken}`,
-                },
+            const exportUrl = `${GDRIVE_BASE_URL}${encodeURIComponent(fileId)}/export?mimeType=text/csv${sharedDriveParams}`;
+            const exportRes = yield axiosClient.get(exportUrl, {
+                headers: { Authorization: `Bearer ${authToken}` },
+                responseType: "text",
             });
-            return parseGoogleSheetsFromRawContentToPlainText(sheetsRes.data);
+            return exportRes.data
+                .split("\n")
+                .map((line) => line.replace(/,+$/, ""))
+                .map((line) => line.replace(/,{2,}/g, ","))
+                .join("\n");
         }
-        catch (sheetsError) {
-            if (isAxiosTimeoutError(sheetsError)) {
-                console.log("Request timed out using Google Sheets API - dont retry");
-                throw new Error("Request timed out using Google Sheets API");
+        catch (exportError) {
+            // Check if it's a 404 or permission error
+            if (exportError && typeof exportError === "object" && "status" in exportError) {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const status = exportError.status;
+                if (status === 404 || status === 403) {
+                    throw new Error(`Spreadsheet not accessible (${status}): ${fileId}`);
+                }
             }
-            else {
-                console.log("Error using Google Sheets API", sheetsError);
-                const exportUrl = `${GDRIVE_BASE_URL}${encodeURIComponent(fileId)}/export?mimeType=text/csv${sharedDriveParams}`;
-                const exportRes = yield axiosClient.get(exportUrl, {
-                    headers: { Authorization: `Bearer ${authToken}` },
-                    responseType: "text",
+            // If CSV export fails, try the Sheets API as fallback (but this is slower)
+            try {
+                const sheetsUrl = `https://sheets.googleapis.com/v4/spreadsheets/${fileId}?includeGridData=true`;
+                const sheetsRes = yield axiosClient.get(sheetsUrl, {
+                    headers: {
+                        Authorization: `Bearer ${authToken}`,
+                    },
                 });
-                return exportRes.data
-                    .split("\n")
-                    .map((line) => line.replace(/,+$/, ""))
-                    .map((line) => line.replace(/,{2,}/g, ","))
-                    .join("\n");
+                return parseGoogleSheetsFromRawContentToPlainText(sheetsRes.data);
+            }
+            catch (sheetsError) {
+                if (isAxiosTimeoutError(sheetsError)) {
+                    throw new Error("Request timed out using Google Sheets API");
+                }
+                throw new Error(`Unable to access spreadsheet content: ${fileId}`);
             }
         }
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@credal/actions",
-  "version": "0.2.142",
+  "version": "0.2.144",
   "type": "module",
   "description": "AI Actions by Credal AI",
   "sideEffects": false,