@credal/actions 0.2.54 → 0.2.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/autogen/templates.js +5 -1
- package/dist/actions/autogen/types.d.ts +7 -0
- package/dist/actions/autogen/types.js +1 -0
- package/dist/actions/providers/confluence/updatePage.js +15 -14
- package/dist/actions/providers/gitlab/searchGroup.d.ts +1 -1
- package/dist/actions/providers/gitlab/searchGroup.js +81 -7
- package/dist/actions/providers/jamf/types.d.ts +8 -0
- package/dist/actions/providers/jamf/types.js +7 -0
- package/package.json +1 -1
- package/dist/actions/providers/generic/fillTemplateAction.d.ts +0 -7
- package/dist/actions/providers/generic/fillTemplateAction.js +0 -18
- package/dist/actions/providers/generic/genericApiCall.d.ts +0 -3
- package/dist/actions/providers/generic/genericApiCall.js +0 -38
- package/dist/actions/providers/google-oauth/getDriveContentById.d.ts +0 -3
- package/dist/actions/providers/google-oauth/getDriveContentById.js +0 -161
- package/dist/actions/providers/google-oauth/searchAndGetDriveContentByKeywords.d.ts +0 -3
- package/dist/actions/providers/google-oauth/searchAndGetDriveContentByKeywords.js +0 -47
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByKeywords.d.ts +0 -3
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByKeywords.js +0 -110
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByQuery.d.ts +0 -3
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByQuery.js +0 -78
- package/dist/actions/providers/google-oauth/utils/extractContentFromDriveFileId.d.ts +0 -15
- package/dist/actions/providers/google-oauth/utils/extractContentFromDriveFileId.js +0 -129
- package/dist/actions/providers/googlemaps/nearbysearch.d.ts +0 -3
- package/dist/actions/providers/googlemaps/nearbysearch.js +0 -96
- package/dist/actions/providers/snowflake/runSnowflakeQueryWriteResultsToS3.d.ts +0 -3
- package/dist/actions/providers/snowflake/runSnowflakeQueryWriteResultsToS3.js +0 -154
- package/dist/actions/providers/x/scrapeTweetDataWithNitter.d.ts +0 -3
- package/dist/actions/providers/x/scrapeTweetDataWithNitter.js +0 -45
|
@@ -9913,7 +9913,7 @@ export const gitlabSearchGroupDefinition = {
|
|
|
9913
9913
|
properties: {
|
|
9914
9914
|
metadata: {
|
|
9915
9915
|
type: "object",
|
|
9916
|
-
required: ["path", "basename", "data", "project_id", "ref", "startline", "filename"],
|
|
9916
|
+
required: ["path", "basename", "data", "project_id", "ref", "startline", "filename", "web_url"],
|
|
9917
9917
|
properties: {
|
|
9918
9918
|
path: {
|
|
9919
9919
|
type: "string",
|
|
@@ -9943,6 +9943,10 @@ export const gitlabSearchGroupDefinition = {
|
|
|
9943
9943
|
type: "string",
|
|
9944
9944
|
description: "The filename of the blob",
|
|
9945
9945
|
},
|
|
9946
|
+
web_url: {
|
|
9947
|
+
type: "string",
|
|
9948
|
+
description: "The URL of the blob",
|
|
9949
|
+
},
|
|
9946
9950
|
},
|
|
9947
9951
|
},
|
|
9948
9952
|
matchedMergeRequests: {
|
|
@@ -6716,11 +6716,13 @@ export declare const gitlabSearchGroupOutputSchema: z.ZodObject<{
|
|
|
6716
6716
|
ref: z.ZodString;
|
|
6717
6717
|
startline: z.ZodNumber;
|
|
6718
6718
|
filename: z.ZodString;
|
|
6719
|
+
web_url: z.ZodString;
|
|
6719
6720
|
}, "strip", z.ZodTypeAny, {
|
|
6720
6721
|
path: string;
|
|
6721
6722
|
data: string;
|
|
6722
6723
|
filename: string;
|
|
6723
6724
|
project_id: number;
|
|
6725
|
+
web_url: string;
|
|
6724
6726
|
basename: string;
|
|
6725
6727
|
ref: string;
|
|
6726
6728
|
startline: number;
|
|
@@ -6729,6 +6731,7 @@ export declare const gitlabSearchGroupOutputSchema: z.ZodObject<{
|
|
|
6729
6731
|
data: string;
|
|
6730
6732
|
filename: string;
|
|
6731
6733
|
project_id: number;
|
|
6734
|
+
web_url: string;
|
|
6732
6735
|
basename: string;
|
|
6733
6736
|
ref: string;
|
|
6734
6737
|
startline: number;
|
|
@@ -6755,6 +6758,7 @@ export declare const gitlabSearchGroupOutputSchema: z.ZodObject<{
|
|
|
6755
6758
|
data: string;
|
|
6756
6759
|
filename: string;
|
|
6757
6760
|
project_id: number;
|
|
6761
|
+
web_url: string;
|
|
6758
6762
|
basename: string;
|
|
6759
6763
|
ref: string;
|
|
6760
6764
|
startline: number;
|
|
@@ -6771,6 +6775,7 @@ export declare const gitlabSearchGroupOutputSchema: z.ZodObject<{
|
|
|
6771
6775
|
data: string;
|
|
6772
6776
|
filename: string;
|
|
6773
6777
|
project_id: number;
|
|
6778
|
+
web_url: string;
|
|
6774
6779
|
basename: string;
|
|
6775
6780
|
ref: string;
|
|
6776
6781
|
startline: number;
|
|
@@ -6812,6 +6817,7 @@ export declare const gitlabSearchGroupOutputSchema: z.ZodObject<{
|
|
|
6812
6817
|
data: string;
|
|
6813
6818
|
filename: string;
|
|
6814
6819
|
project_id: number;
|
|
6820
|
+
web_url: string;
|
|
6815
6821
|
basename: string;
|
|
6816
6822
|
ref: string;
|
|
6817
6823
|
startline: number;
|
|
@@ -6853,6 +6859,7 @@ export declare const gitlabSearchGroupOutputSchema: z.ZodObject<{
|
|
|
6853
6859
|
data: string;
|
|
6854
6860
|
filename: string;
|
|
6855
6861
|
project_id: number;
|
|
6862
|
+
web_url: string;
|
|
6856
6863
|
basename: string;
|
|
6857
6864
|
ref: string;
|
|
6858
6865
|
startline: number;
|
|
@@ -3475,6 +3475,7 @@ export const gitlabSearchGroupOutputSchema = z.object({
|
|
|
3475
3475
|
ref: z.string().describe("The ref of the blob"),
|
|
3476
3476
|
startline: z.number().describe("The start line of the blob"),
|
|
3477
3477
|
filename: z.string().describe("The filename of the blob"),
|
|
3478
|
+
web_url: z.string().describe("The URL of the blob"),
|
|
3478
3479
|
}),
|
|
3479
3480
|
matchedMergeRequests: z
|
|
3480
3481
|
.array(z.object({
|
|
@@ -8,28 +8,30 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
8
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
9
|
});
|
|
10
10
|
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
11
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
const
|
|
13
|
-
function
|
|
14
|
-
|
|
15
|
+
const axios_1 = __importDefault(require("axios"));
|
|
16
|
+
function getConfluenceApi(baseUrl, username, apiToken) {
|
|
17
|
+
const api = axios_1.default.create({
|
|
15
18
|
baseURL: baseUrl,
|
|
16
19
|
headers: {
|
|
17
20
|
Accept: "application/json",
|
|
21
|
+
// Tokens are associated with a specific user.
|
|
18
22
|
Authorization: `Basic ${Buffer.from(`${username}:${apiToken}`).toString("base64")}`,
|
|
19
23
|
},
|
|
20
|
-
};
|
|
24
|
+
});
|
|
25
|
+
return api;
|
|
21
26
|
}
|
|
22
27
|
const confluenceUpdatePage = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
23
|
-
const { pageId, content, title } = params;
|
|
24
|
-
const { baseUrl, authToken
|
|
25
|
-
|
|
26
|
-
throw new Error("Missing required authentication information");
|
|
27
|
-
}
|
|
28
|
-
const config = getConfluenceRequestConfig(baseUrl, username, authToken);
|
|
28
|
+
const { pageId, username, content, title } = params;
|
|
29
|
+
const { baseUrl, authToken } = authParams;
|
|
30
|
+
const api = getConfluenceApi(baseUrl, username, authToken);
|
|
29
31
|
// Get current version number
|
|
30
|
-
const response = yield
|
|
32
|
+
const response = yield api.get(`/api/v2/pages/${pageId}`);
|
|
31
33
|
const currVersion = response.data.version.number;
|
|
32
|
-
|
|
34
|
+
yield api.put(`/api/v2/pages/${pageId}`, {
|
|
33
35
|
id: pageId,
|
|
34
36
|
status: "current",
|
|
35
37
|
title,
|
|
@@ -40,7 +42,6 @@ const confluenceUpdatePage = (_a) => __awaiter(void 0, [_a], void 0, function* (
|
|
|
40
42
|
version: {
|
|
41
43
|
number: currVersion + 1,
|
|
42
44
|
},
|
|
43
|
-
};
|
|
44
|
-
yield axiosClient_1.axiosClient.put(`/api/v2/pages/${pageId}`, payload, config);
|
|
45
|
+
});
|
|
45
46
|
});
|
|
46
47
|
exports.default = confluenceUpdatePage;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { gitlabSearchGroupFunction } from "../../autogen/types.js";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Searches for merge requests and blobs in a GitLab group
|
|
4
4
|
*/
|
|
5
5
|
declare const searchGroup: gitlabSearchGroupFunction;
|
|
6
6
|
export default searchGroup;
|
|
@@ -9,16 +9,75 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
9
9
|
};
|
|
10
10
|
import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
|
|
11
11
|
const GITLAB_API_URL = "https://gitlab.com";
|
|
12
|
+
// Cache for project paths to avoid repeated API calls
|
|
13
|
+
const projectPathCache = new Map();
|
|
12
14
|
function gitlabFetch(endpoint, authToken) {
|
|
13
15
|
return __awaiter(this, void 0, void 0, function* () {
|
|
14
16
|
const res = yield fetch(endpoint, {
|
|
15
|
-
headers: {
|
|
17
|
+
headers: {
|
|
18
|
+
Authorization: `Bearer ${authToken}`,
|
|
19
|
+
},
|
|
16
20
|
});
|
|
17
21
|
if (!res.ok)
|
|
18
22
|
throw new Error(`GitLab API error: ${res.status} ${res.statusText}`);
|
|
19
23
|
return res.json();
|
|
20
24
|
});
|
|
21
25
|
}
|
|
26
|
+
function getProjectPath(projectId, authToken, baseUrl) {
|
|
27
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
28
|
+
// Check cache first
|
|
29
|
+
if (projectPathCache.has(projectId)) {
|
|
30
|
+
return projectPathCache.get(projectId);
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
const project = yield gitlabFetch(`${baseUrl}/projects/${projectId}`, authToken);
|
|
34
|
+
const path = project.path_with_namespace;
|
|
35
|
+
projectPathCache.set(projectId, path);
|
|
36
|
+
return path;
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
console.warn(`Failed to fetch project path for project ${projectId}:`, error);
|
|
40
|
+
// Fallback to project ID if we can't get the path
|
|
41
|
+
return `project-${projectId}`;
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
function constructBlobUrl(input) {
|
|
46
|
+
const { baseUrl, projectPath, ref, path, startline } = input;
|
|
47
|
+
let url = `${baseUrl}/${projectPath}/-/blob/${ref}/${path}`;
|
|
48
|
+
// Add line number anchor if provided
|
|
49
|
+
if (startline && startline > 0) {
|
|
50
|
+
url += `#L${startline}`;
|
|
51
|
+
}
|
|
52
|
+
return url;
|
|
53
|
+
}
|
|
54
|
+
function enhanceBlobWithUrl(blob, authToken, baseUrl, gitlabWebBaseUrl) {
|
|
55
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
56
|
+
try {
|
|
57
|
+
const projectPath = yield getProjectPath(blob.project_id, authToken, baseUrl);
|
|
58
|
+
const web_url = constructBlobUrl({
|
|
59
|
+
baseUrl: gitlabWebBaseUrl,
|
|
60
|
+
projectPath,
|
|
61
|
+
ref: blob.ref,
|
|
62
|
+
path: blob.path,
|
|
63
|
+
startline: blob.startline,
|
|
64
|
+
});
|
|
65
|
+
return Object.assign(Object.assign({}, blob), { web_url });
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
console.warn(`Failed to construct URL for blob in project ${blob.project_id}:`, error);
|
|
69
|
+
// Fallback URL construction
|
|
70
|
+
const fallbackUrl = constructBlobUrl({
|
|
71
|
+
baseUrl: gitlabWebBaseUrl,
|
|
72
|
+
projectPath: `project-${blob.project_id}`,
|
|
73
|
+
ref: blob.ref,
|
|
74
|
+
path: blob.path,
|
|
75
|
+
startline: blob.startline,
|
|
76
|
+
});
|
|
77
|
+
return Object.assign(Object.assign({}, blob), { web_url: fallbackUrl });
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
}
|
|
22
81
|
function globalSearch(input) {
|
|
23
82
|
return __awaiter(this, void 0, void 0, function* () {
|
|
24
83
|
const { scope, query, groupId, authToken, baseUrl } = input;
|
|
@@ -34,29 +93,44 @@ function getMRDiffs(input) {
|
|
|
34
93
|
});
|
|
35
94
|
}
|
|
36
95
|
/**
|
|
37
|
-
*
|
|
96
|
+
* Searches for merge requests and blobs in a GitLab group
|
|
38
97
|
*/
|
|
39
98
|
const searchGroup = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
40
99
|
const { authToken, baseUrl } = authParams;
|
|
41
|
-
const gitlabBaseUrl =
|
|
100
|
+
const gitlabBaseUrl = baseUrl !== null && baseUrl !== void 0 ? baseUrl : GITLAB_API_URL;
|
|
101
|
+
const gitlabBaseApiUrl = `${gitlabBaseUrl}/api/v4`;
|
|
42
102
|
if (!authToken) {
|
|
43
103
|
throw new Error(MISSING_AUTH_TOKEN);
|
|
44
104
|
}
|
|
45
105
|
const { query, groupId } = params;
|
|
46
106
|
const [mrResults, blobResults] = yield Promise.all([
|
|
47
|
-
globalSearch({
|
|
48
|
-
|
|
107
|
+
globalSearch({
|
|
108
|
+
scope: "merge_requests",
|
|
109
|
+
query,
|
|
110
|
+
groupId,
|
|
111
|
+
authToken,
|
|
112
|
+
baseUrl: gitlabBaseApiUrl,
|
|
113
|
+
}),
|
|
114
|
+
globalSearch({
|
|
115
|
+
scope: "blobs",
|
|
116
|
+
query,
|
|
117
|
+
groupId,
|
|
118
|
+
authToken,
|
|
119
|
+
baseUrl: gitlabBaseApiUrl,
|
|
120
|
+
}),
|
|
49
121
|
]);
|
|
50
122
|
const mergeRequests = yield Promise.all(mrResults.map((metadata) => __awaiter(void 0, void 0, void 0, function* () {
|
|
51
123
|
const diffs = yield getMRDiffs({
|
|
52
124
|
projectId: metadata.project_id,
|
|
53
125
|
mrIid: metadata.iid,
|
|
54
126
|
authToken,
|
|
55
|
-
baseUrl:
|
|
127
|
+
baseUrl: gitlabBaseApiUrl,
|
|
56
128
|
});
|
|
57
129
|
return { metadata, diffs };
|
|
58
130
|
})));
|
|
59
|
-
|
|
131
|
+
// Enhance blobs with web URLs
|
|
132
|
+
const blobsWithUrls = yield Promise.all(blobResults.map(blob => enhanceBlobWithUrl(blob, authToken, gitlabBaseApiUrl, gitlabBaseUrl)));
|
|
133
|
+
const blobs = blobsWithUrls.map(blob => {
|
|
60
134
|
const matches = mergeRequests
|
|
61
135
|
.filter(mr => mr.metadata.project_id === blob.project_id && mr.diffs.some(diff => diff.new_path === blob.path))
|
|
62
136
|
.map(mr => {
|
package/package.json
CHANGED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
const fillTemplateAction = (_a) => __awaiter(void 0, [_a], void 0, function* ({ template }) {
|
|
13
|
-
// Simply return the template without any modification
|
|
14
|
-
return {
|
|
15
|
-
result: template,
|
|
16
|
-
};
|
|
17
|
-
});
|
|
18
|
-
exports.default = fillTemplateAction;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const axios_1 = __importDefault(require("axios"));
|
|
16
|
-
const genericApiCall = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, }) {
|
|
17
|
-
try {
|
|
18
|
-
const { endpoint, method, headers, body } = params;
|
|
19
|
-
const response = yield (0, axios_1.default)({
|
|
20
|
-
url: endpoint,
|
|
21
|
-
method,
|
|
22
|
-
headers,
|
|
23
|
-
data: method !== "GET" ? body : undefined,
|
|
24
|
-
});
|
|
25
|
-
return {
|
|
26
|
-
statusCode: response.status,
|
|
27
|
-
headers: response.headers,
|
|
28
|
-
data: response.data,
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
catch (error) {
|
|
32
|
-
if (axios_1.default.isAxiosError(error)) {
|
|
33
|
-
throw Error("Axios Error: " + (error.message || "Failed to make API call"));
|
|
34
|
-
}
|
|
35
|
-
throw Error("Error: " + (error || "Failed to make API call"));
|
|
36
|
-
}
|
|
37
|
-
});
|
|
38
|
-
exports.default = genericApiCall;
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import pdf from "pdf-parse/lib/pdf-parse.js";
|
|
11
|
-
import { axiosClient } from "../../util/axiosClient.js";
|
|
12
|
-
import mammoth from "mammoth";
|
|
13
|
-
import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
|
|
14
|
-
const getDriveFileContentByID = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
15
|
-
if (!authParams.authToken) {
|
|
16
|
-
return { success: false, error: MISSING_AUTH_TOKEN };
|
|
17
|
-
}
|
|
18
|
-
const { fileId, limit } = params;
|
|
19
|
-
try {
|
|
20
|
-
// First, get file metadata to determine the file type
|
|
21
|
-
const metadataUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?fields=name,mimeType,size`;
|
|
22
|
-
const metadataRes = yield axiosClient.get(metadataUrl, {
|
|
23
|
-
headers: {
|
|
24
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
25
|
-
},
|
|
26
|
-
});
|
|
27
|
-
const { name: fileName, mimeType, size } = metadataRes.data;
|
|
28
|
-
// Check if file is too large (50MB limit for safety)
|
|
29
|
-
if (size && parseInt(size) > 50 * 1024 * 1024) {
|
|
30
|
-
return {
|
|
31
|
-
success: false,
|
|
32
|
-
error: "File too large (>50MB)",
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
let content = "";
|
|
36
|
-
// Handle different file types - read content directly
|
|
37
|
-
if (mimeType === "application/vnd.google-apps.document") {
|
|
38
|
-
// Google Docs - download as plain text
|
|
39
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&format=txt`;
|
|
40
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
41
|
-
headers: {
|
|
42
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
43
|
-
},
|
|
44
|
-
responseType: 'text',
|
|
45
|
-
});
|
|
46
|
-
content = downloadRes.data;
|
|
47
|
-
}
|
|
48
|
-
else if (mimeType === "application/vnd.google-apps.spreadsheet") {
|
|
49
|
-
// Google Sheets - download as CSV
|
|
50
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&format=csv`;
|
|
51
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
52
|
-
headers: {
|
|
53
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
54
|
-
},
|
|
55
|
-
responseType: 'text',
|
|
56
|
-
});
|
|
57
|
-
content = downloadRes.data;
|
|
58
|
-
}
|
|
59
|
-
else if (mimeType === "application/vnd.google-apps.presentation") {
|
|
60
|
-
// Google Slides - download as plain text
|
|
61
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&format=txt`;
|
|
62
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
63
|
-
headers: {
|
|
64
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
65
|
-
},
|
|
66
|
-
responseType: 'text',
|
|
67
|
-
});
|
|
68
|
-
content = downloadRes.data;
|
|
69
|
-
}
|
|
70
|
-
else if (mimeType === "application/pdf") {
|
|
71
|
-
// PDF files - use pdf-parse
|
|
72
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`;
|
|
73
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
74
|
-
headers: {
|
|
75
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
76
|
-
},
|
|
77
|
-
responseType: 'arraybuffer',
|
|
78
|
-
});
|
|
79
|
-
try {
|
|
80
|
-
const pdfData = yield pdf(downloadRes.data);
|
|
81
|
-
content = pdfData.text;
|
|
82
|
-
}
|
|
83
|
-
catch (pdfError) {
|
|
84
|
-
return {
|
|
85
|
-
success: false,
|
|
86
|
-
error: `Failed to parse PDF: ${pdfError instanceof Error ? pdfError.message : 'Unknown PDF error'}`,
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
else if (mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
|
|
91
|
-
mimeType === "application/msword") {
|
|
92
|
-
// Word documents (.docx or .doc) - download and extract text using mammoth
|
|
93
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`;
|
|
94
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
95
|
-
headers: {
|
|
96
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
97
|
-
},
|
|
98
|
-
responseType: 'arraybuffer',
|
|
99
|
-
});
|
|
100
|
-
try {
|
|
101
|
-
// mammoth works with .docx files. It will ignore formatting and return raw text
|
|
102
|
-
const result = yield mammoth.extractRawText({ buffer: Buffer.from(downloadRes.data) });
|
|
103
|
-
content = result.value; // raw text
|
|
104
|
-
}
|
|
105
|
-
catch (wordError) {
|
|
106
|
-
return {
|
|
107
|
-
success: false,
|
|
108
|
-
error: `Failed to parse Word document: ${wordError instanceof Error ? wordError.message : 'Unknown Word error'}`,
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
else if (mimeType === "text/plain" ||
|
|
113
|
-
mimeType === "text/html" ||
|
|
114
|
-
mimeType === "application/rtf" ||
|
|
115
|
-
(mimeType === null || mimeType === void 0 ? void 0 : mimeType.startsWith("text/"))) {
|
|
116
|
-
// Text-based files
|
|
117
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`;
|
|
118
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
119
|
-
headers: {
|
|
120
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
121
|
-
},
|
|
122
|
-
responseType: 'text',
|
|
123
|
-
});
|
|
124
|
-
content = downloadRes.data;
|
|
125
|
-
}
|
|
126
|
-
else if (mimeType === null || mimeType === void 0 ? void 0 : mimeType.startsWith("image/")) {
|
|
127
|
-
// Skip images
|
|
128
|
-
return {
|
|
129
|
-
success: false,
|
|
130
|
-
error: "Image files are not supported for text extraction",
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
else {
|
|
134
|
-
// Unsupported file type
|
|
135
|
-
return {
|
|
136
|
-
success: false,
|
|
137
|
-
error: `Unsupported file type: ${mimeType}`,
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
content = content.trim();
|
|
141
|
-
const originalLength = content.length;
|
|
142
|
-
// Naive way to truncate content
|
|
143
|
-
if (limit && content.length > limit) {
|
|
144
|
-
content = content.substring(0, limit);
|
|
145
|
-
}
|
|
146
|
-
return {
|
|
147
|
-
success: true,
|
|
148
|
-
content,
|
|
149
|
-
fileName,
|
|
150
|
-
fileLength: originalLength,
|
|
151
|
-
};
|
|
152
|
-
}
|
|
153
|
-
catch (error) {
|
|
154
|
-
console.error("Error getting Google Drive file content", error);
|
|
155
|
-
return {
|
|
156
|
-
success: false,
|
|
157
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
});
|
|
161
|
-
export default getDriveFileContentByID;
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { axiosClient } from "../../util/axiosClient.js";
|
|
11
|
-
import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
|
|
12
|
-
const searchDriveByKeywords = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
13
|
-
var _b;
|
|
14
|
-
if (!authParams.authToken) {
|
|
15
|
-
return { success: false, error: MISSING_AUTH_TOKEN, files: [] };
|
|
16
|
-
}
|
|
17
|
-
const { keywords, limit } = params;
|
|
18
|
-
// Build the query: fullText contains 'keyword1' or fullText contains 'keyword2' ...
|
|
19
|
-
const query = keywords.map(kw => `fullText contains '${kw.replace(/'/g, "\\'")}'`).join(" or ");
|
|
20
|
-
const url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&supportsAllDrives=true&includeItemsFromAllDrives=true`;
|
|
21
|
-
// 1. Get the file metadata from google drive search
|
|
22
|
-
let files = [];
|
|
23
|
-
try {
|
|
24
|
-
const res = yield axiosClient.get(url, {
|
|
25
|
-
headers: {
|
|
26
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
27
|
-
},
|
|
28
|
-
});
|
|
29
|
-
files =
|
|
30
|
-
((_b = res.data.files) === null || _b === void 0 ? void 0 : _b.map((file) => ({
|
|
31
|
-
id: file.id || "",
|
|
32
|
-
name: file.name || "",
|
|
33
|
-
mimeType: file.mimeType || "",
|
|
34
|
-
url: file.webViewLink || "",
|
|
35
|
-
}))) || [];
|
|
36
|
-
}
|
|
37
|
-
catch (error) {
|
|
38
|
-
console.error("Error searching Google Drive", error);
|
|
39
|
-
return {
|
|
40
|
-
success: false,
|
|
41
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
42
|
-
files: [],
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
files = limit ? files.splice(0, limit) : files;
|
|
46
|
-
});
|
|
47
|
-
export default searchDriveByKeywords;
|
|
@@ -1,110 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { axiosClient } from "../../util/axiosClient.js";
|
|
11
|
-
import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
|
|
12
|
-
import extractContentFromDriveFileId from "./utils/extractContentFromDriveFileId.js";
|
|
13
|
-
import { normalizeText } from "../../../utils/string.js";
|
|
14
|
-
/** Intelligently selects a section of text around the median occurrence of keywords */
|
|
15
|
-
const intelligentSelectByMedianSection = (text, keywords, limit) => {
|
|
16
|
-
if (!text || text.length <= limit)
|
|
17
|
-
return text;
|
|
18
|
-
if (!(keywords === null || keywords === void 0 ? void 0 : keywords.length))
|
|
19
|
-
return text.substring(0, limit);
|
|
20
|
-
// Find all keyword positions (case-insensitive, limited to first 1000 matches)
|
|
21
|
-
const positions = [];
|
|
22
|
-
const normalizedText = normalizeText(text);
|
|
23
|
-
for (const keyword of keywords) {
|
|
24
|
-
if (keyword.length < 3)
|
|
25
|
-
continue; // Skip very short keywords
|
|
26
|
-
let pos = -1;
|
|
27
|
-
const normalizedKeyword = normalizeText(keyword);
|
|
28
|
-
while ((pos = normalizedText.indexOf(normalizedKeyword, pos + 1)) !== -1 && positions.length < 1000) {
|
|
29
|
-
positions.push(pos);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
if (!positions.length)
|
|
33
|
-
return text.substring(0, limit);
|
|
34
|
-
// Find median position (using sort for simplicity, still fast for 1000 elements)
|
|
35
|
-
positions.sort((a, b) => a - b);
|
|
36
|
-
const medianPos = positions[Math.floor(positions.length / 2)];
|
|
37
|
-
// Return window around median
|
|
38
|
-
const half = Math.floor(limit / 2);
|
|
39
|
-
const start = Math.max(0, medianPos - half);
|
|
40
|
-
const end = Math.min(text.length, start + limit);
|
|
41
|
-
return text.substring(start, end);
|
|
42
|
-
};
|
|
43
|
-
const searchDriveAndGetContentByKeywords = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
44
|
-
var _b;
|
|
45
|
-
if (!authParams.authToken) {
|
|
46
|
-
return { success: false, error: MISSING_AUTH_TOKEN, files: [] };
|
|
47
|
-
}
|
|
48
|
-
const { keywords, fileLimit, fileSizeLimit } = params;
|
|
49
|
-
let files = [];
|
|
50
|
-
// 1. Search for files and get their metadata
|
|
51
|
-
// Build the query: fullText contains 'keyword1' or fullText contains 'keyword2' ...
|
|
52
|
-
const query = keywords.map(kw => `fullText contains '${kw.replace(/'/g, "\\'")}'`).join(" or ");
|
|
53
|
-
const url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&supportsAllDrives=true&includeItemsFromAllDrives=true`;
|
|
54
|
-
try {
|
|
55
|
-
const res = yield axiosClient.get(url, {
|
|
56
|
-
headers: {
|
|
57
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
58
|
-
},
|
|
59
|
-
});
|
|
60
|
-
files =
|
|
61
|
-
((_b = res.data.files) === null || _b === void 0 ? void 0 : _b.map((file) => ({
|
|
62
|
-
id: file.id,
|
|
63
|
-
name: file.name,
|
|
64
|
-
mimeType: file.mimeType,
|
|
65
|
-
url: file.webViewLink,
|
|
66
|
-
}))) || [];
|
|
67
|
-
}
|
|
68
|
-
catch (error) {
|
|
69
|
-
console.error("Error searching Google Drive", error);
|
|
70
|
-
return {
|
|
71
|
-
success: false,
|
|
72
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
73
|
-
files: [],
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
files = fileLimit ? files.splice(0, fileLimit) : files;
|
|
77
|
-
// 2. Extract content from files and do some smart range selection
|
|
78
|
-
const processedFiles = yield Promise.all(files
|
|
79
|
-
.filter((file) => file.id && file.mimeType)
|
|
80
|
-
.map((file) => __awaiter(void 0, void 0, void 0, function* () {
|
|
81
|
-
const content = yield extractContentFromDriveFileId({
|
|
82
|
-
params: { fileId: file.id, mimeType: file.mimeType },
|
|
83
|
-
authParams,
|
|
84
|
-
});
|
|
85
|
-
if (content.success) {
|
|
86
|
-
let selectedContent = content.content;
|
|
87
|
-
if (fileSizeLimit && selectedContent && selectedContent.length > fileSizeLimit) {
|
|
88
|
-
selectedContent = intelligentSelectByMedianSection(selectedContent, keywords, fileSizeLimit);
|
|
89
|
-
}
|
|
90
|
-
return {
|
|
91
|
-
id: file.id || "",
|
|
92
|
-
name: file.name || "",
|
|
93
|
-
mimeType: file.mimeType || "",
|
|
94
|
-
url: file.url || "",
|
|
95
|
-
content: selectedContent,
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
else {
|
|
99
|
-
return {
|
|
100
|
-
id: file.id || "",
|
|
101
|
-
name: file.name || "",
|
|
102
|
-
mimeType: file.mimeType || "",
|
|
103
|
-
url: file.url || "",
|
|
104
|
-
error: content.error,
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
})));
|
|
108
|
-
return { success: true, files: processedFiles };
|
|
109
|
-
});
|
|
110
|
-
export default searchDriveAndGetContentByKeywords;
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { axiosClient } from "../../util/axiosClient.js";
|
|
11
|
-
import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
|
|
12
|
-
import extractContentFromDriveFileId from "./utils/extractContentFromDriveFileId.js";
|
|
13
|
-
const searchDriveAndGetContentByQuery = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
14
|
-
var _b;
|
|
15
|
-
if (!authParams.authToken) {
|
|
16
|
-
return { success: false, error: MISSING_AUTH_TOKEN, files: [] };
|
|
17
|
-
}
|
|
18
|
-
const { query, fileLimit, fileSizeLimit } = params;
|
|
19
|
-
let files = [];
|
|
20
|
-
// 1. Search for files and get their metadata
|
|
21
|
-
const url = `https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(query)}&fields=files(id,name,mimeType,webViewLink)&supportsAllDrives=true&includeItemsFromAllDrives=true&corpora=allDrives`;
|
|
22
|
-
try {
|
|
23
|
-
const res = yield axiosClient.get(url, {
|
|
24
|
-
headers: {
|
|
25
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
26
|
-
},
|
|
27
|
-
});
|
|
28
|
-
files =
|
|
29
|
-
((_b = res.data.files) === null || _b === void 0 ? void 0 : _b.map((file) => ({
|
|
30
|
-
id: file.id,
|
|
31
|
-
name: file.name,
|
|
32
|
-
mimeType: file.mimeType,
|
|
33
|
-
url: file.webViewLink,
|
|
34
|
-
}))) || [];
|
|
35
|
-
}
|
|
36
|
-
catch (error) {
|
|
37
|
-
console.error("Error searching Google Drive", error);
|
|
38
|
-
return {
|
|
39
|
-
success: false,
|
|
40
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
41
|
-
files: [],
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
files = fileLimit ? files.splice(0, fileLimit) : files;
|
|
45
|
-
// 2. Extract content from files and do some smart range selection
|
|
46
|
-
const processedFiles = yield Promise.all(files
|
|
47
|
-
.filter((file) => file.id && file.mimeType)
|
|
48
|
-
.map((file) => __awaiter(void 0, void 0, void 0, function* () {
|
|
49
|
-
const content = yield extractContentFromDriveFileId({
|
|
50
|
-
params: { fileId: file.id, mimeType: file.mimeType },
|
|
51
|
-
authParams,
|
|
52
|
-
});
|
|
53
|
-
if (content.success) {
|
|
54
|
-
let selectedContent = content.content;
|
|
55
|
-
if (fileSizeLimit && selectedContent && selectedContent.length > fileSizeLimit) {
|
|
56
|
-
selectedContent = selectedContent.substring(0, fileSizeLimit);
|
|
57
|
-
}
|
|
58
|
-
return {
|
|
59
|
-
id: file.id || "",
|
|
60
|
-
name: file.name || "",
|
|
61
|
-
mimeType: file.mimeType || "",
|
|
62
|
-
url: file.url || "",
|
|
63
|
-
content: selectedContent,
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
return {
|
|
68
|
-
id: file.id || "",
|
|
69
|
-
name: file.name || "",
|
|
70
|
-
mimeType: file.mimeType || "",
|
|
71
|
-
url: file.url || "",
|
|
72
|
-
error: content.error,
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
})));
|
|
76
|
-
return { success: true, files: processedFiles };
|
|
77
|
-
});
|
|
78
|
-
export default searchDriveAndGetContentByQuery;
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { AuthParamsType } from "../../../autogen/types.js";
|
|
2
|
-
export type getDriveFileContentParams = {
|
|
3
|
-
fileId: string;
|
|
4
|
-
mimeType: string;
|
|
5
|
-
};
|
|
6
|
-
export type getDriveFileContentOutput = {
|
|
7
|
-
success: boolean;
|
|
8
|
-
content?: string;
|
|
9
|
-
error?: string;
|
|
10
|
-
};
|
|
11
|
-
declare const extractContentFromDriveFileId: ({ params, authParams, }: {
|
|
12
|
-
params: getDriveFileContentParams;
|
|
13
|
-
authParams: AuthParamsType;
|
|
14
|
-
}) => Promise<getDriveFileContentOutput>;
|
|
15
|
-
export default extractContentFromDriveFileId;
|
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { axiosClient } from "../../../util/axiosClient.js";
|
|
11
|
-
import mammoth from "mammoth";
|
|
12
|
-
import { MISSING_AUTH_TOKEN } from "../../../util/missingAuthConstants.js";
|
|
13
|
-
const extractContentFromDriveFileId = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
14
|
-
if (!authParams.authToken) {
|
|
15
|
-
return { success: false, error: MISSING_AUTH_TOKEN };
|
|
16
|
-
}
|
|
17
|
-
const { fileId, mimeType } = params;
|
|
18
|
-
let content = "";
|
|
19
|
-
try {
|
|
20
|
-
// Handle different file types - read content directly
|
|
21
|
-
if (mimeType === "application/vnd.google-apps.document") {
|
|
22
|
-
// Google Docs - download as plain text
|
|
23
|
-
const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=text/plain`;
|
|
24
|
-
const exportRes = yield axiosClient.get(exportUrl, {
|
|
25
|
-
headers: {
|
|
26
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
27
|
-
},
|
|
28
|
-
responseType: "text",
|
|
29
|
-
});
|
|
30
|
-
content = exportRes.data;
|
|
31
|
-
}
|
|
32
|
-
else if (mimeType === "application/vnd.google-apps.spreadsheet") {
|
|
33
|
-
// Google Sheets - download as CSV
|
|
34
|
-
const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=text/csv`;
|
|
35
|
-
const exportRes = yield axiosClient.get(exportUrl, {
|
|
36
|
-
headers: {
|
|
37
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
38
|
-
},
|
|
39
|
-
responseType: "text",
|
|
40
|
-
});
|
|
41
|
-
// Clean up excessive commas from empty columns
|
|
42
|
-
content = exportRes.data
|
|
43
|
-
.split("\n")
|
|
44
|
-
.map((line) => line.replace(/,+$/, "")) // Remove trailing commas
|
|
45
|
-
.map((line) => line.replace(/,{2,}/g, ",")) // Replace multiple commas with single comma
|
|
46
|
-
.join("\n");
|
|
47
|
-
}
|
|
48
|
-
else if (mimeType === "application/vnd.google-apps.presentation") {
|
|
49
|
-
// Google Slides - download as plain text
|
|
50
|
-
const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=text/plain`;
|
|
51
|
-
const exportRes = yield axiosClient.get(exportUrl, {
|
|
52
|
-
headers: {
|
|
53
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
54
|
-
},
|
|
55
|
-
responseType: "text",
|
|
56
|
-
});
|
|
57
|
-
content = exportRes.data;
|
|
58
|
-
}
|
|
59
|
-
else if (mimeType === "application/pdf") {
|
|
60
|
-
return {
|
|
61
|
-
success: false,
|
|
62
|
-
error: "PDF files are not supported for text extraction",
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
else if (mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
|
|
66
|
-
mimeType === "application/msword") {
|
|
67
|
-
// Word documents (.docx or .doc) - download and extract text using mammoth
|
|
68
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`;
|
|
69
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
70
|
-
headers: {
|
|
71
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
72
|
-
},
|
|
73
|
-
responseType: "arraybuffer",
|
|
74
|
-
});
|
|
75
|
-
try {
|
|
76
|
-
// mammoth works with .docx files. It will ignore formatting and return raw text
|
|
77
|
-
const result = yield mammoth.extractRawText({ buffer: Buffer.from(downloadRes.data) });
|
|
78
|
-
content = result.value; // raw text
|
|
79
|
-
}
|
|
80
|
-
catch (wordError) {
|
|
81
|
-
return {
|
|
82
|
-
success: false,
|
|
83
|
-
error: `Failed to parse Word document: ${wordError instanceof Error ? wordError.message : "Unknown Word error"}`,
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
else if (mimeType === "text/plain" ||
|
|
88
|
-
mimeType === "text/html" ||
|
|
89
|
-
mimeType === "application/rtf" ||
|
|
90
|
-
(mimeType === null || mimeType === void 0 ? void 0 : mimeType.startsWith("text/"))) {
|
|
91
|
-
// Text-based files
|
|
92
|
-
const downloadUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`;
|
|
93
|
-
const downloadRes = yield axiosClient.get(downloadUrl, {
|
|
94
|
-
headers: {
|
|
95
|
-
Authorization: `Bearer ${authParams.authToken}`,
|
|
96
|
-
},
|
|
97
|
-
responseType: "text",
|
|
98
|
-
});
|
|
99
|
-
content = downloadRes.data;
|
|
100
|
-
}
|
|
101
|
-
else if (mimeType === null || mimeType === void 0 ? void 0 : mimeType.startsWith("image/")) {
|
|
102
|
-
// Skip images
|
|
103
|
-
return {
|
|
104
|
-
success: false,
|
|
105
|
-
error: "Image files are not supported for text extraction",
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
// Unsupported file type
|
|
110
|
-
return {
|
|
111
|
-
success: false,
|
|
112
|
-
error: `Unsupported file type: ${mimeType}`,
|
|
113
|
-
};
|
|
114
|
-
}
|
|
115
|
-
content = content.trim();
|
|
116
|
-
return {
|
|
117
|
-
success: true,
|
|
118
|
-
content,
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
catch (error) {
|
|
122
|
-
console.error("Error getting Google Drive file content", error);
|
|
123
|
-
return {
|
|
124
|
-
success: false,
|
|
125
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
});
|
|
129
|
-
export default extractContentFromDriveFileId;
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const axios_1 = __importDefault(require("axios"));
|
|
16
|
-
const types_1 = require("../../autogen/types");
|
|
17
|
-
const INCLUDED_TYPES = [
|
|
18
|
-
"monument",
|
|
19
|
-
"museum",
|
|
20
|
-
"art_gallery",
|
|
21
|
-
"sculpture",
|
|
22
|
-
"cultural_landmark",
|
|
23
|
-
"historical_place",
|
|
24
|
-
"performing_arts_theater",
|
|
25
|
-
"university",
|
|
26
|
-
"aquarium",
|
|
27
|
-
"botanical_garden",
|
|
28
|
-
"comedy_club",
|
|
29
|
-
"park",
|
|
30
|
-
"movie_theater",
|
|
31
|
-
"national_park",
|
|
32
|
-
"garden",
|
|
33
|
-
"night_club",
|
|
34
|
-
"tourist_attraction",
|
|
35
|
-
"water_park",
|
|
36
|
-
"zoo",
|
|
37
|
-
"bar",
|
|
38
|
-
"restaurant",
|
|
39
|
-
"food_court",
|
|
40
|
-
"bakery",
|
|
41
|
-
"cafe",
|
|
42
|
-
"coffee_shop",
|
|
43
|
-
"pub",
|
|
44
|
-
"wine_bar",
|
|
45
|
-
"spa",
|
|
46
|
-
"beach",
|
|
47
|
-
"market",
|
|
48
|
-
"shopping_mall",
|
|
49
|
-
"stadium",
|
|
50
|
-
];
|
|
51
|
-
const nearbysearch = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
52
|
-
const url = `https://places.googleapis.com/v1/places:searchNearby`;
|
|
53
|
-
const fieldMask = [
|
|
54
|
-
"places.displayName",
|
|
55
|
-
"places.formattedAddress",
|
|
56
|
-
"places.priceLevel",
|
|
57
|
-
"places.rating",
|
|
58
|
-
"places.primaryTypeDisplayName",
|
|
59
|
-
"places.editorialSummary",
|
|
60
|
-
"places.regularOpeningHours",
|
|
61
|
-
].join(",");
|
|
62
|
-
const response = yield axios_1.default.post(url, {
|
|
63
|
-
maxResultCount: 20,
|
|
64
|
-
includedTypes: INCLUDED_TYPES,
|
|
65
|
-
locationRestriction: {
|
|
66
|
-
circle: {
|
|
67
|
-
center: {
|
|
68
|
-
latitude: params.latitude,
|
|
69
|
-
longitude: params.longitude,
|
|
70
|
-
},
|
|
71
|
-
radius: 10000,
|
|
72
|
-
},
|
|
73
|
-
},
|
|
74
|
-
}, {
|
|
75
|
-
headers: {
|
|
76
|
-
"X-Goog-Api-Key": authParams.apiKey,
|
|
77
|
-
"X-Goog-FieldMask": fieldMask,
|
|
78
|
-
"Content-Type": "application/json",
|
|
79
|
-
},
|
|
80
|
-
});
|
|
81
|
-
return types_1.googlemapsNearbysearchOutputSchema.parse({
|
|
82
|
-
results: response.data.places.map((place) => {
|
|
83
|
-
var _a, _b;
|
|
84
|
-
return ({
|
|
85
|
-
name: place.displayName.text,
|
|
86
|
-
address: place.formattedAddress,
|
|
87
|
-
priceLevel: place.priceLevel,
|
|
88
|
-
rating: place.rating,
|
|
89
|
-
primaryType: place.primaryTypeDisplayName.text,
|
|
90
|
-
editorialSummary: ((_a = place.editorialSummary) === null || _a === void 0 ? void 0 : _a.text) || "",
|
|
91
|
-
openingHours: ((_b = place.regularOpeningHours) === null || _b === void 0 ? void 0 : _b.weekdayDescriptions.join("\n")) || "",
|
|
92
|
-
});
|
|
93
|
-
}),
|
|
94
|
-
});
|
|
95
|
-
});
|
|
96
|
-
exports.default = nearbysearch;
|
|
@@ -1,154 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const snowflake_sdk_1 = __importDefault(require("snowflake-sdk"));
|
|
16
|
-
const crypto_1 = __importDefault(require("crypto"));
|
|
17
|
-
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
18
|
-
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
|
19
|
-
const uuid_1 = require("uuid");
|
|
20
|
-
// Only log errors.
|
|
21
|
-
snowflake_sdk_1.default.configure({ logLevel: "ERROR" });
|
|
22
|
-
const runSnowflakeQueryWriteResultsToS3 = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
23
|
-
const { databaseName, warehouse, query, user, accountName, s3BucketName, s3Region, outputFormat = "json" } = params;
|
|
24
|
-
const { apiKey: privateKey, awsAccessKeyId, awsSecretAccessKey } = authParams;
|
|
25
|
-
if (!privateKey) {
|
|
26
|
-
throw new Error("Snowflake private key is required");
|
|
27
|
-
}
|
|
28
|
-
if (!awsAccessKeyId || !awsSecretAccessKey) {
|
|
29
|
-
throw new Error("AWS credentials are required");
|
|
30
|
-
}
|
|
31
|
-
if (!accountName || !user || !databaseName || !warehouse || !query || !s3BucketName) {
|
|
32
|
-
throw new Error("Missing required parameters for Snowflake query or S3 destination");
|
|
33
|
-
}
|
|
34
|
-
const getPrivateKeyCorrectFormat = (privateKey) => {
|
|
35
|
-
const buffer = Buffer.from(privateKey);
|
|
36
|
-
const privateKeyObject = crypto_1.default.createPrivateKey({
|
|
37
|
-
key: buffer,
|
|
38
|
-
format: "pem",
|
|
39
|
-
passphrase: "password",
|
|
40
|
-
});
|
|
41
|
-
const privateKeyCorrectFormat = privateKeyObject.export({
|
|
42
|
-
format: "pem",
|
|
43
|
-
type: "pkcs8",
|
|
44
|
-
});
|
|
45
|
-
return privateKeyCorrectFormat.toString();
|
|
46
|
-
};
|
|
47
|
-
const executeQueryAndFormatData = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
48
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
49
|
-
const queryResults = yield new Promise((resolve, reject) => {
|
|
50
|
-
connection.execute({
|
|
51
|
-
sqlText: query,
|
|
52
|
-
complete: (err, stmt, rows) => {
|
|
53
|
-
if (err) {
|
|
54
|
-
return reject(err);
|
|
55
|
-
}
|
|
56
|
-
return resolve(rows || []);
|
|
57
|
-
},
|
|
58
|
-
});
|
|
59
|
-
});
|
|
60
|
-
// Format the results based on the output format
|
|
61
|
-
let formattedData;
|
|
62
|
-
if (outputFormat.toLowerCase() === "csv") {
|
|
63
|
-
if (queryResults.length === 0) {
|
|
64
|
-
formattedData = "";
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
const headers = Object.keys(queryResults[0]).join(",");
|
|
68
|
-
const rows = queryResults.map(row => Object.values(row)
|
|
69
|
-
.map(value => (typeof value === "object" && value !== null ? JSON.stringify(value) : value))
|
|
70
|
-
.join(","));
|
|
71
|
-
formattedData = [headers, ...rows].join("\n");
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
// Default to JSON
|
|
76
|
-
formattedData = JSON.stringify(queryResults, null, 2);
|
|
77
|
-
}
|
|
78
|
-
return { formattedData, resultsLength: queryResults.length };
|
|
79
|
-
});
|
|
80
|
-
const uploadToS3AndGetURL = (formattedData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
81
|
-
// Create S3 client
|
|
82
|
-
const s3Client = new client_s3_1.S3Client({
|
|
83
|
-
region: s3Region,
|
|
84
|
-
credentials: {
|
|
85
|
-
accessKeyId: awsAccessKeyId,
|
|
86
|
-
secretAccessKey: awsSecretAccessKey,
|
|
87
|
-
},
|
|
88
|
-
});
|
|
89
|
-
const contentType = outputFormat.toLowerCase() === "csv" ? "text/csv" : "application/json";
|
|
90
|
-
const fileExtension = outputFormat.toLowerCase() === "csv" ? "csv" : "json";
|
|
91
|
-
const finalKey = `${databaseName}/${(0, uuid_1.v4)()}.${fileExtension}`;
|
|
92
|
-
// Upload to S3 without ACL
|
|
93
|
-
const uploadCommand = new client_s3_1.PutObjectCommand({
|
|
94
|
-
Bucket: s3BucketName,
|
|
95
|
-
Key: finalKey,
|
|
96
|
-
Body: formattedData,
|
|
97
|
-
ContentType: contentType,
|
|
98
|
-
});
|
|
99
|
-
yield s3Client.send(uploadCommand);
|
|
100
|
-
// Generate a presigned URL (valid for an hour)
|
|
101
|
-
const getObjectCommand = new client_s3_1.GetObjectCommand({
|
|
102
|
-
Bucket: s3BucketName,
|
|
103
|
-
Key: finalKey,
|
|
104
|
-
});
|
|
105
|
-
const presignedUrl = yield (0, s3_request_presigner_1.getSignedUrl)(s3Client, getObjectCommand, { expiresIn: 3600 });
|
|
106
|
-
return presignedUrl;
|
|
107
|
-
});
|
|
108
|
-
// Process the private key
|
|
109
|
-
const privateKeyCorrectFormatString = getPrivateKeyCorrectFormat(privateKey);
|
|
110
|
-
// Set up a connection using snowflake-sdk
|
|
111
|
-
const connection = snowflake_sdk_1.default.createConnection({
|
|
112
|
-
account: accountName,
|
|
113
|
-
username: user,
|
|
114
|
-
privateKey: privateKeyCorrectFormatString,
|
|
115
|
-
authenticator: "SNOWFLAKE_JWT",
|
|
116
|
-
role: "ACCOUNTADMIN",
|
|
117
|
-
warehouse: warehouse,
|
|
118
|
-
database: databaseName,
|
|
119
|
-
});
|
|
120
|
-
try {
|
|
121
|
-
// Connect to Snowflake
|
|
122
|
-
yield new Promise((resolve, reject) => {
|
|
123
|
-
connection.connect((err, conn) => {
|
|
124
|
-
if (err) {
|
|
125
|
-
console.error("Unable to connect to Snowflake:", err.message);
|
|
126
|
-
return reject(err);
|
|
127
|
-
}
|
|
128
|
-
resolve(conn);
|
|
129
|
-
});
|
|
130
|
-
});
|
|
131
|
-
const { formattedData, resultsLength } = yield executeQueryAndFormatData();
|
|
132
|
-
const presignedUrl = yield uploadToS3AndGetURL(formattedData);
|
|
133
|
-
// Return fields to match schema definition
|
|
134
|
-
connection.destroy(err => {
|
|
135
|
-
if (err) {
|
|
136
|
-
console.log("Failed to disconnect from Snowflake:", err);
|
|
137
|
-
}
|
|
138
|
-
});
|
|
139
|
-
return {
|
|
140
|
-
bucketUrl: presignedUrl,
|
|
141
|
-
message: `Query results successfully written to S3. URL valid for 1 hour.`,
|
|
142
|
-
rowCount: resultsLength,
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
catch (error) {
|
|
146
|
-
connection.destroy(err => {
|
|
147
|
-
if (err) {
|
|
148
|
-
console.log("Failed to disconnect from Snowflake:", err);
|
|
149
|
-
}
|
|
150
|
-
});
|
|
151
|
-
throw Error(`An error occurred: ${error}`);
|
|
152
|
-
}
|
|
153
|
-
});
|
|
154
|
-
exports.default = runSnowflakeQueryWriteResultsToS3;
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
|
|
16
|
-
const scrapeTweetDataWithNitter = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
17
|
-
const tweetUrlRegex = /^(?:https?:\/\/)?(?:www\.)?(?:twitter\.com|x\.com)\/([a-zA-Z0-9_]+)\/status\/(\d+)(?:\?.*)?$/;
|
|
18
|
-
if (!tweetUrlRegex.test(params.tweetUrl)) {
|
|
19
|
-
throw new Error("Invalid tweet URL. Expected format: https://twitter.com/username/status/id or https://x.com/username/status/id");
|
|
20
|
-
}
|
|
21
|
-
const nitterUrl = params.tweetUrl.replace(/^(?:https?:\/\/)?(?:www\.)?(?:twitter\.com|x\.com)/i, "https://nitter.net");
|
|
22
|
-
// Initialize Firecrawl
|
|
23
|
-
if (!authParams.apiKey) {
|
|
24
|
-
throw new Error("API key is required for X+Nitter+Firecrawl");
|
|
25
|
-
}
|
|
26
|
-
const firecrawl = new firecrawl_js_1.default({
|
|
27
|
-
apiKey: authParams.apiKey,
|
|
28
|
-
});
|
|
29
|
-
try {
|
|
30
|
-
// Scrape the Nitter URL
|
|
31
|
-
const result = yield firecrawl.scrapeUrl(nitterUrl);
|
|
32
|
-
if (!result.success) {
|
|
33
|
-
throw new Error(`Failed to scrape tweet: ${result.error || "Unknown error"}`);
|
|
34
|
-
}
|
|
35
|
-
// Extract the tweet text from the scraped content - simple approach - in practice, you might need more robust parsing based on nitter html structure
|
|
36
|
-
const tweetContent = result.markdown;
|
|
37
|
-
return {
|
|
38
|
-
text: tweetContent || "Error scraping with firecrawl",
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
catch (error) {
|
|
42
|
-
throw new Error(`Error scraping tweet: ${error instanceof Error ? error.message : error}`);
|
|
43
|
-
}
|
|
44
|
-
});
|
|
45
|
-
exports.default = scrapeTweetDataWithNitter;
|