@credal/actions 0.1.78 → 0.1.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/actions/autogen/templates.js +14 -1
- package/dist/actions/autogen/types.d.ts +9 -0
- package/dist/actions/autogen/types.js +12 -1
- package/dist/actions/providers/confluence/updatePage.d.ts +3 -0
- package/dist/actions/providers/confluence/updatePage.js +46 -0
- package/dist/actions/providers/generic/fillTemplateAction.d.ts +7 -0
- package/dist/actions/providers/generic/fillTemplateAction.js +18 -0
- package/dist/actions/providers/generic/genericApiCall.d.ts +3 -0
- package/dist/actions/providers/generic/genericApiCall.js +38 -0
- package/dist/actions/providers/googlemaps/nearbysearch.d.ts +3 -0
- package/dist/actions/providers/googlemaps/nearbysearch.js +96 -0
- package/dist/actions/providers/snowflake/runSnowflakeQuery.js +1 -0
- package/dist/actions/providers/snowflake/runSnowflakeQueryWriteResultsToS3.d.ts +3 -0
- package/dist/actions/providers/snowflake/runSnowflakeQueryWriteResultsToS3.js +154 -0
- package/dist/actions/providers/x/scrapeTweetDataWithNitter.d.ts +3 -0
- package/dist/actions/providers/x/scrapeTweetDataWithNitter.js +45 -0
- package/package.json +1 -1
- package/dist/actions/providers/salesforce/getSalesforceRecordByQuery.d.ts +0 -3
- package/dist/actions/providers/salesforce/getSalesforceRecordByQuery.js +0 -43
package/README.md
CHANGED
@@ -13,7 +13,7 @@ We strongly encourage you to develop actions that rely on oauth based credential
|
|
13
13
|
3. Run `npm run prettier-format` to format the new files
|
14
14
|
4. Create a new provider function in `src/actions/providers/<provider>/<action>.ts` (eg. `src/actions/providers/math/add.ts`) which exports a function using the generated types
|
15
15
|
5. If adding a new action or provider, update `src/actions/actionMapper.ts` and `src/actions/groups.ts`.
|
16
|
-
6. In `package.json
|
16
|
+
6. In `package.json` and `package-lock.json` (which must be updated in two places), bump the version number.
|
17
17
|
7. Run `npm publish --access public` to publish the new version to npm. (Need to be logged in via `npm login`)
|
18
18
|
|
19
19
|
## Usage
|
@@ -1723,6 +1723,15 @@ exports.snowflakeRunSnowflakeQueryDefinition = {
|
|
1723
1723
|
type: "number",
|
1724
1724
|
description: "A limit on the number of rows to return",
|
1725
1725
|
},
|
1726
|
+
codeInterpreterLimit: {
|
1727
|
+
type: "number",
|
1728
|
+
description: "A minimum number of rows required to pass to code interpreter (if enabled)",
|
1729
|
+
},
|
1730
|
+
generateImageViaCodeInterpreter: {
|
1731
|
+
enum: [true, false],
|
1732
|
+
type: "boolean",
|
1733
|
+
description: "Whether we should try to generate an image with results via code interpreter, regardless of if the codeInterpreter limit is reached",
|
1734
|
+
},
|
1726
1735
|
},
|
1727
1736
|
},
|
1728
1737
|
output: {
|
@@ -1742,6 +1751,10 @@ exports.snowflakeRunSnowflakeQueryDefinition = {
|
|
1742
1751
|
type: "number",
|
1743
1752
|
description: "The number of rows returned by the query",
|
1744
1753
|
},
|
1754
|
+
error: {
|
1755
|
+
type: "string",
|
1756
|
+
description: "The error that occurred if the query results failed or were limited",
|
1757
|
+
},
|
1745
1758
|
},
|
1746
1759
|
},
|
1747
1760
|
name: "runSnowflakeQuery",
|
@@ -5671,7 +5684,7 @@ exports.salesforceUpdateRecordDefinition = {
|
|
5671
5684
|
},
|
5672
5685
|
fieldsToUpdate: {
|
5673
5686
|
type: "object",
|
5674
|
-
description: "
|
5687
|
+
description: "A key,value pair where the keys are the fields to update on the record and the values are the new values of those fields.",
|
5675
5688
|
additionalProperties: {
|
5676
5689
|
type: "string",
|
5677
5690
|
},
|
@@ -1660,6 +1660,8 @@ export declare const snowflakeRunSnowflakeQueryParamsSchema: z.ZodObject<{
|
|
1660
1660
|
accountName: z.ZodString;
|
1661
1661
|
outputFormat: z.ZodOptional<z.ZodEnum<["json", "csv"]>>;
|
1662
1662
|
limit: z.ZodOptional<z.ZodNumber>;
|
1663
|
+
codeInterpreterLimit: z.ZodOptional<z.ZodNumber>;
|
1664
|
+
generateImageViaCodeInterpreter: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<true>, z.ZodLiteral<false>]>>;
|
1663
1665
|
}, "strip", z.ZodTypeAny, {
|
1664
1666
|
query: string;
|
1665
1667
|
databaseName: string;
|
@@ -1667,6 +1669,8 @@ export declare const snowflakeRunSnowflakeQueryParamsSchema: z.ZodObject<{
|
|
1667
1669
|
warehouse: string;
|
1668
1670
|
limit?: number | undefined;
|
1669
1671
|
outputFormat?: "json" | "csv" | undefined;
|
1672
|
+
codeInterpreterLimit?: number | undefined;
|
1673
|
+
generateImageViaCodeInterpreter?: boolean | undefined;
|
1670
1674
|
}, {
|
1671
1675
|
query: string;
|
1672
1676
|
databaseName: string;
|
@@ -1674,20 +1678,25 @@ export declare const snowflakeRunSnowflakeQueryParamsSchema: z.ZodObject<{
|
|
1674
1678
|
warehouse: string;
|
1675
1679
|
limit?: number | undefined;
|
1676
1680
|
outputFormat?: "json" | "csv" | undefined;
|
1681
|
+
codeInterpreterLimit?: number | undefined;
|
1682
|
+
generateImageViaCodeInterpreter?: boolean | undefined;
|
1677
1683
|
}>;
|
1678
1684
|
export type snowflakeRunSnowflakeQueryParamsType = z.infer<typeof snowflakeRunSnowflakeQueryParamsSchema>;
|
1679
1685
|
export declare const snowflakeRunSnowflakeQueryOutputSchema: z.ZodObject<{
|
1680
1686
|
format: z.ZodEnum<["json", "csv"]>;
|
1681
1687
|
content: z.ZodString;
|
1682
1688
|
rowCount: z.ZodNumber;
|
1689
|
+
error: z.ZodOptional<z.ZodString>;
|
1683
1690
|
}, "strip", z.ZodTypeAny, {
|
1684
1691
|
content: string;
|
1685
1692
|
format: "json" | "csv";
|
1686
1693
|
rowCount: number;
|
1694
|
+
error?: string | undefined;
|
1687
1695
|
}, {
|
1688
1696
|
content: string;
|
1689
1697
|
format: "json" | "csv";
|
1690
1698
|
rowCount: number;
|
1699
|
+
error?: string | undefined;
|
1691
1700
|
}>;
|
1692
1701
|
export type snowflakeRunSnowflakeQueryOutputType = z.infer<typeof snowflakeRunSnowflakeQueryOutputSchema>;
|
1693
1702
|
export type snowflakeRunSnowflakeQueryFunction = ActionFunction<snowflakeRunSnowflakeQueryParamsType, AuthParamsType, snowflakeRunSnowflakeQueryOutputType>;
|
@@ -526,11 +526,20 @@ exports.snowflakeRunSnowflakeQueryParamsSchema = zod_1.z.object({
|
|
526
526
|
accountName: zod_1.z.string().describe("The name of the Snowflake account"),
|
527
527
|
outputFormat: zod_1.z.enum(["json", "csv"]).describe("The format of the output").optional(),
|
528
528
|
limit: zod_1.z.number().describe("A limit on the number of rows to return").optional(),
|
529
|
+
codeInterpreterLimit: zod_1.z
|
530
|
+
.number()
|
531
|
+
.describe("A minimum number of rows required to pass to code interpreter (if enabled)")
|
532
|
+
.optional(),
|
533
|
+
generateImageViaCodeInterpreter: zod_1.z
|
534
|
+
.union([zod_1.z.literal(true), zod_1.z.literal(false)])
|
535
|
+
.describe("Whether we should try to generate an image with results via code interpreter, regardless of if the codeInterpreter limit is reached")
|
536
|
+
.optional(),
|
529
537
|
});
|
530
538
|
exports.snowflakeRunSnowflakeQueryOutputSchema = zod_1.z.object({
|
531
539
|
format: zod_1.z.enum(["json", "csv"]).describe("The format of the output"),
|
532
540
|
content: zod_1.z.string().describe("The content of the query result (json)"),
|
533
541
|
rowCount: zod_1.z.number().describe("The number of rows returned by the query"),
|
542
|
+
error: zod_1.z.string().describe("The error that occurred if the query results failed or were limited").optional(),
|
534
543
|
});
|
535
544
|
exports.openstreetmapGetLatitudeLongitudeFromLocationParamsSchema = zod_1.z.object({
|
536
545
|
location: zod_1.z.string().describe("The location to get the latitude and longitude of"),
|
@@ -2248,7 +2257,9 @@ exports.ashbyUpdateCandidateOutputSchema = zod_1.z.void();
|
|
2248
2257
|
exports.salesforceUpdateRecordParamsSchema = zod_1.z.object({
|
2249
2258
|
objectType: zod_1.z.string().describe("The Salesforce object type to update (e.g., Lead, Account, Contact)"),
|
2250
2259
|
recordId: zod_1.z.string().describe("The ID of the record to update"),
|
2251
|
-
fieldsToUpdate: zod_1.z
|
2260
|
+
fieldsToUpdate: zod_1.z
|
2261
|
+
.record(zod_1.z.string())
|
2262
|
+
.describe("A key,value pair where the keys are the fields to update on the record and the values are the new values of those fields."),
|
2252
2263
|
});
|
2253
2264
|
exports.salesforceUpdateRecordOutputSchema = zod_1.z.object({
|
2254
2265
|
success: zod_1.z.boolean().describe("Whether the record was successfully updated"),
|
@@ -0,0 +1,46 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
12
|
+
const axiosClient_1 = require("../../util/axiosClient");
|
13
|
+
function getConfluenceRequestConfig(baseUrl, username, apiToken) {
|
14
|
+
return {
|
15
|
+
baseURL: baseUrl,
|
16
|
+
headers: {
|
17
|
+
Accept: "application/json",
|
18
|
+
Authorization: `Basic ${Buffer.from(`${username}:${apiToken}`).toString("base64")}`,
|
19
|
+
},
|
20
|
+
};
|
21
|
+
}
|
22
|
+
const confluenceUpdatePage = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
23
|
+
const { pageId, content, title } = params;
|
24
|
+
const { baseUrl, authToken, username } = authParams;
|
25
|
+
if (!baseUrl || !authToken || !username) {
|
26
|
+
throw new Error("Missing required authentication information");
|
27
|
+
}
|
28
|
+
const config = getConfluenceRequestConfig(baseUrl, username, authToken);
|
29
|
+
// Get current version number
|
30
|
+
const response = yield axiosClient_1.axiosClient.get(`/api/v2/pages/${pageId}`, config);
|
31
|
+
const currVersion = response.data.version.number;
|
32
|
+
const payload = {
|
33
|
+
id: pageId,
|
34
|
+
status: "current",
|
35
|
+
title,
|
36
|
+
body: {
|
37
|
+
representation: "storage",
|
38
|
+
value: content,
|
39
|
+
},
|
40
|
+
version: {
|
41
|
+
number: currVersion + 1,
|
42
|
+
},
|
43
|
+
};
|
44
|
+
yield axiosClient_1.axiosClient.put(`/api/v2/pages/${pageId}`, payload, config);
|
45
|
+
});
|
46
|
+
exports.default = confluenceUpdatePage;
|
@@ -0,0 +1,18 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
12
|
+
const fillTemplateAction = (_a) => __awaiter(void 0, [_a], void 0, function* ({ template }) {
|
13
|
+
// Simply return the template without any modification
|
14
|
+
return {
|
15
|
+
result: template,
|
16
|
+
};
|
17
|
+
});
|
18
|
+
exports.default = fillTemplateAction;
|
@@ -0,0 +1,38 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
13
|
+
};
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
15
|
+
const axios_1 = __importDefault(require("axios"));
|
16
|
+
const genericApiCall = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, }) {
|
17
|
+
try {
|
18
|
+
const { endpoint, method, headers, body } = params;
|
19
|
+
const response = yield (0, axios_1.default)({
|
20
|
+
url: endpoint,
|
21
|
+
method,
|
22
|
+
headers,
|
23
|
+
data: method !== "GET" ? body : undefined,
|
24
|
+
});
|
25
|
+
return {
|
26
|
+
statusCode: response.status,
|
27
|
+
headers: response.headers,
|
28
|
+
data: response.data,
|
29
|
+
};
|
30
|
+
}
|
31
|
+
catch (error) {
|
32
|
+
if (axios_1.default.isAxiosError(error)) {
|
33
|
+
throw Error("Axios Error: " + (error.message || "Failed to make API call"));
|
34
|
+
}
|
35
|
+
throw Error("Error: " + (error || "Failed to make API call"));
|
36
|
+
}
|
37
|
+
});
|
38
|
+
exports.default = genericApiCall;
|
@@ -0,0 +1,96 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
13
|
+
};
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
15
|
+
const axios_1 = __importDefault(require("axios"));
|
16
|
+
const types_1 = require("../../autogen/types");
|
17
|
+
const INCLUDED_TYPES = [
|
18
|
+
"monument",
|
19
|
+
"museum",
|
20
|
+
"art_gallery",
|
21
|
+
"sculpture",
|
22
|
+
"cultural_landmark",
|
23
|
+
"historical_place",
|
24
|
+
"performing_arts_theater",
|
25
|
+
"university",
|
26
|
+
"aquarium",
|
27
|
+
"botanical_garden",
|
28
|
+
"comedy_club",
|
29
|
+
"park",
|
30
|
+
"movie_theater",
|
31
|
+
"national_park",
|
32
|
+
"garden",
|
33
|
+
"night_club",
|
34
|
+
"tourist_attraction",
|
35
|
+
"water_park",
|
36
|
+
"zoo",
|
37
|
+
"bar",
|
38
|
+
"restaurant",
|
39
|
+
"food_court",
|
40
|
+
"bakery",
|
41
|
+
"cafe",
|
42
|
+
"coffee_shop",
|
43
|
+
"pub",
|
44
|
+
"wine_bar",
|
45
|
+
"spa",
|
46
|
+
"beach",
|
47
|
+
"market",
|
48
|
+
"shopping_mall",
|
49
|
+
"stadium",
|
50
|
+
];
|
51
|
+
const nearbysearch = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
52
|
+
const url = `https://places.googleapis.com/v1/places:searchNearby`;
|
53
|
+
const fieldMask = [
|
54
|
+
"places.displayName",
|
55
|
+
"places.formattedAddress",
|
56
|
+
"places.priceLevel",
|
57
|
+
"places.rating",
|
58
|
+
"places.primaryTypeDisplayName",
|
59
|
+
"places.editorialSummary",
|
60
|
+
"places.regularOpeningHours",
|
61
|
+
].join(",");
|
62
|
+
const response = yield axios_1.default.post(url, {
|
63
|
+
maxResultCount: 20,
|
64
|
+
includedTypes: INCLUDED_TYPES,
|
65
|
+
locationRestriction: {
|
66
|
+
circle: {
|
67
|
+
center: {
|
68
|
+
latitude: params.latitude,
|
69
|
+
longitude: params.longitude,
|
70
|
+
},
|
71
|
+
radius: 10000,
|
72
|
+
},
|
73
|
+
},
|
74
|
+
}, {
|
75
|
+
headers: {
|
76
|
+
"X-Goog-Api-Key": authParams.apiKey,
|
77
|
+
"X-Goog-FieldMask": fieldMask,
|
78
|
+
"Content-Type": "application/json",
|
79
|
+
},
|
80
|
+
});
|
81
|
+
return types_1.googlemapsNearbysearchOutputSchema.parse({
|
82
|
+
results: response.data.places.map((place) => {
|
83
|
+
var _a, _b;
|
84
|
+
return ({
|
85
|
+
name: place.displayName.text,
|
86
|
+
address: place.formattedAddress,
|
87
|
+
priceLevel: place.priceLevel,
|
88
|
+
rating: place.rating,
|
89
|
+
primaryType: place.primaryTypeDisplayName.text,
|
90
|
+
editorialSummary: ((_a = place.editorialSummary) === null || _a === void 0 ? void 0 : _a.text) || "",
|
91
|
+
openingHours: ((_b = place.regularOpeningHours) === null || _b === void 0 ? void 0 : _b.weekdayDescriptions.join("\n")) || "",
|
92
|
+
});
|
93
|
+
}),
|
94
|
+
});
|
95
|
+
});
|
96
|
+
exports.default = nearbysearch;
|
@@ -61,6 +61,7 @@ const runSnowflakeQuery = (_a) => __awaiter(void 0, [_a], void 0, function* ({ p
|
|
61
61
|
rowCount: resultsLength,
|
62
62
|
content: formattedData,
|
63
63
|
format: outputFormat,
|
64
|
+
error: limit && limit < resultsLength ? `Query results truncated to ${limit} rows.` : undefined,
|
64
65
|
};
|
65
66
|
}
|
66
67
|
catch (error) {
|
@@ -0,0 +1,154 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
13
|
+
};
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
15
|
+
const snowflake_sdk_1 = __importDefault(require("snowflake-sdk"));
|
16
|
+
const crypto_1 = __importDefault(require("crypto"));
|
17
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
18
|
+
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
19
|
+
const uuid_1 = require("uuid");
|
20
|
+
// Only log errors.
|
21
|
+
snowflake_sdk_1.default.configure({ logLevel: "ERROR" });
|
22
|
+
const runSnowflakeQueryWriteResultsToS3 = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
23
|
+
const { databaseName, warehouse, query, user, accountName, s3BucketName, s3Region, outputFormat = "json" } = params;
|
24
|
+
const { apiKey: privateKey, awsAccessKeyId, awsSecretAccessKey } = authParams;
|
25
|
+
if (!privateKey) {
|
26
|
+
throw new Error("Snowflake private key is required");
|
27
|
+
}
|
28
|
+
if (!awsAccessKeyId || !awsSecretAccessKey) {
|
29
|
+
throw new Error("AWS credentials are required");
|
30
|
+
}
|
31
|
+
if (!accountName || !user || !databaseName || !warehouse || !query || !s3BucketName) {
|
32
|
+
throw new Error("Missing required parameters for Snowflake query or S3 destination");
|
33
|
+
}
|
34
|
+
const getPrivateKeyCorrectFormat = (privateKey) => {
|
35
|
+
const buffer = Buffer.from(privateKey);
|
36
|
+
const privateKeyObject = crypto_1.default.createPrivateKey({
|
37
|
+
key: buffer,
|
38
|
+
format: "pem",
|
39
|
+
passphrase: "password",
|
40
|
+
});
|
41
|
+
const privateKeyCorrectFormat = privateKeyObject.export({
|
42
|
+
format: "pem",
|
43
|
+
type: "pkcs8",
|
44
|
+
});
|
45
|
+
return privateKeyCorrectFormat.toString();
|
46
|
+
};
|
47
|
+
const executeQueryAndFormatData = () => __awaiter(void 0, void 0, void 0, function* () {
|
48
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
49
|
+
const queryResults = yield new Promise((resolve, reject) => {
|
50
|
+
connection.execute({
|
51
|
+
sqlText: query,
|
52
|
+
complete: (err, stmt, rows) => {
|
53
|
+
if (err) {
|
54
|
+
return reject(err);
|
55
|
+
}
|
56
|
+
return resolve(rows || []);
|
57
|
+
},
|
58
|
+
});
|
59
|
+
});
|
60
|
+
// Format the results based on the output format
|
61
|
+
let formattedData;
|
62
|
+
if (outputFormat.toLowerCase() === "csv") {
|
63
|
+
if (queryResults.length === 0) {
|
64
|
+
formattedData = "";
|
65
|
+
}
|
66
|
+
else {
|
67
|
+
const headers = Object.keys(queryResults[0]).join(",");
|
68
|
+
const rows = queryResults.map(row => Object.values(row)
|
69
|
+
.map(value => (typeof value === "object" && value !== null ? JSON.stringify(value) : value))
|
70
|
+
.join(","));
|
71
|
+
formattedData = [headers, ...rows].join("\n");
|
72
|
+
}
|
73
|
+
}
|
74
|
+
else {
|
75
|
+
// Default to JSON
|
76
|
+
formattedData = JSON.stringify(queryResults, null, 2);
|
77
|
+
}
|
78
|
+
return { formattedData, resultsLength: queryResults.length };
|
79
|
+
});
|
80
|
+
const uploadToS3AndGetURL = (formattedData) => __awaiter(void 0, void 0, void 0, function* () {
|
81
|
+
// Create S3 client
|
82
|
+
const s3Client = new client_s3_1.S3Client({
|
83
|
+
region: s3Region,
|
84
|
+
credentials: {
|
85
|
+
accessKeyId: awsAccessKeyId,
|
86
|
+
secretAccessKey: awsSecretAccessKey,
|
87
|
+
},
|
88
|
+
});
|
89
|
+
const contentType = outputFormat.toLowerCase() === "csv" ? "text/csv" : "application/json";
|
90
|
+
const fileExtension = outputFormat.toLowerCase() === "csv" ? "csv" : "json";
|
91
|
+
const finalKey = `${databaseName}/${(0, uuid_1.v4)()}.${fileExtension}`;
|
92
|
+
// Upload to S3 without ACL
|
93
|
+
const uploadCommand = new client_s3_1.PutObjectCommand({
|
94
|
+
Bucket: s3BucketName,
|
95
|
+
Key: finalKey,
|
96
|
+
Body: formattedData,
|
97
|
+
ContentType: contentType,
|
98
|
+
});
|
99
|
+
yield s3Client.send(uploadCommand);
|
100
|
+
// Generate a presigned URL (valid for an hour)
|
101
|
+
const getObjectCommand = new client_s3_1.GetObjectCommand({
|
102
|
+
Bucket: s3BucketName,
|
103
|
+
Key: finalKey,
|
104
|
+
});
|
105
|
+
const presignedUrl = yield (0, s3_request_presigner_1.getSignedUrl)(s3Client, getObjectCommand, { expiresIn: 3600 });
|
106
|
+
return presignedUrl;
|
107
|
+
});
|
108
|
+
// Process the private key
|
109
|
+
const privateKeyCorrectFormatString = getPrivateKeyCorrectFormat(privateKey);
|
110
|
+
// Set up a connection using snowflake-sdk
|
111
|
+
const connection = snowflake_sdk_1.default.createConnection({
|
112
|
+
account: accountName,
|
113
|
+
username: user,
|
114
|
+
privateKey: privateKeyCorrectFormatString,
|
115
|
+
authenticator: "SNOWFLAKE_JWT",
|
116
|
+
role: "ACCOUNTADMIN",
|
117
|
+
warehouse: warehouse,
|
118
|
+
database: databaseName,
|
119
|
+
});
|
120
|
+
try {
|
121
|
+
// Connect to Snowflake
|
122
|
+
yield new Promise((resolve, reject) => {
|
123
|
+
connection.connect((err, conn) => {
|
124
|
+
if (err) {
|
125
|
+
console.error("Unable to connect to Snowflake:", err.message);
|
126
|
+
return reject(err);
|
127
|
+
}
|
128
|
+
resolve(conn);
|
129
|
+
});
|
130
|
+
});
|
131
|
+
const { formattedData, resultsLength } = yield executeQueryAndFormatData();
|
132
|
+
const presignedUrl = yield uploadToS3AndGetURL(formattedData);
|
133
|
+
// Return fields to match schema definition
|
134
|
+
connection.destroy(err => {
|
135
|
+
if (err) {
|
136
|
+
console.log("Failed to disconnect from Snowflake:", err);
|
137
|
+
}
|
138
|
+
});
|
139
|
+
return {
|
140
|
+
bucketUrl: presignedUrl,
|
141
|
+
message: `Query results successfully written to S3. URL valid for 1 hour.`,
|
142
|
+
rowCount: resultsLength,
|
143
|
+
};
|
144
|
+
}
|
145
|
+
catch (error) {
|
146
|
+
connection.destroy(err => {
|
147
|
+
if (err) {
|
148
|
+
console.log("Failed to disconnect from Snowflake:", err);
|
149
|
+
}
|
150
|
+
});
|
151
|
+
throw Error(`An error occurred: ${error}`);
|
152
|
+
}
|
153
|
+
});
|
154
|
+
exports.default = runSnowflakeQueryWriteResultsToS3;
|
@@ -0,0 +1,45 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
+
});
|
10
|
+
};
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
13
|
+
};
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
15
|
+
const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
|
16
|
+
const scrapeTweetDataWithNitter = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
17
|
+
const tweetUrlRegex = /^(?:https?:\/\/)?(?:www\.)?(?:twitter\.com|x\.com)\/([a-zA-Z0-9_]+)\/status\/(\d+)(?:\?.*)?$/;
|
18
|
+
if (!tweetUrlRegex.test(params.tweetUrl)) {
|
19
|
+
throw new Error("Invalid tweet URL. Expected format: https://twitter.com/username/status/id or https://x.com/username/status/id");
|
20
|
+
}
|
21
|
+
const nitterUrl = params.tweetUrl.replace(/^(?:https?:\/\/)?(?:www\.)?(?:twitter\.com|x\.com)/i, "https://nitter.net");
|
22
|
+
// Initialize Firecrawl
|
23
|
+
if (!authParams.apiKey) {
|
24
|
+
throw new Error("API key is required for X+Nitter+Firecrawl");
|
25
|
+
}
|
26
|
+
const firecrawl = new firecrawl_js_1.default({
|
27
|
+
apiKey: authParams.apiKey,
|
28
|
+
});
|
29
|
+
try {
|
30
|
+
// Scrape the Nitter URL
|
31
|
+
const result = yield firecrawl.scrapeUrl(nitterUrl);
|
32
|
+
if (!result.success) {
|
33
|
+
throw new Error(`Failed to scrape tweet: ${result.error || "Unknown error"}`);
|
34
|
+
}
|
35
|
+
// Extract the tweet text from the scraped content - simple approach - in practice, you might need more robust parsing based on nitter html structure
|
36
|
+
const tweetContent = result.markdown;
|
37
|
+
return {
|
38
|
+
text: tweetContent || "Error scraping with firecrawl",
|
39
|
+
};
|
40
|
+
}
|
41
|
+
catch (error) {
|
42
|
+
throw new Error(`Error scraping tweet: ${error instanceof Error ? error.message : error}`);
|
43
|
+
}
|
44
|
+
});
|
45
|
+
exports.default = scrapeTweetDataWithNitter;
|
package/package.json
CHANGED
@@ -1,43 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
9
|
-
});
|
10
|
-
};
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
12
|
-
const axiosClient_1 = require("../../util/axiosClient");
|
13
|
-
const getSalesforceRecordByQuery = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
14
|
-
const { authToken, baseUrl } = authParams;
|
15
|
-
const { query, limit } = params;
|
16
|
-
if (!authToken || !baseUrl) {
|
17
|
-
return {
|
18
|
-
success: false,
|
19
|
-
error: "authToken and baseUrl are required for Salesforce API",
|
20
|
-
};
|
21
|
-
}
|
22
|
-
// The API limits the maximum number of records returned to 2000, the limit lets the user set a smaller custom limit
|
23
|
-
const url = `${baseUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query + " LIMIT " + (limit != undefined && limit <= 2000 ? limit : 2000))}`;
|
24
|
-
try {
|
25
|
-
const response = yield axiosClient_1.axiosClient.get(url, {
|
26
|
-
headers: {
|
27
|
-
Authorization: `Bearer ${authToken}`,
|
28
|
-
},
|
29
|
-
});
|
30
|
-
return {
|
31
|
-
success: true,
|
32
|
-
records: response.data,
|
33
|
-
};
|
34
|
-
}
|
35
|
-
catch (error) {
|
36
|
-
console.error("Error retrieving Salesforce record:", error);
|
37
|
-
return {
|
38
|
-
success: false,
|
39
|
-
error: error instanceof Error ? error.message : "An unknown error occurred",
|
40
|
-
};
|
41
|
-
}
|
42
|
-
});
|
43
|
-
exports.default = getSalesforceRecordByQuery;
|