@credal/actions 0.2.152 → 0.2.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/autogen/templates.js +33 -12
- package/dist/actions/autogen/types.d.ts +163 -56
- package/dist/actions/autogen/types.js +19 -6
- package/dist/actions/providers/confluence/updatePage.js +14 -15
- package/dist/actions/providers/generic/fillTemplateAction.d.ts +7 -0
- package/dist/actions/providers/generic/fillTemplateAction.js +18 -0
- package/dist/actions/providers/generic/genericApiCall.d.ts +3 -0
- package/dist/actions/providers/generic/genericApiCall.js +38 -0
- package/dist/actions/providers/google-oauth/getDriveContentById.d.ts +3 -0
- package/dist/actions/providers/google-oauth/getDriveContentById.js +161 -0
- package/dist/actions/providers/google-oauth/searchAndGetDriveContentByKeywords.d.ts +3 -0
- package/dist/actions/providers/google-oauth/searchAndGetDriveContentByKeywords.js +47 -0
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByKeywords.d.ts +3 -0
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByKeywords.js +110 -0
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByQuery.d.ts +3 -0
- package/dist/actions/providers/google-oauth/searchDriveAndGetContentByQuery.js +78 -0
- package/dist/actions/providers/google-oauth/utils/extractContentFromDriveFileId.d.ts +15 -0
- package/dist/actions/providers/google-oauth/utils/extractContentFromDriveFileId.js +129 -0
- package/dist/actions/providers/googlemaps/nearbysearch.d.ts +3 -0
- package/dist/actions/providers/googlemaps/nearbysearch.js +96 -0
- package/dist/actions/providers/salesforce/searchAllSalesforceRecords.js +19 -8
- package/dist/actions/providers/slackUser/getSlackMessagesInTimeRange.d.ts +3 -0
- package/dist/actions/providers/slackUser/getSlackMessagesInTimeRange.js +81 -0
- package/dist/actions/providers/slackUser/searchSlack.d.ts +11 -0
- package/dist/actions/providers/slackUser/searchSlack.js +67 -24
- package/dist/actions/providers/snowflake/runSnowflakeQueryWriteResultsToS3.d.ts +3 -0
- package/dist/actions/providers/snowflake/runSnowflakeQueryWriteResultsToS3.js +154 -0
- package/dist/actions/providers/x/scrapeTweetDataWithNitter.d.ts +3 -0
- package/dist/actions/providers/x/scrapeTweetDataWithNitter.js +45 -0
- package/package.json +1 -1
- package/dist/actions/providers/jamf/types.d.ts +0 -8
- package/dist/actions/providers/jamf/types.js +0 -7
|
@@ -21,7 +21,7 @@ const searchAllSalesforceRecords = (_a) => __awaiter(void 0, [_a], void 0, funct
|
|
|
21
21
|
.replace(/-/g, "\\-"); // Escape dashes
|
|
22
22
|
let customObject = "";
|
|
23
23
|
if (params.usesLightningKnowledge) {
|
|
24
|
-
customObject = `Knowledge__kav(Article_Body__c, Title),
|
|
24
|
+
customObject = `Knowledge__kav(Article_Body__c, Title, Link_to_Community_Article__c),
|
|
25
25
|
FeedItem(Id, Body, Title, ParentId, Parent.Name, CreatedBy.Name, CreatedDate, CommentCount),
|
|
26
26
|
FeedComment(Id, CommentBody, FeedItemId, ParentId, CreatedBy.Name, CreatedDate),
|
|
27
27
|
EmailMessage(Id, Subject, TextBody, FromAddress, ToAddress, ParentId, CreatedDate, Incoming)`;
|
|
@@ -50,13 +50,24 @@ const searchAllSalesforceRecords = (_a) => __awaiter(void 0, [_a], void 0, funct
|
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
52
|
// Salesforce record types are confusing and non standard
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
53
|
+
return {
|
|
54
|
+
success: true,
|
|
55
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
56
|
+
results: response.data.searchRecords.map((record) => {
|
|
57
|
+
const recordId = record.Id;
|
|
58
|
+
const webUrl = recordId ? `${baseUrl}/lightning/r/${recordId}/view` : undefined;
|
|
59
|
+
// Try common name fields in order of preference, using only what's available
|
|
60
|
+
const displayName = record.Name ||
|
|
61
|
+
record.Title ||
|
|
62
|
+
record.Subject ||
|
|
63
|
+
record.CaseNumber ||
|
|
64
|
+
record.AccountName ||
|
|
65
|
+
record.ContactName ||
|
|
66
|
+
record.Id ||
|
|
67
|
+
webUrl;
|
|
68
|
+
return { name: displayName, url: webUrl, contents: record };
|
|
69
|
+
}),
|
|
70
|
+
};
|
|
60
71
|
}
|
|
61
72
|
catch (error) {
|
|
62
73
|
console.error("Error retrieving Salesforce record:", error);
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { WebClient } from "@slack/web-api";
|
|
11
|
+
import { MISSING_AUTH_TOKEN } from "../../util/missingAuthConstants.js";
|
|
12
|
+
import { extractMessageText, SlackUserCache, normalizeChannelOperand } from "./searchSlack.js";
|
|
13
|
+
/* ===================== Helpers ===================== */
|
|
14
|
+
function searchMessagesInTimeRange(input) {
|
|
15
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
16
|
+
var _a, _b;
|
|
17
|
+
const { client, channel, oldest, latest, limit } = input;
|
|
18
|
+
// Build search query - use wildcard to match all messages
|
|
19
|
+
const parts = [`in:${normalizeChannelOperand(channel)}`];
|
|
20
|
+
if (oldest) {
|
|
21
|
+
parts.push(`after:${oldest}`);
|
|
22
|
+
}
|
|
23
|
+
if (latest) {
|
|
24
|
+
parts.push(`before:${latest}`);
|
|
25
|
+
}
|
|
26
|
+
// Use * as a wildcard to match all messages
|
|
27
|
+
const query = parts.join(" ") + " *";
|
|
28
|
+
const searchRes = yield client.search.messages({ query, count: limit, highlight: true });
|
|
29
|
+
return (_b = (_a = searchRes.messages) === null || _a === void 0 ? void 0 : _a.matches) !== null && _b !== void 0 ? _b : [];
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
/* ===================== MAIN EXPORT ===================== */
|
|
33
|
+
const getSlackMessagesInTimeRange = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
34
|
+
if (!authParams.authToken)
|
|
35
|
+
throw new Error(MISSING_AUTH_TOKEN);
|
|
36
|
+
const client = new WebClient(authParams.authToken);
|
|
37
|
+
const cache = new SlackUserCache(client);
|
|
38
|
+
const { channel, oldest, limit = 50 } = params;
|
|
39
|
+
let { latest } = params;
|
|
40
|
+
// Ensure latest doesn't exceed current time
|
|
41
|
+
const now = Math.floor(Date.now() / 1000);
|
|
42
|
+
if (latest && latest > now) {
|
|
43
|
+
latest = now;
|
|
44
|
+
}
|
|
45
|
+
const { user_id: myUserId } = yield client.auth.test();
|
|
46
|
+
if (!myUserId)
|
|
47
|
+
throw new Error("Failed to get my user ID.");
|
|
48
|
+
const meInfo = yield cache.get(myUserId);
|
|
49
|
+
// Search for messages in the channel using search API
|
|
50
|
+
const matches = yield searchMessagesInTimeRange({ client, channel, oldest, latest, limit });
|
|
51
|
+
// Convert matches to results
|
|
52
|
+
const results = yield Promise.all(matches.map((m) => __awaiter(void 0, void 0, void 0, function* () {
|
|
53
|
+
var _a, _b, _c, _d;
|
|
54
|
+
if (!m.ts || !((_a = m.channel) === null || _a === void 0 ? void 0 : _a.id))
|
|
55
|
+
return null;
|
|
56
|
+
// Get user info for the message author
|
|
57
|
+
const userId = (_b = m.user) !== null && _b !== void 0 ? _b : m.username;
|
|
58
|
+
const userInfo = userId ? yield cache.get(userId) : undefined;
|
|
59
|
+
// Extract text from the match
|
|
60
|
+
const text = (_d = (_c = extractMessageText(m)) !== null && _c !== void 0 ? _c : m.text) !== null && _d !== void 0 ? _d : "";
|
|
61
|
+
return {
|
|
62
|
+
channelId: m.channel.id,
|
|
63
|
+
ts: m.ts,
|
|
64
|
+
text,
|
|
65
|
+
userEmail: userInfo === null || userInfo === void 0 ? void 0 : userInfo.email,
|
|
66
|
+
userName: userInfo === null || userInfo === void 0 ? void 0 : userInfo.name,
|
|
67
|
+
permalink: m.permalink,
|
|
68
|
+
};
|
|
69
|
+
})));
|
|
70
|
+
// Filter out nulls and sort by timestamp descending
|
|
71
|
+
const validResults = results.filter(Boolean).sort((a, b) => Number(b.ts) - Number(a.ts));
|
|
72
|
+
return {
|
|
73
|
+
results: validResults.map(r => ({
|
|
74
|
+
name: r.text || "Untitled",
|
|
75
|
+
url: r.permalink || "",
|
|
76
|
+
contents: r,
|
|
77
|
+
})),
|
|
78
|
+
currentUser: { userId: myUserId, userName: meInfo === null || meInfo === void 0 ? void 0 : meInfo.name, userEmail: meInfo === null || meInfo === void 0 ? void 0 : meInfo.email },
|
|
79
|
+
};
|
|
80
|
+
});
|
|
81
|
+
export default getSlackMessagesInTimeRange;
|
|
@@ -29,6 +29,17 @@ interface SlackMessage {
|
|
|
29
29
|
thread_ts?: string;
|
|
30
30
|
blocks?: KnownBlock[];
|
|
31
31
|
attachments?: Attachment[];
|
|
32
|
+
reactions?: Array<{
|
|
33
|
+
name: string;
|
|
34
|
+
count: number;
|
|
35
|
+
users: string[];
|
|
36
|
+
}>;
|
|
37
|
+
files?: Array<{
|
|
38
|
+
name?: string;
|
|
39
|
+
title?: string;
|
|
40
|
+
mimetype?: string;
|
|
41
|
+
url_private?: string;
|
|
42
|
+
}>;
|
|
32
43
|
}
|
|
33
44
|
/**
|
|
34
45
|
* Extracts all visible text from a Slack message
|
|
@@ -306,6 +306,8 @@ function transformToSlackMessage(message) {
|
|
|
306
306
|
thread_ts: message.thread_ts,
|
|
307
307
|
blocks: message.blocks,
|
|
308
308
|
attachments: message.attachments,
|
|
309
|
+
reactions: message.reactions,
|
|
310
|
+
files: message.files,
|
|
309
311
|
};
|
|
310
312
|
}
|
|
311
313
|
function fetchOneMessage(client, channel, ts) {
|
|
@@ -397,17 +399,45 @@ function searchByTopic(input) {
|
|
|
397
399
|
return (_b = (_a = searchRes.messages) === null || _a === void 0 ? void 0 : _a.matches) !== null && _b !== void 0 ? _b : [];
|
|
398
400
|
});
|
|
399
401
|
}
|
|
402
|
+
/**
|
|
403
|
+
* Deduplicates and merges Slack threads.
|
|
404
|
+
* When multiple search hits point to the same thread (same thread_ts),
|
|
405
|
+
* we merge them into a single result with all unique messages in context.
|
|
406
|
+
*/
|
|
400
407
|
function dedupeAndSort(results) {
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
409
|
+
// Group by thread: channelId + ts (where ts is the root thread_ts)
|
|
410
|
+
const threadMap = new Map();
|
|
411
|
+
for (const result of results) {
|
|
412
|
+
const threadKey = `${result.channelId}-${result.ts}`;
|
|
413
|
+
const existing = threadMap.get(threadKey);
|
|
414
|
+
if (!existing) {
|
|
415
|
+
// First time seeing this thread
|
|
416
|
+
threadMap.set(threadKey, result);
|
|
417
|
+
}
|
|
418
|
+
else {
|
|
419
|
+
// Merge: dedupe context messages by ts
|
|
420
|
+
const existingTsSet = new Set((_b = (_a = existing.context) === null || _a === void 0 ? void 0 : _a.map(m => m.ts)) !== null && _b !== void 0 ? _b : []);
|
|
421
|
+
const newMessages = ((_c = result.context) !== null && _c !== void 0 ? _c : []).filter(m => !existingTsSet.has(m.ts));
|
|
422
|
+
if (newMessages.length > 0) {
|
|
423
|
+
existing.context = [...((_d = existing.context) !== null && _d !== void 0 ? _d : []), ...newMessages].sort((a, b) => Number(a.ts) - Number(b.ts));
|
|
424
|
+
}
|
|
425
|
+
// Update permalink if missing
|
|
426
|
+
if (!existing.permalink && result.permalink) {
|
|
427
|
+
existing.permalink = result.permalink;
|
|
428
|
+
}
|
|
429
|
+
// Merge members if needed (for DMs/MPIMs)
|
|
430
|
+
if (result.members && result.members.length > 0) {
|
|
431
|
+
const existingMemberIds = new Set((_f = (_e = existing.members) === null || _e === void 0 ? void 0 : _e.map(m => m.userId)) !== null && _f !== void 0 ? _f : []);
|
|
432
|
+
const newMembers = result.members.filter(m => !existingMemberIds.has(m.userId));
|
|
433
|
+
if (newMembers.length > 0) {
|
|
434
|
+
existing.members = [...((_g = existing.members) !== null && _g !== void 0 ? _g : []), ...newMembers];
|
|
435
|
+
}
|
|
436
|
+
}
|
|
408
437
|
}
|
|
409
438
|
}
|
|
410
|
-
|
|
439
|
+
// Sort by timestamp descending (most recent first)
|
|
440
|
+
return Array.from(threadMap.values()).sort((a, b) => Number(b.ts) - Number(a.ts));
|
|
411
441
|
}
|
|
412
442
|
/* ===================== MAIN EXPORT ===================== */
|
|
413
443
|
const searchSlack = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
@@ -415,7 +445,7 @@ const searchSlack = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params,
|
|
|
415
445
|
throw new Error(MISSING_AUTH_TOKEN);
|
|
416
446
|
const client = new WebClient(authParams.authToken);
|
|
417
447
|
const cache = new SlackUserCache(client);
|
|
418
|
-
const { emails, topic, timeRange, limit = 20, channel } = params;
|
|
448
|
+
const { emails, topic, timeRange, limit = 20, channel, fetchAdjacentMessages = true } = params;
|
|
419
449
|
const { user_id: myUserId } = yield client.auth.test();
|
|
420
450
|
if (!myUserId)
|
|
421
451
|
throw new Error("Failed to get my user ID.");
|
|
@@ -466,7 +496,7 @@ const searchSlack = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params,
|
|
|
466
496
|
searchResults.forEach(matches => allMatches.push(...matches));
|
|
467
497
|
const channelInfoCache = new Map();
|
|
468
498
|
const expanded = yield Promise.all(allMatches.map(m => limitHit(() => __awaiter(void 0, void 0, void 0, function* () {
|
|
469
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
|
|
499
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _o;
|
|
470
500
|
if (!m.ts || !((_a = m.channel) === null || _a === void 0 ? void 0 : _a.id))
|
|
471
501
|
return null;
|
|
472
502
|
const anchor = yield fetchOneMessage(client, m.channel.id, m.ts);
|
|
@@ -491,10 +521,12 @@ const searchSlack = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params,
|
|
|
491
521
|
yield fetchThread(client, m.channel.id, rootTs),
|
|
492
522
|
(_g = m.permalink) !== null && _g !== void 0 ? _g : (yield getPermalink(client, m.channel.id, rootTs)),
|
|
493
523
|
]
|
|
494
|
-
:
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
524
|
+
: fetchAdjacentMessages
|
|
525
|
+
? [
|
|
526
|
+
yield fetchContextWindow(client, m.channel.id, m.ts),
|
|
527
|
+
(_h = m.permalink) !== null && _h !== void 0 ? _h : (yield getPermalink(client, m.channel.id, m.ts)),
|
|
528
|
+
]
|
|
529
|
+
: [[], (_j = m.permalink) !== null && _j !== void 0 ? _j : (yield getPermalink(client, m.channel.id, m.ts))];
|
|
498
530
|
// filter logic
|
|
499
531
|
let passesFilter = false;
|
|
500
532
|
if (channelInfo.isIm || channelInfo.isMpim) {
|
|
@@ -506,17 +538,28 @@ const searchSlack = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params,
|
|
|
506
538
|
}
|
|
507
539
|
if (filteredTargetIds.length && !passesFilter)
|
|
508
540
|
return null;
|
|
509
|
-
const
|
|
541
|
+
const allContext = yield Promise.all(contextMsgs.map((t) => __awaiter(void 0, void 0, void 0, function* () {
|
|
510
542
|
var _a;
|
|
511
543
|
const u = t.user ? yield cache.get(t.user) : undefined;
|
|
512
544
|
const rawText = extractMessageText(t);
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
545
|
+
// Build interaction description
|
|
546
|
+
const interactions = [];
|
|
547
|
+
if (t.reactions && t.reactions.length > 0) {
|
|
548
|
+
interactions.push(`Reactions: ${t.reactions.map(r => `:${r.name}: (${r.count})`).join(", ")}`);
|
|
549
|
+
}
|
|
550
|
+
if (t.files && t.files.length > 0) {
|
|
551
|
+
interactions.push(`Files: ${t.files.map(f => f.title || f.name || "Untitled").join(", ")}`);
|
|
552
|
+
}
|
|
553
|
+
return Object.assign({ ts: t.ts, text: rawText ? yield expandSlackEntities(cache, rawText) : undefined, userEmail: u === null || u === void 0 ? void 0 : u.email, userName: (_a = u === null || u === void 0 ? void 0 : u.name) !== null && _a !== void 0 ? _a : t.username }, (interactions.length > 0 ? { interactions: interactions.join(" | ") } : {}));
|
|
519
554
|
})));
|
|
555
|
+
// Deduplicate by timestamp - appears the the context array returned can have duplicates
|
|
556
|
+
const seenTs = new Set();
|
|
557
|
+
const context = allContext.filter(msg => {
|
|
558
|
+
if (seenTs.has(msg.ts))
|
|
559
|
+
return false;
|
|
560
|
+
seenTs.add(msg.ts);
|
|
561
|
+
return true;
|
|
562
|
+
});
|
|
520
563
|
const anchorUser = (anchor === null || anchor === void 0 ? void 0 : anchor.user) ? yield cache.get(anchor.user) : undefined;
|
|
521
564
|
const anchorText = extractMessageText(anchor);
|
|
522
565
|
return {
|
|
@@ -524,10 +567,10 @@ const searchSlack = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params,
|
|
|
524
567
|
ts: rootTs,
|
|
525
568
|
text: anchorText ? yield expandSlackEntities(cache, anchorText) : undefined,
|
|
526
569
|
userEmail: anchorUser === null || anchorUser === void 0 ? void 0 : anchorUser.email,
|
|
527
|
-
userName: (
|
|
570
|
+
userName: (_k = anchorUser === null || anchorUser === void 0 ? void 0 : anchorUser.name) !== null && _k !== void 0 ? _k : anchor === null || anchor === void 0 ? void 0 : anchor.username,
|
|
528
571
|
context,
|
|
529
|
-
permalink: (
|
|
530
|
-
members: ((
|
|
572
|
+
permalink: (_l = m.permalink) !== null && _l !== void 0 ? _l : permalink,
|
|
573
|
+
members: ((_o = channelInfo.members) !== null && _o !== void 0 ? _o : []).map(uid => {
|
|
531
574
|
const u = cache.getSync(uid);
|
|
532
575
|
return { userId: uid, userEmail: u === null || u === void 0 ? void 0 : u.email, userName: u === null || u === void 0 ? void 0 : u.name };
|
|
533
576
|
}),
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const snowflake_sdk_1 = __importDefault(require("snowflake-sdk"));
|
|
16
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
17
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
18
|
+
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
|
19
|
+
const uuid_1 = require("uuid");
|
|
20
|
+
// Only log errors.
|
|
21
|
+
snowflake_sdk_1.default.configure({ logLevel: "ERROR" });
|
|
22
|
+
const runSnowflakeQueryWriteResultsToS3 = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
23
|
+
const { databaseName, warehouse, query, user, accountName, s3BucketName, s3Region, outputFormat = "json" } = params;
|
|
24
|
+
const { apiKey: privateKey, awsAccessKeyId, awsSecretAccessKey } = authParams;
|
|
25
|
+
if (!privateKey) {
|
|
26
|
+
throw new Error("Snowflake private key is required");
|
|
27
|
+
}
|
|
28
|
+
if (!awsAccessKeyId || !awsSecretAccessKey) {
|
|
29
|
+
throw new Error("AWS credentials are required");
|
|
30
|
+
}
|
|
31
|
+
if (!accountName || !user || !databaseName || !warehouse || !query || !s3BucketName) {
|
|
32
|
+
throw new Error("Missing required parameters for Snowflake query or S3 destination");
|
|
33
|
+
}
|
|
34
|
+
const getPrivateKeyCorrectFormat = (privateKey) => {
|
|
35
|
+
const buffer = Buffer.from(privateKey);
|
|
36
|
+
const privateKeyObject = crypto_1.default.createPrivateKey({
|
|
37
|
+
key: buffer,
|
|
38
|
+
format: "pem",
|
|
39
|
+
passphrase: "password",
|
|
40
|
+
});
|
|
41
|
+
const privateKeyCorrectFormat = privateKeyObject.export({
|
|
42
|
+
format: "pem",
|
|
43
|
+
type: "pkcs8",
|
|
44
|
+
});
|
|
45
|
+
return privateKeyCorrectFormat.toString();
|
|
46
|
+
};
|
|
47
|
+
const executeQueryAndFormatData = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
48
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
49
|
+
const queryResults = yield new Promise((resolve, reject) => {
|
|
50
|
+
connection.execute({
|
|
51
|
+
sqlText: query,
|
|
52
|
+
complete: (err, stmt, rows) => {
|
|
53
|
+
if (err) {
|
|
54
|
+
return reject(err);
|
|
55
|
+
}
|
|
56
|
+
return resolve(rows || []);
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
// Format the results based on the output format
|
|
61
|
+
let formattedData;
|
|
62
|
+
if (outputFormat.toLowerCase() === "csv") {
|
|
63
|
+
if (queryResults.length === 0) {
|
|
64
|
+
formattedData = "";
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
const headers = Object.keys(queryResults[0]).join(",");
|
|
68
|
+
const rows = queryResults.map(row => Object.values(row)
|
|
69
|
+
.map(value => (typeof value === "object" && value !== null ? JSON.stringify(value) : value))
|
|
70
|
+
.join(","));
|
|
71
|
+
formattedData = [headers, ...rows].join("\n");
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
// Default to JSON
|
|
76
|
+
formattedData = JSON.stringify(queryResults, null, 2);
|
|
77
|
+
}
|
|
78
|
+
return { formattedData, resultsLength: queryResults.length };
|
|
79
|
+
});
|
|
80
|
+
const uploadToS3AndGetURL = (formattedData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
81
|
+
// Create S3 client
|
|
82
|
+
const s3Client = new client_s3_1.S3Client({
|
|
83
|
+
region: s3Region,
|
|
84
|
+
credentials: {
|
|
85
|
+
accessKeyId: awsAccessKeyId,
|
|
86
|
+
secretAccessKey: awsSecretAccessKey,
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
const contentType = outputFormat.toLowerCase() === "csv" ? "text/csv" : "application/json";
|
|
90
|
+
const fileExtension = outputFormat.toLowerCase() === "csv" ? "csv" : "json";
|
|
91
|
+
const finalKey = `${databaseName}/${(0, uuid_1.v4)()}.${fileExtension}`;
|
|
92
|
+
// Upload to S3 without ACL
|
|
93
|
+
const uploadCommand = new client_s3_1.PutObjectCommand({
|
|
94
|
+
Bucket: s3BucketName,
|
|
95
|
+
Key: finalKey,
|
|
96
|
+
Body: formattedData,
|
|
97
|
+
ContentType: contentType,
|
|
98
|
+
});
|
|
99
|
+
yield s3Client.send(uploadCommand);
|
|
100
|
+
// Generate a presigned URL (valid for an hour)
|
|
101
|
+
const getObjectCommand = new client_s3_1.GetObjectCommand({
|
|
102
|
+
Bucket: s3BucketName,
|
|
103
|
+
Key: finalKey,
|
|
104
|
+
});
|
|
105
|
+
const presignedUrl = yield (0, s3_request_presigner_1.getSignedUrl)(s3Client, getObjectCommand, { expiresIn: 3600 });
|
|
106
|
+
return presignedUrl;
|
|
107
|
+
});
|
|
108
|
+
// Process the private key
|
|
109
|
+
const privateKeyCorrectFormatString = getPrivateKeyCorrectFormat(privateKey);
|
|
110
|
+
// Set up a connection using snowflake-sdk
|
|
111
|
+
const connection = snowflake_sdk_1.default.createConnection({
|
|
112
|
+
account: accountName,
|
|
113
|
+
username: user,
|
|
114
|
+
privateKey: privateKeyCorrectFormatString,
|
|
115
|
+
authenticator: "SNOWFLAKE_JWT",
|
|
116
|
+
role: "ACCOUNTADMIN",
|
|
117
|
+
warehouse: warehouse,
|
|
118
|
+
database: databaseName,
|
|
119
|
+
});
|
|
120
|
+
try {
|
|
121
|
+
// Connect to Snowflake
|
|
122
|
+
yield new Promise((resolve, reject) => {
|
|
123
|
+
connection.connect((err, conn) => {
|
|
124
|
+
if (err) {
|
|
125
|
+
console.error("Unable to connect to Snowflake:", err.message);
|
|
126
|
+
return reject(err);
|
|
127
|
+
}
|
|
128
|
+
resolve(conn);
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
const { formattedData, resultsLength } = yield executeQueryAndFormatData();
|
|
132
|
+
const presignedUrl = yield uploadToS3AndGetURL(formattedData);
|
|
133
|
+
// Return fields to match schema definition
|
|
134
|
+
connection.destroy(err => {
|
|
135
|
+
if (err) {
|
|
136
|
+
console.log("Failed to disconnect from Snowflake:", err);
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
return {
|
|
140
|
+
bucketUrl: presignedUrl,
|
|
141
|
+
message: `Query results successfully written to S3. URL valid for 1 hour.`,
|
|
142
|
+
rowCount: resultsLength,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
connection.destroy(err => {
|
|
147
|
+
if (err) {
|
|
148
|
+
console.log("Failed to disconnect from Snowflake:", err);
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
throw Error(`An error occurred: ${error}`);
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
exports.default = runSnowflakeQueryWriteResultsToS3;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
|
|
16
|
+
const scrapeTweetDataWithNitter = (_a) => __awaiter(void 0, [_a], void 0, function* ({ params, authParams, }) {
|
|
17
|
+
const tweetUrlRegex = /^(?:https?:\/\/)?(?:www\.)?(?:twitter\.com|x\.com)\/([a-zA-Z0-9_]+)\/status\/(\d+)(?:\?.*)?$/;
|
|
18
|
+
if (!tweetUrlRegex.test(params.tweetUrl)) {
|
|
19
|
+
throw new Error("Invalid tweet URL. Expected format: https://twitter.com/username/status/id or https://x.com/username/status/id");
|
|
20
|
+
}
|
|
21
|
+
const nitterUrl = params.tweetUrl.replace(/^(?:https?:\/\/)?(?:www\.)?(?:twitter\.com|x\.com)/i, "https://nitter.net");
|
|
22
|
+
// Initialize Firecrawl
|
|
23
|
+
if (!authParams.apiKey) {
|
|
24
|
+
throw new Error("API key is required for X+Nitter+Firecrawl");
|
|
25
|
+
}
|
|
26
|
+
const firecrawl = new firecrawl_js_1.default({
|
|
27
|
+
apiKey: authParams.apiKey,
|
|
28
|
+
});
|
|
29
|
+
try {
|
|
30
|
+
// Scrape the Nitter URL
|
|
31
|
+
const result = yield firecrawl.scrapeUrl(nitterUrl);
|
|
32
|
+
if (!result.success) {
|
|
33
|
+
throw new Error(`Failed to scrape tweet: ${result.error || "Unknown error"}`);
|
|
34
|
+
}
|
|
35
|
+
// Extract the tweet text from the scraped content - simple approach - in practice, you might need more robust parsing based on nitter html structure
|
|
36
|
+
const tweetContent = result.markdown;
|
|
37
|
+
return {
|
|
38
|
+
text: tweetContent || "Error scraping with firecrawl",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
throw new Error(`Error scraping tweet: ${error instanceof Error ? error.message : error}`);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
exports.default = scrapeTweetDataWithNitter;
|
package/package.json
CHANGED