@lobu/runtime-device 24.5.0 → 24.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +32 -17
- package/npm-shrinkwrap.json +140 -89
- package/package.json +2 -1
- package/vendor/cli/dist/bundled-skills/lobu/SKILL.md +57 -9
- package/vendor/cli/dist/commands/_lib/view-bundler.js +9 -1
- package/vendor/cli/dist/connectors/linkedin-identity.ts +43 -0
- package/vendor/cli/dist/connectors/linkedin.ts +154 -3
- package/vendor/cli/dist/connectors/x-like-follow.ts +329 -0
- package/vendor/cli/dist/connectors/x.ts +709 -80
- package/vendor/cli/dist/internal/connector-compiler.js +3 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/compile/index.d.ts +4 -3
- package/vendor/connector-worker/dist/compile/index.js +6 -3
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/contracts/tools/view-host.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/view-host.js +28 -0
- package/vendor/core/dist/contracts/tools/view-runtime.d.ts +2 -4
- package/vendor/core/dist/contracts/tools/view-runtime.js +19 -7
- package/vendor/core/package.json +11 -1
- package/vendor/views/dist/api.d.ts +290 -0
- package/vendor/views/dist/api.js +714 -0
- package/vendor/views/dist/bridge.d.ts +157 -0
- package/vendor/views/dist/bridge.js +372 -0
- package/vendor/views/dist/components.d.ts +172 -0
- package/vendor/views/dist/components.js +376 -0
- package/vendor/views/dist/index.d.ts +13 -0
- package/vendor/views/dist/index.js +3 -0
- package/vendor/views/dist/native.d.ts +327 -0
- package/vendor/views/dist/native.js +57 -0
- package/vendor/views/package.json +53 -0
|
@@ -18,6 +18,9 @@
|
|
|
18
18
|
* - tweets: search by query or track a handle (API v2 or extension search)
|
|
19
19
|
* - my_tweets: authenticated user's posts and replies (API v2 or extension)
|
|
20
20
|
* - liked_tweets: posts the user has liked (API v2 or signed-in extension backfill)
|
|
21
|
+
* - following: accounts the connected account follows (extension only);
|
|
22
|
+
* links each to a person, or a company for X Verified
|
|
23
|
+
* Organizations, walking the list across runs by cursor
|
|
21
24
|
* - bookmarks: posts the user has bookmarked (API v2 or extension)
|
|
22
25
|
* - direct_messages: 1:1 and group DMs (OAuth API; extension fallback on /messages)
|
|
23
26
|
* - live_messages: DMs read live from the signed-in browser without importing
|
|
@@ -63,6 +66,7 @@ import {
|
|
|
63
66
|
truncateHandoffReason as truncateSharedHandoffReason,
|
|
64
67
|
} from "@lobu/connector-sdk";
|
|
65
68
|
import { xDirectMessagesProgram } from "./x-dm-adapter.js";
|
|
69
|
+
import { parseXHandleInput, setXFollow, setXLike } from "./x-like-follow.js";
|
|
66
70
|
|
|
67
71
|
/** What one X sync gathers before it is committed. */
|
|
68
72
|
type XSyncPage = {
|
|
@@ -73,6 +77,20 @@ type XSyncPage = {
|
|
|
73
77
|
|
|
74
78
|
import { normalizeXHandle, X_IDENTITY } from "./x-identity.js";
|
|
75
79
|
|
|
80
|
+
export function xGraphqlInterceptPattern(
|
|
81
|
+
operation:
|
|
82
|
+
| "Search"
|
|
83
|
+
| "UserTweets"
|
|
84
|
+
| "Bookmark"
|
|
85
|
+
| "DM"
|
|
86
|
+
| "Dm"
|
|
87
|
+
| "Message"
|
|
88
|
+
| "Inbox",
|
|
89
|
+
): { regex: string } {
|
|
90
|
+
// X query ids use the URL-safe alphabet, including hyphens.
|
|
91
|
+
return { regex: `/i/api/graphql/[A-Za-z0-9_-]+/.*${operation}` };
|
|
92
|
+
}
|
|
93
|
+
|
|
76
94
|
/** OAuth scopes needed per feed for the API path (not used to pause browser-capable feeds). */
|
|
77
95
|
const X_OAUTH_FEED_SCOPES: Record<string, readonly string[]> = {
|
|
78
96
|
tweets: ["tweet.read", "users.read"],
|
|
@@ -94,6 +112,9 @@ interface XCheckpoint {
|
|
|
94
112
|
likes_oldest_tweet_id?: string;
|
|
95
113
|
likes_oldest_timestamp?: Date | string;
|
|
96
114
|
likes_backfill_completed_at?: Date | string;
|
|
115
|
+
/** Following walk position; absent when the next run starts at the top. */
|
|
116
|
+
following_cursor?: string;
|
|
117
|
+
following_walk_completed_at?: Date | string;
|
|
97
118
|
}
|
|
98
119
|
|
|
99
120
|
type XLikesBackfillStatus = "in_progress" | "complete";
|
|
@@ -232,6 +253,8 @@ interface XApiDmListResponse {
|
|
|
232
253
|
/** x.com origins the dispatched chrome actions are allowed to touch. */
|
|
233
254
|
const X_ALLOWED_ORIGINS = ["x.com", "*.x.com", "twitter.com", "*.twitter.com"];
|
|
234
255
|
const X_LIKES_INTERCEPT_PATTERN = "/i/api/graphql/[^/]+/[^?]*Like";
|
|
256
|
+
// Exactly the `Following` operation; `Followers` and `FollowersYouKnow` differ.
|
|
257
|
+
const X_FOLLOWING_INTERCEPT_PATTERN = "/i/api/graphql/[^/]+/Following\\?";
|
|
235
258
|
const X_RESPONSE_SHAPE_MAX_PATHS = 20;
|
|
236
259
|
const X_RESPONSE_SHAPE_MAX_DEPTH = 6;
|
|
237
260
|
const X_RESPONSE_SHAPE_MAX_SAMPLES = 3;
|
|
@@ -311,6 +334,23 @@ const X_TWEET_AUTHOR_ATTRIBUTIONS: EventAttributionRule[] = [
|
|
|
311
334
|
},
|
|
312
335
|
];
|
|
313
336
|
|
|
337
|
+
/**
|
|
338
|
+
* Home-timeline authors stay link-only unless the connection turns on
|
|
339
|
+
* `create_authors`. The setting is stamped on each event at sync time, so the
|
|
340
|
+
* gate reads it per event and every other tweet feed keeps link-only authors.
|
|
341
|
+
*/
|
|
342
|
+
const X_HOME_FEED_AUTHOR_ATTRIBUTIONS: EventAttributionRule[] = [
|
|
343
|
+
{
|
|
344
|
+
role: "authored_by",
|
|
345
|
+
autoCreate: true,
|
|
346
|
+
target: {
|
|
347
|
+
...X_PERSON_AUTHOR_TARGET,
|
|
348
|
+
createWhen: { path: "metadata.create_authors", equals: true },
|
|
349
|
+
},
|
|
350
|
+
traits: X_PERSON_AUTHOR_TRAITS,
|
|
351
|
+
},
|
|
352
|
+
];
|
|
353
|
+
|
|
314
354
|
/**
|
|
315
355
|
* A liked post has two people with different roles: the connected account that
|
|
316
356
|
* performed the interaction, and the post author. Both are resolved into the
|
|
@@ -362,6 +402,86 @@ const X_LIKED_POST_ATTRIBUTIONS: EventAttributionRule[] = [
|
|
|
362
402
|
},
|
|
363
403
|
];
|
|
364
404
|
|
|
405
|
+
/** The followed account as a person or company, keyed on its immutable id. */
|
|
406
|
+
function xFollowedAccountTarget(
|
|
407
|
+
entityType: "person" | "company",
|
|
408
|
+
isOrganization: boolean,
|
|
409
|
+
): EventAttributionTargetSpec {
|
|
410
|
+
return {
|
|
411
|
+
entityType,
|
|
412
|
+
createWhen: { path: "metadata.is_organization", equals: isOrganization },
|
|
413
|
+
titlePath: "metadata.followed_name",
|
|
414
|
+
identities: [
|
|
415
|
+
{
|
|
416
|
+
namespace: X_IDENTITY.USER_ID,
|
|
417
|
+
eventPath: "metadata.followed_user_id",
|
|
418
|
+
primary: true,
|
|
419
|
+
},
|
|
420
|
+
{ namespace: X_IDENTITY.HANDLE, eventPath: "metadata.followed_handle" },
|
|
421
|
+
],
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
const X_FOLLOWED_ACCOUNT_TRAITS: Record<string, EntityTraitSpec> = {
|
|
426
|
+
x_handle: {
|
|
427
|
+
eventPath: "metadata.followed_handle",
|
|
428
|
+
mergeStrategy: "prefer_non_empty",
|
|
429
|
+
},
|
|
430
|
+
x_display_name: {
|
|
431
|
+
eventPath: "metadata.followed_name",
|
|
432
|
+
mergeStrategy: "prefer_non_empty",
|
|
433
|
+
},
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* A followed account is an explicit personal signal, so it is created on no
|
|
438
|
+
* match: as a `company` when X marks it a Verified Organization (Business or
|
|
439
|
+
* Government), otherwise as a `person`. Identity lookup is type-agnostic and
|
|
440
|
+
* `createWhen` only gates create-on-miss, so an account already known under
|
|
441
|
+
* either type is linked, never duplicated. The followed account comes first
|
|
442
|
+
* so it keeps the event's flat `x_user_id` slot; the connected account is the
|
|
443
|
+
* performer, as on liked posts.
|
|
444
|
+
*/
|
|
445
|
+
const X_FOLLOWING_ATTRIBUTIONS: EventAttributionRule[] = [
|
|
446
|
+
{
|
|
447
|
+
role: "about",
|
|
448
|
+
autoCreate: true,
|
|
449
|
+
target: xFollowedAccountTarget("person", false),
|
|
450
|
+
traits: X_FOLLOWED_ACCOUNT_TRAITS,
|
|
451
|
+
},
|
|
452
|
+
{
|
|
453
|
+
role: "about",
|
|
454
|
+
autoCreate: true,
|
|
455
|
+
target: xFollowedAccountTarget("company", true),
|
|
456
|
+
traits: X_FOLLOWED_ACCOUNT_TRAITS,
|
|
457
|
+
},
|
|
458
|
+
{
|
|
459
|
+
role: "performed_by",
|
|
460
|
+
autoCreate: true,
|
|
461
|
+
target: {
|
|
462
|
+
entityType: "person",
|
|
463
|
+
titlePath: "metadata.followed_by_name",
|
|
464
|
+
identities: [
|
|
465
|
+
{
|
|
466
|
+
namespace: X_IDENTITY.USER_ID,
|
|
467
|
+
eventPath: "metadata.followed_by_id",
|
|
468
|
+
primary: true,
|
|
469
|
+
},
|
|
470
|
+
{
|
|
471
|
+
namespace: X_IDENTITY.HANDLE,
|
|
472
|
+
eventPath: "metadata.followed_by_handle",
|
|
473
|
+
},
|
|
474
|
+
],
|
|
475
|
+
},
|
|
476
|
+
traits: {
|
|
477
|
+
x_handle: {
|
|
478
|
+
eventPath: "metadata.followed_by_handle",
|
|
479
|
+
mergeStrategy: "prefer_non_empty",
|
|
480
|
+
},
|
|
481
|
+
},
|
|
482
|
+
},
|
|
483
|
+
];
|
|
484
|
+
|
|
365
485
|
const X_DM_COUNTERPARTY_ATTRIBUTIONS: EventAttributionRule[] = [
|
|
366
486
|
{
|
|
367
487
|
role: "authored_by",
|
|
@@ -676,6 +796,52 @@ function extractGraphqlUrls(legacy: any): XExpandedUrl[] {
|
|
|
676
796
|
});
|
|
677
797
|
}
|
|
678
798
|
|
|
799
|
+
/** An X account as GraphQL `user_results.result` describes it. */
|
|
800
|
+
interface XGraphqlUser {
|
|
801
|
+
id?: string;
|
|
802
|
+
handle?: string;
|
|
803
|
+
name?: string;
|
|
804
|
+
description?: string;
|
|
805
|
+
createdAt?: string;
|
|
806
|
+
isBlueVerified?: boolean;
|
|
807
|
+
/** X's organization/official verification, e.g. "Business" or "Government". */
|
|
808
|
+
verifiedType?: string;
|
|
809
|
+
profileImageShape?: string;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
function optionalString(value: unknown): string | undefined {
|
|
813
|
+
return typeof value === "string" && value ? value : undefined;
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/**
|
|
817
|
+
* Read one `user_results.result`. X moved the profile fields from `legacy`
|
|
818
|
+
* into `core`, `verification` and `profile_bio`; both shapes are read.
|
|
819
|
+
*/
|
|
820
|
+
function extractGraphqlUser(result: any): XGraphqlUser {
|
|
821
|
+
const id = result?.rest_id ?? result?.id;
|
|
822
|
+
return {
|
|
823
|
+
id: id ? String(id) : undefined,
|
|
824
|
+
handle: optionalString(
|
|
825
|
+
result?.core?.screen_name ?? result?.legacy?.screen_name,
|
|
826
|
+
),
|
|
827
|
+
name: optionalString(result?.core?.name ?? result?.legacy?.name),
|
|
828
|
+
description: optionalString(
|
|
829
|
+
result?.legacy?.description ?? result?.profile_bio?.description,
|
|
830
|
+
),
|
|
831
|
+
createdAt: optionalString(
|
|
832
|
+
result?.core?.created_at ?? result?.legacy?.created_at,
|
|
833
|
+
),
|
|
834
|
+
isBlueVerified:
|
|
835
|
+
typeof result?.is_blue_verified === "boolean"
|
|
836
|
+
? result.is_blue_verified
|
|
837
|
+
: undefined,
|
|
838
|
+
verifiedType: optionalString(
|
|
839
|
+
result?.verification?.verified_type ?? result?.legacy?.verified_type,
|
|
840
|
+
),
|
|
841
|
+
profileImageShape: optionalString(result?.profile_image_shape),
|
|
842
|
+
};
|
|
843
|
+
}
|
|
844
|
+
|
|
679
845
|
/**
|
|
680
846
|
* Extract one tweet from an x.com GraphQL `tweet_results.result` object.
|
|
681
847
|
* Shared by the search and home-timeline parsers so both handle the same edge
|
|
@@ -697,20 +863,16 @@ function extractTweetFromGraphqlResult(result: any): XTweet | null {
|
|
|
697
863
|
const restId = legacy.id_str ?? tweetNode?.rest_id ?? result.rest_id;
|
|
698
864
|
if (!restId) return null;
|
|
699
865
|
|
|
700
|
-
const
|
|
701
|
-
tweetNode?.core?.user_results?.result ?? result.core?.user_results?.result
|
|
702
|
-
|
|
703
|
-
userResult?.core?.screen_name ?? userResult?.legacy?.screen_name ?? "";
|
|
704
|
-
const authorId = userResult?.rest_id ?? userResult?.id;
|
|
705
|
-
const authorDisplayName =
|
|
706
|
-
userResult?.core?.name ?? userResult?.legacy?.name ?? undefined;
|
|
866
|
+
const author = extractGraphqlUser(
|
|
867
|
+
tweetNode?.core?.user_results?.result ?? result.core?.user_results?.result,
|
|
868
|
+
);
|
|
707
869
|
|
|
708
870
|
return {
|
|
709
871
|
id: restId,
|
|
710
872
|
text,
|
|
711
|
-
username:
|
|
712
|
-
authorId:
|
|
713
|
-
authorDisplayName,
|
|
873
|
+
username: author.handle ?? "",
|
|
874
|
+
authorId: author.id,
|
|
875
|
+
authorDisplayName: author.name,
|
|
714
876
|
likes: legacy.favorite_count ?? 0,
|
|
715
877
|
retweets: legacy.retweet_count ?? 0,
|
|
716
878
|
replies: legacy.reply_count ?? 0,
|
|
@@ -870,15 +1032,29 @@ export function extractBottomCursor(instructions: any[]): string | undefined {
|
|
|
870
1032
|
return undefined;
|
|
871
1033
|
}
|
|
872
1034
|
|
|
1035
|
+
/** The first non-empty known instruction array, else the first array. */
|
|
1036
|
+
function timelineInstructions(data: any): any[] | undefined {
|
|
1037
|
+
const candidates = timelineInstructionCandidates(data);
|
|
1038
|
+
return (candidates.find(
|
|
1039
|
+
(candidate) => Array.isArray(candidate) && candidate.length > 0,
|
|
1040
|
+
) ?? candidates.find(Array.isArray)) as any[] | undefined;
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function graphqlErrors(data: any): string[] {
|
|
1044
|
+
return Array.isArray(data?.errors)
|
|
1045
|
+
? data.errors.flatMap((error: any) => {
|
|
1046
|
+
const message = error?.message ?? error?.detail;
|
|
1047
|
+
return typeof message === "string" && message ? [message] : [];
|
|
1048
|
+
})
|
|
1049
|
+
: [];
|
|
1050
|
+
}
|
|
1051
|
+
|
|
873
1052
|
export function parseBrowserTimelinePage(
|
|
874
1053
|
_url: string,
|
|
875
1054
|
json: unknown,
|
|
876
1055
|
): XTimelinePage {
|
|
877
1056
|
const data = json as any;
|
|
878
|
-
const
|
|
879
|
-
const instructions = (candidates.find(
|
|
880
|
-
(candidate) => Array.isArray(candidate) && candidate.length > 0,
|
|
881
|
-
) ?? candidates.find(Array.isArray)) as any[] | undefined;
|
|
1057
|
+
const instructions = timelineInstructions(data);
|
|
882
1058
|
const ownerResult = data?.data?.user?.result;
|
|
883
1059
|
const owner = ownerResult
|
|
884
1060
|
? {
|
|
@@ -897,12 +1073,7 @@ export function parseBrowserTimelinePage(
|
|
|
897
1073
|
: {}),
|
|
898
1074
|
}
|
|
899
1075
|
: undefined;
|
|
900
|
-
const errors =
|
|
901
|
-
? data.errors.flatMap((error: any) => {
|
|
902
|
-
const message = error?.message ?? error?.detail;
|
|
903
|
-
return typeof message === "string" && message ? [message] : [];
|
|
904
|
-
})
|
|
905
|
-
: [];
|
|
1076
|
+
const errors = graphqlErrors(data);
|
|
906
1077
|
return {
|
|
907
1078
|
tweets: instructions ? extractTweetsFromInstructions(instructions) : [],
|
|
908
1079
|
bottomCursor: instructions ? extractBottomCursor(instructions) : undefined,
|
|
@@ -924,6 +1095,128 @@ export function parseBrowserTimelineResponse(
|
|
|
924
1095
|
return parseBrowserTimelinePage(url, json).tweets;
|
|
925
1096
|
}
|
|
926
1097
|
|
|
1098
|
+
/** A followed account with the two fields an event cannot do without. */
|
|
1099
|
+
type XFollowedUser = XGraphqlUser & { id: string; handle: string };
|
|
1100
|
+
|
|
1101
|
+
interface XFollowingPage {
|
|
1102
|
+
users: XFollowedUser[];
|
|
1103
|
+
/**
|
|
1104
|
+
* Every account entry X sent, including unavailable ones. End of list is
|
|
1105
|
+
* decided from these, not from the accounts that could be emitted.
|
|
1106
|
+
*/
|
|
1107
|
+
entryIds: string[];
|
|
1108
|
+
bottomCursor?: string;
|
|
1109
|
+
recognized: boolean;
|
|
1110
|
+
errors: string[];
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
/** Parse one x.com GraphQL `Following` response into the accounts it lists. */
|
|
1114
|
+
function parseBrowserFollowingPage(
|
|
1115
|
+
_url: string,
|
|
1116
|
+
json: unknown,
|
|
1117
|
+
): XFollowingPage {
|
|
1118
|
+
const data = json as any;
|
|
1119
|
+
const instructions = timelineInstructions(data);
|
|
1120
|
+
const users: XFollowedUser[] = [];
|
|
1121
|
+
const entryIds: string[] = [];
|
|
1122
|
+
for (const instruction of instructions ?? []) {
|
|
1123
|
+
for (const entry of instruction?.entries ?? []) {
|
|
1124
|
+
const userResults =
|
|
1125
|
+
entry?.content?.itemContent?.user_results ??
|
|
1126
|
+
entry?.item?.itemContent?.user_results;
|
|
1127
|
+
if (!userResults) continue;
|
|
1128
|
+
entryIds.push(String(entry?.entryId ?? `entry-${entryIds.length}`));
|
|
1129
|
+
const result = userResults.result;
|
|
1130
|
+
if (!result || result.__typename === "UserUnavailable") continue;
|
|
1131
|
+
const user = extractGraphqlUser(result);
|
|
1132
|
+
if (user.id && user.handle)
|
|
1133
|
+
users.push({ ...user, id: user.id, handle: user.handle });
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
return {
|
|
1137
|
+
users,
|
|
1138
|
+
entryIds,
|
|
1139
|
+
bottomCursor: instructions ? extractBottomCursor(instructions) : undefined,
|
|
1140
|
+
recognized: instructions !== undefined,
|
|
1141
|
+
errors: graphqlErrors(data),
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
/** X's Verified Organization types; other verified accounts are people. */
|
|
1146
|
+
const X_ORGANIZATION_VERIFIED_TYPES = new Set(["Business", "Government"]);
|
|
1147
|
+
|
|
1148
|
+
/** X's snowflake epoch, and its launch date as the last-resort timestamp. */
|
|
1149
|
+
const X_SNOWFLAKE_EPOCH_MS = 1288834974657n;
|
|
1150
|
+
const X_LAUNCH_DATE = new Date("2006-03-21T00:00:00.000Z");
|
|
1151
|
+
/** Pre-snowflake user ids were 32-bit; anything at or below is not a snowflake. */
|
|
1152
|
+
const X_MAX_LEGACY_USER_ID = 2n ** 32n;
|
|
1153
|
+
|
|
1154
|
+
/**
|
|
1155
|
+
* A deterministic time for a followed account: its creation time, else the
|
|
1156
|
+
* time encoded in a snowflake user id, else X's launch date. It must never
|
|
1157
|
+
* depend on when the walk ran: ingestion compares `occurred_at`, so a
|
|
1158
|
+
* sync-time value would add a version of the row on every walk.
|
|
1159
|
+
*/
|
|
1160
|
+
function followedAccountTime(user: XFollowedUser): {
|
|
1161
|
+
at: Date;
|
|
1162
|
+
basis: "account_created_at" | "user_id_snowflake" | "platform_launch";
|
|
1163
|
+
} {
|
|
1164
|
+
const created = user.createdAt ? new Date(user.createdAt) : null;
|
|
1165
|
+
if (created && !Number.isNaN(created.getTime()))
|
|
1166
|
+
return { at: created, basis: "account_created_at" };
|
|
1167
|
+
const digits =
|
|
1168
|
+
user.id.length > 0 && [...user.id].every((char) => char >= "0" && char <= "9");
|
|
1169
|
+
if (digits && BigInt(user.id) > X_MAX_LEGACY_USER_ID) {
|
|
1170
|
+
const ms = Number((BigInt(user.id) >> 22n) + X_SNOWFLAKE_EPOCH_MS);
|
|
1171
|
+
if (ms > Number(X_SNOWFLAKE_EPOCH_MS) && ms <= Date.now())
|
|
1172
|
+
return { at: new Date(ms), basis: "user_id_snowflake" };
|
|
1173
|
+
}
|
|
1174
|
+
return { at: X_LAUNCH_DATE, basis: "platform_launch" };
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/**
|
|
1178
|
+
* One event per followed account, keyed on the connected account and the
|
|
1179
|
+
* followed id so a later walk updates it instead of adding another. X does
|
|
1180
|
+
* not say when a follow happened, so `occurred_at` is a stable account time
|
|
1181
|
+
* (`followedAccountTime`) and `timestamp_basis` says which one.
|
|
1182
|
+
*/
|
|
1183
|
+
export function followedAccountToEvent(
|
|
1184
|
+
user: XFollowedUser,
|
|
1185
|
+
follower: { id: string; handle: string },
|
|
1186
|
+
): EventEnvelope {
|
|
1187
|
+
const handle = normalizeXHandle(user.handle) ?? user.handle;
|
|
1188
|
+
const name = user.name ?? `@${user.handle}`;
|
|
1189
|
+
const time = followedAccountTime(user);
|
|
1190
|
+
return {
|
|
1191
|
+
origin_id: `x_following_${follower.id}_${user.id}`,
|
|
1192
|
+
origin_type: "followed_account",
|
|
1193
|
+
payload_text: `Follows @${user.handle}${user.name ? ` (${user.name})` : ""}${
|
|
1194
|
+
user.description ? `: ${user.description}` : ""
|
|
1195
|
+
}`,
|
|
1196
|
+
author_name: name,
|
|
1197
|
+
source_url: `https://x.com/${user.handle}`,
|
|
1198
|
+
occurred_at: time.at,
|
|
1199
|
+
metadata: {
|
|
1200
|
+
followed_user_id: user.id,
|
|
1201
|
+
followed_handle: handle,
|
|
1202
|
+
followed_name: name,
|
|
1203
|
+
...(user.description ? { followed_description: user.description } : {}),
|
|
1204
|
+
is_blue_verified: user.isBlueVerified === true,
|
|
1205
|
+
...(user.verifiedType ? { verified_type: user.verifiedType } : {}),
|
|
1206
|
+
...(user.profileImageShape
|
|
1207
|
+
? { profile_image_shape: user.profileImageShape }
|
|
1208
|
+
: {}),
|
|
1209
|
+
is_organization: X_ORGANIZATION_VERIFIED_TYPES.has(
|
|
1210
|
+
user.verifiedType ?? "",
|
|
1211
|
+
),
|
|
1212
|
+
followed_by_id: follower.id,
|
|
1213
|
+
followed_by_handle: normalizeXHandle(follower.handle) ?? follower.handle,
|
|
1214
|
+
followed_by_name: `@${follower.handle}`,
|
|
1215
|
+
timestamp_basis: time.basis,
|
|
1216
|
+
},
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
|
|
927
1220
|
const TWEET_FIELDS =
|
|
928
1221
|
"author_id,conversation_id,created_at,public_metrics,referenced_tweets";
|
|
929
1222
|
|
|
@@ -1328,31 +1621,37 @@ function readLikesIncrementalPageBudget(
|
|
|
1328
1621
|
return readPositiveInteger(config.incremental_pages, 2, 10);
|
|
1329
1622
|
}
|
|
1330
1623
|
|
|
1331
|
-
|
|
1624
|
+
/** One non-empty string from a GraphQL request URL's `variables` JSON. */
|
|
1625
|
+
function readGraphqlVariable(
|
|
1626
|
+
requestUrl: string,
|
|
1627
|
+
name: string,
|
|
1628
|
+
): string | undefined {
|
|
1332
1629
|
try {
|
|
1333
1630
|
const parsed = new URL(requestUrl);
|
|
1334
1631
|
const raw = parsed.searchParams.get("variables");
|
|
1335
1632
|
if (!raw) return undefined;
|
|
1336
|
-
const
|
|
1337
|
-
return typeof
|
|
1338
|
-
? variables.cursor
|
|
1339
|
-
: undefined;
|
|
1633
|
+
const value = (JSON.parse(raw) as Record<string, unknown>)[name];
|
|
1634
|
+
return typeof value === "string" && value ? value : undefined;
|
|
1340
1635
|
} catch {
|
|
1341
1636
|
return undefined;
|
|
1342
1637
|
}
|
|
1343
1638
|
}
|
|
1344
1639
|
|
|
1640
|
+
export function readGraphqlCursor(requestUrl: string): string | undefined {
|
|
1641
|
+
return readGraphqlVariable(requestUrl, "cursor");
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1345
1644
|
export function replaceGraphqlCursor(
|
|
1346
1645
|
requestUrl: string,
|
|
1347
1646
|
cursor: string,
|
|
1348
1647
|
): string {
|
|
1349
1648
|
const parsed = new URL(requestUrl);
|
|
1350
1649
|
if (!/(^|\.)(?:x|twitter)\.com$/i.test(parsed.hostname)) {
|
|
1351
|
-
throw new Error("X
|
|
1650
|
+
throw new Error("X pagination refused a non-X request URL");
|
|
1352
1651
|
}
|
|
1353
1652
|
const raw = parsed.searchParams.get("variables");
|
|
1354
1653
|
if (!raw) {
|
|
1355
|
-
throw new Error("X
|
|
1654
|
+
throw new Error("X pagination response URL has no GraphQL variables");
|
|
1356
1655
|
}
|
|
1357
1656
|
const variables = JSON.parse(raw) as Record<string, unknown>;
|
|
1358
1657
|
variables.cursor = cursor;
|
|
@@ -1360,8 +1659,20 @@ export function replaceGraphqlCursor(
|
|
|
1360
1659
|
return parsed.toString();
|
|
1361
1660
|
}
|
|
1362
1661
|
|
|
1363
|
-
|
|
1364
|
-
|
|
1662
|
+
/** One parsed cursor page: the item ids it carried and X's bottom cursor. */
|
|
1663
|
+
interface XCursorPage {
|
|
1664
|
+
itemIds: string[];
|
|
1665
|
+
bottomCursor?: string;
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
/**
|
|
1669
|
+
* Tracks a cursor walk over one X GraphQL timeline (Likes, Following) captured
|
|
1670
|
+
* through the extension: which response advances the walk, the request URL to
|
|
1671
|
+
* replay with the next cursor, and whether the list end was reached. Late
|
|
1672
|
+
* duplicates of an earlier page never move the cursor back.
|
|
1673
|
+
*/
|
|
1674
|
+
class XCursorPageTracker {
|
|
1675
|
+
readonly historicalIds = new Set<string>();
|
|
1365
1676
|
recognizedResponses = 0;
|
|
1366
1677
|
requestedPages = 0;
|
|
1367
1678
|
respondedPages = 0;
|
|
@@ -1373,15 +1684,20 @@ class XLikesPageTracker {
|
|
|
1373
1684
|
private readonly startedFromResume: boolean;
|
|
1374
1685
|
private terminalSeen = false;
|
|
1375
1686
|
|
|
1687
|
+
/**
|
|
1688
|
+
* @param endsOnEmptyPage An advancing page without items ends the list even
|
|
1689
|
+
* when X still returns a bottom cursor (Following does this at its end).
|
|
1690
|
+
*/
|
|
1376
1691
|
constructor(
|
|
1377
1692
|
private readonly backfill: boolean,
|
|
1378
1693
|
resumeCursor: string | undefined,
|
|
1694
|
+
private readonly endsOnEmptyPage = false,
|
|
1379
1695
|
) {
|
|
1380
1696
|
this.nextCursor = resumeCursor;
|
|
1381
1697
|
this.startedFromResume = resumeCursor !== undefined;
|
|
1382
1698
|
}
|
|
1383
1699
|
|
|
1384
|
-
recordPage(url: string, page:
|
|
1700
|
+
recordPage(url: string, page: XCursorPage): void {
|
|
1385
1701
|
const isInitialPage = this.recognizedResponses === 0;
|
|
1386
1702
|
this.recognizedResponses += 1;
|
|
1387
1703
|
|
|
@@ -1395,17 +1711,19 @@ class XLikesPageTracker {
|
|
|
1395
1711
|
|
|
1396
1712
|
if (isRequestedPage) {
|
|
1397
1713
|
this.respondedPages += 1;
|
|
1398
|
-
this.recordHistoricalPage(page.
|
|
1714
|
+
this.recordHistoricalPage(page.itemIds);
|
|
1399
1715
|
this.activeRequestedCursor = undefined;
|
|
1400
1716
|
}
|
|
1401
1717
|
|
|
1402
1718
|
if (this.backfill && isInitialPage && !this.startedFromResume) {
|
|
1403
|
-
this.recordHistoricalPage(page.
|
|
1719
|
+
this.recordHistoricalPage(page.itemIds);
|
|
1404
1720
|
}
|
|
1405
1721
|
|
|
1406
|
-
|
|
1722
|
+
const empty = page.itemIds.length === 0;
|
|
1723
|
+
if (!advancesCursor) return;
|
|
1724
|
+
if (page.bottomCursor && !(this.endsOnEmptyPage && empty)) {
|
|
1407
1725
|
this.nextCursor = page.bottomCursor;
|
|
1408
|
-
} else if (
|
|
1726
|
+
} else if (!empty || this.endsOnEmptyPage) {
|
|
1409
1727
|
this.nextCursor = undefined;
|
|
1410
1728
|
this.terminalSeen = true;
|
|
1411
1729
|
}
|
|
@@ -1431,10 +1749,10 @@ class XLikesPageTracker {
|
|
|
1431
1749
|
return !this.backfill || this.terminalSeen ? "complete" : "in_progress";
|
|
1432
1750
|
}
|
|
1433
1751
|
|
|
1434
|
-
private recordHistoricalPage(
|
|
1752
|
+
private recordHistoricalPage(itemIds: string[]): void {
|
|
1435
1753
|
if (!this.backfill) return;
|
|
1436
1754
|
this.pagesRead += 1;
|
|
1437
|
-
for (const
|
|
1755
|
+
for (const id of itemIds) this.historicalIds.add(id);
|
|
1438
1756
|
}
|
|
1439
1757
|
}
|
|
1440
1758
|
|
|
@@ -2031,7 +2349,7 @@ async function syncSearchViaExtension(
|
|
|
2031
2349
|
return syncViaExtension({
|
|
2032
2350
|
ctx,
|
|
2033
2351
|
url: searchUrl,
|
|
2034
|
-
interceptPatterns: [
|
|
2352
|
+
interceptPatterns: [xGraphqlInterceptPattern("Search")],
|
|
2035
2353
|
parseResponse: parseBrowserSearchResponse,
|
|
2036
2354
|
maxScrolls,
|
|
2037
2355
|
checkpoint,
|
|
@@ -2098,7 +2416,7 @@ async function syncMyTweetsViaExtension(
|
|
|
2098
2416
|
return syncViaExtension({
|
|
2099
2417
|
ctx,
|
|
2100
2418
|
url: profileUrl,
|
|
2101
|
-
interceptPatterns: [
|
|
2419
|
+
interceptPatterns: [xGraphqlInterceptPattern("UserTweets")],
|
|
2102
2420
|
parseResponse: parseBrowserTimelineResponse,
|
|
2103
2421
|
maxScrolls,
|
|
2104
2422
|
checkpoint,
|
|
@@ -2106,6 +2424,51 @@ async function syncMyTweetsViaExtension(
|
|
|
2106
2424
|
});
|
|
2107
2425
|
}
|
|
2108
2426
|
|
|
2427
|
+
/**
|
|
2428
|
+
* Pagination for a captured X timeline: replay the page's own GraphQL request
|
|
2429
|
+
* in the page with the tracker's next cursor. Failures are recorded, never
|
|
2430
|
+
* thrown, so pages already captured are kept.
|
|
2431
|
+
*/
|
|
2432
|
+
function cursorReplayTrigger(
|
|
2433
|
+
pages: XCursorPageTracker,
|
|
2434
|
+
label: string,
|
|
2435
|
+
parserErrors: string[],
|
|
2436
|
+
) {
|
|
2437
|
+
return async (
|
|
2438
|
+
_tabId: number,
|
|
2439
|
+
browserDispatcher: ChromeActionDispatcher,
|
|
2440
|
+
sessionId: string,
|
|
2441
|
+
): Promise<void> => {
|
|
2442
|
+
let requestUrl: string | undefined;
|
|
2443
|
+
try {
|
|
2444
|
+
requestUrl = pages.prepareReplay();
|
|
2445
|
+
} catch {
|
|
2446
|
+
parserErrors.push(`X ${label} cursor rewrite failed`);
|
|
2447
|
+
return;
|
|
2448
|
+
}
|
|
2449
|
+
if (!requestUrl) return;
|
|
2450
|
+
let observation: { ok?: boolean; status?: number };
|
|
2451
|
+
try {
|
|
2452
|
+
observation = await browserDispatcher.dispatch<{
|
|
2453
|
+
ok?: boolean;
|
|
2454
|
+
status?: number;
|
|
2455
|
+
}>("network_intercept_replay", {
|
|
2456
|
+
session_id: sessionId,
|
|
2457
|
+
url: requestUrl,
|
|
2458
|
+
allowed_origins: X_ALLOWED_ORIGINS,
|
|
2459
|
+
});
|
|
2460
|
+
} catch {
|
|
2461
|
+
parserErrors.push(`X ${label} cursor request failed (transport_error)`);
|
|
2462
|
+
return;
|
|
2463
|
+
}
|
|
2464
|
+
if (observation.ok !== true) {
|
|
2465
|
+
parserErrors.push(
|
|
2466
|
+
`X ${label} cursor request failed (${observation.status ?? "unknown"})`,
|
|
2467
|
+
);
|
|
2468
|
+
}
|
|
2469
|
+
};
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2109
2472
|
async function syncLikedTweetsViaExtension(
|
|
2110
2473
|
ctx: SyncContext,
|
|
2111
2474
|
config: Record<string, unknown>,
|
|
@@ -2120,7 +2483,7 @@ async function syncLikedTweetsViaExtension(
|
|
|
2120
2483
|
: readLikesBackfillPageBudget(config);
|
|
2121
2484
|
const startsAtFirstPage =
|
|
2122
2485
|
!previouslyComplete && !checkpoint.likes_backfill_cursor;
|
|
2123
|
-
const pages = new
|
|
2486
|
+
const pages = new XCursorPageTracker(
|
|
2124
2487
|
!previouslyComplete,
|
|
2125
2488
|
previouslyComplete ? undefined : checkpoint.likes_backfill_cursor,
|
|
2126
2489
|
);
|
|
@@ -2139,7 +2502,10 @@ async function syncLikedTweetsViaExtension(
|
|
|
2139
2502
|
if (!page.recognized || page.errors.length > 0) {
|
|
2140
2503
|
return [];
|
|
2141
2504
|
}
|
|
2142
|
-
pages.recordPage(url,
|
|
2505
|
+
pages.recordPage(url, {
|
|
2506
|
+
itemIds: page.tweets.map((tweet) => tweet.id),
|
|
2507
|
+
bottomCursor: page.bottomCursor,
|
|
2508
|
+
});
|
|
2143
2509
|
owner = page.owner ?? owner;
|
|
2144
2510
|
|
|
2145
2511
|
return page.tweets.map((tweet) => ({
|
|
@@ -2167,35 +2533,7 @@ async function syncLikedTweetsViaExtension(
|
|
|
2167
2533
|
url: likesUrl,
|
|
2168
2534
|
parseResponse,
|
|
2169
2535
|
checkAuth: (currentUrl) => !isXAuthWall(currentUrl),
|
|
2170
|
-
triggerNextPage:
|
|
2171
|
-
let requestUrl: string | undefined;
|
|
2172
|
-
try {
|
|
2173
|
-
requestUrl = pages.prepareReplay();
|
|
2174
|
-
} catch {
|
|
2175
|
-
parserErrors.push("X likes cursor rewrite failed");
|
|
2176
|
-
return;
|
|
2177
|
-
}
|
|
2178
|
-
if (!requestUrl) return;
|
|
2179
|
-
let observation: { ok?: boolean; status?: number };
|
|
2180
|
-
try {
|
|
2181
|
-
observation = await browserDispatcher.dispatch<{
|
|
2182
|
-
ok?: boolean;
|
|
2183
|
-
status?: number;
|
|
2184
|
-
}>("network_intercept_replay", {
|
|
2185
|
-
session_id: sessionId,
|
|
2186
|
-
url: requestUrl,
|
|
2187
|
-
allowed_origins: X_ALLOWED_ORIGINS,
|
|
2188
|
-
});
|
|
2189
|
-
} catch {
|
|
2190
|
-
parserErrors.push("X likes cursor request failed (transport_error)");
|
|
2191
|
-
return;
|
|
2192
|
-
}
|
|
2193
|
-
if (observation.ok !== true) {
|
|
2194
|
-
parserErrors.push(
|
|
2195
|
-
`X likes cursor request failed (${observation.status ?? "unknown"})`,
|
|
2196
|
-
);
|
|
2197
|
-
}
|
|
2198
|
-
},
|
|
2536
|
+
triggerNextPage: cursorReplayTrigger(pages, "likes", parserErrors),
|
|
2199
2537
|
});
|
|
2200
2538
|
|
|
2201
2539
|
if (pages.recognizedResponses === 0) {
|
|
@@ -2234,11 +2572,115 @@ async function syncLikedTweetsViaExtension(
|
|
|
2234
2572
|
status,
|
|
2235
2573
|
nextCursor: status === "in_progress" ? pages.nextCursor : undefined,
|
|
2236
2574
|
pagesRead: pages.pagesRead,
|
|
2237
|
-
historicalItemsRead: pages.
|
|
2575
|
+
historicalItemsRead: pages.historicalIds.size,
|
|
2238
2576
|
},
|
|
2239
2577
|
);
|
|
2240
2578
|
}
|
|
2241
2579
|
|
|
2580
|
+
/**
|
|
2581
|
+
* Walk the connected account's Following list from the paired browser. Each
|
|
2582
|
+
* run reads the top page (new follows appear first), then continues from the
|
|
2583
|
+
* checkpointed cursor for up to `pages_per_run` pages. When the list ends the
|
|
2584
|
+
* cursor is cleared, so the next run walks again from the top.
|
|
2585
|
+
*/
|
|
2586
|
+
async function syncFollowingViaExtension(
|
|
2587
|
+
ctx: SyncContext,
|
|
2588
|
+
config: Record<string, unknown>,
|
|
2589
|
+
checkpoint: XCheckpoint,
|
|
2590
|
+
): Promise<XSyncPage> {
|
|
2591
|
+
const accountId = requireXAccount(ctx);
|
|
2592
|
+
const dispatcher = requireXBrowser(ctx);
|
|
2593
|
+
const accountHandle = await resolveAccountHandle(config);
|
|
2594
|
+
const followingUrl = `https://x.com/${encodeURIComponent(accountHandle)}/following`;
|
|
2595
|
+
const pageBudget = readPositiveInteger(config.pages_per_run, 5, 30);
|
|
2596
|
+
const resumeCursor = checkpoint.following_cursor;
|
|
2597
|
+
const pages = new XCursorPageTracker(true, resumeCursor, true);
|
|
2598
|
+
const parserErrors: string[] = [];
|
|
2599
|
+
const listOwners = new Set<string>();
|
|
2600
|
+
|
|
2601
|
+
const parseResponse = (url: string, json: unknown): XFollowedUser[] => {
|
|
2602
|
+
const page = parseBrowserFollowingPage(url, json);
|
|
2603
|
+
parserErrors.push(...page.errors);
|
|
2604
|
+
if (!page.recognized || page.errors.length > 0) return [];
|
|
2605
|
+
// The request names whose list this is. Only the connected account's own
|
|
2606
|
+
// list may be attributed to it.
|
|
2607
|
+
const owner = readGraphqlVariable(url, "userId") ?? "unknown";
|
|
2608
|
+
if (owner !== accountId) {
|
|
2609
|
+
listOwners.add(owner);
|
|
2610
|
+
return [];
|
|
2611
|
+
}
|
|
2612
|
+
pages.recordPage(url, {
|
|
2613
|
+
itemIds: page.entryIds,
|
|
2614
|
+
bottomCursor: page.bottomCursor,
|
|
2615
|
+
});
|
|
2616
|
+
return page.users;
|
|
2617
|
+
};
|
|
2618
|
+
|
|
2619
|
+
const result = await extensionNetworkSync<XFollowedUser>({
|
|
2620
|
+
dispatcher,
|
|
2621
|
+
config: {
|
|
2622
|
+
interceptPatterns: [{ regex: X_FOLLOWING_INTERCEPT_PATTERN }],
|
|
2623
|
+
allowedOrigins: X_ALLOWED_ORIGINS,
|
|
2624
|
+
// The page loads the top page itself; a resumed walk spends the whole
|
|
2625
|
+
// budget past it.
|
|
2626
|
+
maxScrolls: Math.max(0, pageBudget - (resumeCursor ? 0 : 1)),
|
|
2627
|
+
scrollDelayMs: 750,
|
|
2628
|
+
responseTimeoutMs: 5000,
|
|
2629
|
+
},
|
|
2630
|
+
url: followingUrl,
|
|
2631
|
+
parseResponse,
|
|
2632
|
+
checkAuth: (currentUrl) => !isXAuthWall(currentUrl),
|
|
2633
|
+
triggerNextPage: cursorReplayTrigger(pages, "following", parserErrors),
|
|
2634
|
+
});
|
|
2635
|
+
|
|
2636
|
+
if (listOwners.size > 0) {
|
|
2637
|
+
throw new Error(
|
|
2638
|
+
`X Following list for account_handle "${accountHandle}" is not the connected account's list; set account_handle to the connected account's own handle`,
|
|
2639
|
+
);
|
|
2640
|
+
}
|
|
2641
|
+
if (pages.recognizedResponses === 0) {
|
|
2642
|
+
throw new Error(
|
|
2643
|
+
`X Following captured ${result.apiCallCount} matching response(s) but no recognizable list (pattern=${X_FOLLOWING_INTERCEPT_PATTERN})${
|
|
2644
|
+
parserErrors.length > 0 ? `: ${parserErrors.join("; ")}` : ""
|
|
2645
|
+
}`,
|
|
2646
|
+
);
|
|
2647
|
+
}
|
|
2648
|
+
|
|
2649
|
+
const observedAt = new Date();
|
|
2650
|
+
const follower = { id: accountId, handle: accountHandle };
|
|
2651
|
+
const seen = new Set<string>();
|
|
2652
|
+
const events = result.items.flatMap((user) => {
|
|
2653
|
+
if (seen.has(user.id)) return [];
|
|
2654
|
+
seen.add(user.id);
|
|
2655
|
+
return [followedAccountToEvent(user, follower)];
|
|
2656
|
+
});
|
|
2657
|
+
const status = pages.status();
|
|
2658
|
+
const nextCheckpoint: XCheckpoint = { ...checkpoint };
|
|
2659
|
+
if (status === "complete") {
|
|
2660
|
+
delete nextCheckpoint.following_cursor;
|
|
2661
|
+
nextCheckpoint.following_walk_completed_at = observedAt;
|
|
2662
|
+
} else if (pages.nextCursor) {
|
|
2663
|
+
nextCheckpoint.following_cursor = pages.nextCursor;
|
|
2664
|
+
}
|
|
2665
|
+
return {
|
|
2666
|
+
events,
|
|
2667
|
+
checkpoint: nextCheckpoint as unknown as Record<string, unknown>,
|
|
2668
|
+
metadata: {
|
|
2669
|
+
items_found: events.length,
|
|
2670
|
+
backend: result.backend,
|
|
2671
|
+
api_calls: result.apiCallCount,
|
|
2672
|
+
account_handle: accountHandle,
|
|
2673
|
+
feed: "following",
|
|
2674
|
+
collection_status: status,
|
|
2675
|
+
pages_requested: pages.requestedPages,
|
|
2676
|
+
pages_received: pages.respondedPages,
|
|
2677
|
+
pages_unconfirmed: pages.unconfirmedPages,
|
|
2678
|
+
parser_errors: parserErrors,
|
|
2679
|
+
next_cursor: nextCheckpoint.following_cursor ?? null,
|
|
2680
|
+
},
|
|
2681
|
+
};
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2242
2684
|
async function syncLikedTweetsViaOAuthApi(
|
|
2243
2685
|
ctx: SyncContext,
|
|
2244
2686
|
config: Record<string, unknown>,
|
|
@@ -2364,7 +2806,7 @@ async function syncBookmarksViaExtension(
|
|
|
2364
2806
|
return syncViaExtension({
|
|
2365
2807
|
ctx,
|
|
2366
2808
|
url: "https://x.com/i/bookmarks",
|
|
2367
|
-
interceptPatterns: [
|
|
2809
|
+
interceptPatterns: [xGraphqlInterceptPattern("Bookmark")],
|
|
2368
2810
|
parseResponse: parseBrowserTimelineResponse,
|
|
2369
2811
|
maxScrolls,
|
|
2370
2812
|
checkpoint,
|
|
@@ -2457,10 +2899,10 @@ async function syncDirectMessagesViaExtension(
|
|
|
2457
2899
|
dispatcher: requireXBrowser(ctx),
|
|
2458
2900
|
config: {
|
|
2459
2901
|
interceptPatterns: [
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2902
|
+
xGraphqlInterceptPattern("DM"),
|
|
2903
|
+
xGraphqlInterceptPattern("Dm"),
|
|
2904
|
+
xGraphqlInterceptPattern("Message"),
|
|
2905
|
+
xGraphqlInterceptPattern("Inbox"),
|
|
2464
2906
|
],
|
|
2465
2907
|
allowedOrigins: X_ALLOWED_ORIGINS,
|
|
2466
2908
|
maxScrolls,
|
|
@@ -2512,12 +2954,21 @@ async function syncHomeFeedViaDomScrape(
|
|
|
2512
2954
|
}
|
|
2513
2955
|
|
|
2514
2956
|
const tweets = buildHomeFeedTweets(rows);
|
|
2515
|
-
|
|
2957
|
+
const page = finalizeSyncResult(tweets, checkpoint, {
|
|
2516
2958
|
backend: "extension-cs-scrape",
|
|
2517
2959
|
items_scraped: rows.length,
|
|
2518
2960
|
scrolls_this_run: maxScrolls,
|
|
2519
2961
|
timeline: "home",
|
|
2520
2962
|
});
|
|
2963
|
+
// Stamp the connection's author setting for the home-feed author gate.
|
|
2964
|
+
const createAuthors = config.create_authors === true;
|
|
2965
|
+
return {
|
|
2966
|
+
...page,
|
|
2967
|
+
events: page.events.map((event) => ({
|
|
2968
|
+
...event,
|
|
2969
|
+
metadata: { ...event.metadata, create_authors: createAuthors },
|
|
2970
|
+
})),
|
|
2971
|
+
};
|
|
2521
2972
|
}
|
|
2522
2973
|
|
|
2523
2974
|
// ── Config schemas ─────────────────────────────────────────────
|
|
@@ -2601,6 +3052,12 @@ const homeFeedConfigSchema = {
|
|
|
2601
3052
|
description:
|
|
2602
3053
|
"Optional lower bound. When set below max_scrolls, each sync picks a uniform random scroll count in [min_scrolls, max_scrolls].",
|
|
2603
3054
|
},
|
|
3055
|
+
create_authors: {
|
|
3056
|
+
type: "boolean",
|
|
3057
|
+
default: false,
|
|
3058
|
+
description:
|
|
3059
|
+
"Create a new person for each author seen on the home timeline (default: false). When off, timeline posts only link to people that already exist.",
|
|
3060
|
+
},
|
|
2604
3061
|
},
|
|
2605
3062
|
};
|
|
2606
3063
|
|
|
@@ -2647,6 +3104,27 @@ const likedTweetsConfigSchema = {
|
|
|
2647
3104
|
},
|
|
2648
3105
|
};
|
|
2649
3106
|
|
|
3107
|
+
const followingConfigSchema = {
|
|
3108
|
+
type: "object",
|
|
3109
|
+
required: ["account_handle"],
|
|
3110
|
+
properties: {
|
|
3111
|
+
account_handle: {
|
|
3112
|
+
type: "string",
|
|
3113
|
+
minLength: 1,
|
|
3114
|
+
description:
|
|
3115
|
+
'The connected account\'s own X handle (e.g. "testuser"). Its Following list is read in the paired browser.',
|
|
3116
|
+
},
|
|
3117
|
+
pages_per_run: {
|
|
3118
|
+
type: "integer",
|
|
3119
|
+
minimum: 1,
|
|
3120
|
+
maximum: 30,
|
|
3121
|
+
default: 5,
|
|
3122
|
+
description:
|
|
3123
|
+
"Maximum Following pages read per run. The walk position is checkpointed so the next run continues; after the end of the list the next run starts again from the top.",
|
|
3124
|
+
},
|
|
3125
|
+
},
|
|
3126
|
+
};
|
|
3127
|
+
|
|
2650
3128
|
const bookmarksConfigSchema = {
|
|
2651
3129
|
type: "object",
|
|
2652
3130
|
properties: {
|
|
@@ -3397,8 +3875,8 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3397
3875
|
key: "x",
|
|
3398
3876
|
name: "X (Twitter)",
|
|
3399
3877
|
description:
|
|
3400
|
-
"Fetches tweets, browser-visible like history, bookmarks, and DMs through the X API v2 or the paired Owletto Chrome extension, reads the chronological Following timeline live through the X API, captures notification references
|
|
3401
|
-
version: "3.
|
|
3878
|
+
"Fetches tweets, browser-visible like history, bookmarks, and DMs through the X API v2 or the paired Owletto Chrome extension, reads the chronological Following timeline live through the X API, captures notification references and the accounts you follow, reads and answers DMs live in the paired browser, and likes posts or follows accounts there. Links social actors into the person and company graph.",
|
|
3879
|
+
version: "3.17.1",
|
|
3402
3880
|
faviconDomain: "x.com",
|
|
3403
3881
|
browser: {
|
|
3404
3882
|
origins: ["https://x.com", "https://twitter.com"],
|
|
@@ -3481,6 +3959,7 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3481
3959
|
{ key: "tweet", label: "Tweet" },
|
|
3482
3960
|
{ key: "reply", label: "Reply" },
|
|
3483
3961
|
{ key: "liked_tweet", label: "Liked Tweet" },
|
|
3962
|
+
{ key: "followed_account", label: "Followed Account" },
|
|
3484
3963
|
{ key: "bookmark", label: "Bookmark" },
|
|
3485
3964
|
{ key: "dm_message", label: "DM Message" },
|
|
3486
3965
|
{ key: "notification", label: "Notification" },
|
|
@@ -3597,6 +4076,20 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3597
4076
|
},
|
|
3598
4077
|
},
|
|
3599
4078
|
},
|
|
4079
|
+
following: {
|
|
4080
|
+
key: "following",
|
|
4081
|
+
name: "Following",
|
|
4082
|
+
sync: (ctx) => this.syncFeed(ctx),
|
|
4083
|
+
description:
|
|
4084
|
+
"Accounts the connected account follows, read from its Following list in the paired Chrome browser (requires a browser-authenticated X connection). Each followed account becomes a person, or a company when X marks it a Verified Organization (Business or Government). Each run reads the newest follows plus up to pages_per_run pages further down, resuming where the last run stopped and starting over from the top after the end of the list.",
|
|
4085
|
+
configSchema: followingConfigSchema,
|
|
4086
|
+
eventKinds: {
|
|
4087
|
+
followed_account: {
|
|
4088
|
+
description: "An account the connected account follows",
|
|
4089
|
+
attributions: X_FOLLOWING_ATTRIBUTIONS,
|
|
4090
|
+
},
|
|
4091
|
+
},
|
|
4092
|
+
},
|
|
3600
4093
|
bookmarks: {
|
|
3601
4094
|
key: "bookmarks",
|
|
3602
4095
|
name: "Bookmarks",
|
|
@@ -3637,13 +4130,100 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3637
4130
|
eventKinds: {
|
|
3638
4131
|
tweet: {
|
|
3639
4132
|
description: "A tweet from your personalized home timeline",
|
|
3640
|
-
metadataSchema:
|
|
3641
|
-
|
|
4133
|
+
metadataSchema: {
|
|
4134
|
+
...engagementMetadataSchema,
|
|
4135
|
+
properties: {
|
|
4136
|
+
...engagementMetadataSchema.properties,
|
|
4137
|
+
create_authors: {
|
|
4138
|
+
type: "boolean",
|
|
4139
|
+
description:
|
|
4140
|
+
"The connection's create_authors setting when the post was read.",
|
|
4141
|
+
},
|
|
4142
|
+
},
|
|
4143
|
+
},
|
|
4144
|
+
attributions: X_HOME_FEED_AUTHOR_ATTRIBUTIONS,
|
|
3642
4145
|
},
|
|
3643
4146
|
},
|
|
3644
4147
|
},
|
|
3645
4148
|
},
|
|
3646
4149
|
actions: {
|
|
4150
|
+
set_like: {
|
|
4151
|
+
key: "set_like",
|
|
4152
|
+
name: "Set like",
|
|
4153
|
+
kind: "write",
|
|
4154
|
+
description:
|
|
4155
|
+
"Requires an X connection authenticated through your paired Chrome browser. Set the Like on one exact X post as the connected account and verify it after reload. An already-matching post is unchanged. Never replies, reposts or posts; selection and budgets belong to the calling Automation or agent.",
|
|
4156
|
+
annotations: {
|
|
4157
|
+
openWorldHint: true,
|
|
4158
|
+
destructiveHint: false,
|
|
4159
|
+
idempotentHint: true,
|
|
4160
|
+
},
|
|
4161
|
+
inputSchema: {
|
|
4162
|
+
type: "object",
|
|
4163
|
+
required: ["liked"],
|
|
4164
|
+
anyOf: [{ required: ["tweet_url"] }, { required: ["tweet_id"] }],
|
|
4165
|
+
additionalProperties: false,
|
|
4166
|
+
properties: {
|
|
4167
|
+
tweet_url: {
|
|
4168
|
+
type: "string",
|
|
4169
|
+
description:
|
|
4170
|
+
'Post URL, e.g. "https://x.com/someone/status/2083959735481716957". twitter.com URLs are accepted.',
|
|
4171
|
+
},
|
|
4172
|
+
tweet_id: {
|
|
4173
|
+
type: "string",
|
|
4174
|
+
description:
|
|
4175
|
+
"Numeric post id when tweet_url is omitted (the `origin_id` on X timeline events).",
|
|
4176
|
+
},
|
|
4177
|
+
liked: { type: "boolean" },
|
|
4178
|
+
},
|
|
4179
|
+
},
|
|
4180
|
+
outputSchema: {
|
|
4181
|
+
type: "object",
|
|
4182
|
+
properties: {
|
|
4183
|
+
tweet_id: { type: "string" },
|
|
4184
|
+
tweet_url: { type: "string" },
|
|
4185
|
+
liked: { type: "boolean" },
|
|
4186
|
+
changed: { type: "boolean" },
|
|
4187
|
+
verified: { type: "boolean" },
|
|
4188
|
+
},
|
|
4189
|
+
},
|
|
4190
|
+
},
|
|
4191
|
+
set_follow: {
|
|
4192
|
+
key: "set_follow",
|
|
4193
|
+
name: "Set follow",
|
|
4194
|
+
kind: "write",
|
|
4195
|
+
description:
|
|
4196
|
+
"Requires an X connection authenticated through your paired Chrome browser. Follow or unfollow one X account as the connected account and verify it after reload. An already-matching account is unchanged; following a protected account leaves a pending request (pending: true). Refuses the connected account itself. Selection and budgets belong to the calling Automation or agent.",
|
|
4197
|
+
annotations: {
|
|
4198
|
+
openWorldHint: true,
|
|
4199
|
+
destructiveHint: false,
|
|
4200
|
+
idempotentHint: true,
|
|
4201
|
+
},
|
|
4202
|
+
inputSchema: {
|
|
4203
|
+
type: "object",
|
|
4204
|
+
required: ["handle", "following"],
|
|
4205
|
+
additionalProperties: false,
|
|
4206
|
+
properties: {
|
|
4207
|
+
handle: {
|
|
4208
|
+
type: "string",
|
|
4209
|
+
description:
|
|
4210
|
+
'X handle with or without @, or the profile URL, e.g. "someone" or "https://x.com/someone".',
|
|
4211
|
+
},
|
|
4212
|
+
following: { type: "boolean" },
|
|
4213
|
+
},
|
|
4214
|
+
},
|
|
4215
|
+
outputSchema: {
|
|
4216
|
+
type: "object",
|
|
4217
|
+
properties: {
|
|
4218
|
+
handle: { type: "string" },
|
|
4219
|
+
user_id: { type: "string" },
|
|
4220
|
+
following: { type: "boolean" },
|
|
4221
|
+
pending: { type: "boolean" },
|
|
4222
|
+
changed: { type: "boolean" },
|
|
4223
|
+
verified: { type: "boolean" },
|
|
4224
|
+
},
|
|
4225
|
+
},
|
|
4226
|
+
},
|
|
3647
4227
|
send_message: {
|
|
3648
4228
|
key: "send_message",
|
|
3649
4229
|
kind: "write",
|
|
@@ -3757,6 +4337,50 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3757
4337
|
|
|
3758
4338
|
async execute(ctx: ActionContext): Promise<ActionResult> {
|
|
3759
4339
|
try {
|
|
4340
|
+
if (ctx.actionKey === "set_like") {
|
|
4341
|
+
if (typeof ctx.input.liked !== "boolean")
|
|
4342
|
+
return { success: false, error: "liked must be a boolean" };
|
|
4343
|
+
const tweetRef =
|
|
4344
|
+
(typeof ctx.input.tweet_url === "string" &&
|
|
4345
|
+
ctx.input.tweet_url.trim()) ||
|
|
4346
|
+
(typeof ctx.input.tweet_id === "string" &&
|
|
4347
|
+
ctx.input.tweet_id.trim()) ||
|
|
4348
|
+
"";
|
|
4349
|
+
const tweetUrl = tweetRef ? normalizeXPostUrl(tweetRef) : null;
|
|
4350
|
+
if (!tweetUrl)
|
|
4351
|
+
return {
|
|
4352
|
+
success: false,
|
|
4353
|
+
error: "tweet_url or tweet_id must identify an exact X post",
|
|
4354
|
+
};
|
|
4355
|
+
const browser = requireXBrowser(ctx);
|
|
4356
|
+
const output = await setXLike(browser, {
|
|
4357
|
+
tweetUrl,
|
|
4358
|
+
tweetId: new URL(tweetUrl).pathname.split("/").at(-1) ?? "",
|
|
4359
|
+
accountId: requireXAccount(ctx),
|
|
4360
|
+
liked: ctx.input.liked,
|
|
4361
|
+
});
|
|
4362
|
+
return { success: true, output: { ...output } };
|
|
4363
|
+
}
|
|
4364
|
+
if (ctx.actionKey === "set_follow") {
|
|
4365
|
+
if (typeof ctx.input.following !== "boolean")
|
|
4366
|
+
return { success: false, error: "following must be a boolean" };
|
|
4367
|
+
const handle =
|
|
4368
|
+
typeof ctx.input.handle === "string"
|
|
4369
|
+
? parseXHandleInput(ctx.input.handle)
|
|
4370
|
+
: null;
|
|
4371
|
+
if (!handle)
|
|
4372
|
+
return {
|
|
4373
|
+
success: false,
|
|
4374
|
+
error: "handle must be an X handle or x.com profile URL",
|
|
4375
|
+
};
|
|
4376
|
+
const browser = requireXBrowser(ctx);
|
|
4377
|
+
const output = await setXFollow(browser, {
|
|
4378
|
+
handle,
|
|
4379
|
+
accountId: requireXAccount(ctx),
|
|
4380
|
+
following: ctx.input.following,
|
|
4381
|
+
});
|
|
4382
|
+
return { success: true, output: { ...output } };
|
|
4383
|
+
}
|
|
3760
4384
|
if (ctx.actionKey === "send_message") {
|
|
3761
4385
|
const dispatcher = requireXBrowser(ctx);
|
|
3762
4386
|
const accountId = requireXAccount(ctx);
|
|
@@ -3842,6 +4466,11 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3842
4466
|
return syncMyTweetsViaExtension(ctx, config, checkpoint);
|
|
3843
4467
|
}
|
|
3844
4468
|
|
|
4469
|
+
// The Following list is read in the browser only.
|
|
4470
|
+
if (feedKey === "following") {
|
|
4471
|
+
return syncFollowingViaExtension(ctx, config, checkpoint);
|
|
4472
|
+
}
|
|
4473
|
+
|
|
3845
4474
|
if (feedKey === "liked_tweets") {
|
|
3846
4475
|
if (resolveSyncBackend(ctx, config, oauthScopes) === "oauth_api") {
|
|
3847
4476
|
return syncOAuthWithOptionalFallback(
|