fieldtheory 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +133 -0
- package/bin/ft.mjs +3 -0
- package/dist/bookmark-classify-llm.js +247 -0
- package/dist/bookmark-classify.js +223 -0
- package/dist/bookmark-media.js +186 -0
- package/dist/bookmarks-db.js +623 -0
- package/dist/bookmarks-service.js +49 -0
- package/dist/bookmarks-viz.js +531 -0
- package/dist/bookmarks.js +190 -0
- package/dist/chrome-cookies.js +146 -0
- package/dist/cli.js +381 -0
- package/dist/config.js +54 -0
- package/dist/db.js +33 -0
- package/dist/fs.js +45 -0
- package/dist/graphql-bookmarks.js +388 -0
- package/dist/paths.js +43 -0
- package/dist/types.js +1 -0
- package/dist/xauth.js +135 -0
- package/package.json +54 -0
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
import { readJsonLines, writeJsonLines, readJson, writeJson, pathExists } from './fs.js';
|
|
2
|
+
import { ensureDataDir, twitterBookmarksCachePath, twitterBackfillStatePath } from './paths.js';
|
|
3
|
+
import { loadChromeSessionConfig } from './config.js';
|
|
4
|
+
import { extractChromeXCookies } from './chrome-cookies.js';
|
|
5
|
+
import { exportBookmarksForSyncSeed } from './bookmarks-db.js';
|
|
6
|
+
const X_PUBLIC_BEARER = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
7
|
+
const BOOKMARKS_QUERY_ID = 'Z9GWmP0kP2dajyckAaDUBw';
|
|
8
|
+
const BOOKMARKS_OPERATION = 'Bookmarks';
|
|
9
|
+
const GRAPHQL_FEATURES = {
|
|
10
|
+
graphql_timeline_v2_bookmark_timeline: true,
|
|
11
|
+
rweb_tipjar_consumption_enabled: true,
|
|
12
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
13
|
+
verified_phone_label_enabled: false,
|
|
14
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
15
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
16
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
17
|
+
communities_web_enable_tweet_community_results_fetch: true,
|
|
18
|
+
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
|
19
|
+
articles_preview_enabled: true,
|
|
20
|
+
responsive_web_edit_tweet_api_enabled: true,
|
|
21
|
+
tweetypie_unmention_optimization_enabled: true,
|
|
22
|
+
responsive_web_uc_gql_enabled: true,
|
|
23
|
+
vibe_api_enabled: true,
|
|
24
|
+
responsive_web_text_conversations_enabled: false,
|
|
25
|
+
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
26
|
+
longform_notetweets_rich_text_read_enabled: true,
|
|
27
|
+
longform_notetweets_inline_media_enabled: true,
|
|
28
|
+
responsive_web_enhance_cards_enabled: false,
|
|
29
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
30
|
+
responsive_web_media_download_video_enabled: false,
|
|
31
|
+
};
|
|
32
|
+
function parseSnowflake(value) {
|
|
33
|
+
if (!value || !/^\d+$/.test(value))
|
|
34
|
+
return null;
|
|
35
|
+
try {
|
|
36
|
+
return BigInt(value);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function parseBookmarkTimestamp(record) {
|
|
43
|
+
const candidates = [record.bookmarkedAt, record.postedAt, record.syncedAt];
|
|
44
|
+
for (const candidate of candidates) {
|
|
45
|
+
if (!candidate)
|
|
46
|
+
continue;
|
|
47
|
+
const parsed = Date.parse(candidate);
|
|
48
|
+
if (Number.isFinite(parsed))
|
|
49
|
+
return parsed;
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
function compareBookmarkChronology(a, b) {
|
|
54
|
+
const aTimestamp = parseBookmarkTimestamp(a);
|
|
55
|
+
const bTimestamp = parseBookmarkTimestamp(b);
|
|
56
|
+
if (aTimestamp != null && bTimestamp != null && aTimestamp !== bTimestamp) {
|
|
57
|
+
return aTimestamp > bTimestamp ? 1 : -1;
|
|
58
|
+
}
|
|
59
|
+
const aId = parseSnowflake(a.tweetId ?? a.id);
|
|
60
|
+
const bId = parseSnowflake(b.tweetId ?? b.id);
|
|
61
|
+
if (aId != null && bId != null && aId !== bId) {
|
|
62
|
+
return aId > bId ? 1 : -1;
|
|
63
|
+
}
|
|
64
|
+
const aStamp = String(a.bookmarkedAt ?? a.postedAt ?? a.syncedAt ?? '');
|
|
65
|
+
const bStamp = String(b.bookmarkedAt ?? b.postedAt ?? b.syncedAt ?? '');
|
|
66
|
+
return aStamp.localeCompare(bStamp);
|
|
67
|
+
}
|
|
68
|
+
async function loadExistingBookmarks() {
|
|
69
|
+
const cachePath = twitterBookmarksCachePath();
|
|
70
|
+
const existing = await readJsonLines(cachePath);
|
|
71
|
+
if (existing.length > 0)
|
|
72
|
+
return existing;
|
|
73
|
+
return exportBookmarksForSyncSeed();
|
|
74
|
+
}
|
|
75
|
+
function buildUrl(cursor) {
|
|
76
|
+
const variables = { count: 20 };
|
|
77
|
+
if (cursor)
|
|
78
|
+
variables.cursor = cursor;
|
|
79
|
+
const params = new URLSearchParams({
|
|
80
|
+
variables: JSON.stringify(variables),
|
|
81
|
+
features: JSON.stringify(GRAPHQL_FEATURES),
|
|
82
|
+
});
|
|
83
|
+
return `https://x.com/i/api/graphql/${BOOKMARKS_QUERY_ID}/${BOOKMARKS_OPERATION}?${params}`;
|
|
84
|
+
}
|
|
85
|
+
function buildHeaders(csrfToken, cookieHeader) {
|
|
86
|
+
return {
|
|
87
|
+
authorization: `Bearer ${X_PUBLIC_BEARER}`,
|
|
88
|
+
'x-csrf-token': csrfToken,
|
|
89
|
+
'x-twitter-auth-type': 'OAuth2Session',
|
|
90
|
+
'x-twitter-active-user': 'yes',
|
|
91
|
+
'content-type': 'application/json',
|
|
92
|
+
cookie: cookieHeader ?? `ct0=${csrfToken}`,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
export function convertTweetToRecord(tweetResult, now) {
|
|
96
|
+
const tweet = tweetResult.tweet ?? tweetResult;
|
|
97
|
+
const legacy = tweet?.legacy;
|
|
98
|
+
if (!legacy)
|
|
99
|
+
return null;
|
|
100
|
+
const tweetId = legacy.id_str ?? tweet?.rest_id;
|
|
101
|
+
if (!tweetId)
|
|
102
|
+
return null;
|
|
103
|
+
const userResult = tweet?.core?.user_results?.result;
|
|
104
|
+
const authorHandle = userResult?.core?.screen_name ?? userResult?.legacy?.screen_name;
|
|
105
|
+
const authorName = userResult?.core?.name ?? userResult?.legacy?.name;
|
|
106
|
+
const authorProfileImageUrl = userResult?.avatar?.image_url ??
|
|
107
|
+
userResult?.legacy?.profile_image_url_https ??
|
|
108
|
+
userResult?.legacy?.profile_image_url;
|
|
109
|
+
const author = userResult
|
|
110
|
+
? {
|
|
111
|
+
id: userResult.rest_id,
|
|
112
|
+
handle: authorHandle,
|
|
113
|
+
name: authorName,
|
|
114
|
+
profileImageUrl: authorProfileImageUrl,
|
|
115
|
+
bio: userResult?.legacy?.description,
|
|
116
|
+
followerCount: userResult?.legacy?.followers_count,
|
|
117
|
+
followingCount: userResult?.legacy?.friends_count,
|
|
118
|
+
isVerified: Boolean(userResult?.is_blue_verified ?? userResult?.legacy?.verified),
|
|
119
|
+
location: typeof userResult?.location === 'object'
|
|
120
|
+
? userResult.location.location
|
|
121
|
+
: userResult?.legacy?.location,
|
|
122
|
+
snapshotAt: now,
|
|
123
|
+
}
|
|
124
|
+
: undefined;
|
|
125
|
+
const mediaEntities = legacy?.extended_entities?.media ?? legacy?.entities?.media ?? [];
|
|
126
|
+
const media = mediaEntities
|
|
127
|
+
.map((m) => m.media_url_https ?? m.media_url)
|
|
128
|
+
.filter(Boolean);
|
|
129
|
+
const mediaObjects = mediaEntities.map((m) => ({
|
|
130
|
+
type: m.type,
|
|
131
|
+
url: m.media_url_https ?? m.media_url,
|
|
132
|
+
expandedUrl: m.expanded_url,
|
|
133
|
+
width: m.original_info?.width,
|
|
134
|
+
height: m.original_info?.height,
|
|
135
|
+
altText: m.ext_alt_text,
|
|
136
|
+
videoVariants: Array.isArray(m.video_info?.variants)
|
|
137
|
+
? m.video_info.variants
|
|
138
|
+
.filter((v) => v.content_type === 'video/mp4')
|
|
139
|
+
.map((v) => ({ bitrate: v.bitrate, url: v.url }))
|
|
140
|
+
: undefined,
|
|
141
|
+
}));
|
|
142
|
+
const urlEntities = legacy?.entities?.urls ?? [];
|
|
143
|
+
const links = urlEntities
|
|
144
|
+
.map((u) => u.expanded_url)
|
|
145
|
+
.filter((u) => u && !u.includes('t.co'));
|
|
146
|
+
return {
|
|
147
|
+
id: tweetId,
|
|
148
|
+
tweetId,
|
|
149
|
+
url: `https://x.com/${authorHandle ?? '_'}/status/${tweetId}`,
|
|
150
|
+
text: legacy.full_text ?? legacy.text ?? '',
|
|
151
|
+
authorHandle,
|
|
152
|
+
authorName,
|
|
153
|
+
authorProfileImageUrl,
|
|
154
|
+
author,
|
|
155
|
+
postedAt: legacy.created_at ?? null,
|
|
156
|
+
bookmarkedAt: null,
|
|
157
|
+
syncedAt: now,
|
|
158
|
+
conversationId: legacy.conversation_id_str,
|
|
159
|
+
inReplyToStatusId: legacy.in_reply_to_status_id_str,
|
|
160
|
+
inReplyToUserId: legacy.in_reply_to_user_id_str,
|
|
161
|
+
quotedStatusId: legacy.quoted_status_id_str,
|
|
162
|
+
language: legacy.lang,
|
|
163
|
+
sourceApp: legacy.source,
|
|
164
|
+
possiblySensitive: legacy.possibly_sensitive,
|
|
165
|
+
engagement: {
|
|
166
|
+
likeCount: legacy.favorite_count,
|
|
167
|
+
repostCount: legacy.retweet_count,
|
|
168
|
+
replyCount: legacy.reply_count,
|
|
169
|
+
quoteCount: legacy.quote_count,
|
|
170
|
+
bookmarkCount: legacy.bookmark_count,
|
|
171
|
+
viewCount: tweet?.views?.count ? Number(tweet.views.count) : undefined,
|
|
172
|
+
},
|
|
173
|
+
media,
|
|
174
|
+
mediaObjects,
|
|
175
|
+
links,
|
|
176
|
+
tags: [],
|
|
177
|
+
ingestedVia: 'graphql',
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
export function parseBookmarksResponse(json, now) {
|
|
181
|
+
const ts = now ?? new Date().toISOString();
|
|
182
|
+
const instructions = json?.data?.bookmark_timeline_v2?.timeline?.instructions ?? [];
|
|
183
|
+
const entries = [];
|
|
184
|
+
for (const inst of instructions) {
|
|
185
|
+
if (inst.type === 'TimelineAddEntries' && Array.isArray(inst.entries)) {
|
|
186
|
+
entries.push(...inst.entries);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
const records = [];
|
|
190
|
+
let nextCursor;
|
|
191
|
+
for (const entry of entries) {
|
|
192
|
+
if (entry.entryId?.startsWith('cursor-bottom')) {
|
|
193
|
+
nextCursor = entry.content?.value;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const tweetResult = entry?.content?.itemContent?.tweet_results?.result;
|
|
197
|
+
if (!tweetResult)
|
|
198
|
+
continue;
|
|
199
|
+
const record = convertTweetToRecord(tweetResult, ts);
|
|
200
|
+
if (record)
|
|
201
|
+
records.push(record);
|
|
202
|
+
}
|
|
203
|
+
return { records, nextCursor };
|
|
204
|
+
}
|
|
205
|
+
async function fetchPageWithRetry(csrfToken, cursor, cookieHeader) {
|
|
206
|
+
let lastError;
|
|
207
|
+
for (let attempt = 0; attempt < 4; attempt++) {
|
|
208
|
+
const response = await fetch(buildUrl(cursor), { headers: buildHeaders(csrfToken, cookieHeader) });
|
|
209
|
+
if (response.status === 429) {
|
|
210
|
+
const waitSec = Math.min(15 * Math.pow(2, attempt), 120);
|
|
211
|
+
lastError = new Error(`Rate limited (429) on attempt ${attempt + 1}`);
|
|
212
|
+
await new Promise((r) => setTimeout(r, waitSec * 1000));
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (response.status >= 500) {
|
|
216
|
+
lastError = new Error(`Server error (${response.status}) on attempt ${attempt + 1}`);
|
|
217
|
+
await new Promise((r) => setTimeout(r, 5000 * (attempt + 1)));
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (!response.ok) {
|
|
221
|
+
const text = await response.text();
|
|
222
|
+
throw new Error(`GraphQL Bookmarks API returned ${response.status}.\n` +
|
|
223
|
+
`Response: ${text.slice(0, 300)}\n\n` +
|
|
224
|
+
(response.status === 401 || response.status === 403
|
|
225
|
+
? 'Fix: Your X session may have expired. Open Chrome, go to https://x.com, and make sure you are logged in. Then retry.'
|
|
226
|
+
: 'This may be a temporary issue. Try again in a few minutes.'));
|
|
227
|
+
}
|
|
228
|
+
const json = await response.json();
|
|
229
|
+
return parseBookmarksResponse(json);
|
|
230
|
+
}
|
|
231
|
+
throw lastError ?? new Error('GraphQL Bookmarks API: all retry attempts failed. Try again later.');
|
|
232
|
+
}
|
|
233
|
+
export function scoreRecord(record) {
|
|
234
|
+
let score = 0;
|
|
235
|
+
if (record.postedAt)
|
|
236
|
+
score += 2;
|
|
237
|
+
if (record.authorProfileImageUrl)
|
|
238
|
+
score += 2;
|
|
239
|
+
if (record.author)
|
|
240
|
+
score += 3;
|
|
241
|
+
if (record.engagement)
|
|
242
|
+
score += 3;
|
|
243
|
+
if ((record.mediaObjects?.length ?? 0) > 0)
|
|
244
|
+
score += 3;
|
|
245
|
+
if ((record.links?.length ?? 0) > 0)
|
|
246
|
+
score += 2;
|
|
247
|
+
return score;
|
|
248
|
+
}
|
|
249
|
+
export function mergeBookmarkRecord(existing, incoming) {
|
|
250
|
+
if (!existing)
|
|
251
|
+
return incoming;
|
|
252
|
+
return scoreRecord(incoming) >= scoreRecord(existing)
|
|
253
|
+
? { ...existing, ...incoming }
|
|
254
|
+
: { ...incoming, ...existing };
|
|
255
|
+
}
|
|
256
|
+
export function mergeRecords(existing, incoming) {
|
|
257
|
+
const byId = new Map(existing.map((r) => [r.id, r]));
|
|
258
|
+
let added = 0;
|
|
259
|
+
for (const record of incoming) {
|
|
260
|
+
const prev = byId.get(record.id);
|
|
261
|
+
if (!prev)
|
|
262
|
+
added += 1;
|
|
263
|
+
byId.set(record.id, mergeBookmarkRecord(prev, record));
|
|
264
|
+
}
|
|
265
|
+
const merged = Array.from(byId.values());
|
|
266
|
+
merged.sort((a, b) => compareBookmarkChronology(b, a));
|
|
267
|
+
return { merged, added };
|
|
268
|
+
}
|
|
269
|
+
function updateState(prev, input) {
|
|
270
|
+
return {
|
|
271
|
+
provider: 'twitter',
|
|
272
|
+
lastRunAt: new Date().toISOString(),
|
|
273
|
+
totalRuns: prev.totalRuns + 1,
|
|
274
|
+
totalAdded: prev.totalAdded + input.added,
|
|
275
|
+
lastAdded: input.added,
|
|
276
|
+
lastSeenIds: input.seenIds.slice(-20),
|
|
277
|
+
stopReason: input.stopReason,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
export function formatSyncResult(result) {
|
|
281
|
+
return [
|
|
282
|
+
'Sync complete.',
|
|
283
|
+
`- bookmarks added: ${result.added}`,
|
|
284
|
+
`- total bookmarks: ${result.totalBookmarks}`,
|
|
285
|
+
`- pages fetched: ${result.pages}`,
|
|
286
|
+
`- stop reason: ${result.stopReason}`,
|
|
287
|
+
`- cache: ${result.cachePath}`,
|
|
288
|
+
`- state: ${result.statePath}`,
|
|
289
|
+
].join('\n');
|
|
290
|
+
}
|
|
291
|
+
export async function syncBookmarksGraphQL(options = {}) {
|
|
292
|
+
const incremental = options.incremental ?? true;
|
|
293
|
+
const maxPages = options.maxPages ?? 500;
|
|
294
|
+
const delayMs = options.delayMs ?? 600;
|
|
295
|
+
const maxMinutes = options.maxMinutes ?? 30;
|
|
296
|
+
const stalePageLimit = options.stalePageLimit ?? 3;
|
|
297
|
+
const checkpointEvery = options.checkpointEvery ?? 25;
|
|
298
|
+
let csrfToken;
|
|
299
|
+
let cookieHeader;
|
|
300
|
+
if (options.csrfToken) {
|
|
301
|
+
csrfToken = options.csrfToken;
|
|
302
|
+
cookieHeader = options.cookieHeader;
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
const chromeConfig = loadChromeSessionConfig();
|
|
306
|
+
const chromeDir = options.chromeUserDataDir ?? chromeConfig.chromeUserDataDir;
|
|
307
|
+
const chromeProfile = options.chromeProfileDirectory ?? chromeConfig.chromeProfileDirectory;
|
|
308
|
+
const cookies = extractChromeXCookies(chromeDir, chromeProfile);
|
|
309
|
+
csrfToken = cookies.csrfToken;
|
|
310
|
+
cookieHeader = cookies.cookieHeader;
|
|
311
|
+
}
|
|
312
|
+
ensureDataDir();
|
|
313
|
+
const cachePath = twitterBookmarksCachePath();
|
|
314
|
+
const statePath = twitterBackfillStatePath();
|
|
315
|
+
let existing = await loadExistingBookmarks();
|
|
316
|
+
const newestKnownId = incremental
|
|
317
|
+
? existing.slice().sort((a, b) => compareBookmarkChronology(b, a))[0]?.id
|
|
318
|
+
: undefined;
|
|
319
|
+
const prevState = (await pathExists(statePath))
|
|
320
|
+
? await readJson(statePath)
|
|
321
|
+
: { provider: 'twitter', totalRuns: 0, totalAdded: 0, lastAdded: 0, lastSeenIds: [] };
|
|
322
|
+
const started = Date.now();
|
|
323
|
+
let page = 0;
|
|
324
|
+
let totalAdded = 0;
|
|
325
|
+
let stalePages = 0;
|
|
326
|
+
let cursor;
|
|
327
|
+
const allSeenIds = [];
|
|
328
|
+
let stopReason = 'unknown';
|
|
329
|
+
while (page < maxPages) {
|
|
330
|
+
if (Date.now() - started > maxMinutes * 60_000) {
|
|
331
|
+
stopReason = 'max runtime reached';
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
const result = await fetchPageWithRetry(csrfToken, cursor, cookieHeader);
|
|
335
|
+
page += 1;
|
|
336
|
+
if (result.records.length === 0 && !result.nextCursor) {
|
|
337
|
+
stopReason = 'end of bookmarks';
|
|
338
|
+
break;
|
|
339
|
+
}
|
|
340
|
+
const { merged, added } = mergeRecords(existing, result.records);
|
|
341
|
+
existing = merged;
|
|
342
|
+
totalAdded += added;
|
|
343
|
+
result.records.forEach((r) => allSeenIds.push(r.id));
|
|
344
|
+
const reachedLatestStored = Boolean(newestKnownId) && result.records.some((record) => record.id === newestKnownId);
|
|
345
|
+
stalePages = added === 0 ? stalePages + 1 : 0;
|
|
346
|
+
options.onProgress?.({
|
|
347
|
+
page,
|
|
348
|
+
totalFetched: allSeenIds.length,
|
|
349
|
+
newAdded: totalAdded,
|
|
350
|
+
running: true,
|
|
351
|
+
done: false,
|
|
352
|
+
});
|
|
353
|
+
if (options.targetAdds && totalAdded >= options.targetAdds) {
|
|
354
|
+
stopReason = 'target additions reached';
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
if (incremental && reachedLatestStored) {
|
|
358
|
+
stopReason = 'caught up to newest stored bookmark';
|
|
359
|
+
break;
|
|
360
|
+
}
|
|
361
|
+
if (incremental && stalePages >= stalePageLimit) {
|
|
362
|
+
stopReason = 'no new bookmarks (stale)';
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
if (!result.nextCursor) {
|
|
366
|
+
stopReason = 'end of bookmarks';
|
|
367
|
+
break;
|
|
368
|
+
}
|
|
369
|
+
if (page % checkpointEvery === 0)
|
|
370
|
+
await writeJsonLines(cachePath, existing);
|
|
371
|
+
cursor = result.nextCursor;
|
|
372
|
+
if (page < maxPages)
|
|
373
|
+
await new Promise((r) => setTimeout(r, delayMs));
|
|
374
|
+
}
|
|
375
|
+
if (stopReason === 'unknown')
|
|
376
|
+
stopReason = page >= maxPages ? 'max pages reached' : 'unknown';
|
|
377
|
+
await writeJsonLines(cachePath, existing);
|
|
378
|
+
await writeJson(statePath, updateState(prevState, { added: totalAdded, seenIds: allSeenIds.slice(-20), stopReason }));
|
|
379
|
+
options.onProgress?.({
|
|
380
|
+
page,
|
|
381
|
+
totalFetched: allSeenIds.length,
|
|
382
|
+
newAdded: totalAdded,
|
|
383
|
+
running: false,
|
|
384
|
+
done: true,
|
|
385
|
+
stopReason,
|
|
386
|
+
});
|
|
387
|
+
return { added: totalAdded, totalBookmarks: existing.length, pages: page, stopReason, cachePath, statePath };
|
|
388
|
+
}
|
package/dist/paths.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
export function dataDir() {
|
|
5
|
+
const override = process.env.FT_DATA_DIR;
|
|
6
|
+
if (override)
|
|
7
|
+
return override;
|
|
8
|
+
return path.join(os.homedir(), '.ft-bookmarks');
|
|
9
|
+
}
|
|
10
|
+
function ensureDirSync(dir) {
|
|
11
|
+
if (!fs.existsSync(dir)) {
|
|
12
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export function ensureDataDir() {
|
|
16
|
+
const dir = dataDir();
|
|
17
|
+
ensureDirSync(dir);
|
|
18
|
+
return dir;
|
|
19
|
+
}
|
|
20
|
+
export function twitterBookmarksCachePath() {
|
|
21
|
+
return path.join(dataDir(), 'bookmarks.jsonl');
|
|
22
|
+
}
|
|
23
|
+
export function twitterBookmarksMetaPath() {
|
|
24
|
+
return path.join(dataDir(), 'bookmarks-meta.json');
|
|
25
|
+
}
|
|
26
|
+
export function twitterOauthTokenPath() {
|
|
27
|
+
return path.join(dataDir(), 'oauth-token.json');
|
|
28
|
+
}
|
|
29
|
+
export function twitterBackfillStatePath() {
|
|
30
|
+
return path.join(dataDir(), 'bookmarks-backfill-state.json');
|
|
31
|
+
}
|
|
32
|
+
export function bookmarkMediaDir() {
|
|
33
|
+
return path.join(dataDir(), 'media');
|
|
34
|
+
}
|
|
35
|
+
export function bookmarkMediaManifestPath() {
|
|
36
|
+
return path.join(dataDir(), 'media-manifest.json');
|
|
37
|
+
}
|
|
38
|
+
export function twitterBookmarksIndexPath() {
|
|
39
|
+
return path.join(dataDir(), 'bookmarks.db');
|
|
40
|
+
}
|
|
41
|
+
export function isFirstRun() {
|
|
42
|
+
return !fs.existsSync(twitterBookmarksCachePath());
|
|
43
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/xauth.js
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import http from 'node:http';
|
|
3
|
+
import { URL } from 'node:url';
|
|
4
|
+
import { pathExists, readJson, writeJson } from './fs.js';
|
|
5
|
+
import { ensureDataDir, twitterOauthTokenPath } from './paths.js';
|
|
6
|
+
import { loadXApiConfig } from './config.js';
|
|
7
|
+
function base64Url(input) {
|
|
8
|
+
return input.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '');
|
|
9
|
+
}
|
|
10
|
+
function createPkce() {
|
|
11
|
+
const verifier = base64Url(crypto.randomBytes(32));
|
|
12
|
+
const challenge = base64Url(crypto.createHash('sha256').update(verifier).digest());
|
|
13
|
+
const state = base64Url(crypto.randomBytes(16));
|
|
14
|
+
return { verifier, challenge, state };
|
|
15
|
+
}
|
|
16
|
+
export function buildTwitterOAuthUrl() {
|
|
17
|
+
const cfg = loadXApiConfig();
|
|
18
|
+
if (!cfg.callbackUrl) {
|
|
19
|
+
throw new Error('Missing X_CALLBACK_URL in .env.local');
|
|
20
|
+
}
|
|
21
|
+
const { verifier, challenge, state } = createPkce();
|
|
22
|
+
const url = new URL('https://twitter.com/i/oauth2/authorize');
|
|
23
|
+
url.searchParams.set('response_type', 'code');
|
|
24
|
+
url.searchParams.set('client_id', cfg.clientId);
|
|
25
|
+
url.searchParams.set('redirect_uri', cfg.callbackUrl);
|
|
26
|
+
url.searchParams.set('scope', 'tweet.read users.read bookmark.read offline.access');
|
|
27
|
+
url.searchParams.set('state', state);
|
|
28
|
+
url.searchParams.set('code_challenge', challenge);
|
|
29
|
+
url.searchParams.set('code_challenge_method', 'S256');
|
|
30
|
+
return { url: url.toString(), state, verifier };
|
|
31
|
+
}
|
|
32
|
+
async function exchangeCodeForToken(code, verifier) {
|
|
33
|
+
const cfg = loadXApiConfig();
|
|
34
|
+
if (!cfg.callbackUrl) {
|
|
35
|
+
throw new Error('Missing X_CALLBACK_URL in .env.local');
|
|
36
|
+
}
|
|
37
|
+
const basic = Buffer.from(`${cfg.clientId}:${cfg.clientSecret}`).toString('base64');
|
|
38
|
+
const body = new URLSearchParams({
|
|
39
|
+
grant_type: 'authorization_code',
|
|
40
|
+
code,
|
|
41
|
+
redirect_uri: cfg.callbackUrl,
|
|
42
|
+
code_verifier: verifier,
|
|
43
|
+
client_id: cfg.clientId,
|
|
44
|
+
});
|
|
45
|
+
const response = await fetch('https://api.x.com/2/oauth2/token', {
|
|
46
|
+
method: 'POST',
|
|
47
|
+
headers: {
|
|
48
|
+
Authorization: `Basic ${basic}`,
|
|
49
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
50
|
+
},
|
|
51
|
+
body,
|
|
52
|
+
});
|
|
53
|
+
const text = await response.text();
|
|
54
|
+
const parsed = JSON.parse(text);
|
|
55
|
+
if (!response.ok) {
|
|
56
|
+
throw new Error(`Token exchange failed (${response.status}): ${text}`);
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
access_token: parsed.access_token,
|
|
60
|
+
refresh_token: parsed.refresh_token,
|
|
61
|
+
expires_in: parsed.expires_in,
|
|
62
|
+
scope: parsed.scope,
|
|
63
|
+
token_type: parsed.token_type,
|
|
64
|
+
obtained_at: new Date().toISOString(),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export async function saveTwitterOAuthToken(token) {
|
|
68
|
+
ensureDataDir();
|
|
69
|
+
const tokenPath = twitterOauthTokenPath();
|
|
70
|
+
await writeJson(tokenPath, token);
|
|
71
|
+
// Restrict permissions — OAuth tokens should only be readable by the owner
|
|
72
|
+
const { chmod } = await import('node:fs/promises');
|
|
73
|
+
await chmod(tokenPath, 0o600);
|
|
74
|
+
return tokenPath;
|
|
75
|
+
}
|
|
76
|
+
export async function loadTwitterOAuthToken() {
|
|
77
|
+
const tokenPath = twitterOauthTokenPath();
|
|
78
|
+
if (!(await pathExists(tokenPath)))
|
|
79
|
+
return null;
|
|
80
|
+
return readJson(tokenPath);
|
|
81
|
+
}
|
|
82
|
+
export async function runTwitterOAuthFlow() {
|
|
83
|
+
const cfg = loadXApiConfig();
|
|
84
|
+
if (!cfg.callbackUrl) {
|
|
85
|
+
throw new Error('Missing X_CALLBACK_URL in .env.local');
|
|
86
|
+
}
|
|
87
|
+
const { url, state, verifier } = buildTwitterOAuthUrl();
|
|
88
|
+
const callback = new URL(cfg.callbackUrl);
|
|
89
|
+
const port = Number(callback.port || 80);
|
|
90
|
+
const pathname = callback.pathname;
|
|
91
|
+
const code = await new Promise((resolve, reject) => {
|
|
92
|
+
const server = http.createServer((req, res) => {
|
|
93
|
+
try {
|
|
94
|
+
const reqUrl = new URL(req.url ?? '/', `http://127.0.0.1:${port}`);
|
|
95
|
+
if (reqUrl.pathname !== pathname) {
|
|
96
|
+
res.statusCode = 404;
|
|
97
|
+
res.end('Not found');
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
const returnedState = reqUrl.searchParams.get('state');
|
|
101
|
+
const returnedCode = reqUrl.searchParams.get('code');
|
|
102
|
+
const error = reqUrl.searchParams.get('error');
|
|
103
|
+
if (error) {
|
|
104
|
+
res.statusCode = 400;
|
|
105
|
+
res.end(`OAuth error: ${error}`);
|
|
106
|
+
server.close();
|
|
107
|
+
reject(new Error(`OAuth error: ${error}`));
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
if (!returnedCode || returnedState !== state) {
|
|
111
|
+
res.statusCode = 400;
|
|
112
|
+
res.end('Invalid OAuth callback');
|
|
113
|
+
server.close();
|
|
114
|
+
reject(new Error('Invalid OAuth callback state/code'));
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
res.statusCode = 200;
|
|
118
|
+
res.end('ft auth complete. You can close this tab.');
|
|
119
|
+
server.close();
|
|
120
|
+
resolve(returnedCode);
|
|
121
|
+
}
|
|
122
|
+
catch (err) {
|
|
123
|
+
server.close();
|
|
124
|
+
reject(err);
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
server.listen(port, '127.0.0.1', () => {
|
|
128
|
+
console.log('Open this URL in your browser to authorize X bookmarks access:');
|
|
129
|
+
console.log(url);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
const token = await exchangeCodeForToken(code, verifier);
|
|
133
|
+
const tokenPath = await saveTwitterOAuthToken(token);
|
|
134
|
+
return { tokenPath, scope: token.scope };
|
|
135
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "fieldtheory",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Field Theory CLI. Self-custody for your X/Twitter bookmarks. Local sync, full-text search, classification, and terminal dashboards.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"ft": "bin/ft.mjs"
|
|
8
|
+
},
|
|
9
|
+
"scripts": {
|
|
10
|
+
"build": "tsc -p tsconfig.json",
|
|
11
|
+
"test": "tsx --test tests/**/*.test.ts",
|
|
12
|
+
"dev": "tsx src/cli.ts",
|
|
13
|
+
"start": "node dist/cli.js",
|
|
14
|
+
"prepublishOnly": "npm run build"
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"bin/",
|
|
18
|
+
"dist/",
|
|
19
|
+
"LICENSE",
|
|
20
|
+
"README.md"
|
|
21
|
+
],
|
|
22
|
+
"engines": {
|
|
23
|
+
"node": ">=20"
|
|
24
|
+
},
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"repository": {
|
|
27
|
+
"type": "git",
|
|
28
|
+
"url": "git+https://github.com/afar1/fieldtheory-cli.git"
|
|
29
|
+
},
|
|
30
|
+
"keywords": [
|
|
31
|
+
"bookmarks",
|
|
32
|
+
"twitter",
|
|
33
|
+
"x",
|
|
34
|
+
"search",
|
|
35
|
+
"fts5",
|
|
36
|
+
"sqlite",
|
|
37
|
+
"local-first",
|
|
38
|
+
"self-custody",
|
|
39
|
+
"cli"
|
|
40
|
+
],
|
|
41
|
+
"dependencies": {
|
|
42
|
+
"commander": "^14.0.3",
|
|
43
|
+
"dotenv": "^17.3.1",
|
|
44
|
+
"sql.js": "^1.14.1",
|
|
45
|
+
"sql.js-fts5": "^1.4.0",
|
|
46
|
+
"zod": "^4.3.6"
|
|
47
|
+
},
|
|
48
|
+
"devDependencies": {
|
|
49
|
+
"@types/node": "^25.5.0",
|
|
50
|
+
"@types/sql.js": "^1.4.11",
|
|
51
|
+
"tsx": "^4.21.0",
|
|
52
|
+
"typescript": "^6.0.2"
|
|
53
|
+
}
|
|
54
|
+
}
|