fieldtheory-cli-windowsport 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,58 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { constants as fsConstants } from 'node:fs';
4
+ function pathEntries() {
5
+ return (process.env.PATH ?? '')
6
+ .split(path.delimiter)
7
+ .map((entry) => entry.trim())
8
+ .filter(Boolean);
9
+ }
10
+ function commandExtensions() {
11
+ if (process.platform !== 'win32')
12
+ return [''];
13
+ const raw = process.env.PATHEXT ?? '.COM;.EXE;.BAT;.CMD';
14
+ return raw
15
+ .split(';')
16
+ .map((entry) => entry.trim().toLowerCase())
17
+ .filter(Boolean);
18
+ }
19
+ function candidateNames(command) {
20
+ if (process.platform !== 'win32')
21
+ return [command];
22
+ const lower = command.toLowerCase();
23
+ const hasKnownExtension = commandExtensions().some((ext) => lower.endsWith(ext));
24
+ if (hasKnownExtension)
25
+ return [command];
26
+ return ['', ...commandExtensions()].map((ext) => `${command}${ext}`);
27
+ }
28
+ function isExecutable(filePath) {
29
+ if (!fs.existsSync(filePath))
30
+ return false;
31
+ if (process.platform === 'win32')
32
+ return true;
33
+ try {
34
+ fs.accessSync(filePath, fsConstants.X_OK);
35
+ return true;
36
+ }
37
+ catch {
38
+ return false;
39
+ }
40
+ }
41
+ export function resolveCommandPath(command) {
42
+ const includesSeparator = command.includes('/') || command.includes('\\');
43
+ if (includesSeparator) {
44
+ const absolute = path.resolve(command);
45
+ return isExecutable(absolute) ? absolute : null;
46
+ }
47
+ for (const dir of pathEntries()) {
48
+ for (const name of candidateNames(command)) {
49
+ const fullPath = path.join(dir, name);
50
+ if (isExecutable(fullPath))
51
+ return fullPath;
52
+ }
53
+ }
54
+ return null;
55
+ }
56
+ export function isCommandAvailable(command) {
57
+ return resolveCommandPath(command) !== null;
58
+ }
package/dist/config.js ADDED
@@ -0,0 +1,54 @@
1
+ import { config as loadDotenv } from 'dotenv';
2
+ import path from 'node:path';
3
+ import os from 'node:os';
4
+ import { dataDir } from './paths.js';
5
+ export function loadEnv() {
6
+ const dir = dataDir();
7
+ const candidatePaths = [
8
+ path.join(process.cwd(), '.env.local'),
9
+ path.join(process.cwd(), '.env'),
10
+ path.join(dir, '.env.local'),
11
+ path.join(dir, '.env'),
12
+ ];
13
+ for (const envPath of candidatePaths) {
14
+ loadDotenv({ path: envPath, quiet: true });
15
+ }
16
+ }
17
+ function detectChromeUserDataDir() {
18
+ const platform = os.platform();
19
+ const home = os.homedir();
20
+ if (platform === 'darwin')
21
+ return path.join(home, 'Library', 'Application Support', 'Google', 'Chrome');
22
+ if (platform === 'linux')
23
+ return path.join(home, '.config', 'google-chrome');
24
+ if (platform === 'win32')
25
+ return path.join(home, 'AppData', 'Local', 'Google', 'Chrome', 'User Data');
26
+ return undefined;
27
+ }
28
+ export function loadChromeSessionConfig() {
29
+ loadEnv();
30
+ const dir = process.env.FTX_CHROME_USER_DATA_DIR ?? process.env.FT_CHROME_USER_DATA_DIR ?? detectChromeUserDataDir();
31
+ if (!dir) {
32
+ throw new Error('Could not detect Chrome user-data directory.\n' +
33
+ 'Set FTX_CHROME_USER_DATA_DIR in .env or pass --chrome-user-data-dir.');
34
+ }
35
+ return {
36
+ chromeUserDataDir: dir,
37
+ chromeProfileDirectory: process.env.FTX_CHROME_PROFILE_DIRECTORY ?? process.env.FT_CHROME_PROFILE_DIRECTORY ?? 'Default',
38
+ };
39
+ }
40
+ export function loadXApiConfig() {
41
+ loadEnv();
42
+ const apiKey = process.env.X_API_KEY ?? process.env.X_CONSUMER_KEY;
43
+ const apiSecret = process.env.X_API_SECRET ?? process.env.X_SECRET_KEY;
44
+ const clientId = process.env.X_CLIENT_ID;
45
+ const clientSecret = process.env.X_CLIENT_SECRET;
46
+ const bearerToken = process.env.X_BEARER_TOKEN;
47
+ const callbackUrl = process.env.X_CALLBACK_URL ?? 'http://127.0.0.1:3000/callback';
48
+ if (!apiKey || !apiSecret || !clientId || !clientSecret) {
49
+ throw new Error('Missing X API credentials for API sync.\n' +
50
+ 'Set X_API_KEY, X_API_SECRET, X_CLIENT_ID, and X_CLIENT_SECRET in .env.\n' +
51
+ 'These are only needed for --api mode. Default sync uses your Chrome session.');
52
+ }
53
+ return { apiKey, apiSecret, clientId, clientSecret, bearerToken, callbackUrl };
54
+ }
package/dist/db.js ADDED
@@ -0,0 +1,33 @@
1
+ import { createRequire } from 'node:module';
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+ const require = createRequire(import.meta.url);
5
+ let sqlPromise;
6
+ function getSql() {
7
+ if (!sqlPromise) {
8
+ const initSqlJs = require('sql.js-fts5');
9
+ const wasmPath = require.resolve('sql.js-fts5/dist/sql-wasm.wasm');
10
+ const wasmBinary = fs.readFileSync(wasmPath);
11
+ sqlPromise = initSqlJs({ wasmBinary });
12
+ }
13
+ return sqlPromise;
14
+ }
15
+ export async function openDb(filePath) {
16
+ const SQL = await getSql();
17
+ if (fs.existsSync(filePath)) {
18
+ const buf = fs.readFileSync(filePath);
19
+ return new SQL.Database(buf);
20
+ }
21
+ return new SQL.Database();
22
+ }
23
+ export async function createDb() {
24
+ const SQL = await getSql();
25
+ return new SQL.Database();
26
+ }
27
+ export function saveDb(db, filePath) {
28
+ const dir = path.dirname(filePath);
29
+ if (!fs.existsSync(dir))
30
+ fs.mkdirSync(dir, { recursive: true });
31
+ const data = db.export();
32
+ fs.writeFileSync(filePath, Buffer.from(data));
33
+ }
package/dist/fs.js ADDED
@@ -0,0 +1,45 @@
1
+ import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
2
+ export async function ensureDir(dirPath) {
3
+ await mkdir(dirPath, { recursive: true });
4
+ }
5
+ export async function pathExists(filePath) {
6
+ try {
7
+ await access(filePath);
8
+ return true;
9
+ }
10
+ catch {
11
+ return false;
12
+ }
13
+ }
14
+ export async function listFiles(dirPath) {
15
+ try {
16
+ return await readdir(dirPath);
17
+ }
18
+ catch {
19
+ return [];
20
+ }
21
+ }
22
+ export async function writeJson(filePath, value) {
23
+ await writeFile(filePath, JSON.stringify(value, null, 2), 'utf8');
24
+ }
25
+ export async function readJson(filePath) {
26
+ const raw = await readFile(filePath, 'utf8');
27
+ return JSON.parse(raw);
28
+ }
29
+ export async function writeJsonLines(filePath, rows) {
30
+ const content = rows.map((row) => JSON.stringify(row)).join('\n') + (rows.length ? '\n' : '');
31
+ await writeFile(filePath, content, 'utf8');
32
+ }
33
+ export async function readJsonLines(filePath) {
34
+ try {
35
+ const raw = await readFile(filePath, 'utf8');
36
+ return raw
37
+ .split('\n')
38
+ .map((line) => line.trim())
39
+ .filter(Boolean)
40
+ .map((line) => JSON.parse(line));
41
+ }
42
+ catch {
43
+ return [];
44
+ }
45
+ }
@@ -0,0 +1,398 @@
1
+ import { readJsonLines, writeJsonLines, readJson, writeJson, pathExists } from './fs.js';
2
+ import { ensureDataDir, twitterBookmarksCachePath, twitterBackfillStatePath } from './paths.js';
3
+ import { loadChromeSessionConfig } from './config.js';
4
+ import { extractChromeXCookies } from './chrome-cookies.js';
5
+ import { exportBookmarksForSyncSeed } from './bookmarks-db.js';
6
+ const X_PUBLIC_BEARER = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
7
+ const BOOKMARKS_QUERY_ID = 'Z9GWmP0kP2dajyckAaDUBw';
8
+ const BOOKMARKS_OPERATION = 'Bookmarks';
9
+ const GRAPHQL_FEATURES = {
10
+ graphql_timeline_v2_bookmark_timeline: true,
11
+ rweb_tipjar_consumption_enabled: true,
12
+ responsive_web_graphql_exclude_directive_enabled: true,
13
+ verified_phone_label_enabled: false,
14
+ creator_subscriptions_tweet_preview_api_enabled: true,
15
+ responsive_web_graphql_timeline_navigation_enabled: true,
16
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
17
+ communities_web_enable_tweet_community_results_fetch: true,
18
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
19
+ articles_preview_enabled: true,
20
+ responsive_web_edit_tweet_api_enabled: true,
21
+ tweetypie_unmention_optimization_enabled: true,
22
+ responsive_web_uc_gql_enabled: true,
23
+ vibe_api_enabled: true,
24
+ responsive_web_text_conversations_enabled: false,
25
+ freedom_of_speech_not_reach_fetch_enabled: true,
26
+ longform_notetweets_rich_text_read_enabled: true,
27
+ longform_notetweets_inline_media_enabled: true,
28
+ responsive_web_enhance_cards_enabled: false,
29
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
30
+ responsive_web_media_download_video_enabled: false,
31
+ };
32
+ function parseSnowflake(value) {
33
+ if (!value || !/^\d+$/.test(value))
34
+ return null;
35
+ try {
36
+ return BigInt(value);
37
+ }
38
+ catch {
39
+ return null;
40
+ }
41
+ }
42
+ function parseBookmarkTimestamp(record) {
43
+ const candidates = [record.bookmarkedAt, record.postedAt, record.syncedAt];
44
+ for (const candidate of candidates) {
45
+ if (!candidate)
46
+ continue;
47
+ const parsed = Date.parse(candidate);
48
+ if (Number.isFinite(parsed))
49
+ return parsed;
50
+ }
51
+ return null;
52
+ }
53
+ function compareBookmarkChronology(a, b) {
54
+ const aTimestamp = parseBookmarkTimestamp(a);
55
+ const bTimestamp = parseBookmarkTimestamp(b);
56
+ if (aTimestamp != null && bTimestamp != null && aTimestamp !== bTimestamp) {
57
+ return aTimestamp > bTimestamp ? 1 : -1;
58
+ }
59
+ const aId = parseSnowflake(a.tweetId ?? a.id);
60
+ const bId = parseSnowflake(b.tweetId ?? b.id);
61
+ if (aId != null && bId != null && aId !== bId) {
62
+ return aId > bId ? 1 : -1;
63
+ }
64
+ const aStamp = String(a.bookmarkedAt ?? a.postedAt ?? a.syncedAt ?? '');
65
+ const bStamp = String(b.bookmarkedAt ?? b.postedAt ?? b.syncedAt ?? '');
66
+ return aStamp.localeCompare(bStamp);
67
+ }
68
+ async function loadExistingBookmarks() {
69
+ const cachePath = twitterBookmarksCachePath();
70
+ const existing = await readJsonLines(cachePath);
71
+ if (existing.length > 0)
72
+ return existing;
73
+ // On first run, no JSONL and no DB — return empty
74
+ try {
75
+ return await exportBookmarksForSyncSeed();
76
+ }
77
+ catch {
78
+ return [];
79
+ }
80
+ }
81
+ function buildUrl(cursor) {
82
+ const variables = { count: 20 };
83
+ if (cursor)
84
+ variables.cursor = cursor;
85
+ const params = new URLSearchParams({
86
+ variables: JSON.stringify(variables),
87
+ features: JSON.stringify(GRAPHQL_FEATURES),
88
+ });
89
+ return `https://x.com/i/api/graphql/${BOOKMARKS_QUERY_ID}/${BOOKMARKS_OPERATION}?${params}`;
90
+ }
91
+ function buildHeaders(csrfToken, cookieHeader) {
92
+ const userAgent = process.platform === 'win32'
93
+ ? 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
94
+ : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36';
95
+ return {
96
+ authorization: `Bearer ${X_PUBLIC_BEARER}`,
97
+ 'x-csrf-token': csrfToken,
98
+ 'x-twitter-auth-type': 'OAuth2Session',
99
+ 'x-twitter-active-user': 'yes',
100
+ 'content-type': 'application/json',
101
+ 'user-agent': userAgent,
102
+ cookie: cookieHeader ?? `ct0=${csrfToken}`,
103
+ };
104
+ }
105
+ export function convertTweetToRecord(tweetResult, now) {
106
+ const tweet = tweetResult.tweet ?? tweetResult;
107
+ const legacy = tweet?.legacy;
108
+ if (!legacy)
109
+ return null;
110
+ const tweetId = legacy.id_str ?? tweet?.rest_id;
111
+ if (!tweetId)
112
+ return null;
113
+ const userResult = tweet?.core?.user_results?.result;
114
+ const authorHandle = userResult?.core?.screen_name ?? userResult?.legacy?.screen_name;
115
+ const authorName = userResult?.core?.name ?? userResult?.legacy?.name;
116
+ const authorProfileImageUrl = userResult?.avatar?.image_url ??
117
+ userResult?.legacy?.profile_image_url_https ??
118
+ userResult?.legacy?.profile_image_url;
119
+ const author = userResult
120
+ ? {
121
+ id: userResult.rest_id,
122
+ handle: authorHandle,
123
+ name: authorName,
124
+ profileImageUrl: authorProfileImageUrl,
125
+ bio: userResult?.legacy?.description,
126
+ followerCount: userResult?.legacy?.followers_count,
127
+ followingCount: userResult?.legacy?.friends_count,
128
+ isVerified: Boolean(userResult?.is_blue_verified ?? userResult?.legacy?.verified),
129
+ location: typeof userResult?.location === 'object'
130
+ ? userResult.location.location
131
+ : userResult?.legacy?.location,
132
+ snapshotAt: now,
133
+ }
134
+ : undefined;
135
+ const mediaEntities = legacy?.extended_entities?.media ?? legacy?.entities?.media ?? [];
136
+ const media = mediaEntities
137
+ .map((m) => m.media_url_https ?? m.media_url)
138
+ .filter(Boolean);
139
+ const mediaObjects = mediaEntities.map((m) => ({
140
+ type: m.type,
141
+ url: m.media_url_https ?? m.media_url,
142
+ expandedUrl: m.expanded_url,
143
+ width: m.original_info?.width,
144
+ height: m.original_info?.height,
145
+ altText: m.ext_alt_text,
146
+ videoVariants: Array.isArray(m.video_info?.variants)
147
+ ? m.video_info.variants
148
+ .filter((v) => v.content_type === 'video/mp4')
149
+ .map((v) => ({ bitrate: v.bitrate, url: v.url }))
150
+ : undefined,
151
+ }));
152
+ const urlEntities = legacy?.entities?.urls ?? [];
153
+ const links = urlEntities
154
+ .map((u) => u.expanded_url)
155
+ .filter((u) => u && !u.includes('t.co'));
156
+ return {
157
+ id: tweetId,
158
+ tweetId,
159
+ url: `https://x.com/${authorHandle ?? '_'}/status/${tweetId}`,
160
+ text: legacy.full_text ?? legacy.text ?? '',
161
+ authorHandle,
162
+ authorName,
163
+ authorProfileImageUrl,
164
+ author,
165
+ postedAt: legacy.created_at ?? null,
166
+ bookmarkedAt: null,
167
+ syncedAt: now,
168
+ conversationId: legacy.conversation_id_str,
169
+ inReplyToStatusId: legacy.in_reply_to_status_id_str,
170
+ inReplyToUserId: legacy.in_reply_to_user_id_str,
171
+ quotedStatusId: legacy.quoted_status_id_str,
172
+ language: legacy.lang,
173
+ sourceApp: legacy.source,
174
+ possiblySensitive: legacy.possibly_sensitive,
175
+ engagement: {
176
+ likeCount: legacy.favorite_count,
177
+ repostCount: legacy.retweet_count,
178
+ replyCount: legacy.reply_count,
179
+ quoteCount: legacy.quote_count,
180
+ bookmarkCount: legacy.bookmark_count,
181
+ viewCount: tweet?.views?.count ? Number(tweet.views.count) : undefined,
182
+ },
183
+ media,
184
+ mediaObjects,
185
+ links,
186
+ tags: [],
187
+ ingestedVia: 'graphql',
188
+ };
189
+ }
190
+ export function parseBookmarksResponse(json, now) {
191
+ const ts = now ?? new Date().toISOString();
192
+ const instructions = json?.data?.bookmark_timeline_v2?.timeline?.instructions ?? [];
193
+ const entries = [];
194
+ for (const inst of instructions) {
195
+ if (inst.type === 'TimelineAddEntries' && Array.isArray(inst.entries)) {
196
+ entries.push(...inst.entries);
197
+ }
198
+ }
199
+ const records = [];
200
+ let nextCursor;
201
+ for (const entry of entries) {
202
+ if (entry.entryId?.startsWith('cursor-bottom')) {
203
+ nextCursor = entry.content?.value;
204
+ continue;
205
+ }
206
+ const tweetResult = entry?.content?.itemContent?.tweet_results?.result;
207
+ if (!tweetResult)
208
+ continue;
209
+ const record = convertTweetToRecord(tweetResult, ts);
210
+ if (record)
211
+ records.push(record);
212
+ }
213
+ return { records, nextCursor };
214
+ }
215
+ async function fetchPageWithRetry(csrfToken, cursor, cookieHeader) {
216
+ let lastError;
217
+ for (let attempt = 0; attempt < 4; attempt++) {
218
+ const response = await fetch(buildUrl(cursor), { headers: buildHeaders(csrfToken, cookieHeader) });
219
+ if (response.status === 429) {
220
+ const waitSec = Math.min(15 * Math.pow(2, attempt), 120);
221
+ lastError = new Error(`Rate limited (429) on attempt ${attempt + 1}`);
222
+ await new Promise((r) => setTimeout(r, waitSec * 1000));
223
+ continue;
224
+ }
225
+ if (response.status >= 500) {
226
+ lastError = new Error(`Server error (${response.status}) on attempt ${attempt + 1}`);
227
+ await new Promise((r) => setTimeout(r, 5000 * (attempt + 1)));
228
+ continue;
229
+ }
230
+ if (!response.ok) {
231
+ const text = await response.text();
232
+ throw new Error(`GraphQL Bookmarks API returned ${response.status}.\n` +
233
+ `Response: ${text.slice(0, 300)}\n\n` +
234
+ (response.status === 401 || response.status === 403
235
+ ? 'Fix: Your X session may have expired. Open Chrome, go to https://x.com, and make sure you are logged in. Then retry.'
236
+ : 'This may be a temporary issue. Try again in a few minutes.'));
237
+ }
238
+ const json = await response.json();
239
+ return parseBookmarksResponse(json);
240
+ }
241
+ throw lastError ?? new Error('GraphQL Bookmarks API: all retry attempts failed. Try again later.');
242
+ }
243
+ export function scoreRecord(record) {
244
+ let score = 0;
245
+ if (record.postedAt)
246
+ score += 2;
247
+ if (record.authorProfileImageUrl)
248
+ score += 2;
249
+ if (record.author)
250
+ score += 3;
251
+ if (record.engagement)
252
+ score += 3;
253
+ if ((record.mediaObjects?.length ?? 0) > 0)
254
+ score += 3;
255
+ if ((record.links?.length ?? 0) > 0)
256
+ score += 2;
257
+ return score;
258
+ }
259
+ export function mergeBookmarkRecord(existing, incoming) {
260
+ if (!existing)
261
+ return incoming;
262
+ return scoreRecord(incoming) >= scoreRecord(existing)
263
+ ? { ...existing, ...incoming }
264
+ : { ...incoming, ...existing };
265
+ }
266
+ export function mergeRecords(existing, incoming) {
267
+ const byId = new Map(existing.map((r) => [r.id, r]));
268
+ let added = 0;
269
+ for (const record of incoming) {
270
+ const prev = byId.get(record.id);
271
+ if (!prev)
272
+ added += 1;
273
+ byId.set(record.id, mergeBookmarkRecord(prev, record));
274
+ }
275
+ const merged = Array.from(byId.values());
276
+ merged.sort((a, b) => compareBookmarkChronology(b, a));
277
+ return { merged, added };
278
+ }
279
+ function updateState(prev, input) {
280
+ return {
281
+ provider: 'twitter',
282
+ lastRunAt: new Date().toISOString(),
283
+ totalRuns: prev.totalRuns + 1,
284
+ totalAdded: prev.totalAdded + input.added,
285
+ lastAdded: input.added,
286
+ lastSeenIds: input.seenIds.slice(-20),
287
+ stopReason: input.stopReason,
288
+ };
289
+ }
290
+ export function formatSyncResult(result) {
291
+ return [
292
+ 'Sync complete.',
293
+ `- bookmarks added: ${result.added}`,
294
+ `- total bookmarks: ${result.totalBookmarks}`,
295
+ `- pages fetched: ${result.pages}`,
296
+ `- stop reason: ${result.stopReason}`,
297
+ `- cache: ${result.cachePath}`,
298
+ `- state: ${result.statePath}`,
299
+ ].join('\n');
300
+ }
301
+ export async function syncBookmarksGraphQL(options = {}) {
302
+ const incremental = options.incremental ?? true;
303
+ const maxPages = options.maxPages ?? 500;
304
+ const delayMs = options.delayMs ?? 600;
305
+ const maxMinutes = options.maxMinutes ?? 30;
306
+ const stalePageLimit = options.stalePageLimit ?? 3;
307
+ const checkpointEvery = options.checkpointEvery ?? 25;
308
+ let csrfToken;
309
+ let cookieHeader;
310
+ if (options.csrfToken) {
311
+ csrfToken = options.csrfToken;
312
+ cookieHeader = options.cookieHeader;
313
+ }
314
+ else {
315
+ const chromeConfig = loadChromeSessionConfig();
316
+ const chromeDir = options.chromeUserDataDir ?? chromeConfig.chromeUserDataDir;
317
+ const chromeProfile = options.chromeProfileDirectory ?? chromeConfig.chromeProfileDirectory;
318
+ const cookies = await extractChromeXCookies(chromeDir, chromeProfile);
319
+ csrfToken = cookies.csrfToken;
320
+ cookieHeader = cookies.cookieHeader;
321
+ }
322
+ ensureDataDir();
323
+ const cachePath = twitterBookmarksCachePath();
324
+ const statePath = twitterBackfillStatePath();
325
+ let existing = await loadExistingBookmarks();
326
+ const newestKnownId = incremental
327
+ ? existing.slice().sort((a, b) => compareBookmarkChronology(b, a))[0]?.id
328
+ : undefined;
329
+ const prevState = (await pathExists(statePath))
330
+ ? await readJson(statePath)
331
+ : { provider: 'twitter', totalRuns: 0, totalAdded: 0, lastAdded: 0, lastSeenIds: [] };
332
+ const started = Date.now();
333
+ let page = 0;
334
+ let totalAdded = 0;
335
+ let stalePages = 0;
336
+ let cursor;
337
+ const allSeenIds = [];
338
+ let stopReason = 'unknown';
339
+ while (page < maxPages) {
340
+ if (Date.now() - started > maxMinutes * 60_000) {
341
+ stopReason = 'max runtime reached';
342
+ break;
343
+ }
344
+ const result = await fetchPageWithRetry(csrfToken, cursor, cookieHeader);
345
+ page += 1;
346
+ if (result.records.length === 0 && !result.nextCursor) {
347
+ stopReason = 'end of bookmarks';
348
+ break;
349
+ }
350
+ const { merged, added } = mergeRecords(existing, result.records);
351
+ existing = merged;
352
+ totalAdded += added;
353
+ result.records.forEach((r) => allSeenIds.push(r.id));
354
+ const reachedLatestStored = Boolean(newestKnownId) && result.records.some((record) => record.id === newestKnownId);
355
+ stalePages = added === 0 ? stalePages + 1 : 0;
356
+ options.onProgress?.({
357
+ page,
358
+ totalFetched: allSeenIds.length,
359
+ newAdded: totalAdded,
360
+ running: true,
361
+ done: false,
362
+ });
363
+ if (options.targetAdds && totalAdded >= options.targetAdds) {
364
+ stopReason = 'target additions reached';
365
+ break;
366
+ }
367
+ if (incremental && reachedLatestStored) {
368
+ stopReason = 'caught up to newest stored bookmark';
369
+ break;
370
+ }
371
+ if (incremental && stalePages >= stalePageLimit) {
372
+ stopReason = 'no new bookmarks (stale)';
373
+ break;
374
+ }
375
+ if (!result.nextCursor) {
376
+ stopReason = 'end of bookmarks';
377
+ break;
378
+ }
379
+ if (page % checkpointEvery === 0)
380
+ await writeJsonLines(cachePath, existing);
381
+ cursor = result.nextCursor;
382
+ if (page < maxPages)
383
+ await new Promise((r) => setTimeout(r, delayMs));
384
+ }
385
+ if (stopReason === 'unknown')
386
+ stopReason = page >= maxPages ? 'max pages reached' : 'unknown';
387
+ await writeJsonLines(cachePath, existing);
388
+ await writeJson(statePath, updateState(prevState, { added: totalAdded, seenIds: allSeenIds.slice(-20), stopReason }));
389
+ options.onProgress?.({
390
+ page,
391
+ totalFetched: allSeenIds.length,
392
+ newAdded: totalAdded,
393
+ running: false,
394
+ done: true,
395
+ stopReason,
396
+ });
397
+ return { added: totalAdded, totalBookmarks: existing.length, pages: page, stopReason, cachePath, statePath };
398
+ }
package/dist/paths.js ADDED
@@ -0,0 +1,43 @@
1
+ import path from 'node:path';
2
+ import os from 'node:os';
3
+ import fs from 'node:fs';
4
+ export function dataDir() {
5
+ const override = process.env.FTX_DATA_DIR ?? process.env.FT_DATA_DIR;
6
+ if (override)
7
+ return override;
8
+ return path.join(os.homedir(), '.ftx-bookmarks');
9
+ }
10
+ function ensureDirSync(dir) {
11
+ if (!fs.existsSync(dir)) {
12
+ fs.mkdirSync(dir, { recursive: true });
13
+ }
14
+ }
15
+ export function ensureDataDir() {
16
+ const dir = dataDir();
17
+ ensureDirSync(dir);
18
+ return dir;
19
+ }
20
+ export function twitterBookmarksCachePath() {
21
+ return path.join(dataDir(), 'bookmarks.jsonl');
22
+ }
23
+ export function twitterBookmarksMetaPath() {
24
+ return path.join(dataDir(), 'bookmarks-meta.json');
25
+ }
26
+ export function twitterOauthTokenPath() {
27
+ return path.join(dataDir(), 'oauth-token.json');
28
+ }
29
+ export function twitterBackfillStatePath() {
30
+ return path.join(dataDir(), 'bookmarks-backfill-state.json');
31
+ }
32
+ export function bookmarkMediaDir() {
33
+ return path.join(dataDir(), 'media');
34
+ }
35
+ export function bookmarkMediaManifestPath() {
36
+ return path.join(dataDir(), 'media-manifest.json');
37
+ }
38
+ export function twitterBookmarksIndexPath() {
39
+ return path.join(dataDir(), 'bookmarks.db');
40
+ }
41
+ export function isFirstRun() {
42
+ return !fs.existsSync(twitterBookmarksCachePath());
43
+ }
package/dist/types.js ADDED
@@ -0,0 +1 @@
1
+ export {};