fieldtheory-cli-windowsport 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +161 -0
- package/bin/ft.mjs +15 -0
- package/dist/bookmark-classify-llm.js +247 -0
- package/dist/bookmark-classify.js +223 -0
- package/dist/bookmark-media.js +186 -0
- package/dist/bookmarks-db.js +644 -0
- package/dist/bookmarks-service.js +49 -0
- package/dist/bookmarks-viz.js +597 -0
- package/dist/bookmarks.js +190 -0
- package/dist/chrome-cookies.js +239 -0
- package/dist/cli.js +642 -0
- package/dist/command-path.js +58 -0
- package/dist/config.js +54 -0
- package/dist/db.js +33 -0
- package/dist/fs.js +45 -0
- package/dist/graphql-bookmarks.js +398 -0
- package/dist/paths.js +43 -0
- package/dist/types.js +1 -0
- package/dist/xauth.js +135 -0
- package/package.json +63 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import { syncTwitterBookmarks } from './bookmarks.js';
|
|
4
|
+
import { getBookmarkStatusView, formatBookmarkStatus } from './bookmarks-service.js';
|
|
5
|
+
import { runTwitterOAuthFlow } from './xauth.js';
|
|
6
|
+
import { syncBookmarksGraphQL } from './graphql-bookmarks.js';
|
|
7
|
+
import { fetchBookmarkMediaBatch } from './bookmark-media.js';
|
|
8
|
+
import { buildIndex, searchBookmarks, formatSearchResults, getStats, classifyAndRebuild, getCategoryCounts, sampleByCategory, getDomainCounts, listBookmarks, getBookmarkById, } from './bookmarks-db.js';
|
|
9
|
+
import { formatClassificationSummary } from './bookmark-classify.js';
|
|
10
|
+
import { classifyWithLlm, classifyDomainsWithLlm, detectAvailableEngines } from './bookmark-classify-llm.js';
|
|
11
|
+
import { loadChromeSessionConfig } from './config.js';
|
|
12
|
+
import { renderViz } from './bookmarks-viz.js';
|
|
13
|
+
import { dataDir, ensureDataDir, isFirstRun, twitterBookmarksIndexPath } from './paths.js';
|
|
14
|
+
import fs from 'node:fs';
|
|
15
|
+
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
16
|
+
const SPINNER = ['\u280b', '\u2819', '\u2839', '\u2838', '\u283c', '\u2834', '\u2826', '\u2827', '\u2807', '\u280f'];
|
|
17
|
+
let spinnerIdx = 0;
|
|
18
|
+
function renderProgress(status, startTime) {
|
|
19
|
+
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
20
|
+
const spin = SPINNER[spinnerIdx++ % SPINNER.length];
|
|
21
|
+
const line = ` ${spin} Syncing bookmarks... ${status.newAdded} new \u2502 page ${status.page} \u2502 ${elapsed}s`;
|
|
22
|
+
process.stderr.write(`\r\x1b[K${line}`);
|
|
23
|
+
}
|
|
24
|
+
const FRIENDLY_STOP_REASONS = {
|
|
25
|
+
'caught up to newest stored bookmark': 'All caught up \u2014 no new bookmarks since last sync.',
|
|
26
|
+
'no new bookmarks (stale)': 'Sync complete \u2014 reached the end of new bookmarks.',
|
|
27
|
+
'end of bookmarks': 'Sync complete \u2014 all bookmarks fetched.',
|
|
28
|
+
'max runtime reached': 'Paused after 30 minutes. Run again to continue.',
|
|
29
|
+
'max pages reached': 'Paused after reaching page limit. Run again to continue.',
|
|
30
|
+
'target additions reached': 'Reached target bookmark count.',
|
|
31
|
+
};
|
|
32
|
+
function friendlyStopReason(raw) {
|
|
33
|
+
if (!raw)
|
|
34
|
+
return 'Sync complete.';
|
|
35
|
+
return FRIENDLY_STOP_REASONS[raw] ?? `Sync complete \u2014 ${raw}`;
|
|
36
|
+
}
|
|
37
|
+
const LOGO = `
|
|
38
|
+
\x1b[2m\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\x1b[0m
|
|
39
|
+
\x1b[2m\u2502\x1b[0m \x1b[1mFieldTheory for Windows\x1b[0m \x1b[2m\u2502\x1b[0m
|
|
40
|
+
\x1b[2m\u2502\x1b[0m \x1b[2mby Shango Bashi\x1b[0m \x1b[2m\u2502\x1b[0m
|
|
41
|
+
\x1b[2m\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\x1b[0m`;
|
|
42
|
+
function selectedEngine(optionValue) {
|
|
43
|
+
const normalized = String(optionValue ?? 'auto').trim().toLowerCase();
|
|
44
|
+
if (normalized === 'claude' || normalized === 'codex')
|
|
45
|
+
return normalized;
|
|
46
|
+
return 'auto';
|
|
47
|
+
}
|
|
48
|
+
export function showWelcome() {
|
|
49
|
+
console.log(LOGO);
|
|
50
|
+
console.log(`
|
|
51
|
+
Save a local copy of your X/Twitter bookmarks. Search them,
|
|
52
|
+
classify them, and make them available to Codex or any
|
|
53
|
+
shell-access agent on Windows.
|
|
54
|
+
Your data never leaves your machine.
|
|
55
|
+
|
|
56
|
+
Get started:
|
|
57
|
+
|
|
58
|
+
1. Open Google Chrome and log into x.com
|
|
59
|
+
2. Run: ftx sync
|
|
60
|
+
|
|
61
|
+
Data will be stored at: ${dataDir()}
|
|
62
|
+
`);
|
|
63
|
+
}
|
|
64
|
+
export async function showDashboard() {
|
|
65
|
+
console.log(LOGO);
|
|
66
|
+
try {
|
|
67
|
+
const view = await getBookmarkStatusView();
|
|
68
|
+
const ago = view.lastUpdated ? timeAgo(view.lastUpdated) : 'never';
|
|
69
|
+
console.log(`
|
|
70
|
+
\x1b[1m${view.bookmarkCount.toLocaleString()}\x1b[0m bookmarks \x1b[2m\u2502\x1b[0m last synced \x1b[1m${ago}\x1b[0m \x1b[2m\u2502\x1b[0m ${dataDir()}
|
|
71
|
+
`);
|
|
72
|
+
if (fs.existsSync(twitterBookmarksIndexPath())) {
|
|
73
|
+
const counts = await getCategoryCounts();
|
|
74
|
+
const cats = Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 7);
|
|
75
|
+
if (cats.length > 0) {
|
|
76
|
+
const catLine = cats.map(([c, n]) => `${c} (${n})`).join(' \u00b7 ');
|
|
77
|
+
console.log(` \x1b[2m${catLine}\x1b[0m`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
console.log(`
|
|
81
|
+
\x1b[2mSync now:\x1b[0m ftx sync
|
|
82
|
+
\x1b[2mSearch:\x1b[0m ftx search "query"
|
|
83
|
+
\x1b[2mExplore:\x1b[0m ftx viz
|
|
84
|
+
\x1b[2mCheck setup:\x1b[0m ftx doctor
|
|
85
|
+
\x1b[2mAll commands:\x1b[0m ftx --help
|
|
86
|
+
`);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
console.log(`
|
|
90
|
+
Data: ${dataDir()}
|
|
91
|
+
|
|
92
|
+
Run: ftx sync
|
|
93
|
+
`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
function timeAgo(dateStr) {
|
|
97
|
+
const ms = Date.now() - new Date(dateStr).getTime();
|
|
98
|
+
const mins = Math.floor(ms / 60000);
|
|
99
|
+
if (mins < 1)
|
|
100
|
+
return 'just now';
|
|
101
|
+
if (mins < 60)
|
|
102
|
+
return `${mins}m ago`;
|
|
103
|
+
const hours = Math.floor(mins / 60);
|
|
104
|
+
if (hours < 24)
|
|
105
|
+
return `${hours}h ago`;
|
|
106
|
+
const days = Math.floor(hours / 24);
|
|
107
|
+
if (days < 30)
|
|
108
|
+
return `${days}d ago`;
|
|
109
|
+
return `${Math.floor(days / 30)}mo ago`;
|
|
110
|
+
}
|
|
111
|
+
function showSyncWelcome() {
|
|
112
|
+
console.log(`
|
|
113
|
+
Make sure Google Chrome is open and logged into x.com.
|
|
114
|
+
Your Chrome session is used to authenticate \u2014 no passwords
|
|
115
|
+
are stored or transmitted. On Windows, close Chrome before
|
|
116
|
+
syncing if the cookies database is locked.
|
|
117
|
+
`);
|
|
118
|
+
}
|
|
119
|
+
/** Check that bookmarks have been synced. Returns true if data exists. */
|
|
120
|
+
function requireData() {
|
|
121
|
+
if (isFirstRun()) {
|
|
122
|
+
console.log(`
|
|
123
|
+
No bookmarks synced yet.
|
|
124
|
+
|
|
125
|
+
Get started:
|
|
126
|
+
|
|
127
|
+
1. Open Google Chrome and log into x.com
|
|
128
|
+
2. Run: ftx sync
|
|
129
|
+
`);
|
|
130
|
+
process.exitCode = 1;
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
/** Check that the search index exists. Returns true if it does. */
|
|
136
|
+
function requireIndex() {
|
|
137
|
+
if (!requireData())
|
|
138
|
+
return false;
|
|
139
|
+
if (!fs.existsSync(twitterBookmarksIndexPath())) {
|
|
140
|
+
console.log(`
|
|
141
|
+
Search index not built yet.
|
|
142
|
+
|
|
143
|
+
Run: ftx index
|
|
144
|
+
`);
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
/** Wrap an async action with graceful error handling. */
|
|
151
|
+
function safe(fn) {
|
|
152
|
+
return async (...args) => {
|
|
153
|
+
try {
|
|
154
|
+
await fn(...args);
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
const msg = err.message;
|
|
158
|
+
console.error(`\n Error: ${msg}\n`);
|
|
159
|
+
process.exitCode = 1;
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
// ── CLI ─────────────────────────────────────────────────────────────────────
|
|
164
|
+
export function buildCli() {
|
|
165
|
+
const program = new Command();
|
|
166
|
+
async function rebuildIndex(added) {
|
|
167
|
+
if (added <= 0)
|
|
168
|
+
return 0;
|
|
169
|
+
process.stderr.write(' Building search index...\n');
|
|
170
|
+
const idx = await buildIndex();
|
|
171
|
+
process.stderr.write(` \u2713 ${idx.recordCount} bookmarks indexed (${idx.newRecords} new)\n`);
|
|
172
|
+
return idx.newRecords;
|
|
173
|
+
}
|
|
174
|
+
async function classifyNew(engine = 'auto') {
|
|
175
|
+
const start = Date.now();
|
|
176
|
+
process.stderr.write(' Classifying new bookmarks (categories)...\n');
|
|
177
|
+
const catResult = await classifyWithLlm({
|
|
178
|
+
engine,
|
|
179
|
+
onBatch: (done, total) => {
|
|
180
|
+
const pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
|
181
|
+
const elapsed = Math.round((Date.now() - start) / 1000);
|
|
182
|
+
process.stderr.write(` Categories: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`);
|
|
183
|
+
},
|
|
184
|
+
});
|
|
185
|
+
if (catResult.classified > 0) {
|
|
186
|
+
process.stderr.write(` \u2713 ${catResult.classified} categorized\n`);
|
|
187
|
+
}
|
|
188
|
+
const domStart = Date.now();
|
|
189
|
+
process.stderr.write(' Classifying new bookmarks (domains)...\n');
|
|
190
|
+
const domResult = await classifyDomainsWithLlm({
|
|
191
|
+
all: false,
|
|
192
|
+
engine,
|
|
193
|
+
onBatch: (done, total) => {
|
|
194
|
+
const pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
|
195
|
+
const elapsed = Math.round((Date.now() - domStart) / 1000);
|
|
196
|
+
process.stderr.write(` Domains: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`);
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
if (domResult.classified > 0) {
|
|
200
|
+
process.stderr.write(` \u2713 ${domResult.classified} domains assigned\n`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
program
|
|
204
|
+
.name('ftx')
|
|
205
|
+
.description('FieldTheory for Windows by Shango Bashi. Sync, search, classify, and explore X/Twitter bookmarks locally.')
|
|
206
|
+
.version('0.1.0')
|
|
207
|
+
.showHelpAfterError()
|
|
208
|
+
.hook('preAction', () => {
|
|
209
|
+
console.log(LOGO);
|
|
210
|
+
});
|
|
211
|
+
// ── sync ────────────────────────────────────────────────────────────────
|
|
212
|
+
program
|
|
213
|
+
.command('sync')
|
|
214
|
+
.description('Sync bookmarks from X into your local database')
|
|
215
|
+
.option('--api', 'Use OAuth v2 API instead of Chrome session', false)
|
|
216
|
+
.option('--full', 'Full crawl instead of incremental sync', false)
|
|
217
|
+
.option('--classify', 'Classify new bookmarks with LLM after syncing', false)
|
|
218
|
+
.option('--engine <engine>', 'Classification engine: auto, codex, claude', 'auto')
|
|
219
|
+
.option('--max-pages <n>', 'Max pages to fetch', (v) => Number(v), 500)
|
|
220
|
+
.option('--target-adds <n>', 'Stop after N new bookmarks', (v) => Number(v))
|
|
221
|
+
.option('--delay-ms <n>', 'Delay between requests in ms', (v) => Number(v), 600)
|
|
222
|
+
.option('--max-minutes <n>', 'Max runtime in minutes', (v) => Number(v), 30)
|
|
223
|
+
.option('--chrome-user-data-dir <path>', 'Chrome user-data directory')
|
|
224
|
+
.option('--chrome-profile-directory <name>', 'Chrome profile name')
|
|
225
|
+
.option('--csrf-token <token>', 'Direct CSRF token override (skips Chrome cookie extraction)')
|
|
226
|
+
.option('--cookie-header <header>', 'Direct cookie header override (used with --csrf-token)')
|
|
227
|
+
.action(async (options) => {
|
|
228
|
+
const firstRun = isFirstRun();
|
|
229
|
+
if (firstRun)
|
|
230
|
+
showSyncWelcome();
|
|
231
|
+
ensureDataDir();
|
|
232
|
+
const engine = selectedEngine(options.engine);
|
|
233
|
+
try {
|
|
234
|
+
const useApi = Boolean(options.api);
|
|
235
|
+
const mode = Boolean(options.full) ? 'full' : 'incremental';
|
|
236
|
+
if (useApi) {
|
|
237
|
+
const result = await syncTwitterBookmarks(mode, {
|
|
238
|
+
targetAdds: typeof options.targetAdds === 'number' && !Number.isNaN(options.targetAdds) ? options.targetAdds : undefined,
|
|
239
|
+
});
|
|
240
|
+
console.log(`\n \u2713 ${result.added} new bookmarks synced (${result.totalBookmarks} total)`);
|
|
241
|
+
console.log(` \u2713 Data: ${dataDir()}\n`);
|
|
242
|
+
const newCount = await rebuildIndex(result.added);
|
|
243
|
+
if (options.classify && newCount > 0) {
|
|
244
|
+
await classifyNew(engine);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
const startTime = Date.now();
|
|
249
|
+
const result = await syncBookmarksGraphQL({
|
|
250
|
+
incremental: !Boolean(options.full),
|
|
251
|
+
maxPages: Number(options.maxPages) || 500,
|
|
252
|
+
targetAdds: typeof options.targetAdds === 'number' && !Number.isNaN(options.targetAdds) ? options.targetAdds : undefined,
|
|
253
|
+
delayMs: Number(options.delayMs) || 600,
|
|
254
|
+
maxMinutes: Number(options.maxMinutes) || 30,
|
|
255
|
+
chromeUserDataDir: options.chromeUserDataDir ? String(options.chromeUserDataDir) : undefined,
|
|
256
|
+
chromeProfileDirectory: options.chromeProfileDirectory ? String(options.chromeProfileDirectory) : undefined,
|
|
257
|
+
csrfToken: options.csrfToken ? String(options.csrfToken) : undefined,
|
|
258
|
+
cookieHeader: options.cookieHeader ? String(options.cookieHeader) : undefined,
|
|
259
|
+
onProgress: (status) => {
|
|
260
|
+
renderProgress(status, startTime);
|
|
261
|
+
if (status.done)
|
|
262
|
+
process.stderr.write('\n');
|
|
263
|
+
},
|
|
264
|
+
});
|
|
265
|
+
console.log(`\n \u2713 ${result.added} new bookmarks synced (${result.totalBookmarks} total)`);
|
|
266
|
+
console.log(` ${friendlyStopReason(result.stopReason)}`);
|
|
267
|
+
console.log(` \u2713 Data: ${dataDir()}\n`);
|
|
268
|
+
const newCount = await rebuildIndex(result.added);
|
|
269
|
+
if (options.classify && newCount > 0) {
|
|
270
|
+
await classifyNew(engine);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
if (firstRun) {
|
|
274
|
+
console.log(`\n Next steps:`);
|
|
275
|
+
console.log(` ftx classify Classify by category and domain (LLM)`);
|
|
276
|
+
console.log(` ftx classify --regex Classify by category (simple)`);
|
|
277
|
+
console.log(`\n Explore:`);
|
|
278
|
+
console.log(` ftx search "machine learning"`);
|
|
279
|
+
console.log(` ftx viz`);
|
|
280
|
+
console.log(` ftx categories`);
|
|
281
|
+
console.log(`\n Ask Codex to use the ftx CLI to search and explore your bookmarks.`);
|
|
282
|
+
console.log(` FieldTheory for Windows by Shango Bashi.\n`);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
catch (err) {
|
|
286
|
+
const msg = err.message;
|
|
287
|
+
if (firstRun && (msg.includes('cookie') || msg.includes('Cookie') || msg.includes('Keychain'))) {
|
|
288
|
+
console.log(`
|
|
289
|
+
Couldn't connect to your Chrome session.
|
|
290
|
+
|
|
291
|
+
To sync your bookmarks:
|
|
292
|
+
|
|
293
|
+
1. Open Google Chrome
|
|
294
|
+
2. Go to x.com and make sure you're logged in
|
|
295
|
+
3. Close Chrome completely
|
|
296
|
+
4. Run: ftx sync
|
|
297
|
+
|
|
298
|
+
If you use multiple Chrome profiles, specify which one:
|
|
299
|
+
ftx sync --chrome-profile-directory "Profile 1"
|
|
300
|
+
`);
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
console.error(`\n Error: ${msg}\n`);
|
|
304
|
+
}
|
|
305
|
+
process.exitCode = 1;
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
// ── search ──────────────────────────────────────────────────────────────
|
|
309
|
+
program
|
|
310
|
+
.command('search')
|
|
311
|
+
.description('Full-text search across bookmarks')
|
|
312
|
+
.argument('<query>', 'Search query (supports FTS5 syntax: AND, OR, NOT, "exact phrase")')
|
|
313
|
+
.option('--author <handle>', 'Filter by author handle')
|
|
314
|
+
.option('--after <date>', 'Bookmarks posted after this date (YYYY-MM-DD)')
|
|
315
|
+
.option('--before <date>', 'Bookmarks posted before this date (YYYY-MM-DD)')
|
|
316
|
+
.option('--limit <n>', 'Max results', (v) => Number(v), 20)
|
|
317
|
+
.action(safe(async (query, options) => {
|
|
318
|
+
if (!requireIndex())
|
|
319
|
+
return;
|
|
320
|
+
const results = await searchBookmarks({
|
|
321
|
+
query,
|
|
322
|
+
author: options.author ? String(options.author) : undefined,
|
|
323
|
+
after: options.after ? String(options.after) : undefined,
|
|
324
|
+
before: options.before ? String(options.before) : undefined,
|
|
325
|
+
limit: Number(options.limit) || 20,
|
|
326
|
+
});
|
|
327
|
+
console.log(formatSearchResults(results));
|
|
328
|
+
}));
|
|
329
|
+
// ── list ────────────────────────────────────────────────────────────────
|
|
330
|
+
program
|
|
331
|
+
.command('list')
|
|
332
|
+
.description('List bookmarks with filters')
|
|
333
|
+
.option('--query <query>', 'Text query (FTS5 syntax)')
|
|
334
|
+
.option('--author <handle>', 'Filter by author handle')
|
|
335
|
+
.option('--after <date>', 'Posted after (YYYY-MM-DD)')
|
|
336
|
+
.option('--before <date>', 'Posted before (YYYY-MM-DD)')
|
|
337
|
+
.option('--category <category>', 'Filter by category')
|
|
338
|
+
.option('--domain <domain>', 'Filter by domain')
|
|
339
|
+
.option('--limit <n>', 'Max results', (v) => Number(v), 30)
|
|
340
|
+
.option('--offset <n>', 'Offset into results', (v) => Number(v), 0)
|
|
341
|
+
.option('--json', 'JSON output')
|
|
342
|
+
.action(safe(async (options) => {
|
|
343
|
+
if (!requireIndex())
|
|
344
|
+
return;
|
|
345
|
+
const items = await listBookmarks({
|
|
346
|
+
query: options.query ? String(options.query) : undefined,
|
|
347
|
+
author: options.author ? String(options.author) : undefined,
|
|
348
|
+
after: options.after ? String(options.after) : undefined,
|
|
349
|
+
before: options.before ? String(options.before) : undefined,
|
|
350
|
+
category: options.category ? String(options.category) : undefined,
|
|
351
|
+
domain: options.domain ? String(options.domain) : undefined,
|
|
352
|
+
limit: Number(options.limit) || 30,
|
|
353
|
+
offset: Number(options.offset) || 0,
|
|
354
|
+
});
|
|
355
|
+
if (options.json) {
|
|
356
|
+
console.log(JSON.stringify(items, null, 2));
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
for (const item of items) {
|
|
360
|
+
const tags = [item.primaryCategory, item.primaryDomain].filter(Boolean).join(' \u00b7 ');
|
|
361
|
+
const summary = item.text.length > 120 ? `${item.text.slice(0, 117)}...` : item.text;
|
|
362
|
+
console.log(`${item.id} ${item.authorHandle ? `@${item.authorHandle}` : '@?'} ${item.postedAt?.slice(0, 10) ?? '?'}${tags ? ` ${tags}` : ''}`);
|
|
363
|
+
console.log(` ${summary}`);
|
|
364
|
+
console.log(` ${item.url}`);
|
|
365
|
+
console.log();
|
|
366
|
+
}
|
|
367
|
+
}));
|
|
368
|
+
// ── show ─────────────────────────────────────────────────────────────────
|
|
369
|
+
program
|
|
370
|
+
.command('show')
|
|
371
|
+
.description('Show one bookmark in detail')
|
|
372
|
+
.argument('<id>', 'Bookmark id')
|
|
373
|
+
.option('--json', 'JSON output')
|
|
374
|
+
.action(safe(async (id, options) => {
|
|
375
|
+
if (!requireIndex())
|
|
376
|
+
return;
|
|
377
|
+
const item = await getBookmarkById(String(id));
|
|
378
|
+
if (!item) {
|
|
379
|
+
console.log(` Bookmark not found: ${String(id)}`);
|
|
380
|
+
process.exitCode = 1;
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
if (options.json) {
|
|
384
|
+
console.log(JSON.stringify(item, null, 2));
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
console.log(`${item.id} \u00b7 ${item.authorHandle ? `@${item.authorHandle}` : '@?'}`);
|
|
388
|
+
console.log(item.url);
|
|
389
|
+
console.log(item.text);
|
|
390
|
+
if (item.links.length)
|
|
391
|
+
console.log(`links: ${item.links.join(', ')}`);
|
|
392
|
+
if (item.categories)
|
|
393
|
+
console.log(`categories: ${item.categories}`);
|
|
394
|
+
if (item.domains)
|
|
395
|
+
console.log(`domains: ${item.domains}`);
|
|
396
|
+
}));
|
|
397
|
+
// ── stats ───────────────────────────────────────────────────────────────
|
|
398
|
+
program
|
|
399
|
+
.command('stats')
|
|
400
|
+
.description('Aggregate statistics from your bookmarks')
|
|
401
|
+
.action(safe(async () => {
|
|
402
|
+
if (!requireIndex())
|
|
403
|
+
return;
|
|
404
|
+
const stats = await getStats();
|
|
405
|
+
console.log(`Bookmarks: ${stats.totalBookmarks}`);
|
|
406
|
+
console.log(`Unique authors: ${stats.uniqueAuthors}`);
|
|
407
|
+
console.log(`Date range: ${stats.dateRange.earliest?.slice(0, 10) ?? '?'} to ${stats.dateRange.latest?.slice(0, 10) ?? '?'}`);
|
|
408
|
+
console.log(`\nTop authors:`);
|
|
409
|
+
for (const a of stats.topAuthors)
|
|
410
|
+
console.log(` @${a.handle}: ${a.count}`);
|
|
411
|
+
console.log(`\nLanguages:`);
|
|
412
|
+
for (const l of stats.languageBreakdown)
|
|
413
|
+
console.log(` ${l.language}: ${l.count}`);
|
|
414
|
+
}));
|
|
415
|
+
// ── viz ─────────────────────────────────────────────────────────────────
|
|
416
|
+
program
|
|
417
|
+
.command('viz')
|
|
418
|
+
.description('Visual dashboard of your bookmarking patterns')
|
|
419
|
+
.action(safe(async () => {
|
|
420
|
+
if (!requireIndex())
|
|
421
|
+
return;
|
|
422
|
+
console.log(await renderViz());
|
|
423
|
+
}));
|
|
424
|
+
// ── classify ────────────────────────────────────────────────────────────
|
|
425
|
+
program
|
|
426
|
+
.command('classify')
|
|
427
|
+
.description('Classify bookmarks by category and domain using Codex or Claude')
|
|
428
|
+
.option('--regex', 'Use simple regex classification instead of LLM')
|
|
429
|
+
.option('--engine <engine>', 'LLM engine: auto, codex, claude', 'auto')
|
|
430
|
+
.action(safe(async (options) => {
|
|
431
|
+
if (!requireData())
|
|
432
|
+
return;
|
|
433
|
+
if (options.regex) {
|
|
434
|
+
process.stderr.write('Classifying bookmarks (regex)...\n');
|
|
435
|
+
const result = await classifyAndRebuild();
|
|
436
|
+
console.log(`Indexed ${result.recordCount} bookmarks \u2192 ${result.dbPath}`);
|
|
437
|
+
console.log(formatClassificationSummary(result.summary));
|
|
438
|
+
}
|
|
439
|
+
else {
|
|
440
|
+
let catStart = Date.now();
|
|
441
|
+
process.stderr.write('Classifying categories with LLM (batches of 50, ~2 min per batch)...\n');
|
|
442
|
+
const catResult = await classifyWithLlm({
|
|
443
|
+
engine: selectedEngine(options.engine),
|
|
444
|
+
onBatch: (done, total) => {
|
|
445
|
+
const pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
|
446
|
+
const elapsed = Math.round((Date.now() - catStart) / 1000);
|
|
447
|
+
process.stderr.write(` Categories: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`);
|
|
448
|
+
},
|
|
449
|
+
});
|
|
450
|
+
console.log(`\nEngine: ${catResult.engine}`);
|
|
451
|
+
console.log(`Categories: ${catResult.classified}/${catResult.totalUnclassified} classified`);
|
|
452
|
+
let domStart = Date.now();
|
|
453
|
+
process.stderr.write('\nClassifying domains with LLM (batches of 50, ~2 min per batch)...\n');
|
|
454
|
+
const domResult = await classifyDomainsWithLlm({
|
|
455
|
+
all: false,
|
|
456
|
+
engine: selectedEngine(options.engine),
|
|
457
|
+
onBatch: (done, total) => {
|
|
458
|
+
const pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
|
459
|
+
const elapsed = Math.round((Date.now() - domStart) / 1000);
|
|
460
|
+
process.stderr.write(` Domains: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`);
|
|
461
|
+
},
|
|
462
|
+
});
|
|
463
|
+
console.log(`\nDomains: ${domResult.classified}/${domResult.totalUnclassified} classified`);
|
|
464
|
+
}
|
|
465
|
+
}));
|
|
466
|
+
// ── classify-domains ────────────────────────────────────────────────────
|
|
467
|
+
program
|
|
468
|
+
.command('classify-domains')
|
|
469
|
+
.description('Classify bookmarks by subject domain using LLM (ai, finance, etc.)')
|
|
470
|
+
.option('--all', 'Re-classify all bookmarks, not just missing')
|
|
471
|
+
.option('--engine <engine>', 'LLM engine: auto, codex, claude', 'auto')
|
|
472
|
+
.action(safe(async (options) => {
|
|
473
|
+
if (!requireData())
|
|
474
|
+
return;
|
|
475
|
+
const start = Date.now();
|
|
476
|
+
process.stderr.write('Classifying bookmark domains with LLM (batches of 50, ~2 min per batch)...\n');
|
|
477
|
+
const result = await classifyDomainsWithLlm({
|
|
478
|
+
all: options.all ?? false,
|
|
479
|
+
engine: selectedEngine(options.engine),
|
|
480
|
+
onBatch: (done, total) => {
|
|
481
|
+
const pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
|
482
|
+
const elapsed = Math.round((Date.now() - start) / 1000);
|
|
483
|
+
process.stderr.write(` Domains: ${done}/${total} (${pct}%) \u2502 ${elapsed}s elapsed\n`);
|
|
484
|
+
},
|
|
485
|
+
});
|
|
486
|
+
console.log(`\nDomains: ${result.classified}/${result.totalUnclassified} classified`);
|
|
487
|
+
}));
|
|
488
|
+
// ── categories ──────────────────────────────────────────────────────────
|
|
489
|
+
program
|
|
490
|
+
.command('categories')
|
|
491
|
+
.description('Show category distribution')
|
|
492
|
+
.action(safe(async () => {
|
|
493
|
+
if (!requireIndex())
|
|
494
|
+
return;
|
|
495
|
+
const counts = await getCategoryCounts();
|
|
496
|
+
if (Object.keys(counts).length === 0) {
|
|
497
|
+
console.log(' No categories found. Run: ftx classify');
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
|
501
|
+
for (const [cat, count] of Object.entries(counts).sort((a, b) => b[1] - a[1])) {
|
|
502
|
+
const pct = ((count / total) * 100).toFixed(1);
|
|
503
|
+
console.log(` ${cat.padEnd(14)} ${String(count).padStart(5)} (${pct}%)`);
|
|
504
|
+
}
|
|
505
|
+
}));
|
|
506
|
+
// ── domains ─────────────────────────────────────────────────────────────
|
|
507
|
+
program
|
|
508
|
+
.command('domains')
|
|
509
|
+
.description('Show domain distribution')
|
|
510
|
+
.action(safe(async () => {
|
|
511
|
+
if (!requireIndex())
|
|
512
|
+
return;
|
|
513
|
+
const counts = await getDomainCounts();
|
|
514
|
+
if (Object.keys(counts).length === 0) {
|
|
515
|
+
console.log(' No domains found. Run: ftx classify-domains');
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
|
519
|
+
for (const [dom, count] of Object.entries(counts).sort((a, b) => b[1] - a[1])) {
|
|
520
|
+
const pct = ((count / total) * 100).toFixed(1);
|
|
521
|
+
console.log(` ${dom.padEnd(20)} ${String(count).padStart(5)} (${pct}%)`);
|
|
522
|
+
}
|
|
523
|
+
}));
|
|
524
|
+
// ── index ───────────────────────────────────────────────────────────────
|
|
525
|
+
program
|
|
526
|
+
.command('index')
|
|
527
|
+
.description('Rebuild the SQLite search index from the JSONL cache')
|
|
528
|
+
.option('--force', 'Drop and rebuild from scratch (loses classifications)')
|
|
529
|
+
.action(safe(async (options) => {
|
|
530
|
+
if (!requireData())
|
|
531
|
+
return;
|
|
532
|
+
process.stderr.write('Building search index...\n');
|
|
533
|
+
const result = await buildIndex({ force: Boolean(options.force) });
|
|
534
|
+
console.log(`Indexed ${result.recordCount} bookmarks (${result.newRecords} new) \u2192 ${result.dbPath}`);
|
|
535
|
+
}));
|
|
536
|
+
// ── auth ────────────────────────────────────────────────────────────────
|
|
537
|
+
program
|
|
538
|
+
.command('auth')
|
|
539
|
+
.description('Set up OAuth for API-based sync (optional, needed for ftx sync --api)')
|
|
540
|
+
.action(safe(async () => {
|
|
541
|
+
const result = await runTwitterOAuthFlow();
|
|
542
|
+
console.log(`Saved token to ${result.tokenPath}`);
|
|
543
|
+
if (result.scope)
|
|
544
|
+
console.log(`Scope: ${result.scope}`);
|
|
545
|
+
}));
|
|
546
|
+
// ── status ──────────────────────────────────────────────────────────────
|
|
547
|
+
program
|
|
548
|
+
.command('status')
|
|
549
|
+
.description('Show sync status and data location')
|
|
550
|
+
.action(safe(async () => {
|
|
551
|
+
if (!requireData())
|
|
552
|
+
return;
|
|
553
|
+
const view = await getBookmarkStatusView();
|
|
554
|
+
console.log(formatBookmarkStatus(view));
|
|
555
|
+
}));
|
|
556
|
+
// ── path ────────────────────────────────────────────────────────────────
|
|
557
|
+
program
|
|
558
|
+
.command('path')
|
|
559
|
+
.description('Print the data directory path')
|
|
560
|
+
.action(() => { console.log(dataDir()); });
|
|
561
|
+
program
|
|
562
|
+
.command('doctor')
|
|
563
|
+
.description('Check local Windows, Chrome, and LLM prerequisites')
|
|
564
|
+
.action(safe(async () => {
|
|
565
|
+
const engines = detectAvailableEngines();
|
|
566
|
+
let chromeDir = 'not detected';
|
|
567
|
+
let chromeStatus = 'unavailable';
|
|
568
|
+
try {
|
|
569
|
+
const config = loadChromeSessionConfig();
|
|
570
|
+
chromeDir = config.chromeUserDataDir;
|
|
571
|
+
chromeStatus = fs.existsSync(config.chromeUserDataDir)
|
|
572
|
+
? `ok (${config.chromeProfileDirectory ?? 'Default'})`
|
|
573
|
+
: 'configured path not found';
|
|
574
|
+
}
|
|
575
|
+
catch (error) {
|
|
576
|
+
chromeStatus = error.message.split('\n')[0] ?? 'unavailable';
|
|
577
|
+
}
|
|
578
|
+
console.log(`Platform: ${process.platform}`);
|
|
579
|
+
console.log(`Node: ${process.version}`);
|
|
580
|
+
console.log(`Data directory: ${dataDir()}`);
|
|
581
|
+
console.log(`Chrome user data: ${chromeDir}`);
|
|
582
|
+
console.log(`Chrome status: ${chromeStatus}`);
|
|
583
|
+
console.log(`LLM engines: ${engines.length ? engines.join(', ') : 'none found'}`);
|
|
584
|
+
console.log('Project: FieldTheory for Windows by Shango Bashi');
|
|
585
|
+
}));
|
|
586
|
+
// ── sample ──────────────────────────────────────────────────────────────
|
|
587
|
+
program
|
|
588
|
+
.command('sample')
|
|
589
|
+
.description('Sample bookmarks by category')
|
|
590
|
+
.argument('<category>', 'Category: tool, security, technique, launch, research, opinion, commerce')
|
|
591
|
+
.option('--limit <n>', 'Max results', (v) => Number(v), 10)
|
|
592
|
+
.action(safe(async (category, options) => {
|
|
593
|
+
if (!requireIndex())
|
|
594
|
+
return;
|
|
595
|
+
const results = await sampleByCategory(category, Number(options.limit) || 10);
|
|
596
|
+
if (results.length === 0) {
|
|
597
|
+
console.log(` No bookmarks found with category "${category}". Run: ftx classify`);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
for (const r of results) {
|
|
601
|
+
const text = r.text.length > 120 ? r.text.slice(0, 120) + '...' : r.text;
|
|
602
|
+
console.log(`[@${r.authorHandle ?? '?'}] ${text}`);
|
|
603
|
+
console.log(` ${r.url} [${r.categories}]`);
|
|
604
|
+
if (r.githubUrls)
|
|
605
|
+
console.log(` github: ${r.githubUrls}`);
|
|
606
|
+
console.log();
|
|
607
|
+
}
|
|
608
|
+
}));
|
|
609
|
+
// ── fetch-media ─────────────────────────────────────────────────────────
|
|
610
|
+
program
|
|
611
|
+
.command('fetch-media')
|
|
612
|
+
.description('Download media assets for bookmarks (static images only)')
|
|
613
|
+
.option('--limit <n>', 'Max bookmarks to process', (v) => Number(v), 100)
|
|
614
|
+
.option('--max-bytes <n>', 'Per-asset byte limit', (v) => Number(v), 50 * 1024 * 1024)
|
|
615
|
+
.action(safe(async (options) => {
|
|
616
|
+
if (!requireData())
|
|
617
|
+
return;
|
|
618
|
+
const result = await fetchBookmarkMediaBatch({
|
|
619
|
+
limit: Number(options.limit) || 100,
|
|
620
|
+
maxBytes: Number(options.maxBytes) || 50 * 1024 * 1024,
|
|
621
|
+
});
|
|
622
|
+
console.log(JSON.stringify(result, null, 2));
|
|
623
|
+
}));
|
|
624
|
+
// ── hidden backward-compat aliases ────────────────────────────────────
|
|
625
|
+
const bookmarksAlias = program.command('bookmarks').description('(alias) Bookmark commands').helpOption(false);
|
|
626
|
+
for (const cmd of ['sync', 'search', 'list', 'show', 'stats', 'viz', 'classify', 'classify-domains',
|
|
627
|
+
'categories', 'domains', 'index', 'auth', 'status', 'path', 'doctor', 'sample', 'fetch-media']) {
|
|
628
|
+
bookmarksAlias.command(cmd).description(`Alias for: ftx ${cmd}`).allowUnknownOption(true)
|
|
629
|
+
.action(async () => {
|
|
630
|
+
const args = ['node', 'ftx', cmd, ...process.argv.slice(4)];
|
|
631
|
+
await program.parseAsync(args);
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
bookmarksAlias.command('enable').description('Alias for: ftx sync').action(async () => {
|
|
635
|
+
const args = ['node', 'ftx', 'sync', ...process.argv.slice(4)];
|
|
636
|
+
await program.parseAsync(args);
|
|
637
|
+
});
|
|
638
|
+
return program;
|
|
639
|
+
}
|
|
640
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
641
|
+
await buildCli().parseAsync(process.argv);
|
|
642
|
+
}
|