fieldtheory 1.0.0

package/dist/bookmarks-db.js

@@ -0,0 +1,623 @@
+import { openDb, saveDb } from './db.js';
+import { readJsonLines } from './fs.js';
+import { twitterBookmarksCachePath, twitterBookmarksIndexPath } from './paths.js';
+import { classifyCorpus } from './bookmark-classify.js';
+const SCHEMA_VERSION = 3;
+function parseJsonArray(value) {
+    if (typeof value !== 'string' || !value.trim())
+        return [];
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed.filter((entry) => typeof entry === 'string') : [];
+    }
+    catch {
+        return [];
+    }
+}
+function parseCsv(value) {
+    if (typeof value !== 'string' || !value.trim())
+        return [];
+    return value
+        .split(',')
+        .map((entry) => entry.trim())
+        .filter(Boolean);
+}
+function mapTimelineRow(row) {
+    return {
+        id: row[0],
+        tweetId: row[1],
+        url: row[2],
+        text: row[3],
+        authorHandle: row[4] ?? undefined,
+        authorName: row[5] ?? undefined,
+        authorProfileImageUrl: row[6] ?? undefined,
+        postedAt: row[7] ?? null,
+        bookmarkedAt: row[8] ?? null,
+        categories: parseCsv(row[9]),
+        primaryCategory: row[10] ?? null,
+        domains: parseCsv(row[11]),
+        primaryDomain: row[12] ?? null,
+        githubUrls: parseJsonArray(row[13]),
+        links: parseJsonArray(row[14]),
+        mediaCount: Number(row[15] ?? 0),
+        linkCount: Number(row[16] ?? 0),
+        likeCount: row[17],
+        repostCount: row[18],
+        replyCount: row[19],
+        quoteCount: row[20],
+        bookmarkCount: row[21],
+        viewCount: row[22],
+    };
+}
+function buildBookmarkWhereClause(filters) {
+    const conditions = [];
+    const params = [];
+    if (filters.query) {
+        conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`);
+        params.push(filters.query);
+    }
+    if (filters.author) {
+        conditions.push(`b.author_handle = ? COLLATE NOCASE`);
+        params.push(filters.author);
+    }
+    if (filters.after) {
+        conditions.push(`COALESCE(b.posted_at, b.bookmarked_at) >= ?`);
+        params.push(filters.after);
+    }
+    if (filters.before) {
+        conditions.push(`COALESCE(b.posted_at, b.bookmarked_at) <= ?`);
+        params.push(filters.before);
+    }
+    if (filters.category) {
+        conditions.push(`b.categories LIKE ?`);
+        params.push(`%${filters.category}%`);
+    }
+    if (filters.domain) {
+        conditions.push(`b.domains LIKE ?`);
+        params.push(`%${filters.domain}%`);
+    }
+    return {
+        where: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '',
+        params,
+    };
+}
+function bookmarkSortClause(direction = 'desc') {
+    const normalized = direction === 'asc' ? 'ASC' : 'DESC';
+    return `
+    ORDER BY
+      CASE
+        WHEN b.bookmarked_at GLOB '____-__-__*' THEN b.bookmarked_at
+        WHEN b.posted_at GLOB '____-__-__*' THEN b.posted_at
+        ELSE ''
+      END ${normalized},
+      CAST(b.tweet_id AS INTEGER) ${normalized}
+  `;
+}
+function initSchema(db) {
+    db.run(`CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)`);
+    db.run(`CREATE TABLE IF NOT EXISTS bookmarks (
+    id TEXT PRIMARY KEY,
+    tweet_id TEXT NOT NULL,
+    url TEXT NOT NULL,
+    text TEXT NOT NULL,
+    author_handle TEXT,
+    author_name TEXT,
+    author_profile_image_url TEXT,
+    posted_at TEXT,
+    bookmarked_at TEXT,
+    synced_at TEXT NOT NULL,
+    conversation_id TEXT,
+    in_reply_to_status_id TEXT,
+    quoted_status_id TEXT,
+    language TEXT,
+    like_count INTEGER,
+    repost_count INTEGER,
+    reply_count INTEGER,
+    quote_count INTEGER,
+    bookmark_count INTEGER,
+    view_count INTEGER,
+    media_count INTEGER DEFAULT 0,
+    link_count INTEGER DEFAULT 0,
+    links_json TEXT,
+    tags_json TEXT,
+    ingested_via TEXT,
+    categories TEXT,
+    primary_category TEXT,
+    github_urls TEXT,
+    domains TEXT,
+    primary_domain TEXT
+  )`);
+    db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_author ON bookmarks(author_handle)`);
+    db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_posted ON bookmarks(posted_at)`);
+    db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_language ON bookmarks(language)`);
+    db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_category ON bookmarks(primary_category)`);
+    db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_domain ON bookmarks(primary_domain)`);
+    db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS bookmarks_fts USING fts5(
+    text,
+    author_handle,
+    author_name,
+    content=bookmarks,
+    content_rowid=rowid,
+    tokenize='porter unicode61'
+  )`);
+    db.run(`REPLACE INTO meta VALUES ('schema_version', '${SCHEMA_VERSION}')`);
+}
+function ensureMigrations(db) {
+    const rows = db.exec("SELECT value FROM meta WHERE key = 'schema_version'");
+    const version = rows.length ? Number(rows[0].values[0]?.[0] ?? 0) : 0;
+    if (version < 3) {
+        try {
+            db.run('ALTER TABLE bookmarks ADD COLUMN domains TEXT');
+        }
+        catch { /* already exists */ }
+        try {
+            db.run('ALTER TABLE bookmarks ADD COLUMN primary_domain TEXT');
+        }
+        catch { /* already exists */ }
+        db.run('CREATE INDEX IF NOT EXISTS idx_bookmarks_domain ON bookmarks(primary_domain)');
+        db.run("REPLACE INTO meta VALUES ('schema_version', '3')");
+    }
+}
+function insertRecord(db, r) {
+    // Extract GitHub URLs (kept inline — no LLM needed for URL parsing)
+    const text = r.text ?? '';
+    const githubMatches = text.match(/github\.com\/[\w.-]+\/[\w.-]+/gi) ?? [];
+    const githubFromLinks = (r.links ?? []).filter((l) => /github\.com/i.test(l));
+    const githubUrls = [...new Set([...githubMatches.map((m) => `https://${m}`), ...githubFromLinks])];
+    db.run(`INSERT OR REPLACE INTO bookmarks VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, [
+        r.id,
+        r.tweetId,
+        r.url,
+        r.text,
+        r.authorHandle ?? null,
+        r.authorName ?? null,
+        r.authorProfileImageUrl ?? null,
+        r.postedAt ?? null,
+        r.bookmarkedAt ?? null,
+        r.syncedAt,
+        r.conversationId ?? null,
+        r.inReplyToStatusId ?? null,
+        r.quotedStatusId ?? null,
+        r.language ?? null,
+        r.engagement?.likeCount ?? null,
+        r.engagement?.repostCount ?? null,
+        r.engagement?.replyCount ?? null,
+        r.engagement?.quoteCount ?? null,
+        r.engagement?.bookmarkCount ?? null,
+        r.engagement?.viewCount ?? null,
+        r.media?.length ?? 0,
+        r.links?.length ?? 0,
+        r.links?.length ? JSON.stringify(r.links) : null,
+        r.tags?.length ? JSON.stringify(r.tags) : null,
+        r.ingestedVia ?? null,
+        null, // categories — populated by classify pass
+        'unclassified', // primary_category
+        githubUrls.length ? JSON.stringify(githubUrls) : null,
+        null, // domains — populated by classify-domains pass
+        null, // primary_domain
+    ]);
+}
+export async function buildIndex() {
+    const cachePath = twitterBookmarksCachePath();
+    const dbPath = twitterBookmarksIndexPath();
+    const records = await readJsonLines(cachePath);
+    const db = await openDb(dbPath);
+    try {
+        // Drop and recreate for a clean rebuild
+        db.run('DROP TABLE IF EXISTS bookmarks_fts');
+        db.run('DROP TABLE IF EXISTS bookmarks');
+        db.run('DROP TABLE IF EXISTS meta');
+        initSchema(db);
+        db.run('BEGIN TRANSACTION');
+        for (const record of records) {
+            insertRecord(db, record);
+        }
+        // Rebuild FTS index from content table
+        db.run(`INSERT INTO bookmarks_fts(bookmarks_fts) VALUES('rebuild')`);
+        db.run('COMMIT');
+        saveDb(db, dbPath);
+        return { dbPath, recordCount: records.length };
+    }
+    finally {
+        db.close();
+    }
+}
+export async function searchBookmarks(options) {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    const limit = options.limit ?? 20;
+    try {
+        const conditions = [];
+        const params = [];
+        if (options.query) {
+            conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`);
+            params.push(options.query);
+        }
+        if (options.author) {
+            conditions.push(`b.author_handle = ? COLLATE NOCASE`);
+            params.push(options.author);
+        }
+        if (options.after) {
+            conditions.push(`b.posted_at >= ?`);
+            params.push(options.after);
+        }
+        if (options.before) {
+            conditions.push(`b.posted_at <= ?`);
+            params.push(options.before);
+        }
+        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        // If we have an FTS query, use bm25 for ranking; otherwise sort by posted_at
+        const orderBy = options.query
+            ? `ORDER BY bm25(bookmarks_fts, 5.0, 1.0, 1.0) ASC`
+            : `ORDER BY b.posted_at DESC`;
+        // For FTS ranking we need to join with the FTS table for bm25
+        let sql;
+        if (options.query) {
+            sql = `
+        SELECT b.id, b.url, b.text, b.author_handle, b.author_name, b.posted_at,
+               bm25(bookmarks_fts, 5.0, 1.0, 1.0) as score
+        FROM bookmarks b
+        JOIN bookmarks_fts ON bookmarks_fts.rowid = b.rowid
+        ${where}
+        ${orderBy}
+        LIMIT ?
+      `;
+        }
+        else {
+            sql = `
+        SELECT b.id, b.url, b.text, b.author_handle, b.author_name, b.posted_at,
+               0 as score
+        FROM bookmarks b
+        ${where}
+        ORDER BY b.posted_at DESC
+        LIMIT ?
+      `;
+        }
+        params.push(limit);
+        const rows = db.exec(sql, params);
+        if (!rows.length)
+            return [];
+        return rows[0].values.map((row) => ({
+            id: row[0],
+            url: row[1],
+            text: row[2],
+            authorHandle: row[3],
+            authorName: row[4],
+            postedAt: row[5],
+            score: row[6],
+        }));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function listBookmarks(filters = {}) {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    const limit = filters.limit ?? 30;
+    const offset = filters.offset ?? 0;
+    try {
+        const { where, params } = buildBookmarkWhereClause(filters);
+        const sql = `
+      SELECT
+        b.id,
+        b.tweet_id,
+        b.url,
+        b.text,
+        b.author_handle,
+        b.author_name,
+        b.author_profile_image_url,
+        b.posted_at,
+        b.bookmarked_at,
+        b.categories,
+        b.primary_category,
+        b.domains,
+        b.primary_domain,
+        b.github_urls,
+        b.links_json,
+        b.media_count,
+        b.link_count,
+        b.like_count,
+        b.repost_count,
+        b.reply_count,
+        b.quote_count,
+        b.bookmark_count,
+        b.view_count
+      FROM bookmarks b
+      ${where}
+      ${bookmarkSortClause(filters.sort)}
+      LIMIT ?
+      OFFSET ?
+    `;
+        params.push(limit, offset);
+        const rows = db.exec(sql, params);
+        if (!rows.length)
+            return [];
+        return rows[0].values.map((row) => mapTimelineRow(row));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function countBookmarks(filters = {}) {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const { where, params } = buildBookmarkWhereClause(filters);
+        const sql = `
+      SELECT COUNT(*)
+      FROM bookmarks b
+      ${where}
+    `;
+        const rows = db.exec(sql, params);
+        return Number(rows[0]?.values?.[0]?.[0] ?? 0);
+    }
+    finally {
+        db.close();
+    }
+}
+export async function exportBookmarksForSyncSeed() {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const sql = `
+      SELECT
+        b.id,
+        b.tweet_id,
+        b.url,
+        b.text,
+        b.author_handle,
+        b.author_name,
+        b.author_profile_image_url,
+        b.posted_at,
+        b.bookmarked_at,
+        b.synced_at,
+        b.conversation_id,
+        b.in_reply_to_status_id,
+        b.quoted_status_id,
+        b.language,
+        b.like_count,
+        b.repost_count,
+        b.reply_count,
+        b.quote_count,
+        b.bookmark_count,
+        b.view_count,
+        b.links_json
+      FROM bookmarks b
+      ${bookmarkSortClause('desc')}
+    `;
+        const rows = db.exec(sql);
+        if (!rows.length)
+            return [];
+        return rows[0].values.map((row) => ({
+            id: String(row[0]),
+            tweetId: String(row[1]),
+            url: String(row[2]),
+            text: String(row[3] ?? ''),
+            authorHandle: row[4] ?? undefined,
+            authorName: row[5] ?? undefined,
+            authorProfileImageUrl: row[6] ?? undefined,
+            postedAt: row[7] ?? null,
+            bookmarkedAt: row[8] ?? null,
+            syncedAt: String(row[9] ?? row[8] ?? row[7] ?? new Date(0).toISOString()),
+            conversationId: row[10] ?? undefined,
+            inReplyToStatusId: row[11] ?? undefined,
+            quotedStatusId: row[12] ?? undefined,
+            language: row[13] ?? undefined,
+            engagement: {
+                likeCount: row[14],
+                repostCount: row[15],
+                replyCount: row[16],
+                quoteCount: row[17],
+                bookmarkCount: row[18],
+                viewCount: row[19],
+            },
+            links: parseJsonArray(row[20]),
+            tags: [],
+            ingestedVia: 'graphql',
+        }));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function getBookmarkById(id) {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const rows = db.exec(`SELECT
+        b.id,
+        b.tweet_id,
+        b.url,
+        b.text,
+        b.author_handle,
+        b.author_name,
+        b.author_profile_image_url,
+        b.posted_at,
+        b.bookmarked_at,
+        b.categories,
+        b.primary_category,
+        b.domains,
+        b.primary_domain,
+        b.github_urls,
+        b.links_json,
+        b.media_count,
+        b.link_count,
+        b.like_count,
+        b.repost_count,
+        b.reply_count,
+        b.quote_count,
+        b.bookmark_count,
+        b.view_count
+      FROM bookmarks b
+      WHERE b.id = ?
+      LIMIT 1`, [id]);
+        const row = rows[0]?.values?.[0];
+        return row ? mapTimelineRow(row) : null;
+    }
+    finally {
+        db.close();
+    }
+}
+export async function getStats() {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    try {
+        const total = db.exec('SELECT COUNT(*) FROM bookmarks')[0]?.values[0]?.[0];
+        const authors = db.exec('SELECT COUNT(DISTINCT author_handle) FROM bookmarks')[0]?.values[0]?.[0];
+        const range = db.exec('SELECT MIN(posted_at), MAX(posted_at) FROM bookmarks WHERE posted_at IS NOT NULL')[0]?.values[0];
+        const topAuthorsRows = db.exec(`SELECT author_handle, COUNT(*) as c FROM bookmarks
+       WHERE author_handle IS NOT NULL
+       GROUP BY author_handle ORDER BY c DESC LIMIT 15`);
+        const topAuthors = (topAuthorsRows[0]?.values ?? []).map((r) => ({
+            handle: r[0],
+            count: r[1],
+        }));
+        const langRows = db.exec(`SELECT language, COUNT(*) as c FROM bookmarks
+       WHERE language IS NOT NULL
+       GROUP BY language ORDER BY c DESC LIMIT 10`);
+        const languageBreakdown = (langRows[0]?.values ?? []).map((r) => ({
+            language: r[0],
+            count: r[1],
+        }));
+        return {
+            totalBookmarks: total,
+            uniqueAuthors: authors,
+            dateRange: { earliest: range?.[0] ?? null, latest: range?.[1] ?? null },
+            topAuthors,
+            languageBreakdown,
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+// ── Classification ───────────────────────────────────────────────────────
+export async function classifyAndRebuild() {
+    const cachePath = twitterBookmarksCachePath();
+    const dbPath = twitterBookmarksIndexPath();
+    const records = await readJsonLines(cachePath);
+    const { results, summary } = classifyCorpus(records);
+    // Rebuild index then apply regex classifications
+    const buildResult = await buildIndex();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const stmt = db.prepare(`UPDATE bookmarks SET categories = ?, primary_category = ?, github_urls = ? WHERE id = ?`);
+        for (const [id, r] of results) {
+            if (r.categories.length > 0) {
+                stmt.run([r.categories.join(','), r.primary, r.githubUrls.length ? JSON.stringify(r.githubUrls) : null, id]);
+            }
+        }
+        stmt.free();
+        saveDb(db, dbPath);
+    }
+    finally {
+        db.close();
+    }
+    return { ...buildResult, summary };
+}
+export async function sampleByCategory(category, limit) {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    try {
+        const rows = db.exec(`SELECT id, url, text, author_handle, categories, github_urls, links_json
+       FROM bookmarks
+       WHERE categories LIKE ?
+       ORDER BY RANDOM()
+       LIMIT ?`, [`%${category}%`, limit]);
+        if (!rows.length)
+            return [];
+        return rows[0].values.map((r) => ({
+            id: r[0],
+            url: r[1],
+            text: r[2],
+            authorHandle: r[3] ?? undefined,
+            categories: r[4] ?? '',
+            githubUrls: r[5] ?? undefined,
+            links: r[6] ?? undefined,
+        }));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function getCategoryCounts() {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const rows = db.exec(`SELECT primary_category, COUNT(*) as c FROM bookmarks
+       WHERE primary_category IS NOT NULL
+       GROUP BY primary_category ORDER BY c DESC`);
+        const counts = {};
+        for (const row of rows[0]?.values ?? []) {
+            counts[row[0]] = row[1];
+        }
+        return counts;
+    }
+    finally {
+        db.close();
+    }
+}
+export async function getDomainCounts() {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const rows = db.exec(`SELECT primary_domain, COUNT(*) as c FROM bookmarks
+       WHERE primary_domain IS NOT NULL
+       GROUP BY primary_domain ORDER BY c DESC`);
+        const counts = {};
+        for (const row of rows[0]?.values ?? []) {
+            counts[row[0]] = row[1];
+        }
+        return counts;
+    }
+    finally {
+        db.close();
+    }
+}
+export async function sampleByDomain(domain, limit) {
+    const dbPath = twitterBookmarksIndexPath();
+    const db = await openDb(dbPath);
+    ensureMigrations(db);
+    try {
+        const rows = db.exec(`SELECT id, url, text, author_handle, categories, github_urls, links_json
+       FROM bookmarks
+       WHERE domains LIKE ?
+       ORDER BY RANDOM()
+       LIMIT ?`, [`%${domain}%`, limit]);
+        if (!rows.length)
+            return [];
+        return rows[0].values.map((r) => ({
+            id: r[0],
+            url: r[1],
+            text: r[2],
+            authorHandle: r[3] ?? undefined,
+            categories: r[4] ?? '',
+            githubUrls: r[5] ?? undefined,
+            links: r[6] ?? undefined,
+        }));
+    }
+    finally {
+        db.close();
+    }
+}
+export function formatSearchResults(results) {
+    if (results.length === 0)
+        return 'No results found.';
+    return results
+        .map((r, i) => {
+        const author = r.authorHandle ? `@${r.authorHandle}` : 'unknown';
+        const date = r.postedAt ? r.postedAt.slice(0, 10) : '?';
+        const text = r.text.length > 140 ? r.text.slice(0, 140) + '...' : r.text;
+        return `${i + 1}. [${date}] ${author}\n   ${text}\n   ${r.url}`;
+    })
+        .join('\n\n');
+}

package/dist/bookmarks-service.js

@@ -0,0 +1,49 @@
+import { getTwitterBookmarksStatus } from './bookmarks.js';
+import { buildIndex } from './bookmarks-db.js';
+import { loadTwitterOAuthToken } from './xauth.js';
+import { syncBookmarksGraphQL } from './graphql-bookmarks.js';
+export async function enableBookmarks() {
+    const syncResult = await syncBookmarksGraphQL({
+        onProgress: (status) => {
+            if (status.page % 25 === 0 || status.done) {
+                process.stderr.write(`\r[sync] page ${status.page} | ${status.totalFetched} fetched | ${status.newAdded} new${status.done ? ` | ${status.stopReason}\n` : ''}`);
+            }
+        },
+    });
+    const indexResult = await buildIndex();
+    return {
+        synced: true,
+        bookmarkCount: syncResult.totalBookmarks,
+        indexedCount: indexResult.recordCount,
+        cachePath: syncResult.cachePath,
+        messageLines: [
+            'Bookmarks enabled.',
+            `- sync completed: ${syncResult.totalBookmarks} bookmarks (${syncResult.added} new)`,
+            `- indexed: ${indexResult.recordCount} records → ${indexResult.dbPath}`,
+            `- cache: ${syncResult.cachePath}`,
+        ],
+    };
+}
+export async function getBookmarkStatusView() {
+    const token = await loadTwitterOAuthToken();
+    const status = await getTwitterBookmarksStatus();
+    return {
+        connected: Boolean(token?.access_token),
+        bookmarkCount: status.totalBookmarks,
+        lastUpdated: status.lastIncrementalSyncAt ?? status.lastFullSyncAt ?? null,
+        mode: token?.access_token ? 'Incremental by default (GraphQL + API available)' : 'Incremental by default (GraphQL)',
+        cachePath: status.cachePath,
+    };
+}
+export function formatBookmarkStatus(view) {
+    return [
+        'Bookmarks',
+        `  bookmarks: ${view.bookmarkCount}`,
+        `  last updated: ${view.lastUpdated ?? 'never'}`,
+        `  sync mode: ${view.mode}`,
+        `  cache: ${view.cachePath}`,
+    ].join('\n');
+}
+export function formatBookmarkSummary(view) {
+    return `bookmarks=${view.bookmarkCount} updated=${view.lastUpdated ?? 'never'} mode="${view.mode}"`;
+}