@houtini/better-search-console 1.0.0

package/dist/core/Database.js

@@ -0,0 +1,205 @@
+import BetterSqlite3 from 'better-sqlite3';
+export class Database {
+    db;
+    insertStmt = null;
+    constructor(dbPath) {
+        this.db = new BetterSqlite3(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('busy_timeout = 5000'); // wait up to 5s for concurrent writers
+        this.db.pragma('cache_size = -65536'); // 64MB cache (default is ~2MB)
+        this.db.pragma('temp_store = MEMORY'); // temp tables in RAM
+        this.db.pragma('mmap_size = 4294967296'); // 4GB mmap for large DBs
+        this.initializeTables();
+    }
+    initializeTables() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS property_meta (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        site_url TEXT UNIQUE NOT NULL,
+        permission_level TEXT,
+        last_synced_at TEXT,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS search_analytics (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        date TEXT NOT NULL,
+        query TEXT,
+        page TEXT,
+        device TEXT,
+        country TEXT,
+        search_appearance TEXT,
+        clicks INTEGER NOT NULL DEFAULT 0,
+        impressions INTEGER NOT NULL DEFAULT 0,
+        ctr REAL NOT NULL DEFAULT 0,
+        position REAL NOT NULL DEFAULT 0,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+    `);
+        this.db.exec(`
+      -- Primary uniqueness constraint (also serves as composite index for date-first lookups)
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_unique
+        ON search_analytics(date, query, page, device, country);
+
+      -- Single-column indexes for standalone filtering
+      CREATE INDEX IF NOT EXISTS idx_sa_date ON search_analytics(date);
+      CREATE INDEX IF NOT EXISTS idx_sa_query ON search_analytics(query);
+      CREATE INDEX IF NOT EXISTS idx_sa_page ON search_analytics(page);
+      CREATE INDEX IF NOT EXISTS idx_sa_clicks ON search_analytics(clicks DESC);
+      CREATE INDEX IF NOT EXISTS idx_sa_impressions ON search_analytics(impressions DESC);
+
+      -- Composite indexes for dashboard query patterns
+      CREATE INDEX IF NOT EXISTS idx_sa_date_query ON search_analytics(date, query, clicks, impressions, position);
+      CREATE INDEX IF NOT EXISTS idx_sa_date_page ON search_analytics(date, page, clicks, impressions, position);
+      CREATE INDEX IF NOT EXISTS idx_sa_date_country ON search_analytics(date, country, clicks, impressions);
+      CREATE INDEX IF NOT EXISTS idx_sa_query_date ON search_analytics(query, date);
+
+      -- Covering index for summary aggregations
+      CREATE INDEX IF NOT EXISTS idx_sa_date_metrics ON search_analytics(date, clicks, impressions, ctr, position);
+    `);
+        // Update query planner statistics only when needed
+        // sqlite_stat1 may not exist if ANALYZE has never run, so check safely
+        const statExists = this.db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sqlite_stat1' LIMIT 1`).get();
+        const hasStats = statExists && this.db.prepare(`SELECT 1 FROM sqlite_stat1 WHERE tbl = 'search_analytics' AND idx = 'idx_sa_date_metrics' LIMIT 1`).get();
+        if (!hasStats) {
+            this.db.exec('ANALYZE;');
+        }
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS sync_log (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        sync_type TEXT NOT NULL,
+        dimensions TEXT NOT NULL,
+        date_from TEXT,
+        date_to TEXT,
+        rows_fetched INTEGER DEFAULT 0,
+        rows_inserted INTEGER DEFAULT 0,
+        status TEXT DEFAULT 'running',
+        error_message TEXT,
+        started_at TEXT DEFAULT (datetime('now')),
+        completed_at TEXT
+      );
+    `);
+    }
+    // --- Property Meta ---
+    upsertPropertyMeta(siteUrl, permissionLevel) {
+        this.db.prepare(`
+      INSERT INTO property_meta (site_url, permission_level)
+      VALUES (?, ?)
+      ON CONFLICT(site_url) DO UPDATE SET permission_level = excluded.permission_level
+    `).run(siteUrl, permissionLevel);
+    }
+    updateLastSynced(siteUrl) {
+        this.db.prepare(`
+      UPDATE property_meta SET last_synced_at = datetime('now') WHERE site_url = ?
+    `).run(siteUrl);
+    }
+    getPropertyMeta(siteUrl) {
+        const row = this.db.prepare(`
+      SELECT site_url, permission_level, last_synced_at FROM property_meta WHERE site_url = ?
+    `).get(siteUrl);
+        if (!row)
+            return null;
+        return {
+            siteUrl: row.site_url,
+            permissionLevel: row.permission_level,
+            lastSyncedAt: row.last_synced_at,
+        };
+    }
+    getLastSyncDate(siteUrl) {
+        const row = this.db.prepare(`
+      SELECT MAX(date) as max_date FROM search_analytics
+    `).get();
+        return row?.max_date ?? null;
+    }
+    // --- Search Analytics ---
+    getInsertStmt() {
+        if (!this.insertStmt) {
+            this.insertStmt = this.db.prepare(`
+        INSERT OR REPLACE INTO search_analytics
+          (date, query, page, device, country, search_appearance, clicks, impressions, ctr, position)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+        }
+        return this.insertStmt;
+    }
+    insertSearchAnalyticsBatch(rows) {
+        let inserted = 0;
+        const stmt = this.getInsertStmt();
+        const transaction = this.db.transaction((rows) => {
+            for (const row of rows) {
+                stmt.run(row.date, row.query, row.page, row.device, row.country, row.searchAppearance, row.clicks, row.impressions, row.ctr, row.position);
+                inserted++;
+            }
+        });
+        transaction(rows);
+        return inserted;
+    }
+    getRowCount() {
+        const result = this.db.prepare('SELECT COUNT(*) as count FROM search_analytics').get();
+        return result.count;
+    }
+    getDateRange() {
+        const result = this.db.prepare('SELECT MIN(date) as min_date, MAX(date) as max_date FROM search_analytics').get();
+        if (!result || !result.min_date)
+            return null;
+        return { minDate: result.min_date, maxDate: result.max_date };
+    }
+    // --- Sync Log ---
+    createSyncLog(entry) {
+        const result = this.db.prepare(`
+      INSERT INTO sync_log (sync_type, dimensions, date_from, date_to, rows_fetched, rows_inserted, status, error_message)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(entry.syncType, entry.dimensions, entry.dateFrom, entry.dateTo, entry.rowsFetched, entry.rowsInserted, entry.status, entry.errorMessage);
+        return Number(result.lastInsertRowid);
+    }
+    updateSyncLog(id, updates) {
+        const fields = [];
+        const values = [];
+        if (updates.rowsFetched !== undefined) {
+            fields.push('rows_fetched = ?');
+            values.push(updates.rowsFetched);
+        }
+        if (updates.rowsInserted !== undefined) {
+            fields.push('rows_inserted = ?');
+            values.push(updates.rowsInserted);
+        }
+        if (updates.status !== undefined) {
+            fields.push('status = ?');
+            values.push(updates.status);
+        }
+        if (updates.errorMessage !== undefined) {
+            fields.push('error_message = ?');
+            values.push(updates.errorMessage);
+        }
+        if (updates.status === 'completed' || updates.status === 'error') {
+            fields.push("completed_at = datetime('now')");
+        }
+        if (fields.length === 0)
+            return;
+        values.push(id);
+        this.db.prepare(`UPDATE sync_log SET ${fields.join(', ')} WHERE id = ?`).run(...values);
+    }
+    // --- Raw Query (read-only) ---
+    executeReadOnlyQuery(sql, params = [], maxRows = 10000) {
+        const forbidden = /\b(INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|REPLACE|ATTACH|DETACH|REINDEX|VACUUM|PRAGMA)\b/i;
+        if (forbidden.test(sql)) {
+            throw new Error('Only SELECT queries are allowed. Write operations and PRAGMA are blocked.');
+        }
+        // Enforce a result size limit to prevent memory exhaustion
+        const hasLimit = /\bLIMIT\b/i.test(sql);
+        const safeSql = hasLimit ? sql : `${sql} LIMIT ${maxRows}`;
+        return this.db.prepare(safeSql).all(...params);
+    }
+    // --- Generic query helper for insights ---
+    query(sql, params = []) {
+        return this.db.prepare(sql).all(...params);
+    }
+    queryOne(sql, params = []) {
+        return this.db.prepare(sql).get(...params);
+    }
+    close() {
+        this.db.close();
+    }
+}
+//# sourceMappingURL=Database.js.map

package/dist/core/Database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Database.js","sourceRoot":"","sources":["../../src/core/Database.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,gBAAgB,CAAC;AAG3C,MAAM,OAAO,QAAQ;IACX,EAAE,CAAyB;IAC3B,UAAU,GAAmC,IAAI,CAAC;IAE1D,YAAY,MAAc;QACxB,IAAI,CAAC,EAAE,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAK,uCAAuC;QAClF,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAK,+BAA+B;QAC1E,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAK,qBAAqB;QAChE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAE,yBAAyB;QACpE,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;KAQZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;KAeZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;KAoBZ,CAAC,CAAC;QAEH,mDAAmD;QACnD,uEAAuE;QACvE,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAChC,oFAAoF,CACrF,CAAC,GAAG,EAAE,CAAC;QACR,MAAM,QAAQ,GAAG,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAC5C,mGAAmG,CACpG,CAAC,GAAG,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;KAcZ,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IAExB,kBAAkB,CAAC,OAAe,EAAE,eAAuB;QACzD,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAIf,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnC,CAAC;IAED,gBAAgB,CAAC,OAAe;QAC9B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAEf,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClB,CAAC;IAED,eAAe,CAAC,OAAe;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE3B,CAAC,CAAC,GAAG,CAAC,OAAO,CAAQ,CAAC;QACvB,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,QAAQ;YACrB,eAAe,EAAE,GAAG,CAAC,gBAAgB;YACrC,YAAY,EAAE,GAAG,CAAC,cAAc;SACjC,CAAC;IACJ,CAAC;IAED,eAAe,CAAC,OAAe;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE3B,CAAC,CAAC,GAAG,EAAS,CAAC;QAChB,OAAO,GAAG,EAAE,QAAQ,IAAI,IAAI,CAAC;IAC/B,CAAC;IAED,2BAA2B;IAEnB,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIjC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,0BAA0B,CAAC,IAA0B;QACnD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAA0B,EAAE,EAAE;YACrE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC,GAAG,CACN,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,KAAK,EACT,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,OAAO,EACX,GAAG,CAAC,gBAAgB,EACpB,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,GAAG,EACP,GAAG,CAAC,QAAQ,CACb,CAAC;gBACF,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC,CAAC,CAAC;QACH,WAAW,CAAC,IAAI,CAAC,CAAC;QAClB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,WAAW;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,EAAS,CAAC;QAC9F,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,YAAY;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC5B,2EAA2E,CAC5E,CAAC,GAAG,EAAS,CAAC;QACf,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;IAChE,CAAC;IAED,mBAAmB;IAEnB,aAAa,CAAC,KAA6D;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG9B,CAAC,CAAC,GAAG,CACJ,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,YAAY,CACnB,CAAC;QACF,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAED,aAAa,CAAC,EAAU,EAAE,OAA8B;QACtD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAU,EAAE,CAAC;QAEzB,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAAC,CAAC;QAC7G,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAC,CAAC;QAChH,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAC7F,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAC,CAAC;QAChH,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAChC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uBAAuB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC1F,CAAC;IAED,gCAAgC;IAEhC,oBAAoB,CAAC,GAAW,EAAE,SAAgB,EAAE,EAAE,UAAkB,KAAK;QAC3E,MAAM,SAAS,GAAG,2FAA2F,CAAC;QAC9G,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,2DAA2D;QAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,OAAO,EAAE,CAAC;QAC3D,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,4CAA4C;IAE5C,KAAK,CAAC,GAAW,EAAE,SAAgB,EAAE;QACnC,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED,QAAQ,CAAC,GAAW,EAAE,SAAgB,EAAE;QACtC,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}

package/dist/core/GscClient.d.ts

@@ -0,0 +1,23 @@
+import type { GscProperty, GscApiRow, FetchOptions } from '../types/index.js';
+export interface PageResult {
+    rows: GscApiRow[];
+    totalSoFar: number;
+    startRow: number;
+}
+export declare class GscClient {
+    private auth;
+    private searchconsole;
+    constructor(credentialsPath: string);
+    listProperties(): Promise<GscProperty[]>;
+    /**
+     * Fetch search analytics with streaming page callback.
+     *
+     * Each 25k-row page from the API triggers the onPage callback
+     * so callers can commit to the database immediately rather than
+     * accumulating everything in memory.
+     *
+     * Returns the total number of rows fetched across all pages.
+     */
+    fetchSearchAnalytics(siteUrl: string, options: FetchOptions, signal?: AbortSignal, onPage?: (page: PageResult) => void): Promise<number>;
+}
+//# sourceMappingURL=GscClient.d.ts.map

package/dist/core/GscClient.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GscClient.d.ts","sourceRoot":"","sources":["../../src/core/GscClient.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAuB9E,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,IAAI,CAAM;IAClB,OAAO,CAAC,aAAa,CAAM;gBAEf,eAAe,EAAE,MAAM;IAQ7B,cAAc,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAS9C;;;;;;;;OAQG;IACG,oBAAoB,CACxB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,YAAY,EACrB,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC;CAiDnB"}

package/dist/core/GscClient.js

@@ -0,0 +1,92 @@
+import { google } from 'googleapis';
+const ROW_LIMIT = 25000; // GSC API max per request
+const DEFAULT_DIMENSIONS = ['query', 'page', 'date', 'device', 'country'];
+const MAX_RETRIES = 3;
+const RETRYABLE_STATUS_CODES = [429, 500, 502, 503];
+async function withRetry(fn, retries = MAX_RETRIES) {
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            const status = err?.code || err?.response?.status || err?.status;
+            const isRetryable = RETRYABLE_STATUS_CODES.includes(Number(status));
+            if (!isRetryable || attempt === retries)
+                throw err;
+            const delay = Math.min(1000 * Math.pow(2, attempt), 30000);
+            console.error(`[GSC] Retryable error (${status}), attempt ${attempt + 1}/${retries}, waiting ${delay}ms...`);
+            await new Promise(resolve => setTimeout(resolve, delay));
+        }
+    }
+    throw new Error('Unreachable');
+}
+export class GscClient {
+    auth;
+    searchconsole;
+    constructor(credentialsPath) {
+        this.auth = new google.auth.GoogleAuth({
+            keyFile: credentialsPath,
+            scopes: ['https://www.googleapis.com/auth/webmasters.readonly'],
+        });
+        this.searchconsole = google.searchconsole({ version: 'v1', auth: this.auth });
+    }
+    async listProperties() {
+        const response = await withRetry(() => this.searchconsole.sites.list());
+        const sites = response.data.siteEntry || [];
+        return sites.map((site) => ({
+            siteUrl: site.siteUrl,
+            permissionLevel: site.permissionLevel,
+        }));
+    }
+    /**
+     * Fetch search analytics with streaming page callback.
+     *
+     * Each 25k-row page from the API triggers the onPage callback
+     * so callers can commit to the database immediately rather than
+     * accumulating everything in memory.
+     *
+     * Returns the total number of rows fetched across all pages.
+     */
+    async fetchSearchAnalytics(siteUrl, options, signal, onPage) {
+        const dimensions = options.dimensions || DEFAULT_DIMENSIONS;
+        let totalRows = 0;
+        let startRow = 0;
+        while (true) {
+            if (signal?.aborted) {
+                console.error(`[GSC] Sync aborted after ${totalRows} rows`);
+                break;
+            }
+            const response = await withRetry(() => this.searchconsole.searchanalytics.query({
+                siteUrl,
+                requestBody: {
+                    startDate: options.startDate,
+                    endDate: options.endDate,
+                    dimensions,
+                    rowLimit: options.rowLimit || ROW_LIMIT,
+                    startRow,
+                    dataState: options.dataState || 'all',
+                    ...(options.searchType ? { type: options.searchType } : {}),
+                },
+            }));
+            const rows = (response.data.rows || []).map((row) => ({
+                keys: row.keys,
+                clicks: row.clicks,
+                impressions: row.impressions,
+                ctr: row.ctr,
+                position: row.position,
+            }));
+            totalRows += rows.length;
+            // Fire callback so caller can commit this page immediately
+            if (onPage && rows.length > 0) {
+                onPage({ rows, totalSoFar: totalRows, startRow });
+            }
+            if (rows.length < (options.rowLimit || ROW_LIMIT)) {
+                break;
+            }
+            startRow += rows.length;
+            console.error(`[GSC] Fetched ${totalRows} rows so far (page at startRow=${startRow})...`);
+        }
+        return totalRows;
+    }
+}
+//# sourceMappingURL=GscClient.js.map

package/dist/core/GscClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GscClient.js","sourceRoot":"","sources":["../../src/core/GscClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGpC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,0BAA0B;AACnD,MAAM,kBAAkB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;AAC1E,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAEpD,KAAK,UAAU,SAAS,CAAI,EAAoB,EAAE,OAAO,GAAG,WAAW;IACrE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,MAAM,GAAG,GAAG,EAAE,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,EAAE,MAAM,CAAC;YACjE,MAAM,WAAW,GAAG,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACpE,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,OAAO;gBAAE,MAAM,GAAG,CAAC;YACnD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;YAC3D,OAAO,CAAC,KAAK,CAAC,0BAA0B,MAAM,cAAc,OAAO,GAAG,CAAC,IAAI,OAAO,aAAa,KAAK,OAAO,CAAC,CAAC;YAC7G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;AACjC,CAAC;AAQD,MAAM,OAAO,SAAS;IACZ,IAAI,CAAM;IACV,aAAa,CAAM;IAE3B,YAAY,eAAuB;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YACrC,OAAO,EAAE,eAAe;YACxB,MAAM,EAAE,CAAC,qDAAqD,CAAC;SAChE,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,MAAM,QAAQ,GAAQ,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;YAC/B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,eAAe,EAAE,IAAI,CAAC,eAAe;SACtC,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,oBAAoB,CACxB,OAAe,EACf,OAAqB,EACrB,MAAoB,EACpB,MAAmC;QAEnC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;QAC5D,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpB,OAAO,CAAC,KAAK,CAAC,4BAA4B,SAAS,OAAO,CAAC,CAAC;gBAC5D,MAAM;YACR,CAAC;YAED,MAAM,QAAQ,GAAQ,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,KAAK,CAAC;gBACnF,OAAO;gBACP,WAAW,EAAE;oBACX,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,UAAU;oBACV,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS;oBACvC,QAAQ;oBACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK;oBACrC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC5D;aACF,CAAC,CAAC,CAAC;YAEJ,MAAM,IAAI,GAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;gBACtE,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC,CAAC;YAEJ,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC;YAEzB,2DAA2D;YAC3D,IAAI,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;YACpD,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,EAAE,CAAC;gBAClD,MAAM;YACR,CAAC;YAED,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,iBAAiB,SAAS,kCAAkC,QAAQ,MAAM,CAAC,CAAC;QAC5F,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}

package/dist/core/SyncManager.d.ts

@@ -0,0 +1,66 @@
+import { GscClient } from './GscClient.js';
+export type SyncJobStatus = 'queued' | 'syncing' | 'completed' | 'failed' | 'cancelled';
+export interface SyncJobResult {
+    siteUrl: string;
+    status: 'completed' | 'failed' | 'skipped' | 'cancelled';
+    rowsFetched: number;
+    rowsInserted: number;
+    durationMs: number;
+    error?: string;
+    pruned?: {
+        rowsDeleted: number;
+        rowsAfter: number;
+        spaceSavedMB: number;
+    };
+}
+export interface SyncStatus {
+    jobId: string;
+    status: SyncJobStatus;
+    totalProperties: number;
+    completedProperties: number;
+    currentProperty: string | null;
+    rowsFetched: number;
+    estimatedTotalRows: number | null;
+    apiCallsMade: number;
+    startedAt: string;
+    elapsedMs: number;
+    results: SyncJobResult[];
+    error?: string;
+}
+export declare class SyncManager {
+    private gscClient;
+    private jobs;
+    private jobOrder;
+    constructor(gscClient: GscClient);
+    startSync(args: {
+        siteUrl: string;
+        startDate?: string;
+        endDate?: string;
+        dimensions?: string[];
+        searchType?: 'web' | 'discover' | 'googleNews' | 'image' | 'video';
+    }): string;
+    startSyncAll(args: {
+        startDate?: string;
+        endDate?: string;
+        dimensions?: string[];
+        searchType?: 'web' | 'discover' | 'googleNews' | 'image' | 'video';
+    }): Promise<string>;
+    getStatus(jobId?: string): SyncStatus | SyncStatus[];
+    cancelJob(jobId: string): boolean;
+    private createJob;
+    private pruneHistory;
+    /**
+     * Run the job in the background with parallel property syncing.
+     * Up to PROPERTY_CONCURRENCY properties sync concurrently.
+     */
+    private runJob;
+    /**
+     * Sync a single property with parallel chunk fetching.
+     * Up to CHUNK_CONCURRENCY date-range chunks fetch concurrently.
+     * Each chunk's DB writes are serialized (SQLite is single-writer)
+     * but API fetches overlap.
+     */
+    private syncOneProperty;
+    private jobToStatus;
+}
+//# sourceMappingURL=SyncManager.d.ts.map

package/dist/core/SyncManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SyncManager.d.ts","sourceRoot":"","sources":["../../src/core/SyncManager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAa3C,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAExF,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA0BD,qBAAa,WAAW;IAIV,OAAO,CAAC,SAAS;IAH7B,OAAO,CAAC,IAAI,CAA8B;IAC1C,OAAO,CAAC,QAAQ,CAAgB;gBAEZ,SAAS,EAAE,SAAS;IAExC,SAAS,CAAC,IAAI,EAAE;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,UAAU,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC;KACpE,GAAG,MAAM;IAMJ,YAAY,CAAC,IAAI,EAAE;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,UAAU,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC;KACpE,GAAG,OAAO,CAAC,MAAM,CAAC;IAcnB,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,UAAU,EAAE;IAwBpD,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAajC,OAAO,CAAC,SAAS;IAuBjB,OAAO,CAAC,YAAY;IAapB;;;OAGG;YACW,MAAM;IAiDpB;;;;;OAKG;YACW,eAAe;IA6J7B,OAAO,CAAC,WAAW;CAgBpB"}