@antodevs/groundtruth 0.1.0

package/src/search.js

@@ -0,0 +1,157 @@
+/**
+ * @module search
+ * @description Logica di scraping web su DuckDuckGo tramite cheerio e linkedom.
+ */
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import { Readability } from '@mozilla/readability';
+import { DOMParser } from 'linkedom';
+import { searchCache } from './cache.js';
+import { CircuitBreaker } from './circuit-breaker.js';
+import { httpAgent, httpsAgent } from './http-agent.js';
+
+// ─── Config & Cache ──────────────────────────────────
+
+// Evitiamo IP bans ruotando UA comuni in Chrome desktop
+const USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+];
+
+/**
+ * @description Seleziona uno User-Agent rnd dall'array disponibile
+ * @returns {string} Stringa di uno User Agent
+ */
+function getRandomUA() {
+    return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}
+
+const ddgCircuit = new CircuitBreaker({ failureThreshold: 3, resetTimeout: 30000 });
+
+/**
+ * @description Decodifica link mascherati DuckDuckGo recuperando `uddg` querystring.
+ * @param   {string} href - Url incapsulato proveniente da nodeDDG
+ * @returns {string} Url reale target in chiaro
+ */
+export function resolveDDGUrl(href) {
+    try {
+        const url = new URL(href, 'https://duckduckgo.com');
+        const uddg = url.searchParams.get('uddg');
+        return uddg ? decodeURIComponent(uddg) : href;
+    } catch {
+        return href;
+    }
+}
+
+/**
+ * @description Esegue chiamata http reale su node DDG.
+ * @param   {string} query - Ricerca DDG formattata
+ * @returns {Promise<Object>} { results, userAgent }
+ * @throws  {Error} Fallimento http DDG request
+ */
+async function doSearch(query) {
+    const userAgent = getRandomUA();
+    // Fetch DDG raw HTML search endpoint ignoring CSS/JS payloads
+    const searchRes = await fetch(
+        `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
+        { signal: AbortSignal.timeout(5000), headers: { 'User-Agent': userAgent }, agent: httpsAgent }
+    );
+    if (!searchRes.ok) throw new Error(`DDG ${searchRes.status}`);
+
+    const $ = cheerio.load(await searchRes.text());
+    let results = [];
+    $('.result__body').each((i, el) => {
+        const title = $(el).find('.result__title').text().trim();
+        const snippet = $(el).find('.result__snippet').text().trim();
+        let rawUrl = $(el).find('.result__url').attr('href') || $(el).find('a.result__url').attr('href');
+        const resultUrl = rawUrl ? resolveDDGUrl(rawUrl) : '';
+        if (title && resultUrl) results.push({ title, snippet, url: resultUrl });
+    });
+
+    const seen = new Set();
+    results = results.filter(r => r.url && !seen.has(r.url) && seen.add(r.url)).slice(0, 3);
+
+    if (results.length === 0) throw new Error('No DDG results');
+    return { results, userAgent };
+}
+
+/**
+ * @description Punto d'accesso caching+retry orchestrator web.
+ * @param   {string}  query    - Input utente di ricerca convertibile web
+ * @param   {boolean} parallel - Promise.all fast per multiple page scraping
+ * @returns {Promise<Object>} Oggetto risultati + pageText formattato str
+ */
+export async function webSearch(query, parallel = false) {
+    const now = Date.now();
+    // In cache mode skip costose chiamate network
+    const cached = searchCache.get(query);
+    if (cached) {
+        return { results: cached.results, pageText: cached.pageText };
+    }
+
+    let results, userAgent;
+    try {
+        const res = await ddgCircuit.execute(() => doSearch(query));
+        results = res.results;
+        userAgent = res.userAgent;
+    } catch (err) {
+        throw err;
+    }
+
+    let pageText = '';
+    // Se claude-code usa parallel mode; altrimenti solo primo link (antigravity)
+    if (parallel) {
+        const pages = await Promise.all(results.map(async (r) => {
+            try {
+                const pageRes = await fetch(r.url, {
+                    signal: AbortSignal.timeout(5000),
+                    headers: { 'User-Agent': userAgent },
+                    agent: r.url.startsWith('https:') ? httpsAgent : httpAgent
+                });
+                if (pageRes.ok) {
+                    const document = new DOMParser().parseFromString(await pageRes.text(), 'text/html');
+                    let text = '';
+                    try {
+                        const article = new Readability(document).parse();
+                        text = article?.textContent || '';
+                    } catch (_) {
+                        text = document.body?.textContent || '';
+                    }
+                    if (text) return text.replace(/\s+/g, ' ').slice(0, 4000);
+                }
+            } catch (_) { // fail silenzioso parallelo tollerato per timeout link third-party
+            }
+            return '';
+        }));
+        pageText = pages.filter(Boolean).join('\n\n');
+    } else {
+        try {
+            if (results[0]) {
+                const pageRes = await fetch(results[0].url, {
+                    signal: AbortSignal.timeout(5000), // node-fetch hang timeout catch
+                    headers: { 'User-Agent': userAgent },
+                    agent: results[0].url.startsWith('https:') ? httpsAgent : httpAgent
+                });
+                if (pageRes.ok) {
+                    const document = new DOMParser().parseFromString(await pageRes.text(), 'text/html');
+                    let text = '';
+                    try {
+                        const article = new Readability(document).parse();
+                        text = article?.textContent || '';
+                    } catch (_) {
+                        text = document.body?.textContent || '';
+                    }
+                    if (text) {
+                        pageText = text.replace(/\s+/g, ' ').slice(0, 4000);
+                    }
+                }
+            }
+        } catch (_) { // bypass errore url target: fallback al contesto vuoto
+        }
+    }
+
+    const resultData = { results, pageText };
+    searchCache.set(query, resultData);
+    return resultData;
+}

package/src/state.js

@@ -0,0 +1,37 @@
+/**
+ * @module state
+ * @description Persiste la memoria di antigravity prev-hash per fault tolleranza riavvii.
+ */
+import { readFile, writeFile, mkdir } from 'fs/promises';
+import { existsSync } from 'fs';
+import path from 'path';
+import os from 'os';
+
+const STATE_DIR = path.join(os.homedir(), '.groundtruth');
+const STATE_FILE = path.join(STATE_DIR, 'watcher-state.json');
+
+/**
+ * @description Carica gli hash validati e memorizzati dallo schedule storage locale.
+ * @returns {Promise<Map>} Restituisce le hash map entries persistite del cron logic stream precedente.
+ */
+export async function loadBatchState() {
+    try {
+        if (!existsSync(STATE_FILE)) return new Map();
+        const data = await readFile(STATE_FILE, 'utf8');
+        const parsed = JSON.parse(data);
+        return new Map(Object.entries(parsed));
+    } catch {
+        return new Map();
+    }
+}
+
+/**
+ * @description Sincronizza hash batches per fault tolerance cross process
+ * @param {Map} map - Oggetto dei blocchi hashati validi in mem persist state map
+ * @returns {Promise<void>} 
+ */
+export async function saveBatchState(map) {
+    await mkdir(STATE_DIR, { recursive: true });
+    const obj = Object.fromEntries(map);
+    await writeFile(STATE_FILE, JSON.stringify(obj, null, 2), 'utf8');
+}

package/src/utils/atomic-write.js

@@ -0,0 +1,58 @@
+/**
+ * @module atomic-write
+ * @description Scrittura file atomica cross-platform con fallback.
+ */
+import { writeFile, rename, unlink, copyFile } from 'fs/promises';
+import { existsSync } from 'fs';
+import { tmpdir } from 'os';
+import path from 'path';
+
+/**
+ * @description Scrive file in modo atomico con backup automatico opzionale.
+ * @param {string} filePath - Path destinazione.
+ * @param {string} content - Contenuto da scrivere.
+ * @param {Object} options - { backup: boolean, mode: number }
+ * @returns {Promise<Object>} Esito operazione e path backup
+ * @throws {Error} In caso di fallimento filesystem
+ */
+export async function atomicWrite(filePath, content, options = {}) {
+    const { backup = true, mode = 0o644 } = options;
+    const tempFile = path.join(tmpdir(), `.gt-${Date.now()}-${Math.random().toString(36).slice(2)}.tmp`);
+    const backupPath = `${filePath}.bak`;
+
+    try {
+        // Scrittura su file temporaneo
+        await writeFile(tempFile, content, { mode, encoding: 'utf8' });
+
+        // Backup esistente se richiesto
+        if (backup && existsSync(filePath)) {
+            await copyFile(filePath, backupPath);
+        }
+
+        // Atomic rename (POSIX) o safe best-effort (Windows)
+        try {
+            await rename(tempFile, filePath);
+        } catch (renameErr) {
+            if (process.platform === 'win32' && (renameErr.code === 'EACCES' || renameErr.code === 'EPERM' || renameErr.code === 'EBUSY')) {
+                let success = false;
+                for (let i = 0; i < 5; i++) {
+                    await new Promise(r => setTimeout(r, 100 * (2 ** i)));
+                    try {
+                        await rename(tempFile, filePath);
+                        success = true;
+                        break;
+                    } catch (_) { }
+                }
+                if (!success) throw new Error(`Rename failed on Windows after 5 retries`);
+            } else {
+                throw renameErr;
+            }
+        }
+
+        return { success: true, backupPath: backup ? backupPath : null };
+    } catch (err) {
+        // Cleanup temp in caso di errore catch
+        await unlink(tempFile).catch(() => { });
+        throw err;
+    }
+}

package/src/watcher.js

@@ -0,0 +1,146 @@
+/**
+ * @module watcher
+ * @description Timer poll di Antigravity update locale skill inject doc rules, ora con caching a batch blocchi separati.
+ */
+import os from 'os';
+import path from 'path';
+import { webSearch } from './search.js';
+import { readPackageDeps, buildQuery, groupIntoBatches, batchHash } from './packages.js';
+import { updateGeminiFiles, removeStaleBlocks } from './inject.js';
+import { chalk, label, log, LOG_WARN, LOG_REFRESH } from './logger.js';
+import { version } from './cli.js';
+import { loadBatchState, saveBatchState } from './state.js';
+import { httpsAgent } from './http-agent.js';
+
+// ─── Scheduler Watcher Instance ──────────────────────
+
+export function startWatcher({ intervalMinutes, usePackageJson, batchSize }) {
+    const homeDir = os.homedir();
+    const globalPath = path.join(homeDir, '.gemini', 'GEMINI.md');
+    const workspacePath = path.join(process.cwd(), '.gemini', 'GEMINI.md');
+
+    const globalSkillFilePretty = '~/.gemini/GEMINI.md';
+    const skillFilePretty = '.gemini/GEMINI.md';
+
+    console.log();
+    console.log(`  ${chalk.white.bold('GroundTruth')}  ${chalk.gray(`v${version}`)}  ${chalk.gray('[antigravity mode]')}`);
+    console.log();
+    console.log(label('◆', 'global', globalSkillFilePretty));
+    console.log(label('◆', 'workspace', skillFilePretty));
+    console.log(label('◆', 'interval', `every ${intervalMinutes} min`));
+    console.log(label('◆', 'batch_size', `chunk limit ${batchSize}`));
+    console.log(label('◆', 'context', 'DuckDuckGo → live'));
+    console.log();
+    console.log(`  ${chalk.cyan('✻')} Running. Antigravity will load context automatically.`);
+    console.log();
+
+    let previousBatchHashes = new Map();
+
+    async function updateSkill() {
+        if (previousBatchHashes.size === 0) {
+            previousBatchHashes = await loadBatchState();
+        }
+        const deps = await readPackageDeps(); // tutte le deps
+        if (!deps || deps.length === 0) {
+            return; // fall back to something default or just skip 
+        }
+
+        const batches = groupIntoBatches(deps, batchSize);
+        const activeBlockIds = new Set();
+        let updatedCount = 0;
+        let skippedCount = 0;
+        let failedCount = 0;
+
+        const maxConcurrency = 3;
+        const executing = new Set();
+
+        for (const batch of batches) {
+            const promise = (async () => {
+                const blockId = batchHash(batch);
+                activeBlockIds.add(blockId);
+
+                const currentHash = batchHash(batch);
+                if (previousBatchHashes.get(blockId) === currentHash) {
+                    skippedCount++;
+                    return;
+                }
+
+                const query = buildQuery(batch);
+                try {
+                    const { results, pageText } = await webSearch(query, false);
+                    const badSignals = ['403', 'captcha', 'blocked', 'access denied', 'forbidden'];
+                    const isBad = !pageText || pageText.length < 200 || badSignals.some(s => pageText.toLowerCase().includes(s));
+                    if (isBad && previousBatchHashes.has(blockId)) {
+                        log(LOG_WARN, chalk.yellow, `low quality result for block ${blockId} → keeping previous context`);
+                        failedCount++;
+                        return;
+                    }
+
+                    const now = new Date();
+                    const nowStr = now.toLocaleString('it-IT');
+                    const batchTitle = batch.map(b => b.split(' ')[0]).join(', ');
+
+                    let globalMd = `## Live Context — ${batchTitle} (${nowStr})\n`;
+                    globalMd += `**Query:** ${query}\n\n`;
+                    if (results.length > 0) {
+                        globalMd += `### ${results[0].title}\n`;
+                        globalMd += `${results[0].snippet.slice(0, 300)} — ${results[0].url}\n`;
+                    }
+
+                    let md = `## Live Context — ${batchTitle} (${nowStr})\n`;
+                    md += `**Query:** ${query}\n\n`;
+                    for (const r of results) {
+                        md += `### ${r.title}\n${r.snippet} — ${r.url}\n\n`;
+                    }
+                    if (pageText) {
+                        md += `FULL TEXT: ${pageText}\n`;
+                    }
+
+                    await updateGeminiFiles([{
+                        blockId,
+                        globalContent: globalMd,
+                        workspaceContent: md
+                    }]);
+
+                    previousBatchHashes.set(blockId, currentHash);
+                    updatedCount++;
+                    log(LOG_REFRESH, chalk.cyan, `block ${blockId} updated → ${batch.join(', ')}`);
+                } catch (e) {
+                    failedCount++;
+                    log(LOG_WARN, chalk.yellow, `block ${blockId} fetch failed → keeping previous`);
+                }
+            })().then(() => executing.delete(promise));
+
+            executing.add(promise);
+            if (executing.size >= maxConcurrency) {
+                await Promise.race(executing);
+            }
+        }
+        await Promise.all(executing);
+
+        await removeStaleBlocks(globalPath, activeBlockIds);
+        await removeStaleBlocks(workspacePath, activeBlockIds);
+
+        await saveBatchState(previousBatchHashes);
+
+        log(LOG_REFRESH, chalk.gray, `cycle done → ${activeBlockIds.size} blocks active, ${updatedCount} updated, ${skippedCount} skipped, ${failedCount} errors`);
+    }
+
+    let cycleCount = 0;
+
+    // Periodical state persistence on process exit to avoid total crash data loss
+    process.on('SIGINT', async () => {
+        await saveBatchState(previousBatchHashes);
+        process.exit(0);
+    });
+
+    // Lancio a startup immediato
+    updateSkill();
+    setInterval(() => {
+        cycleCount++;
+        if (cycleCount % 10 === 0) {
+            httpsAgent.destroy(); // Forza chiusura idle connections
+        }
+        updateSkill();
+    }, intervalMinutes * 60 * 1000);
+}