@taurgis/bonsai 1.0.6 → 3.0.1

package/dist/commands/research.js

@@ -1,304 +0,0 @@
-import { Args, Flags } from '@oclif/core';
-import { mkdtempSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { BaseCommand } from '../base-command.js';
-import { normalizeUrl } from '../lib/research/url.js';
-import { deriveCacheKey } from '../lib/research/cache-key.js';
-import { locateArtifact, writeArtifact, getArtifactPath } from '../lib/research/storage.js';
-import { loadStoreRoots } from '../lib/research/store-roots.js';
-import { writeArtifactSecurely } from '../lib/research/secure-write.js';
-import { evaluateFreshness, parseTtlToMs, checkMaxAgeExpired } from '../lib/research/freshness.js';
-import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
-import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
-import { fetchRenderedHtml } from '../lib/research/browser.js';
-import { capturePage } from '../lib/research/capture.js';
-import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
-import { detectSite } from '../sites/index.js';
-const CAPTURE_DEPS = {
-    fetchStatic: (url) => fetchStaticHtml(url),
-    fetchRendered: (url) => fetchRenderedHtml(url),
-    fetchText: (url) => fetchText(url),
-};
-export default class Research extends BaseCommand {
-    static id = 'research';
-    static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
-    static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.';
-    static examples = [
-        {
-            description: 'Research a URL with detailed output, tagged with topic and tags',
-            command: '<%= config.bin %> research https://docs.nestjs.com/ --topic "Backend Frameworks" --tags "Node" --tags "NestJS" --format detailed --ttl 30d',
-        },
-        {
-            description: 'Research a volatile page with compressed output and short TTL, returned as JSON',
-            command: '<%= config.bin %> research https://news.ycombinator.com/ --format compressed --ttl 2h --json',
-        },
-    ];
-    static args = {
-        url: Args.string({
-            required: true,
-            description: 'The full HTTP/HTTPS URL of the web page to research.',
-        }),
-    };
-    static flags = {
-        topic: Flags.string({
-            char: 't',
-            description: 'The main category/topic of the research for metadata tagging.',
-        }),
-        tags: Flags.string({
-            char: 'g',
-            description: 'Taxonomic tags for this research (can be repeated).',
-            multiple: true,
-        }),
-        format: Flags.option({
-            char: 'f',
-            description: 'Desired data density.',
-            options: ['compressed', 'detailed'],
-            default: 'compressed',
-        })(),
-        tier: Flags.option({
-            description: 'Freshness tier policy.',
-            options: ['stable', 'standard', 'volatile'],
-            default: 'standard',
-        })(),
-        ttl: Flags.string({
-            char: 'l',
-            description: 'Predicted lifespan of the data (e.g., "2h", "7d", "3m").',
-        }),
-        'max-age': Flags.string({
-            description: 'Maximum age of cache to accept (e.g., "1d", "30d").',
-        }),
-        force: Flags.boolean({
-            description: 'Force a fresh fetch, ignoring any cached entries.',
-            default: false,
-        }),
-        'dry-run': Flags.boolean({
-            description: 'Perform validation and fetch without saving to local cache.',
-            default: false,
-        }),
-        'allow-stale': Flags.boolean({
-            description: 'Allow serving stale cache if the remote server is unreachable.',
-            default: false,
-        }),
-        rendered: Flags.boolean({
-            description: 'Force using a browser-rendered scraping path for dynamic pages.',
-            default: false,
-        }),
-        storage: Flags.option({
-            description: 'Override where this result is cached (overrides configured default). Secret-bearing pages are always stored globally.',
-            options: ['global', 'project'],
-        })(),
-    };
-    static stdoutIsPrimaryData = true;
-    async init() {
-        await super.init();
-        const { args, flags } = await this.parse(Research);
-        this.args = args;
-        this.flags = flags;
-    }
-    async executeCacheHit(cached, targetDir, currentTime) {
-        const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
-        const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
-        const freshnessState = isExpired
-            ? 'stale_expired'
-            : evaluateFreshness(cached.metadata, currentTime, ttl);
-        if (freshnessState === 'fresh') {
-            return { cacheStatus: 'hit', freshnessState, artifact: cached };
-        }
-        const revalResult = await revalidateCache(targetDir, cached, currentTime, {
-            allowStale: Boolean(allowStale),
-            ttlOverride: ttl,
-            rendered: Boolean(rendered),
-        });
-        handleStaleRevalidationResult(this, revalResult);
-        return {
-            cacheStatus: revalResult.status,
-            freshnessState,
-            artifact: revalResult.artifact,
-        };
-    }
-    async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
-        const { topic, tags, tier, ttl, rendered } = this.flags;
-        const siteModule = detectSite(normalizedUrl);
-        const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
-        let fetchResult;
-        let extraction;
-        let capture = null;
-        if (siteModule?.fetchPage) {
-            // Custom site modules own their fetch/extract strategy; the generic capture path is skipped.
-            ({ fetchResult, extraction } = await siteModule.fetchPage(normalizedUrl));
-        }
-        else {
-            capture = await capturePage(normalizedUrl, { forceRendered: useRendered }, CAPTURE_DEPS);
-            fetchResult = capture.fetchResult;
-            extraction = capture.extraction;
-        }
-        const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
-        artifact.metadata.topic = topic || null;
-        artifact.metadata.tags = tags || [];
-        artifact.metadata.site_module_id = siteModule?.id ?? null;
-        if (capture) {
-            applyCaptureMetadata(artifact, capture);
-        }
-        else if (useRendered) {
-            artifact.metadata.capture_method = 'browser_fallback';
-        }
-        return artifact;
-    }
-    buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
-        return {
-            schemaVersion: 1,
-            command: 'research',
-            cache: {
-                key: cacheKey,
-                status: cacheStatus,
-                freshness: freshnessState,
-                path: getArtifactPath(storageDir, cacheKey),
-                storage: storageMode,
-                redirectedToGlobal,
-            },
-            source: {
-                url,
-                normalizedUrl,
-                captureMethod: artifact.metadata.capture_method,
-                extractionStatus: artifact.metadata.extraction_status,
-                extractionConfidence: artifact.metadata.extraction_confidence,
-                qualityNotes: artifact.metadata.quality_notes,
-                fetchedAt: artifact.metadata.fetched_at,
-                validatedAt: artifact.metadata.validated_at,
-                staleAfter: artifact.metadata.stale_after,
-            },
-            artifactType: artifact.metadata.artifact_type,
-            docsEngine: artifact.metadata.docs_engine,
-            docsFramework: artifact.metadata.docs_framework,
-            sourceDocUrl: artifact.metadata.source_doc_url,
-            searchProvider: artifact.metadata.search_provider,
-            format,
-            tokenEstimate: format === 'compressed'
-                ? artifact.metadata.token_estimate.compressed
-                : artifact.metadata.token_estimate.detailed,
-            content,
-        };
-    }
-    // Validates the duration flags up front, exiting with code 2 on a malformed value.
-    validateDurationFlags(ttl, maxAge) {
-        for (const [label, value] of [
-            ['TTL', ttl],
-            ['max-age', maxAge],
-        ]) {
-            if (!value)
-                continue;
-            try {
-                parseTtlToMs(value);
-            }
-            catch (err) {
-                this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
-            }
-        }
-    }
-    // Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
-    // back project→global; the resolved artifact is returned with the dir it lives in / landed in.
-    async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime) {
-        const located = this.flags.force ? null : locateArtifact(roots.readRoots, cacheKey);
-        if (located) {
-            // Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
-            // never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
-            // gains a secret is not re-routed — only first-time project writes are scanned.
-            const revalDir = tmpDir ?? located.dataDir;
-            const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
-            return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
-        }
-        const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
-        const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
-        return { cacheStatus: 'miss', freshnessState: 'stale_expired', artifact, storageDir: dir, redirectedToGlobal };
-    }
-    // Writes a freshly fetched artifact, honoring dry-run (throwaway dir) and the secret-safety
-    // redirect (project→global). Returns the data dir reported to the user and whether a redirect
-    // occurred (so the JSON envelope mirrors `research import`).
-    persistFreshArtifact(roots, tmpDir, cacheKey, artifact) {
-        if (tmpDir) {
-            writeArtifact(tmpDir, cacheKey, artifact);
-            // Report the real would-be location, not the throwaway dir that is about to be deleted.
-            return { dir: roots.writeRoot, redirectedToGlobal: false };
-        }
-        const result = writeArtifactSecurely(roots, cacheKey, artifact);
-        if (result.redirected) {
-            this.warn(`Detected ${result.secretLabel} in the page content; stored in the global cache instead of the project to avoid committing secrets.`);
-        }
-        return { dir: result.dataDir, redirectedToGlobal: result.redirected };
-    }
-    // Long references are split into searchable/inspectable section children whenever the page artifact
-    // is freshly written (T-22). Best-effort: never let chunking break the main result.
-    persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
-        if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
-            return;
-        try {
-            persistSectionArtifacts(targetDir, artifact, currentTime);
-        }
-        catch {
-            /* section generation is non-essential; ignore failures */
-        }
-    }
-    async execute() {
-        const { url } = this.args;
-        const { format, ttl, 'max-age': maxAge, 'dry-run': dryRun } = this.flags;
-        this.validateDurationFlags(ttl, maxAge);
-        let normalizedUrl;
-        let cacheKey;
-        try {
-            normalizedUrl = normalizeUrl(url);
-            cacheKey = deriveCacheKey(normalizedUrl);
-        }
-        catch (err) {
-            this.error(`Invalid URL: ${err.message}`, { exit: 2 });
-        }
-        const roots = loadStoreRoots({
-            configDir: this.config.configDir,
-            cwd: process.cwd(),
-            dataDir: this.config.dataDir,
-            flagOverride: this.flags.storage,
-        });
-        const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
-        const currentTime = new Date();
-        try {
-            const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime);
-            this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
-            const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
-            const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
-            if (!this.requestedJson()) {
-                this.log(content);
-            }
-            return resultData;
-        }
-        finally {
-            if (tmpDir) {
-                rmSync(tmpDir, { recursive: true, force: true });
-            }
-        }
-    }
-}
-// Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
-function applyCaptureMetadata(artifact, capture) {
-    const meta = artifact.metadata;
-    meta.capture_method = capture.captureMethod;
-    meta.docs_engine = capture.capabilities.docsEngine ?? null;
-    meta.docs_framework = capture.capabilities.framework ?? null;
-    meta.source_doc_url = capture.sourceDocUrl;
-    meta.search_provider = capture.capabilities.search?.provider ?? null;
-    if (capture.extraction.isIndexHub)
-        meta.artifact_type = 'index';
-    if (capture.sourceDocUrl && !meta.source_urls.includes(capture.sourceDocUrl)) {
-        meta.source_urls.push(capture.sourceDocUrl);
-    }
-}
-function handleStaleRevalidationResult(command, revalResult) {
-    if (revalResult.status !== 'stale')
-        return;
-    if (revalResult.allowed) {
-        command.warn(`Serving stale content: revalidation failed: ${revalResult.error}`);
-    }
-    else {
-        command.warn(`Serving stale content (exit 5): revalidation failed: ${revalResult.error}`);
-        process.exitCode = 5;
-    }
-}

package/dist/commands/search.d.ts

@@ -1,35 +0,0 @@
-import { BaseCommand } from '../base-command.js';
-export default class ResearchSearch extends BaseCommand<typeof ResearchSearch> {
-    static id: string;
-    static summary: string;
-    static description: string;
-    static examples: {
-        description: string;
-        command: string;
-    }[];
-    static args: {
-        query: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
-    };
-    static flags: {
-        topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        'artifact-type': import("@oclif/core/interfaces").OptionFlag<"source" | "research_note" | "index" | "section" | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        limit: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
-        'include-stale': import("@oclif/core/interfaces").BooleanFlag<boolean>;
-        domain: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        remote: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-    };
-    static stdoutIsPrimaryData: boolean;
-    private validateSearchFlags;
-    private getSearchQueryTerms;
-    private matchesFilters;
-    private phraseMatchBonus;
-    private calculateScore;
-    private makeSnippet;
-    private scanCacheDirForResults;
-    private logSearchResults;
-    private printResults;
-    private executeSiteSearch;
-    private executeRemoteSearch;
-    run(): Promise<unknown>;
-}