@taurgis/bonsai 1.0.3 → 1.0.4

package/README.md

@@ -101,7 +101,7 @@ npx @taurgis/bonsai <url> [flags]
 Common flags:
 
 * `--topic`, `-t`: Primary category/topic for metadata tagging.
-* `--tags`, `-g`: Searchable tags. Can be repeated.
+* `--tags`, `-g`: Searchable tags. Can be repeated. When omitted, tags are auto-generated from the content by keyword detection (recorded with an `auto-generated tags via keyword extraction` quality note); passing `--tags` always takes precedence.
 * `--format`, `-f`: Output density: `compressed` or `detailed`.
 * `--tier`: Freshness tier: `stable`, `standard`, or `volatile`.
 * `--ttl`, `-l`: Custom predicted lifespan, such as `2h`, `7d`, or `30d`.
@@ -114,7 +114,7 @@ Common flags:
 
 ### `import [url]`
 
-Save custom Markdown notes directly to the cache.
+Save custom Markdown notes directly to the cache. As with `fetch`, when `--tags` is omitted the tags are auto-generated from the note content by keyword detection.
 
 ```bash
 npx @taurgis/bonsai import [url] --stdin [flags]
@@ -146,13 +146,19 @@ npx @taurgis/bonsai search "<keywords>" [flags]
 
 ### `config`
 
-Choose global or project-local cache storage.
+Two keys are configurable:
+
+| Key | Values | Default | Effect |
+| --- | --- | --- | --- |
+| `storage` | `global`, `project` | `global` | Where cache files are stored. |
+| `summary` | `conservative`, `balanced`, `aggressive` | `conservative` | How aggressively the `compressed` format is shortened when structural compression alone leaves it close to `detailed`. Prose is condensed extractively (no LLM); headings, code blocks, tables, and lists are always preserved. |
 
 ```bash
 npx @taurgis/bonsai config set storage project --local
+npx @taurgis/bonsai config set summary balanced
 ```
 
-Project config is stored in `.bonsai.json`; project cache files are stored under `.bonsai/research/`.
+Both keys can also be overridden per-invocation via the `BONSAI_STORAGE` and `BONSAI_SUMMARY` environment variables. Project config is stored in `.bonsai.json`; project cache files are stored under `.bonsai/research/`.
 
 ---
 

package/dist/commands/fetch.js

@@ -5,6 +5,7 @@ import { join } from 'node:path';
 import { BaseCommand } from '../base-command.js';
 import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
 import { writeArtifactSecurely } from '../lib/research/secure-write.js';
+import { loadSummaryLevel } from '../lib/config/index.js';
 import { evaluateFreshness, parseTtlToMs, checkMaxAgeExpired } from '../lib/research/freshness.js';
 import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
 import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
@@ -12,6 +13,7 @@ import { fetchRenderedHtml } from '../lib/research/browser.js';
 import { capturePage } from '../lib/research/capture.js';
 import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
 import { resolveResearchTarget } from '../lib/research/resolve-target.js';
+import { applyAutoTags } from '../lib/research/keywords.js';
 import { detectSite } from '../sites/index.js';
 const CAPTURE_DEPS = {
     fetchStatic: (url) => fetchStaticHtml(url),
@@ -92,7 +94,7 @@ export default class FetchCommand extends BaseCommand {
     envelopeCommandId() {
         return this.config.bin;
     }
-    async executeCacheHit(cached, targetDir, currentTime) {
+    async executeCacheHit(cached, targetDir, currentTime, summaryLevel) {
         const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
         const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
         const freshnessState = isExpired
@@ -105,6 +107,7 @@ export default class FetchCommand extends BaseCommand {
             allowStale: Boolean(allowStale),
             ttlOverride: ttl,
             rendered: Boolean(rendered),
+            summaryLevel,
         });
         handleStaleRevalidationResult(this, revalResult);
         return {
@@ -113,7 +116,7 @@ export default class FetchCommand extends BaseCommand {
             artifact: revalResult.artifact,
         };
     }
-    async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
+    async executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel) {
         const { topic, tags, tier, ttl, rendered } = this.flags;
         const siteModule = detectSite(normalizedUrl);
         const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
@@ -129,7 +132,7 @@ export default class FetchCommand extends BaseCommand {
             fetchResult = capture.fetchResult;
             extraction = capture.extraction;
         }
-        const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
+        const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime, summaryLevel);
         artifact.metadata.topic = topic || null;
         artifact.metadata.tags = tags || [];
         artifact.metadata.site_module_id = siteModule?.id ?? null;
@@ -139,7 +142,9 @@ export default class FetchCommand extends BaseCommand {
         else if (useRendered) {
             artifact.metadata.capture_method = 'browser_fallback';
         }
-        return artifact;
+        // Auto-tag from the fetched content when the caller supplied none, keeping cached pages
+        // searchable by keyword. Explicit --tags always win (handled in applyAutoTags).
+        return applyAutoTags(artifact);
     }
     buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
         return {
@@ -194,16 +199,16 @@ export default class FetchCommand extends BaseCommand {
     }
     // Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
     // back project→global; the resolved artifact is returned with the dir it lives in / landed in.
-    async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located) {
+    async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel) {
         if (located) {
             // Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
             // never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
             // gains a secret is not re-routed — only first-time project writes are scanned.
             const revalDir = tmpDir ?? located.dataDir;
-            const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
+            const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime, summaryLevel);
             return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
         }
-        const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
+        const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel);
         const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
         return {
             cacheStatus: 'miss',
@@ -230,11 +235,11 @@ export default class FetchCommand extends BaseCommand {
     }
     // Long references are split into searchable/inspectable section children whenever the page artifact
     // is freshly written (T-22). Best-effort: never let chunking break the main result.
-    persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
+    persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus, summaryLevel) {
         if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
             return;
         try {
-            persistSectionArtifacts(targetDir, artifact, currentTime);
+            persistSectionArtifacts(targetDir, artifact, currentTime, summaryLevel);
         }
         catch {
             /* section generation is non-essential; ignore failures */
@@ -259,11 +264,12 @@ export default class FetchCommand extends BaseCommand {
             this.error(`Invalid URL: ${err.message}`, { exit: 2 });
         }
         const { cacheKey, located, normalizedUrl, roots } = target;
+        const summaryLevel = loadSummaryLevel(this.config.configDir, process.cwd());
         const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
         const currentTime = new Date();
         try {
-            const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located);
-            this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
+            const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel);
+            this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus, summaryLevel);
             const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
             const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
             if (!this.jsonEnabled()) {

package/dist/commands/import.js

@@ -8,8 +8,10 @@ import { deriveCacheKey } from '../lib/research/cache-key.js';
 import { getArtifactPath } from '../lib/research/storage.js';
 import { loadStoreRoots } from '../lib/research/store-roots.js';
 import { writeArtifactSecurely } from '../lib/research/secure-write.js';
+import { loadSummaryLevel } from '../lib/config/index.js';
 import { getPolicy } from '../lib/research/freshness.js';
-import { compressMarkdown } from '../lib/research/compress.js';
+import { buildCompressed } from '../lib/research/compress.js';
+import { applyAutoTags } from '../lib/research/keywords.js';
 import { estimateTokens } from '../lib/research/token-estimate.js';
 export default class ResearchImport extends BaseCommand {
     static id = 'import';
@@ -200,7 +202,11 @@ export default class ResearchImport extends BaseCommand {
         const staleAfterTime = new Date(currentTime.getTime() + freshWindowMs);
         const inputFormat = this.flags['input-format'];
         const detailed = rawInput;
-        const compressed = inputFormat === 'detailed' ? compressMarkdown(rawInput) : rawInput;
+        // A caller-supplied compressed input is trusted as-is; only a detailed import is condensed here,
+        // and it shares the same buildCompressed policy (structural pass + extractive fallback) as fetch.
+        const compressed = inputFormat === 'detailed'
+            ? buildCompressed(rawInput, loadSummaryLevel(this.config.configDir, process.cwd()))
+            : rawInput;
         const contentHash = createHash('sha256').update(detailed).digest('hex');
         const metadata = {
             schema_version: 1,
@@ -240,13 +246,15 @@ export default class ResearchImport extends BaseCommand {
             section_anchor: null,
             section_heading_path: null,
         };
-        return {
+        // Auto-tag from the content when the caller supplied none, so imported notes stay searchable by
+        // keyword without forcing manual tagging. Explicit --tags always win (handled in applyAutoTags).
+        return applyAutoTags({
             metadata,
             summary: `Synthesized research for ${hasSingle ? singleNormalizedUrl : this.flags.topic}`,
             compressed,
             detailed,
             provenance: `Imported via agent-supplied research at ${currentTime.toISOString()}`,
-        };
+        });
     }
     async run() {
         const hasSingle = Boolean(this.args.url);
@@ -292,7 +300,7 @@ export default class ResearchImport extends BaseCommand {
                 captureMethod: 'agent_supplied',
                 extractionStatus: 'agent_supplied',
                 extractionConfidence: 'high',
-                qualityNotes: ['agent-supplied research import'],
+                qualityNotes: artifact.metadata.quality_notes,
                 fetchedAt: null,
                 validatedAt: artifact.metadata.validated_at,
                 staleAfter: artifact.metadata.stale_after,

package/dist/commands/search.js

@@ -1,6 +1,6 @@
 import { Args, Flags } from '@oclif/core';
 import { BaseCommand } from '../base-command.js';
-import { scanCacheDirs } from '../lib/research/storage.js';
+import { loadSearchableArtifacts } from '../lib/research/search-index.js';
 import { loadStoreRoots } from '../lib/research/store-roots.js';
 import { evaluateFreshness } from '../lib/research/freshness.js';
 import { detectSite } from '../sites/index.js';
@@ -150,21 +150,22 @@ export default class ResearchSearch extends BaseCommand {
         return snippet;
     }
     scanCacheDirForResults(readRoots, queryTerms, currentTime) {
-        return scanCacheDirs(readRoots, (artifact, filePath) => {
+        const results = [];
+        for (const { artifact, filePath } of loadSearchableArtifacts(readRoots)) {
             if (artifact.metadata.status !== 'active')
-                return null;
+                continue;
             const freshness = evaluateFreshness(artifact.metadata, currentTime, null);
             if (!this.matchesFilters(artifact.metadata, freshness, this.flags.topic, this.flags.tags, this.flags['artifact-type'], Boolean(this.flags['include-stale'])))
-                return null;
+                continue;
             let score = this.calculateScore(artifact.metadata, artifact.summary, artifact.compressed, queryTerms, freshness);
             if (score <= 0)
-                return null;
+                continue;
             // Section children are more precise than the whole page, so rank a section hit slightly
             // above its parent for the same query (T-22).
             if (artifact.metadata.artifact_type === 'section')
                 score += 15;
             const snippetText = [artifact.summary, artifact.compressed].filter(Boolean).join('\n');
-            return {
+            results.push({
                 cacheKey: artifact.metadata.cache_key,
                 path: filePath,
                 artifactType: artifact.metadata.artifact_type,
@@ -177,8 +178,9 @@ export default class ResearchSearch extends BaseCommand {
                 snippet: this.makeSnippet(snippetText, queryTerms),
                 siteModuleId: artifact.metadata.site_module_id,
                 score,
-            };
-        });
+            });
+        }
+        return results;
     }
     logSearchResults(finalResults) {
         if (this.jsonEnabled())

package/dist/lib/config/index.d.ts

@@ -1,12 +1,17 @@
-export { type ConfigValues, type ResolvedConfig, type ConfigKey, type StorageMode, type KeyMeta, SCHEMA_VERSION, STORAGE_MODES, ALL_KEYS, BUILT_IN_DEFAULTS, KEY_META, isKnownKey, suggestKey, validKeysHint, } from './schema.js';
+export { type ConfigValues, type ResolvedConfig, type ConfigKey, type StorageMode, type SummaryLevel, type KeyMeta, SCHEMA_VERSION, STORAGE_MODES, SUMMARY_LEVELS, ALL_KEYS, BUILT_IN_DEFAULTS, KEY_META, isKnownKey, suggestKey, validKeysHint, } from './schema.js';
 export { readUserConfig, readProjectConfig, writeUserConfig, writeProjectConfig, USER_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, } from './io.js';
-export { resolveStorageMode, parseEnvStorage, STORAGE_ENV_VAR, type ResolveStorageInput, } from './resolve.js';
-import type { ConfigValues, ResolvedConfig, StorageMode } from './schema.js';
+export { resolveStorageMode, resolveSummaryLevel, parseEnvStorage, parseEnvSummary, STORAGE_ENV_VAR, SUMMARY_ENV_VAR, type ResolveStorageInput, type ResolveSummaryInput, } from './resolve.js';
+import type { ConfigValues, ResolvedConfig, StorageMode, SummaryLevel } from './schema.js';
 /**
  * Read both config files once and resolve the effective storage mode.
  * `flag` (a per-invocation `--storage` override) wins over everything else.
  */
 export declare function loadStorageMode(configDir: string | undefined, cwd: string, flag?: StorageMode, env?: Record<string, string | undefined>): StorageMode;
+/**
+ * Read both config files once and resolve the effective summary level.
+ * `flag` (a per-invocation override) wins over everything else.
+ */
+export declare function loadSummaryLevel(configDir: string | undefined, cwd: string, flag?: SummaryLevel, env?: Record<string, string | undefined>): SummaryLevel;
 /** The resolved value of every config key, factoring file + env layers. Extend as keys are added. */
 export declare function effectiveConfig(configDir: string | undefined, cwd: string): ResolvedConfig;
 export type ConfigScope = 'global' | 'local' | 'effective';

package/dist/lib/config/index.js

@@ -1,8 +1,8 @@
-export { SCHEMA_VERSION, STORAGE_MODES, ALL_KEYS, BUILT_IN_DEFAULTS, KEY_META, isKnownKey, suggestKey, validKeysHint, } from './schema.js';
+export { SCHEMA_VERSION, STORAGE_MODES, SUMMARY_LEVELS, ALL_KEYS, BUILT_IN_DEFAULTS, KEY_META, isKnownKey, suggestKey, validKeysHint, } from './schema.js';
 export { readUserConfig, readProjectConfig, writeUserConfig, writeProjectConfig, USER_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, } from './io.js';
-export { resolveStorageMode, parseEnvStorage, STORAGE_ENV_VAR, } from './resolve.js';
+export { resolveStorageMode, resolveSummaryLevel, parseEnvStorage, parseEnvSummary, STORAGE_ENV_VAR, SUMMARY_ENV_VAR, } from './resolve.js';
 import { readUserConfig, readProjectConfig } from './io.js';
-import { resolveStorageMode } from './resolve.js';
+import { resolveStorageMode, resolveSummaryLevel } from './resolve.js';
 /**
  * Read both config files once and resolve the effective storage mode.
  * `flag` (a per-invocation `--storage` override) wins over everything else.
@@ -15,9 +15,24 @@ export function loadStorageMode(configDir, cwd, flag, env = process.env) {
         userConfig: readUserConfig(configDir),
     });
 }
+/**
+ * Read both config files once and resolve the effective summary level.
+ * `flag` (a per-invocation override) wins over everything else.
+ */
+export function loadSummaryLevel(configDir, cwd, flag, env = process.env) {
+    return resolveSummaryLevel({
+        flag,
+        env,
+        projectConfig: readProjectConfig(cwd),
+        userConfig: readUserConfig(configDir),
+    });
+}
 /** The resolved value of every config key, factoring file + env layers. Extend as keys are added. */
 export function effectiveConfig(configDir, cwd) {
-    return { storage: loadStorageMode(configDir, cwd) };
+    return {
+        storage: loadStorageMode(configDir, cwd),
+        summary: loadSummaryLevel(configDir, cwd),
+    };
 }
 /**
  * The config values for a requested scope: the user file (`global`), the project file (`local`),

package/dist/lib/config/resolve.d.ts

@@ -1,7 +1,9 @@
-import type { ConfigValues, StorageMode } from './schema.js';
+import type { ConfigValues, StorageMode, SummaryLevel } from './schema.js';
 /** Scoped env override, following oclif's `<BIN>_*` env-var convention for this CLI. */
 export declare const STORAGE_ENV_VAR = "BONSAI_STORAGE";
+export declare const SUMMARY_ENV_VAR = "BONSAI_SUMMARY";
 export declare function parseEnvStorage(env: Record<string, string | undefined>): StorageMode | undefined;
+export declare function parseEnvSummary(env: Record<string, string | undefined>): SummaryLevel | undefined;
 export interface ResolveStorageInput {
     flag?: StorageMode;
     env: Record<string, string | undefined>;
@@ -13,3 +15,14 @@ export interface ResolveStorageInput {
  * flag > env > project file > user file > built-in default.
  */
 export declare function resolveStorageMode(input: ResolveStorageInput): StorageMode;
+export interface ResolveSummaryInput {
+    flag?: SummaryLevel;
+    env: Record<string, string | undefined>;
+    projectConfig: Partial<ConfigValues>;
+    userConfig: Partial<ConfigValues>;
+}
+/**
+ * Resolve the effective summary level in precedence order:
+ * flag > env > project file > user file > built-in default.
+ */
+export declare function resolveSummaryLevel(input: ResolveSummaryInput): SummaryLevel;

package/dist/lib/config/resolve.js

@@ -1,12 +1,19 @@
-import { BUILT_IN_DEFAULTS, STORAGE_MODES } from './schema.js';
+import { BUILT_IN_DEFAULTS, STORAGE_MODES, SUMMARY_LEVELS } from './schema.js';
 /** Scoped env override, following oclif's `<BIN>_*` env-var convention for this CLI. */
 export const STORAGE_ENV_VAR = 'BONSAI_STORAGE';
+export const SUMMARY_ENV_VAR = 'BONSAI_SUMMARY';
 export function parseEnvStorage(env) {
     const raw = env[STORAGE_ENV_VAR]?.trim();
     return STORAGE_MODES.includes(raw ?? '')
         ? raw
         : undefined;
 }
+export function parseEnvSummary(env) {
+    const raw = env[SUMMARY_ENV_VAR]?.trim();
+    return SUMMARY_LEVELS.includes(raw ?? '')
+        ? raw
+        : undefined;
+}
 /**
  * Resolve the effective storage mode in precedence order:
  * flag > env > project file > user file > built-in default.
@@ -18,3 +25,14 @@ export function resolveStorageMode(input) {
         input.userConfig.storage ??
         BUILT_IN_DEFAULTS.storage);
 }
+/**
+ * Resolve the effective summary level in precedence order:
+ * flag > env > project file > user file > built-in default.
+ */
+export function resolveSummaryLevel(input) {
+    return (input.flag ??
+        parseEnvSummary(input.env) ??
+        input.projectConfig.summary ??
+        input.userConfig.summary ??
+        BUILT_IN_DEFAULTS.summary);
+}

package/dist/lib/config/schema.d.ts

@@ -4,13 +4,16 @@
  * where the research cache lives.
  */
 export type StorageMode = 'global' | 'project';
+export type SummaryLevel = 'conservative' | 'balanced' | 'aggressive';
 export interface ConfigValues {
     storage?: StorageMode;
+    summary?: SummaryLevel;
 }
 export type ResolvedConfig = Required<ConfigValues>;
 export type ConfigKey = keyof ConfigValues;
 export declare const SCHEMA_VERSION = 1;
 export declare const STORAGE_MODES: readonly StorageMode[];
+export declare const SUMMARY_LEVELS: readonly SummaryLevel[];
 export declare const ALL_KEYS: readonly ConfigKey[];
 export declare const BUILT_IN_DEFAULTS: ResolvedConfig;
 export interface KeyMeta {

package/dist/lib/config/schema.js

@@ -5,10 +5,13 @@
  */
 export const SCHEMA_VERSION = 1;
 export const STORAGE_MODES = ['global', 'project'];
-export const ALL_KEYS = ['storage'];
+export const SUMMARY_LEVELS = ['conservative', 'balanced', 'aggressive'];
+export const ALL_KEYS = ['storage', 'summary'];
 export const BUILT_IN_DEFAULTS = {
     // Default to global so existing installs keep using the OCLIF data dir unchanged.
     storage: 'global',
+    // Default to the gentlest summarization so cached content keeps the most detail out of the box.
+    summary: 'conservative',
 };
 export const KEY_META = {
     storage: {
@@ -18,6 +21,13 @@ export const KEY_META = {
         isValid: (v) => STORAGE_MODES.includes(String(v)),
         format: (v) => String(v),
     },
+    summary: {
+        description: `How aggressively the compressed format is summarized (${SUMMARY_LEVELS.join('|')}).`,
+        values: SUMMARY_LEVELS,
+        parseValue: (raw) => raw.trim(),
+        isValid: (v) => SUMMARY_LEVELS.includes(String(v)),
+        format: (v) => String(v),
+    },
 };
 export function isKnownKey(key) {
     return ALL_KEYS.includes(key);

package/dist/lib/research/artifact.d.ts

@@ -12,9 +12,20 @@ export declare function serializeArtifact(artifact: ResearchArtifact): string;
  */
 export declare function parseMetadata(lines: string[]): ResearchArtifactMetadata;
 /**
- * Extracts a named markdown section (e.g. ## Summary) from body text.
+ * Extracts a named markdown section (e.g. ## Summary) from body text. The section ends at the next
+ * known section header (`nextSection`), matched as the exact serialized separator `\n\n## <name>` —
+ * NOT at the next `## ` heading, because section bodies legitimately contain their own `##` headings
+ * (a compressed/detailed page often *starts* with one). Bounding on a generic heading would truncate
+ * such content to nothing. Omit `nextSection` for the final section to read to end-of-body.
  */
-export declare function extractSection(body: string, sectionName: string): string;
+export declare function extractSection(body: string, sectionName: string, nextSection?: string): string;
+/**
+ * Parses an artifact for search/listing without materializing the (often large) Detailed and
+ * Provenance bodies. Truncates at the Detailed section boundary before parsing so megabyte-scale
+ * detailed content is never split or sliced. `detailed` and `provenance` come back empty — callers
+ * that need them must use {@link parseArtifact}. Shares all parsing rules with parseArtifact.
+ */
+export declare function parseArtifactShallow(content: string): ResearchArtifact;
 /**
  * Parses full Markdown content into a structured ResearchArtifact.
  */

package/dist/lib/research/artifact.js

@@ -181,18 +181,38 @@ export function parseMetadata(lines) {
     return state.meta;
 }
 /**
- * Extracts a named markdown section (e.g. ## Summary) from body text.
+ * Extracts a named markdown section (e.g. ## Summary) from body text. The section ends at the next
+ * known section header (`nextSection`), matched as the exact serialized separator `\n\n## <name>` —
+ * NOT at the next `## ` heading, because section bodies legitimately contain their own `##` headings
+ * (a compressed/detailed page often *starts* with one). Bounding on a generic heading would truncate
+ * such content to nothing. Omit `nextSection` for the final section to read to end-of-body.
  */
-export function extractSection(body, sectionName) {
+export function extractSection(body, sectionName, nextSection) {
     const header = `## ${sectionName}`;
     const idx = body.indexOf(header);
     if (idx === -1)
         return '';
     const start = idx + header.length;
-    const nextHeadingIdx = body.indexOf('\n## ', start);
-    const rawText = nextHeadingIdx === -1 ? body.slice(start) : body.slice(start, nextHeadingIdx);
+    const endIdx = nextSection ? body.indexOf(`\n\n## ${nextSection}`, start) : -1;
+    const rawText = endIdx === -1 ? body.slice(start) : body.slice(start, endIdx);
     return rawText.trim();
 }
+/**
+ * Parses an artifact for search/listing without materializing the (often large) Detailed and
+ * Provenance bodies. Truncates at the Detailed section boundary before parsing so megabyte-scale
+ * detailed content is never split or sliced. `detailed` and `provenance` come back empty — callers
+ * that need them must use {@link parseArtifact}. Shares all parsing rules with parseArtifact.
+ */
+export function parseArtifactShallow(content) {
+    // Only look for the Detailed boundary in the body (after the closing frontmatter fence) and match
+    // the exact serialized separator `\n\n## Detailed`, so neither a frontmatter value nor an earlier
+    // body heading named "Detailed" can truncate the content prematurely.
+    const bodyStart = content.indexOf('\n---', 3);
+    const searchFrom = bodyStart === -1 ? 0 : bodyStart + 4;
+    const detailedIdx = content.indexOf('\n\n## Detailed', searchFrom);
+    const sliced = detailedIdx === -1 ? content : content.slice(0, detailedIdx);
+    return parseArtifact(sliced);
+}
 /**
  * Parses full Markdown content into a structured ResearchArtifact.
  */
@@ -209,9 +229,9 @@ export function parseArtifact(content) {
     const bodyLines = parts.slice(closingIdx + 1);
     const bodyText = bodyLines.join('\n');
     const metadata = parseMetadata(frontmatterLines);
-    const summary = extractSection(bodyText, 'Summary');
-    const compressed = extractSection(bodyText, 'Compressed');
-    const detailed = extractSection(bodyText, 'Detailed');
+    const summary = extractSection(bodyText, 'Summary', 'Compressed');
+    const compressed = extractSection(bodyText, 'Compressed', 'Detailed');
+    const detailed = extractSection(bodyText, 'Detailed', 'Provenance');
     const provenance = extractSection(bodyText, 'Provenance');
     return { metadata, summary, compressed, detailed, provenance };
 }

package/dist/lib/research/compress.d.ts

@@ -1,5 +1,24 @@
+import { type SummaryLevel } from './summarize.js';
 /**
  * Compresses Markdown text by removing images, simplifying links from [text](url) to [text],
  * and collapsing multiple consecutive blank lines.
  */
 export declare function compressMarkdown(markdown: string): string;
+/**
+ * Produces the `compressed` form of fetched content. Structural compression (strip images,
+ * simplify links, collapse blank lines) runs first, then the dependency-free extractive summarizer
+ * condenses prose to the chosen aggressiveness while keeping headings, code, tables, and lists
+ * intact. The summarizer self-guards: documents below the token floor, or with no prose to drop
+ * (e.g. pure structure once links/images are stripped), come back unchanged. Running it
+ * unconditionally — rather than only as a fallback when structural compression underperforms — is
+ * what makes the `summary` level matter even on link-heavy pages that structural compression
+ * already shrinks. This is the single place the "how do we build compressed" policy lives.
+ *
+ * A trust guard backs all of this: `compressed` is only useful if it is a non-empty, no-larger view
+ * of `detailed`. Both steps only ever remove content, so that should always hold — but rather than
+ * assume it, we verify and fall back through progressively safer forms (summarized → structural →
+ * detailed). This means the cache can never end up serving a `compressed` that is larger than
+ * `detailed`, or one that is empty while `detailed` has content — if the auto-compaction can't be
+ * trusted, the caller transparently gets the (verified) structural form or the detailed text itself.
+ */
+export declare function buildCompressed(detailed: string, level: SummaryLevel): string;

package/dist/lib/research/compress.js

@@ -1,3 +1,6 @@
+import { summarizeMarkdown } from './summarize.js';
+// Below this token count a page is never extractively summarized — there is nothing to gain.
+const SUMMARY_FLOOR_TOKENS = 200;
 /**
  * Compresses Markdown text by removing images, simplifying links from [text](url) to [text],
  * and collapsing multiple consecutive blank lines.
@@ -15,3 +18,31 @@ export function compressMarkdown(markdown) {
     result = result.replace(/\n{3,}/g, '\n\n');
     return result.trim();
 }
+/**
+ * Produces the `compressed` form of fetched content. Structural compression (strip images,
+ * simplify links, collapse blank lines) runs first, then the dependency-free extractive summarizer
+ * condenses prose to the chosen aggressiveness while keeping headings, code, tables, and lists
+ * intact. The summarizer self-guards: documents below the token floor, or with no prose to drop
+ * (e.g. pure structure once links/images are stripped), come back unchanged. Running it
+ * unconditionally — rather than only as a fallback when structural compression underperforms — is
+ * what makes the `summary` level matter even on link-heavy pages that structural compression
+ * already shrinks. This is the single place the "how do we build compressed" policy lives.
+ *
+ * A trust guard backs all of this: `compressed` is only useful if it is a non-empty, no-larger view
+ * of `detailed`. Both steps only ever remove content, so that should always hold — but rather than
+ * assume it, we verify and fall back through progressively safer forms (summarized → structural →
+ * detailed). This means the cache can never end up serving a `compressed` that is larger than
+ * `detailed`, or one that is empty while `detailed` has content — if the auto-compaction can't be
+ * trusted, the caller transparently gets the (verified) structural form or the detailed text itself.
+ */
+export function buildCompressed(detailed, level) {
+    const structural = compressMarkdown(detailed);
+    const summarized = summarizeMarkdown(structural, { level, minTokens: SUMMARY_FLOOR_TOKENS });
+    // Trustworthy = not larger than detailed, and not empty unless detailed is itself empty.
+    const trustworthy = (candidate) => candidate.length <= detailed.length && (candidate.trim() !== '' || detailed.trim() === '');
+    if (trustworthy(summarized))
+        return summarized;
+    if (trustworthy(structural))
+        return structural;
+    return detailed;
+}

package/dist/lib/research/docs/code-blocks.js

@@ -42,6 +42,12 @@ function reconstructLines(code) {
 function removeCopyButtons(document) {
     const candidates = document.querySelectorAll('button, [class*="copy" i], [class*="Copy"], [aria-label*="copy" i]');
     for (const el of candidates) {
+        // Never remove a container that wraps a code block. Some highlighters put a copy-related class
+        // on the <pre>'s wrapper (e.g. Elementor's `<div class="copy-to-clipboard"><pre>…`); removing
+        // it would delete the entire code example, not just the button. Guard on <pre> specifically: a
+        // real copy button never contains a <pre>, but may contain a decorative <code> icon.
+        if (el.querySelector?.('pre'))
+            continue;
         const cls = (el.getAttribute('class') || '').toString();
         const aria = el.getAttribute('aria-label') || '';
         const text = (el.textContent || '').trim();
@@ -50,6 +56,23 @@ function removeCopyButtons(document) {
         }
     }
 }
+// Some highlighters (e.g. Elementor's code-highlight widget) wrap the authored source in a
+// deprecated <xmp>/<plaintext> element and pretty-print the markup, so <code> is not <pre>'s first
+// child. Turndown's fenced-code rule requires <code> as the first child; without this the block
+// collapses onto a single inline span. Flatten to a clean <pre><code>…</code></pre>.
+function flattenCodeStructure(pre, code) {
+    if (code === pre)
+        return;
+    const rawHolder = code.querySelector?.('xmp, plaintext');
+    const raw = rawHolder?.textContent ?? '';
+    if (raw)
+        code.textContent = raw;
+    if (pre.firstChild !== code || pre.childNodes.length > 1) {
+        while (pre.firstChild)
+            pre.removeChild(pre.firstChild);
+        pre.appendChild(code);
+    }
+}
 /**
  * Reconstructs newline-delimited text for every highlighted code block in the document and tags it
  * with a `language-x` class when one is detectable. Idempotent and a no-op on plain `<pre>` blocks.
@@ -65,5 +88,6 @@ export function normalizeCodeBlocks(document) {
             const existing = (code.getAttribute('class') || '').replace(/language-[\w-]+/gi, '').trim();
             code.setAttribute('class', `${existing} language-${language}`.trim());
         }
+        flattenCodeStructure(pre, code);
     }
 }

package/dist/lib/research/docs/section-artifacts.d.ts

@@ -1,12 +1,13 @@
 import type { ResearchArtifact } from '../schema.js';
+import type { SummaryLevel } from '../../config/schema.js';
 /**
  * Derives section child artifacts from a parent page artifact. Returns [] when the page is short
  * or has too few headings to chunk.
  */
-export declare function buildSectionArtifacts(parent: ResearchArtifact, currentTime: Date): ResearchArtifact[];
+export declare function buildSectionArtifacts(parent: ResearchArtifact, currentTime: Date, summaryLevel: SummaryLevel): ResearchArtifact[];
 /**
  * Regenerates a page's section artifacts: writes fresh children and archives any prior child whose
  * heading no longer exists, so a revalidated parent never leaves orphaned sections active. Returns
  * the number of active section children written.
  */
-export declare function persistSectionArtifacts(dataDir: string, parent: ResearchArtifact, currentTime: Date): number;
+export declare function persistSectionArtifacts(dataDir: string, parent: ResearchArtifact, currentTime: Date, summaryLevel: SummaryLevel): number;