@hraness/sponge 0.6.1 → 0.6.3

package/dist/src/app/article.d.ts

@@ -1,6 +1,7 @@
 import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
 import { type CaptureRequest, type CaptureOutcome, type CaptureConnection } from '../acquisition/index.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
+import { type PdfCaptureRequest, type CaptureOutcome as PdfCaptureOutcome } from '../pdf/capture.js';
 interface ArticleResult {
     ok: true;
     itemId: string;
@@ -17,6 +18,7 @@ export interface ArticleOptions {
     signal?: AbortSignal;
     allowPrivateNetwork?: boolean;
     capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
+    capturePdf?: (request: PdfCaptureRequest) => Promise<PdfCaptureOutcome>;
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestArticle(store: LibraryStore, token: AttemptToken, options: ArticleOptions): Promise<ArticleOutcome>;

package/dist/src/app/article.js

@@ -7,6 +7,8 @@ import { bundleBytes, bundleText } from './bundle-files.js';
 import { selectedMetadata } from '../metadata/article.js';
 import { localizeMarkdown } from './markdown.js';
 import { SpongeError } from '../errors.js';
+import { capturePdf } from '../pdf/capture.js';
+import { storePdfCapture } from '../pdf/handler.js';
 const classify = classifyErrors([
     'integrity',
     'limit-exceeded',
@@ -41,7 +43,45 @@ export function ingestArticle(store, token, options) {
         validateCapture(capture);
         if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
             throw new SpongeError('integrity');
-        const original = await bundleText(result.bundleRoot, capture.markdown), localized = localizeMarkdown(original, capture.assets);
+        const original = await bundleText(result.bundleRoot, capture.markdown);
+        const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
+        const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
+        const paperUrl = proseWords(original) < landingPageWords ? fullTextPdf(capture.metadata, source.locator) : undefined;
+        if (paperUrl) {
+            const paper = await (options.capturePdf ?? capturePdf)({
+                runProcess: options.runProcess,
+                stagingRoot: join(store.root, 'staging'),
+                sourceUrl: paperUrl,
+                requestId: token.attemptId,
+                signal: options.signal,
+                allowPrivateNetwork: options.allowPrivateNetwork,
+            });
+            if (paper.ok) {
+                attempt.hold({ dispose: () => Promise.all([result.dispose(), paper.dispose()]).then(() => undefined) });
+                const pdf = await storePdfCapture(store, token, paper, paperUrl, options.signal);
+                const pageFields = new Set(capture.metadata.map((claim) => claim.field));
+                const bundleHash = sha256(JSON.stringify(['article-pdf-full-text-v1', capture.manifestSha256, pdf.capture.manifestSha256]));
+                await options.checkpoint?.('promoting');
+                const revision = store.revisions.commit({
+                    ...token,
+                    itemId: source.itemId,
+                    bundleHash,
+                    markdown: pdf.markdown,
+                    title: title ?? pdf.title,
+                    extractor: pdf.capture.extractor,
+                    status: pdf.status,
+                    metadata: [...capture.metadata, ...pdf.capture.metadata.filter((claim) => !pageFields.has(claim.field))],
+                    metadataWarnings: capture.metadataWarnings,
+                    assets: pdf.assets,
+                    warnings: pdf.warnings,
+                });
+                return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status: pdf.status };
+            }
+            // Without the paper, the landing page is still what the reader saved.
+            if (paper.code === 'cleanup-required')
+                attempt.cleanup = 'pending';
+        }
+        const localized = localizeMarkdown(original, capture.assets);
         const assets = [];
         for (const asset of capture.assets) {
             if (options.signal?.aborted)
@@ -53,8 +93,6 @@ export function ingestArticle(store, token, options) {
             store.queue.cancel(token.jobId);
             throw new SpongeError('cancelled');
         }
-        const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
-        const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
         const status = localized.missingImages ? 'partial' : capture.status;
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
         const bundleHash = sha256(JSON.stringify(['article-local-markdown-v2', capture.manifestSha256]));
@@ -75,3 +113,34 @@ export function ingestArticle(store, token, options) {
         return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
     });
 }
+/** Prose words under which a page that names its paper's PDF is only a landing page: a title, authors and abstract. */
+const landingPageWords = 1000;
+function proseWords(markdown) {
+    const body = markdown.startsWith('---\n') ? markdown.slice(Math.max(0, markdown.indexOf('\n---', 4))) : markdown;
+    const prose = body.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1');
+    return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length;
+}
+/**
+ * The full text a scholarly landing page declares in `citation_pdf_url` (arXiv, NBER, OSF and most journals), when
+ * it is served from the page's own site or a subdomain relation of it; other sites are never fetched on its word.
+ */
+function fullTextPdf(metadata, pageUrl) {
+    const page = new URL(pageUrl).hostname;
+    for (const claim of metadata) {
+        if (claim.field !== 'alternate' || !claim.locator?.includes('citation_pdf_url'))
+            continue;
+        for (const value of Array.isArray(claim.value) ? claim.value : [claim.value]) {
+            let url;
+            try {
+                url = new URL(value);
+            }
+            catch {
+                continue;
+            }
+            const host = url.hostname;
+            if (url.protocol === 'https:' && (host === page || host.endsWith('.' + page) || page.endsWith('.' + host)))
+                return url.href;
+        }
+    }
+    return undefined;
+}

package/dist/src/cli/commands.js

@@ -8,6 +8,7 @@ import { queueAction } from './queue.js';
 import { portabilityAction } from './portability.js';
 import { restoreLibrary } from '../portability/backup.js';
 import { workerAction, serveWorker, hasDueAcquisition, workerDoctor } from './worker.js';
+import { libraryStatus, statusText } from './status.js';
 import { nativeSetupPlan, provisionNative, inspectNative, optionalNative } from '../models/runtime.js';
 import { digestModel, digestSetupPlan, inspectDigest, provisionDigest } from '../models/digest.js';
 import { withDigestServer } from '../digest/server.js';
@@ -423,11 +424,44 @@ export const commands = [
                     states[kind] = publicCode(error, 'runtime-missing');
                 }
             }
+            let digest;
+            try {
+                await inspectDigest(c.nativeRoot, store.digests.settings().model ?? undefined);
+                digest = 'ready';
+            }
+            catch (error) {
+                digest = publicCode(error, 'runtime-missing');
+            }
             const cleanup = store.queue.pendingCleanup(), worker = await workerDoctor(store, c.version), workerBroken = !['ready', 'not-installed', 'unsupported'].includes(worker.worker);
-            c.emit({ library: 'ready', capture, pdf, ...states, pendingCleanup: cleanup, worker }, `Library: ready\nCapture: ${capture}\nPDF: ${pdf === 'ready' ? 'ready' : 'Poppler not found (brew install poppler)'}\nASR: ${states.asr}\nEmbedding: ${states.embedding}\nPending cleanup: ${cleanup}\nWorker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`);
+            // Each runtime that is not ready names the command that installs it.
+            const line = (label, state, fix) => `${label}: ${state}${state === 'runtime-missing' ? ` (${fix})` : ''}`;
+            c.emit({ library: 'ready', capture, pdf, ...states, digest, pendingCleanup: cleanup, worker }, [
+                'Library: ready',
+                `Capture: ${capture}`,
+                `PDF: ${pdf === 'ready' ? 'ready' : 'Poppler not found (brew install poppler)'}`,
+                line('Media', states.media, 'sponge setup --capture'),
+                line('Speech-to-text', states.asr, 'sponge setup --asr'),
+                line('Embedding', states.embedding, 'sponge setup --embedding'),
+                line('Digest model', digest, 'sponge setup --digest'),
+                `Pending cleanup: ${cleanup}${cleanup ? ' (sponge queue recover)' : ''}`,
+                `Worker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`,
+            ].join('\n'));
             return cleanup || workerBroken ? 4 : 0;
         },
     },
+    {
+        name: 'status',
+        summary: 'Overview of items, index, digests and queue',
+        usage: [['status', 'Overview of items, index, digests and queue']],
+        positional: 0,
+        flags: {},
+        library: 'open',
+        handler: async (c) => {
+            const status = libraryStatus(c.store), worker = (await workerDoctor(c.store, c.version)).worker;
+            c.emit({ ...status, worker }, statusText(status, worker));
+            return 0;
+        },
+    },
     {
         name: 'export',
         summary: 'Export Markdown and local assets',
@@ -802,7 +836,7 @@ const row = ([synopsis, description]) => !description
 const notes = `Global flags: --library /absolute/path --json --debug (or SPONGE_DEBUG=1: print error causes to stderr)
 Items are public by default only when captured from a public source.
 Use -- before an option-like search query. Capture setup is explicit.
-Digest endpoints must be literal loopback HTTP (127.0.0.1 or [::1]); Sponge never starts a model server.
+Digest endpoints must be literal loopback HTTP (127.0.0.1 or [::1]); --draft starts Sponge's own server there.
 Media: --language en enables local English ASR; --force-local bypasses captions.
 These flags apply to --wait, run --drain, or worker serve.
 worker install --language en saves the English ASR choice for background work.

package/dist/src/cli/status.d.ts

@@ -0,0 +1,26 @@
+import type { LibraryStore } from '../store/index.js';
+/** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
+export declare function libraryStatus(store: LibraryStore): {
+    items: {
+        total: number;
+    };
+    index: {
+        embedded: number;
+        expected: number;
+        pending: number;
+    } | null;
+    digests: {
+        auto: boolean;
+        accepted: number;
+        rejected: number;
+        draft: number;
+        none: number;
+    };
+    jobs: {
+        operation: string;
+        state: string;
+        reason: string | null;
+    }[];
+    pendingCleanup: number;
+};
+export declare function statusText(s: ReturnType<typeof libraryStatus>, worker: string): string;

package/dist/src/cli/status.js

@@ -0,0 +1,63 @@
+const plural = (n, one, many = one + 's') => `${n} ${n === 1 ? one : many}`;
+/** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
+export function libraryStatus(store) {
+    const items = store.list(), kinds = {}, digests = { accepted: 0, rejected: 0, draft: 0, none: 0 };
+    for (const item of items) {
+        kinds[item.kind] = (kinds[item.kind] ?? 0) + 1;
+        const gist = store.derivations.current('digest', item.itemId)?.review.gist;
+        digests[gist === undefined ? 'none' : gist === 'accepted' || gist === 'rejected' ? gist : 'draft']++;
+    }
+    // Settled jobs are history; what is left is waiting, running or needs the reader.
+    // Only each target's newest job counts: a failure that a later capture of the item (or index job) fixed is history.
+    const newest = new Map();
+    for (const job of [...store.queue.list()].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at))))
+        newest.set(job.operation === 'index' ? 'index' : `${job.operation} ${job.item_id}`, job);
+    const jobs = [...newest.values()]
+        .filter((job) => !['succeeded', 'cancelled'].includes(String(job.state)))
+        .map((job) => ({
+        operation: String(job.operation),
+        state: String(job.state),
+        reason: job.reason ?? null,
+    }));
+    const { generation } = store.index.overview();
+    return {
+        items: { total: items.length, ...kinds },
+        index: generation
+            ? { embedded: generation.completed, expected: generation.expected, pending: generation.pending }
+            : null,
+        digests: { ...digests, auto: store.digests.settings().auto },
+        jobs,
+        pendingCleanup: store.queue.pendingCleanup(),
+    };
+}
+export function statusText(s, worker) {
+    const { total, ...kinds } = s.items, names = {
+        article: ['article', 'articles'],
+        pdf: ['PDF', 'PDFs'],
+        media: ['media item', 'media items'],
+    };
+    const counts = {};
+    for (const job of s.jobs) {
+        const key = `${job.operation} ${job.state}${job.reason ? ` (${job.reason})` : ''}`;
+        counts[key] = (counts[key] ?? 0) + 1;
+    }
+    const failed = s.jobs.some((job) => !['queued', 'running'].includes(job.state));
+    const d = s.digests, drafted = d.accepted + d.rejected + d.draft;
+    return [
+        `Items: ${total}${total
+            ? ` (${Object.entries(kinds)
+                .map(([kind, n]) => plural(n, ...(names[kind] ?? [kind, kind])))
+                .join(', ')})`
+            : ''}`,
+        `Search index: ${s.index ? `${s.index.embedded} of ${s.index.expected} chunks and images embedded${s.index.pending ? `; run sponge index run or let the worker` : ''}` : 'none; run sponge setup --embedding, then sponge index create'}`,
+        `Digests: ${drafted} of ${total} items (${d.accepted} accepted, ${d.rejected} rejected, ${d.draft} unreviewed); auto-digest ${d.auto ? 'on' : 'off'}`,
+        `Queue: ${s.jobs.length
+            ? Object.entries(counts)
+                .map(([key, n]) => `${n} ${key}`)
+                .join(', ')
+            : 'nothing waiting'}`,
+        ...(failed ? ['  See sponge queue status; sponge queue retry JOB once the cause is fixed.'] : []),
+        ...(s.pendingCleanup ? [`Pending cleanup: ${s.pendingCleanup}; run sponge queue recover`] : []),
+        `Worker: ${worker}`,
+    ].join('\n');
+}

package/dist/src/importers/apply.js

@@ -3,6 +3,7 @@ import { runAttempt } from '../ingest/attempt.js';
 import { bundleBytes, bundleText } from '../app/bundle-files.js';
 import { localizeMarkdown } from '../app/markdown.js';
 import { selectedMetadata } from '../metadata/article.js';
+import { pdfTitle } from '../pdf/title.js';
 import { drainQueue } from '../worker/engine.js';
 import { readImportManifest, importBundleRoot } from './manifest.js';
 import { inspectImportBundle, ownedBundleVerifier } from './inspect.js';
@@ -40,7 +41,7 @@ function promote(context, bundle, namespace, entry, sourceHash) {
         if (signal.aborted)
             throw new SpongeError('cancelled');
         await checkpoint('promoting');
-        const title = selectedMetadata(bundle.metadata, 'title')?.value;
+        const title = bundle.kind === 'pdf' ? pdfTitle(bundle.metadata, markdown) : selectedMetadata(bundle.metadata, 'title')?.value;
         store.revisions.commit({
             ...attempt,
             itemId: source.itemId,

package/dist/src/pdf/handler.d.ts

@@ -1,4 +1,4 @@
-import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
+import type { CaptureResult, ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
 import { type CaptureConnection } from '../acquisition/index.js';
 import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
@@ -21,4 +21,25 @@ export interface PdfOptions {
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestPdf(store: LibraryStore, token: AttemptToken, options: PdfOptions): Promise<PdfOutcome>;
+/**
+ * A verified PDF capture of `sourceUrl` in its stored form: blobs put (the source PDF among them), page markers as
+ * headings, images localized. The caller commits the revision.
+ */
+export declare function storePdfCapture(store: LibraryStore, token: AttemptToken, result: {
+    capture: CaptureResult;
+    bundleRoot: string;
+}, sourceUrl: string, signal?: AbortSignal): Promise<{
+    capture: CaptureResult;
+    markdown: string;
+    assets: {
+        mediaType: string;
+        relationship: string;
+        sha256: string;
+        bytes: number;
+        relativePath: string;
+    }[];
+    title: string | undefined;
+    status: "partial" | "complete";
+    warnings: string[];
+}>;
 export {};

package/dist/src/pdf/handler.js

@@ -4,9 +4,8 @@ import { capturePdf } from './capture.js';
 import { runAttempt, classifyErrors } from '../ingest/attempt.js';
 import { sha256 } from '../util/hash.js';
 import { bundleBytes, bundleText } from '../app/bundle-files.js';
-import { selectedMetadata } from '../metadata/article.js';
 import { localizeMarkdown } from '../app/markdown.js';
-import { pdfHeadingTitle } from './title.js';
+import { pdfTitle } from './title.js';
 import { SpongeError } from '../errors.js';
 const classify = classifyErrors([
     'integrity',
@@ -38,46 +37,58 @@ export function ingestPdf(store, token, options) {
         }
         attempt.hold(result);
         await options.checkpoint?.('extracting');
-        const capture = result.capture;
-        validateCapture(capture);
-        if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
-            throw new SpongeError('integrity');
-        const original = await bundleText(result.bundleRoot, capture.markdown);
-        const allAssets = [...capture.assets, capture.sourcePdf];
-        const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
-        const assets = [];
-        for (const asset of allAssets) {
-            if (options.signal?.aborted)
-                throw new SpongeError('cancelled');
-            const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
-            assets.push({
-                ...blob,
-                mediaType: asset.mediaType,
-                relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image',
-            });
-        }
-        if (options.signal?.aborted) {
-            store.queue.cancel(token.jobId);
-            throw new SpongeError('cancelled');
-        }
-        const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
-        const title = typeof selectedTitle === 'string' ? selectedTitle : pdfHeadingTitle(original);
-        const status = localized.missingImages ? 'partial' : capture.status;
+        const pdf = await storePdfCapture(store, token, result, source.locator, options.signal);
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
-        const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256]));
+        const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', pdf.capture.manifestSha256]));
         await options.checkpoint?.('promoting');
         const revision = store.revisions.commit({
             ...token,
             itemId: source.itemId,
             bundleHash,
-            markdown: localized.markdown,
-            title,
-            extractor: capture.extractor,
-            status,
-            metadata: capture.metadata,
-            assets,
-            warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
+            markdown: pdf.markdown,
+            title: pdf.title,
+            extractor: pdf.capture.extractor,
+            status: pdf.status,
+            metadata: pdf.capture.metadata,
+            assets: pdf.assets,
+            warnings: pdf.warnings,
         });
-        return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
+        return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status: pdf.status };
     });
 }
+/**
+ * A verified PDF capture of `sourceUrl` in its stored form: blobs put (the source PDF among them), page markers as
+ * headings, images localized. The caller commits the revision.
+ */
+export async function storePdfCapture(store, token, result, sourceUrl, signal) {
+    const { capture } = result;
+    validateCapture(capture);
+    if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== sourceUrl)
+        throw new SpongeError('integrity');
+    const original = await bundleText(result.bundleRoot, capture.markdown);
+    const allAssets = [...capture.assets, capture.sourcePdf];
+    const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
+    const assets = [];
+    for (const asset of allAssets) {
+        if (signal?.aborted)
+            throw new SpongeError('cancelled');
+        const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
+        assets.push({
+            ...blob,
+            mediaType: asset.mediaType,
+            relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image',
+        });
+    }
+    if (signal?.aborted) {
+        store.queue.cancel(token.jobId);
+        throw new SpongeError('cancelled');
+    }
+    return {
+        capture,
+        markdown: localized.markdown,
+        assets,
+        title: pdfTitle(capture.metadata, original),
+        status: localized.missingImages ? 'partial' : capture.status,
+        warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
+    };
+}

package/dist/src/pdf/title.d.ts

@@ -1,5 +1,8 @@
+import type { MetadataClaim } from '../../contracts/index.js';
 /**
  * The first heading Wordcell detected on page 1, for a PDF whose metadata has no title.
  * Wordcell writes body headings at level 2 or deeper and escapes Markdown punctuation in them.
  */
 export declare function pdfHeadingTitle(markdown: string): string | undefined;
+/** A PDF's captured title: its metadata title, else the first page-1 heading, else the file name its bundle records. */
+export declare function pdfTitle(metadata: readonly MetadataClaim[], markdown: string): string | undefined;

package/dist/src/pdf/title.js

@@ -1,3 +1,4 @@
+import { selectedMetadata } from '../metadata/article.js';
 // Section names that open many papers but never name one.
 const generic = /^(?:abstract|contents|table of contents|introduction|preface|foreword|acknowledge?ments?|summary|executive summary|references)$/i;
 /**
@@ -13,11 +14,21 @@ export function pdfHeadingTitle(markdown) {
         .replace(/\\([\\`*_[\]{}<>#|])/g, '$1')
         .replace(/\s+/g, ' ')
         .trim());
-    const lowercase = (text = '') => /^\p{Ll}/u.test(text);
-    for (const [index, title] of headings.entries()) {
-        // Text Wordcell read from an image, and prose wrapped across headings such as a copyright notice.
-        if (/^Text visible in (?:.+ )?image$/.test(title) || lowercase(title) || lowercase(headings[index + 1]))
+    // A lowercase first word with no capital in it: rStar and iPhone open titles, "with" and "reproduce" open prose.
+    const lowercase = (text = '') => /^\p{Ll}/u.test(text) && !/\p{Lu}/u.test(text.split(' ')[0]);
+    for (let index = 0; index < headings.length; index++) {
+        let title = headings[index];
+        // Text Wordcell read from an image.
+        if (/^Text visible in (?:.+ )?image$/.test(title) || lowercase(title))
             continue;
+        if (lowercase(headings[index + 1])) {
+            // Prose wrapped across headings, such as a copyright notice, breaks after a function word or a comma.
+            if (/(?:,|\b(?:a|an|and|by|for|from|in|of|on|or|the|to|with))$/i.test(title))
+                continue;
+            // A title wrapped across two headings continues in lowercase.
+            while (lowercase(headings[index + 1]))
+                title += ' ' + headings[++index];
+        }
         if (title.length < 3 || title.length > 300)
             continue;
         // Body sections have begun, so the title was never detected as a heading.
@@ -26,3 +37,21 @@ export function pdfHeadingTitle(markdown) {
         return title;
     }
 }
+/** The bundle's own frontmatter title: Sponge and Wordcell both write the source file name there. */
+function frontmatterTitle(markdown) {
+    const quoted = /^---\n(?:.*\n)*?title: (".*")\n(?:.*\n)*?---\n/.exec(markdown)?.[1];
+    try {
+        const title = quoted === undefined ? undefined : JSON.parse(quoted);
+        return typeof title === 'string' && title.trim() ? title.trim() : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** A PDF's captured title: its metadata title, else the first page-1 heading, else the file name its bundle records. */
+export function pdfTitle(metadata, markdown) {
+    const selected = selectedMetadata(metadata, 'title')?.value;
+    if (typeof selected === 'string' && selected.trim())
+        return selected;
+    return pdfHeadingTitle(markdown) ?? frontmatterTitle(markdown);
+}

package/dist/src/search/hybrid.js

@@ -1,14 +1,14 @@
 import { chunkDocument } from '../indexing/index.js';
-import { lexicalSearch } from './lexical.js';
+import { anyWordMatch, lexicalChannel, phraseMatch } from './lexical.js';
 import { pinSearchGeneration, vectorSearch } from './vector.js';
 import { SpongeError } from '../errors.js';
-/** Reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. */
-function fuse(lexicalHits, semanticHits, limit) {
+/**
+ * Weighted reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. The any-word channel
+ * counts a quarter: at full weight its loose matches outranked exact phrases and close meanings (scripts/eval/search.ts).
+ */
+function fuse(channels, limit) {
     const hits = new Map();
-    for (const [method, rows] of [
-        ['lexical', lexicalHits],
-        ['semantic', semanticHits],
-    ]) {
+    for (const { method, rows, weight } of channels) {
         const seen = new Set();
         let rank = 0;
         for (const row of rows) {
@@ -26,8 +26,9 @@ function fuse(lexicalHits, semanticHits, limit) {
                 }
                 : { kind: 'document' };
             if (previous) {
-                previous.score += 1 / (60 + rank);
-                previous.methods.push(method);
+                previous.score += weight / (60 + rank);
+                if (!previous.methods.includes(method))
+                    previous.methods.push(method);
                 if (vector)
                     previous.anchor = anchor;
             }
@@ -36,7 +37,7 @@ function fuse(lexicalHits, semanticHits, limit) {
                     itemId: row.itemId,
                     revisionId: row.revisionId,
                     title: row.title,
-                    score: 1 / (60 + rank),
+                    score: weight / (60 + rank),
                     methods: [method],
                     anchor,
                 });
@@ -84,7 +85,7 @@ function projectAnchors(db, hits) {
     }
 }
 export async function hybridSearch(db, query, options = {}) {
-    const limit = options.limit ?? 20, timeout = options.semanticTimeoutMs ?? 10000;
+    const limit = options.limit ?? 20, depth = Math.min(500, limit * 5), timeout = options.semanticTimeoutMs ?? 10000;
     if (!query.trim() ||
         Buffer.byteLength(query) > 4096 ||
         !Number.isSafeInteger(limit) ||
@@ -130,14 +131,18 @@ export async function hybridSearch(db, query, options = {}) {
         let semantic = [];
         if (pin && encoded) {
             try {
-                semantic = vectorSearch(db, pin, encoded, Math.min(500, limit * 5));
+                semantic = vectorSearch(db, pin, encoded, depth);
             }
             catch {
                 warnings.push('semantic-unavailable');
             }
         }
         const result = {
-            hits: fuse(lexicalSearch(db, query, Math.min(500, limit * 5)), semantic, limit),
+            hits: fuse([
+                { method: 'lexical', rows: lexicalChannel(db, phraseMatch(query), depth), weight: 1 },
+                { method: 'lexical', rows: lexicalChannel(db, anyWordMatch(query), depth), weight: 0.25 },
+                { method: 'semantic', rows: semantic, weight: 1 },
+            ], limit),
             warnings,
             generation: pin,
         };

package/dist/src/search/lexical.d.ts

@@ -7,5 +7,14 @@ export interface LexicalHit {
 /** Contentless FTS rows are keyed by readable_fts_rows.id; items.rowid is not VACUUM-stable. Callers own the transaction. */
 export declare function indexReadable(db: DatabaseSync, itemId: string, revisionId: string, title: string, markdown: string): void;
 export declare function unindexReadable(db: DatabaseSync, itemId: string): void;
-/** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
+/** The query as one literal phrase; surrounding double quotes are the user's way of asking for exactly that. */
+export declare function phraseMatch(query: string): string;
+/** Any content word of the query; undefined for a quoted query or one of function words only. */
+export declare function anyWordMatch(query: string): string | undefined;
+/** bm25 with title matches weighted five times body, then item id; excludes tombstones and superseded revisions. */
+export declare function lexicalChannel(db: DatabaseSync, expression: string | undefined, limit: number): LexicalHit[];
+/**
+ * Items with the exact phrase first, then items with any of its content words. The phrase alone missed most vague
+ * queries, and any word alone buried the precise matches (scripts/eval/search.ts).
+ */
 export declare function lexicalSearch(db: DatabaseSync, query: string, limit: number): LexicalHit[];

package/dist/src/search/lexical.js

@@ -11,10 +11,39 @@ export function unindexReadable(db, itemId) {
     if (row)
         db.prepare('DELETE FROM readable_fts WHERE rowid=?').run(row.id);
 }
-/** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
-export function lexicalSearch(db, query, limit) {
-    const literal = '"' + query.replaceAll('"', '""') + '"';
+// Function words match nearly every item, so they never decide whether one matches.
+const stopwords = new Set('a an and are as at be but by can do does for from had has have how i if in into is it its me my no not of on or our so than that the their them then there these they this to too us was we were what when where which who why will with you your'.split(' '));
+const literal = (text) => '"' + text.replaceAll('"', '""') + '"';
+/** The query as one literal phrase; surrounding double quotes are the user's way of asking for exactly that. */
+export function phraseMatch(query) {
+    const trimmed = query.trim();
+    return literal(/^"[^"]+"$/.test(trimmed) ? trimmed.slice(1, -1) : trimmed);
+}
+/** Any content word of the query; undefined for a quoted query or one of function words only. */
+export function anyWordMatch(query) {
+    const trimmed = query.trim();
+    if (/^"[^"]+"$/.test(trimmed))
+        return undefined;
+    const words = [...new Set(trimmed.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])].filter((w) => !stopwords.has(w));
+    return words.length ? words.map(literal).join(' OR ') : undefined;
+}
+/** bm25 with title matches weighted five times body, then item id; excludes tombstones and superseded revisions. */
+export function lexicalChannel(db, expression, limit) {
+    if (expression === undefined)
+        return [];
     return db
-        .prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY rank,i.id LIMIT ?')
-        .all(literal, limit);
+        .prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY bm25(readable_fts,5.0,1.0),i.id LIMIT ?')
+        .all(expression, limit);
+}
+/**
+ * Items with the exact phrase first, then items with any of its content words. The phrase alone missed most vague
+ * queries, and any word alone buried the precise matches (scripts/eval/search.ts).
+ */
+export function lexicalSearch(db, query, limit) {
+    const hits = new Map();
+    for (const expression of [phraseMatch(query), anyWordMatch(query)])
+        for (const hit of lexicalChannel(db, expression, limit))
+            if (!hits.has(hit.itemId))
+                hits.set(hit.itemId, hit);
+    return [...hits.values()].slice(0, limit);
 }

package/native/runtime-manifest.json

@@ -33,16 +33,16 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "17934fedff0c472a1a11f93849b20ec142334a25e035d5dfad40c9151b72c9eb",
+    "asrSourceSha256": "a094bd8c3363c466f2c17c8822f6780d2db04f2a6ebcacebf9a0b4d5ca3cf940",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "ca8d8bc22b03a3ca65a8ff4fb43f88d3d0cb3bd93591fe5842fd0106601aa6f1",
-    "mediaSourceSha256": "dc775291679fd69e554e28ee4cbab457f4e79080fb58542d92a83bc0004cac82",
+    "embeddingSourceSha256": "c3340bff6e1c2d3e7733c6e2491e272202644d138f01ac528a7679909ce26e3b",
+    "mediaSourceSha256": "297f87728d318d62852a6f0f8f8b20d4fa65ffb0486761fd22e48728e7ae84b2",
     "rustcVersion": "rustc 1.97.1 (8bab26f4f 2026-07-14)",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.6.1",
+  "packageVersion": "0.6.3",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "e5785b1c0df60222bf5be92d24ecefeab921712d"
+  "sourceCommit": "f0db55223b258afe0b29cca01e489a70d10bca82"
 }