@hraness/sponge 0.6.2 → 0.6.3

package/dist/src/app/article.d.ts

@@ -1,6 +1,7 @@
 import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
 import { type CaptureRequest, type CaptureOutcome, type CaptureConnection } from '../acquisition/index.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
+import { type PdfCaptureRequest, type CaptureOutcome as PdfCaptureOutcome } from '../pdf/capture.js';
 interface ArticleResult {
     ok: true;
     itemId: string;
@@ -17,6 +18,7 @@ export interface ArticleOptions {
     signal?: AbortSignal;
     allowPrivateNetwork?: boolean;
     capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
+    capturePdf?: (request: PdfCaptureRequest) => Promise<PdfCaptureOutcome>;
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestArticle(store: LibraryStore, token: AttemptToken, options: ArticleOptions): Promise<ArticleOutcome>;

package/dist/src/app/article.js

@@ -7,6 +7,8 @@ import { bundleBytes, bundleText } from './bundle-files.js';
 import { selectedMetadata } from '../metadata/article.js';
 import { localizeMarkdown } from './markdown.js';
 import { SpongeError } from '../errors.js';
+import { capturePdf } from '../pdf/capture.js';
+import { storePdfCapture } from '../pdf/handler.js';
 const classify = classifyErrors([
     'integrity',
     'limit-exceeded',
@@ -41,7 +43,45 @@ export function ingestArticle(store, token, options) {
         validateCapture(capture);
         if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
             throw new SpongeError('integrity');
-        const original = await bundleText(result.bundleRoot, capture.markdown), localized = localizeMarkdown(original, capture.assets);
+        const original = await bundleText(result.bundleRoot, capture.markdown);
+        const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
+        const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
+        const paperUrl = proseWords(original) < landingPageWords ? fullTextPdf(capture.metadata, source.locator) : undefined;
+        if (paperUrl) {
+            const paper = await (options.capturePdf ?? capturePdf)({
+                runProcess: options.runProcess,
+                stagingRoot: join(store.root, 'staging'),
+                sourceUrl: paperUrl,
+                requestId: token.attemptId,
+                signal: options.signal,
+                allowPrivateNetwork: options.allowPrivateNetwork,
+            });
+            if (paper.ok) {
+                attempt.hold({ dispose: () => Promise.all([result.dispose(), paper.dispose()]).then(() => undefined) });
+                const pdf = await storePdfCapture(store, token, paper, paperUrl, options.signal);
+                const pageFields = new Set(capture.metadata.map((claim) => claim.field));
+                const bundleHash = sha256(JSON.stringify(['article-pdf-full-text-v1', capture.manifestSha256, pdf.capture.manifestSha256]));
+                await options.checkpoint?.('promoting');
+                const revision = store.revisions.commit({
+                    ...token,
+                    itemId: source.itemId,
+                    bundleHash,
+                    markdown: pdf.markdown,
+                    title: title ?? pdf.title,
+                    extractor: pdf.capture.extractor,
+                    status: pdf.status,
+                    metadata: [...capture.metadata, ...pdf.capture.metadata.filter((claim) => !pageFields.has(claim.field))],
+                    metadataWarnings: capture.metadataWarnings,
+                    assets: pdf.assets,
+                    warnings: pdf.warnings,
+                });
+                return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status: pdf.status };
+            }
+            // Without the paper, the landing page is still what the reader saved.
+            if (paper.code === 'cleanup-required')
+                attempt.cleanup = 'pending';
+        }
+        const localized = localizeMarkdown(original, capture.assets);
         const assets = [];
         for (const asset of capture.assets) {
             if (options.signal?.aborted)
@@ -53,8 +93,6 @@ export function ingestArticle(store, token, options) {
             store.queue.cancel(token.jobId);
             throw new SpongeError('cancelled');
         }
-        const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
-        const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
         const status = localized.missingImages ? 'partial' : capture.status;
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
         const bundleHash = sha256(JSON.stringify(['article-local-markdown-v2', capture.manifestSha256]));
@@ -75,3 +113,34 @@ export function ingestArticle(store, token, options) {
         return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
     });
 }
+/** Prose words under which a page that names its paper's PDF is only a landing page: a title, authors and abstract. */
+const landingPageWords = 1000;
+function proseWords(markdown) {
+    const body = markdown.startsWith('---\n') ? markdown.slice(Math.max(0, markdown.indexOf('\n---', 4))) : markdown;
+    const prose = body.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1');
+    return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length;
+}
+/**
+ * The full text a scholarly landing page declares in `citation_pdf_url` (arXiv, NBER, OSF and most journals), when
+ * it is served from the page's own site or a subdomain relation of it; other sites are never fetched on its word.
+ */
+function fullTextPdf(metadata, pageUrl) {
+    const page = new URL(pageUrl).hostname;
+    for (const claim of metadata) {
+        if (claim.field !== 'alternate' || !claim.locator?.includes('citation_pdf_url'))
+            continue;
+        for (const value of Array.isArray(claim.value) ? claim.value : [claim.value]) {
+            let url;
+            try {
+                url = new URL(value);
+            }
+            catch {
+                continue;
+            }
+            const host = url.hostname;
+            if (url.protocol === 'https:' && (host === page || host.endsWith('.' + page) || page.endsWith('.' + host)))
+                return url.href;
+        }
+    }
+    return undefined;
+}

package/dist/src/cli/commands.js

@@ -8,6 +8,7 @@ import { queueAction } from './queue.js';
 import { portabilityAction } from './portability.js';
 import { restoreLibrary } from '../portability/backup.js';
 import { workerAction, serveWorker, hasDueAcquisition, workerDoctor } from './worker.js';
+import { libraryStatus, statusText } from './status.js';
 import { nativeSetupPlan, provisionNative, inspectNative, optionalNative } from '../models/runtime.js';
 import { digestModel, digestSetupPlan, inspectDigest, provisionDigest } from '../models/digest.js';
 import { withDigestServer } from '../digest/server.js';
@@ -423,11 +424,44 @@ export const commands = [
                     states[kind] = publicCode(error, 'runtime-missing');
                 }
             }
+            let digest;
+            try {
+                await inspectDigest(c.nativeRoot, store.digests.settings().model ?? undefined);
+                digest = 'ready';
+            }
+            catch (error) {
+                digest = publicCode(error, 'runtime-missing');
+            }
             const cleanup = store.queue.pendingCleanup(), worker = await workerDoctor(store, c.version), workerBroken = !['ready', 'not-installed', 'unsupported'].includes(worker.worker);
-            c.emit({ library: 'ready', capture, pdf, ...states, pendingCleanup: cleanup, worker }, `Library: ready\nCapture: ${capture}\nPDF: ${pdf === 'ready' ? 'ready' : 'Poppler not found (brew install poppler)'}\nASR: ${states.asr}\nEmbedding: ${states.embedding}\nPending cleanup: ${cleanup}\nWorker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`);
+            // Each runtime that is not ready names the command that installs it.
+            const line = (label, state, fix) => `${label}: ${state}${state === 'runtime-missing' ? ` (${fix})` : ''}`;
+            c.emit({ library: 'ready', capture, pdf, ...states, digest, pendingCleanup: cleanup, worker }, [
+                'Library: ready',
+                `Capture: ${capture}`,
+                `PDF: ${pdf === 'ready' ? 'ready' : 'Poppler not found (brew install poppler)'}`,
+                line('Media', states.media, 'sponge setup --capture'),
+                line('Speech-to-text', states.asr, 'sponge setup --asr'),
+                line('Embedding', states.embedding, 'sponge setup --embedding'),
+                line('Digest model', digest, 'sponge setup --digest'),
+                `Pending cleanup: ${cleanup}${cleanup ? ' (sponge queue recover)' : ''}`,
+                `Worker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`,
+            ].join('\n'));
             return cleanup || workerBroken ? 4 : 0;
         },
     },
+    {
+        name: 'status',
+        summary: 'Overview of items, index, digests and queue',
+        usage: [['status', 'Overview of items, index, digests and queue']],
+        positional: 0,
+        flags: {},
+        library: 'open',
+        handler: async (c) => {
+            const status = libraryStatus(c.store), worker = (await workerDoctor(c.store, c.version)).worker;
+            c.emit({ ...status, worker }, statusText(status, worker));
+            return 0;
+        },
+    },
     {
         name: 'export',
         summary: 'Export Markdown and local assets',
@@ -802,7 +836,7 @@ const row = ([synopsis, description]) => !description
 const notes = `Global flags: --library /absolute/path --json --debug (or SPONGE_DEBUG=1: print error causes to stderr)
 Items are public by default only when captured from a public source.
 Use -- before an option-like search query. Capture setup is explicit.
-Digest endpoints must be literal loopback HTTP (127.0.0.1 or [::1]); Sponge never starts a model server.
+Digest endpoints must be literal loopback HTTP (127.0.0.1 or [::1]); --draft starts Sponge's own server there.
 Media: --language en enables local English ASR; --force-local bypasses captions.
 These flags apply to --wait, run --drain, or worker serve.
 worker install --language en saves the English ASR choice for background work.

package/dist/src/cli/status.d.ts

@@ -0,0 +1,26 @@
+import type { LibraryStore } from '../store/index.js';
+/** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
+export declare function libraryStatus(store: LibraryStore): {
+    items: {
+        total: number;
+    };
+    index: {
+        embedded: number;
+        expected: number;
+        pending: number;
+    } | null;
+    digests: {
+        auto: boolean;
+        accepted: number;
+        rejected: number;
+        draft: number;
+        none: number;
+    };
+    jobs: {
+        operation: string;
+        state: string;
+        reason: string | null;
+    }[];
+    pendingCleanup: number;
+};
+export declare function statusText(s: ReturnType<typeof libraryStatus>, worker: string): string;

package/dist/src/cli/status.js

@@ -0,0 +1,63 @@
+const plural = (n, one, many = one + 's') => `${n} ${n === 1 ? one : many}`;
+/** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
+export function libraryStatus(store) {
+    const items = store.list(), kinds = {}, digests = { accepted: 0, rejected: 0, draft: 0, none: 0 };
+    for (const item of items) {
+        kinds[item.kind] = (kinds[item.kind] ?? 0) + 1;
+        const gist = store.derivations.current('digest', item.itemId)?.review.gist;
+        digests[gist === undefined ? 'none' : gist === 'accepted' || gist === 'rejected' ? gist : 'draft']++;
+    }
+    // Settled jobs are history; what is left is waiting, running or needs the reader.
+    // Only each target's newest job counts: a failure that a later capture of the item (or index job) fixed is history.
+    const newest = new Map();
+    for (const job of [...store.queue.list()].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at))))
+        newest.set(job.operation === 'index' ? 'index' : `${job.operation} ${job.item_id}`, job);
+    const jobs = [...newest.values()]
+        .filter((job) => !['succeeded', 'cancelled'].includes(String(job.state)))
+        .map((job) => ({
+        operation: String(job.operation),
+        state: String(job.state),
+        reason: job.reason ?? null,
+    }));
+    const { generation } = store.index.overview();
+    return {
+        items: { total: items.length, ...kinds },
+        index: generation
+            ? { embedded: generation.completed, expected: generation.expected, pending: generation.pending }
+            : null,
+        digests: { ...digests, auto: store.digests.settings().auto },
+        jobs,
+        pendingCleanup: store.queue.pendingCleanup(),
+    };
+}
+export function statusText(s, worker) {
+    const { total, ...kinds } = s.items, names = {
+        article: ['article', 'articles'],
+        pdf: ['PDF', 'PDFs'],
+        media: ['media item', 'media items'],
+    };
+    const counts = {};
+    for (const job of s.jobs) {
+        const key = `${job.operation} ${job.state}${job.reason ? ` (${job.reason})` : ''}`;
+        counts[key] = (counts[key] ?? 0) + 1;
+    }
+    const failed = s.jobs.some((job) => !['queued', 'running'].includes(job.state));
+    const d = s.digests, drafted = d.accepted + d.rejected + d.draft;
+    return [
+        `Items: ${total}${total
+            ? ` (${Object.entries(kinds)
+                .map(([kind, n]) => plural(n, ...(names[kind] ?? [kind, kind])))
+                .join(', ')})`
+            : ''}`,
+        `Search index: ${s.index ? `${s.index.embedded} of ${s.index.expected} chunks and images embedded${s.index.pending ? `; run sponge index run or let the worker` : ''}` : 'none; run sponge setup --embedding, then sponge index create'}`,
+        `Digests: ${drafted} of ${total} items (${d.accepted} accepted, ${d.rejected} rejected, ${d.draft} unreviewed); auto-digest ${d.auto ? 'on' : 'off'}`,
+        `Queue: ${s.jobs.length
+            ? Object.entries(counts)
+                .map(([key, n]) => `${n} ${key}`)
+                .join(', ')
+            : 'nothing waiting'}`,
+        ...(failed ? ['  See sponge queue status; sponge queue retry JOB once the cause is fixed.'] : []),
+        ...(s.pendingCleanup ? [`Pending cleanup: ${s.pendingCleanup}; run sponge queue recover`] : []),
+        `Worker: ${worker}`,
+    ].join('\n');
+}

package/dist/src/pdf/handler.d.ts

@@ -1,4 +1,4 @@
-import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
+import type { CaptureResult, ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
 import { type CaptureConnection } from '../acquisition/index.js';
 import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
@@ -21,4 +21,25 @@ export interface PdfOptions {
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestPdf(store: LibraryStore, token: AttemptToken, options: PdfOptions): Promise<PdfOutcome>;
+/**
+ * A verified PDF capture of `sourceUrl` in its stored form: blobs put (the source PDF among them), page markers as
+ * headings, images localized. The caller commits the revision.
+ */
+export declare function storePdfCapture(store: LibraryStore, token: AttemptToken, result: {
+    capture: CaptureResult;
+    bundleRoot: string;
+}, sourceUrl: string, signal?: AbortSignal): Promise<{
+    capture: CaptureResult;
+    markdown: string;
+    assets: {
+        mediaType: string;
+        relationship: string;
+        sha256: string;
+        bytes: number;
+        relativePath: string;
+    }[];
+    title: string | undefined;
+    status: "partial" | "complete";
+    warnings: string[];
+}>;
 export {};

package/dist/src/pdf/handler.js

@@ -37,45 +37,58 @@ export function ingestPdf(store, token, options) {
         }
         attempt.hold(result);
         await options.checkpoint?.('extracting');
-        const capture = result.capture;
-        validateCapture(capture);
-        if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
-            throw new SpongeError('integrity');
-        const original = await bundleText(result.bundleRoot, capture.markdown);
-        const allAssets = [...capture.assets, capture.sourcePdf];
-        const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
-        const assets = [];
-        for (const asset of allAssets) {
-            if (options.signal?.aborted)
-                throw new SpongeError('cancelled');
-            const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
-            assets.push({
-                ...blob,
-                mediaType: asset.mediaType,
-                relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image',
-            });
-        }
-        if (options.signal?.aborted) {
-            store.queue.cancel(token.jobId);
-            throw new SpongeError('cancelled');
-        }
-        const title = pdfTitle(capture.metadata, original);
-        const status = localized.missingImages ? 'partial' : capture.status;
+        const pdf = await storePdfCapture(store, token, result, source.locator, options.signal);
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
-        const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256]));
+        const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', pdf.capture.manifestSha256]));
         await options.checkpoint?.('promoting');
         const revision = store.revisions.commit({
             ...token,
             itemId: source.itemId,
             bundleHash,
-            markdown: localized.markdown,
-            title,
-            extractor: capture.extractor,
-            status,
-            metadata: capture.metadata,
-            assets,
-            warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
+            markdown: pdf.markdown,
+            title: pdf.title,
+            extractor: pdf.capture.extractor,
+            status: pdf.status,
+            metadata: pdf.capture.metadata,
+            assets: pdf.assets,
+            warnings: pdf.warnings,
         });
-        return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
+        return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status: pdf.status };
     });
 }
+/**
+ * A verified PDF capture of `sourceUrl` in its stored form: blobs put (the source PDF among them), page markers as
+ * headings, images localized. The caller commits the revision.
+ */
+export async function storePdfCapture(store, token, result, sourceUrl, signal) {
+    const { capture } = result;
+    validateCapture(capture);
+    if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== sourceUrl)
+        throw new SpongeError('integrity');
+    const original = await bundleText(result.bundleRoot, capture.markdown);
+    const allAssets = [...capture.assets, capture.sourcePdf];
+    const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
+    const assets = [];
+    for (const asset of allAssets) {
+        if (signal?.aborted)
+            throw new SpongeError('cancelled');
+        const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
+        assets.push({
+            ...blob,
+            mediaType: asset.mediaType,
+            relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image',
+        });
+    }
+    if (signal?.aborted) {
+        store.queue.cancel(token.jobId);
+        throw new SpongeError('cancelled');
+    }
+    return {
+        capture,
+        markdown: localized.markdown,
+        assets,
+        title: pdfTitle(capture.metadata, original),
+        status: localized.missingImages ? 'partial' : capture.status,
+        warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
+    };
+}

package/dist/src/pdf/title.js

@@ -14,11 +14,21 @@ export function pdfHeadingTitle(markdown) {
         .replace(/\\([\\`*_[\]{}<>#|])/g, '$1')
         .replace(/\s+/g, ' ')
         .trim());
-    const lowercase = (text = '') => /^\p{Ll}/u.test(text);
-    for (const [index, title] of headings.entries()) {
-        // Text Wordcell read from an image, and prose wrapped across headings such as a copyright notice.
-        if (/^Text visible in (?:.+ )?image$/.test(title) || lowercase(title) || lowercase(headings[index + 1]))
+    // A lowercase first word with no capital in it: rStar and iPhone open titles, "with" and "reproduce" open prose.
+    const lowercase = (text = '') => /^\p{Ll}/u.test(text) && !/\p{Lu}/u.test(text.split(' ')[0]);
+    for (let index = 0; index < headings.length; index++) {
+        let title = headings[index];
+        // Text Wordcell read from an image.
+        if (/^Text visible in (?:.+ )?image$/.test(title) || lowercase(title))
             continue;
+        if (lowercase(headings[index + 1])) {
+            // Prose wrapped across headings, such as a copyright notice, breaks after a function word or a comma.
+            if (/(?:,|\b(?:a|an|and|by|for|from|in|of|on|or|the|to|with))$/i.test(title))
+                continue;
+            // A title wrapped across two headings continues in lowercase.
+            while (lowercase(headings[index + 1]))
+                title += ' ' + headings[++index];
+        }
         if (title.length < 3 || title.length > 300)
             continue;
         // Body sections have begun, so the title was never detected as a heading.

package/native/runtime-manifest.json

@@ -33,16 +33,16 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "a90a44536ce014501d83964253fac06913e5cea442b930efcb9e394de191e39e",
+    "asrSourceSha256": "a094bd8c3363c466f2c17c8822f6780d2db04f2a6ebcacebf9a0b4d5ca3cf940",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "0ecdc89d1e3591febbd1e2eab1acc85af10ddb00fa96077f52a627adff3fa28b",
-    "mediaSourceSha256": "a9cb79f88d235350ce857460b322cc5501839b886dbff2e8af9d55f5700f92eb",
+    "embeddingSourceSha256": "c3340bff6e1c2d3e7733c6e2491e272202644d138f01ac528a7679909ce26e3b",
+    "mediaSourceSha256": "297f87728d318d62852a6f0f8f8b20d4fa65ffb0486761fd22e48728e7ae84b2",
     "rustcVersion": "rustc 1.97.1 (8bab26f4f 2026-07-14)",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.6.2",
+  "packageVersion": "0.6.3",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "7dad1c9a3501ac00c361f8247da8d4f655d9952e"
+  "sourceCommit": "f0db55223b258afe0b29cca01e489a70d10bca82"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hraness/sponge",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "A local library for captured articles and verified offline assets.",
   "repository": {
     "type": "git",