@hraness/sponge 0.6.1 → 0.6.2

package/bin/sponge.js

@@ -5,6 +5,10 @@ if (major < 24 || (major === 24 && minor < 20)) {
   process.stderr.write('Sponge requires Node.js 24.20 or newer.\n');
   process.exitCode = 2;
 } else {
+  // A reader that closes early (`sponge show ITEM | head`) ends the output, not the command: it still finishes and closes the library.
+  process.stdout.on('error', (error) => {
+    if (error.code !== 'EPIPE') throw error;
+  });
   const { version } = createRequire(import.meta.url)('../package.json');
   const { main } = await import('../dist/src/cli/main.js');
   process.exitCode = await main(process.argv.slice(2), version);

package/dist/src/importers/apply.js

@@ -3,6 +3,7 @@ import { runAttempt } from '../ingest/attempt.js';
 import { bundleBytes, bundleText } from '../app/bundle-files.js';
 import { localizeMarkdown } from '../app/markdown.js';
 import { selectedMetadata } from '../metadata/article.js';
+import { pdfTitle } from '../pdf/title.js';
 import { drainQueue } from '../worker/engine.js';
 import { readImportManifest, importBundleRoot } from './manifest.js';
 import { inspectImportBundle, ownedBundleVerifier } from './inspect.js';
@@ -40,7 +41,7 @@ function promote(context, bundle, namespace, entry, sourceHash) {
         if (signal.aborted)
             throw new SpongeError('cancelled');
         await checkpoint('promoting');
-        const title = selectedMetadata(bundle.metadata, 'title')?.value;
+        const title = bundle.kind === 'pdf' ? pdfTitle(bundle.metadata, markdown) : selectedMetadata(bundle.metadata, 'title')?.value;
         store.revisions.commit({
             ...attempt,
             itemId: source.itemId,

package/dist/src/pdf/handler.js

@@ -4,9 +4,8 @@ import { capturePdf } from './capture.js';
 import { runAttempt, classifyErrors } from '../ingest/attempt.js';
 import { sha256 } from '../util/hash.js';
 import { bundleBytes, bundleText } from '../app/bundle-files.js';
-import { selectedMetadata } from '../metadata/article.js';
 import { localizeMarkdown } from '../app/markdown.js';
-import { pdfHeadingTitle } from './title.js';
+import { pdfTitle } from './title.js';
 import { SpongeError } from '../errors.js';
 const classify = classifyErrors([
     'integrity',
@@ -60,8 +59,7 @@ export function ingestPdf(store, token, options) {
             store.queue.cancel(token.jobId);
             throw new SpongeError('cancelled');
         }
-        const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
-        const title = typeof selectedTitle === 'string' ? selectedTitle : pdfHeadingTitle(original);
+        const title = pdfTitle(capture.metadata, original);
         const status = localized.missingImages ? 'partial' : capture.status;
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
         const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256]));

package/dist/src/pdf/title.d.ts

@@ -1,5 +1,8 @@
+import type { MetadataClaim } from '../../contracts/index.js';
 /**
  * The first heading Wordcell detected on page 1, for a PDF whose metadata has no title.
  * Wordcell writes body headings at level 2 or deeper and escapes Markdown punctuation in them.
  */
 export declare function pdfHeadingTitle(markdown: string): string | undefined;
+/** A PDF's captured title: its metadata title, else the first page-1 heading, else the file name its bundle records. */
+export declare function pdfTitle(metadata: readonly MetadataClaim[], markdown: string): string | undefined;

package/dist/src/pdf/title.js

@@ -1,3 +1,4 @@
+import { selectedMetadata } from '../metadata/article.js';
 // Section names that open many papers but never name one.
 const generic = /^(?:abstract|contents|table of contents|introduction|preface|foreword|acknowledge?ments?|summary|executive summary|references)$/i;
 /**
@@ -26,3 +27,21 @@ export function pdfHeadingTitle(markdown) {
         return title;
     }
 }
+/** The bundle's own frontmatter title: Sponge and Wordcell both write the source file name there. */
+function frontmatterTitle(markdown) {
+    const quoted = /^---\n(?:.*\n)*?title: (".*")\n(?:.*\n)*?---\n/.exec(markdown)?.[1];
+    try {
+        const title = quoted === undefined ? undefined : JSON.parse(quoted);
+        return typeof title === 'string' && title.trim() ? title.trim() : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** A PDF's captured title: its metadata title, else the first page-1 heading, else the file name its bundle records. */
+export function pdfTitle(metadata, markdown) {
+    const selected = selectedMetadata(metadata, 'title')?.value;
+    if (typeof selected === 'string' && selected.trim())
+        return selected;
+    return pdfHeadingTitle(markdown) ?? frontmatterTitle(markdown);
+}

package/dist/src/search/hybrid.js

@@ -1,14 +1,14 @@
 import { chunkDocument } from '../indexing/index.js';
-import { lexicalSearch } from './lexical.js';
+import { anyWordMatch, lexicalChannel, phraseMatch } from './lexical.js';
 import { pinSearchGeneration, vectorSearch } from './vector.js';
 import { SpongeError } from '../errors.js';
-/** Reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. */
-function fuse(lexicalHits, semanticHits, limit) {
+/**
+ * Weighted reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. The any-word channel
+ * counts a quarter: at full weight its loose matches outranked exact phrases and close meanings (scripts/eval/search.ts).
+ */
+function fuse(channels, limit) {
     const hits = new Map();
-    for (const [method, rows] of [
-        ['lexical', lexicalHits],
-        ['semantic', semanticHits],
-    ]) {
+    for (const { method, rows, weight } of channels) {
         const seen = new Set();
         let rank = 0;
         for (const row of rows) {
@@ -26,8 +26,9 @@ function fuse(lexicalHits, semanticHits, limit) {
                 }
                 : { kind: 'document' };
             if (previous) {
-                previous.score += 1 / (60 + rank);
-                previous.methods.push(method);
+                previous.score += weight / (60 + rank);
+                if (!previous.methods.includes(method))
+                    previous.methods.push(method);
                 if (vector)
                     previous.anchor = anchor;
             }
@@ -36,7 +37,7 @@ function fuse(lexicalHits, semanticHits, limit) {
                     itemId: row.itemId,
                     revisionId: row.revisionId,
                     title: row.title,
-                    score: 1 / (60 + rank),
+                    score: weight / (60 + rank),
                     methods: [method],
                     anchor,
                 });
@@ -84,7 +85,7 @@ function projectAnchors(db, hits) {
     }
 }
 export async function hybridSearch(db, query, options = {}) {
-    const limit = options.limit ?? 20, timeout = options.semanticTimeoutMs ?? 10000;
+    const limit = options.limit ?? 20, depth = Math.min(500, limit * 5), timeout = options.semanticTimeoutMs ?? 10000;
     if (!query.trim() ||
         Buffer.byteLength(query) > 4096 ||
         !Number.isSafeInteger(limit) ||
@@ -130,14 +131,18 @@ export async function hybridSearch(db, query, options = {}) {
         let semantic = [];
         if (pin && encoded) {
             try {
-                semantic = vectorSearch(db, pin, encoded, Math.min(500, limit * 5));
+                semantic = vectorSearch(db, pin, encoded, depth);
             }
             catch {
                 warnings.push('semantic-unavailable');
             }
         }
         const result = {
-            hits: fuse(lexicalSearch(db, query, Math.min(500, limit * 5)), semantic, limit),
+            hits: fuse([
+                { method: 'lexical', rows: lexicalChannel(db, phraseMatch(query), depth), weight: 1 },
+                { method: 'lexical', rows: lexicalChannel(db, anyWordMatch(query), depth), weight: 0.25 },
+                { method: 'semantic', rows: semantic, weight: 1 },
+            ], limit),
             warnings,
             generation: pin,
         };

package/dist/src/search/lexical.d.ts

@@ -7,5 +7,14 @@ export interface LexicalHit {
 /** Contentless FTS rows are keyed by readable_fts_rows.id; items.rowid is not VACUUM-stable. Callers own the transaction. */
 export declare function indexReadable(db: DatabaseSync, itemId: string, revisionId: string, title: string, markdown: string): void;
 export declare function unindexReadable(db: DatabaseSync, itemId: string): void;
-/** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
+/** The query as one literal phrase; surrounding double quotes are the user's way of asking for exactly that. */
+export declare function phraseMatch(query: string): string;
+/** Any content word of the query; undefined for a quoted query or one of function words only. */
+export declare function anyWordMatch(query: string): string | undefined;
+/** bm25 with title matches weighted five times body, then item id; excludes tombstones and superseded revisions. */
+export declare function lexicalChannel(db: DatabaseSync, expression: string | undefined, limit: number): LexicalHit[];
+/**
+ * Items with the exact phrase first, then items with any of its content words. The phrase alone missed most vague
+ * queries, and any word alone buried the precise matches (scripts/eval/search.ts).
+ */
 export declare function lexicalSearch(db: DatabaseSync, query: string, limit: number): LexicalHit[];

package/dist/src/search/lexical.js

@@ -11,10 +11,39 @@ export function unindexReadable(db, itemId) {
     if (row)
         db.prepare('DELETE FROM readable_fts WHERE rowid=?').run(row.id);
 }
-/** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
-export function lexicalSearch(db, query, limit) {
-    const literal = '"' + query.replaceAll('"', '""') + '"';
+// Function words match nearly every item, so they never decide whether one matches.
+const stopwords = new Set('a an and are as at be but by can do does for from had has have how i if in into is it its me my no not of on or our so than that the their them then there these they this to too us was we were what when where which who why will with you your'.split(' '));
+const literal = (text) => '"' + text.replaceAll('"', '""') + '"';
+/** The query as one literal phrase; surrounding double quotes are the user's way of asking for exactly that. */
+export function phraseMatch(query) {
+    const trimmed = query.trim();
+    return literal(/^"[^"]+"$/.test(trimmed) ? trimmed.slice(1, -1) : trimmed);
+}
+/** Any content word of the query; undefined for a quoted query or one of function words only. */
+export function anyWordMatch(query) {
+    const trimmed = query.trim();
+    if (/^"[^"]+"$/.test(trimmed))
+        return undefined;
+    const words = [...new Set(trimmed.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])].filter((w) => !stopwords.has(w));
+    return words.length ? words.map(literal).join(' OR ') : undefined;
+}
+/** bm25 with title matches weighted five times body, then item id; excludes tombstones and superseded revisions. */
+export function lexicalChannel(db, expression, limit) {
+    if (expression === undefined)
+        return [];
     return db
-        .prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY rank,i.id LIMIT ?')
-        .all(literal, limit);
+        .prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY bm25(readable_fts,5.0,1.0),i.id LIMIT ?')
+        .all(expression, limit);
+}
+/**
+ * Items with the exact phrase first, then items with any of its content words. The phrase alone missed most vague
+ * queries, and any word alone buried the precise matches (scripts/eval/search.ts).
+ */
+export function lexicalSearch(db, query, limit) {
+    const hits = new Map();
+    for (const expression of [phraseMatch(query), anyWordMatch(query)])
+        for (const hit of lexicalChannel(db, expression, limit))
+            if (!hits.has(hit.itemId))
+                hits.set(hit.itemId, hit);
+    return [...hits.values()].slice(0, limit);
 }

package/native/runtime-manifest.json

@@ -33,16 +33,16 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "17934fedff0c472a1a11f93849b20ec142334a25e035d5dfad40c9151b72c9eb",
+    "asrSourceSha256": "a90a44536ce014501d83964253fac06913e5cea442b930efcb9e394de191e39e",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "ca8d8bc22b03a3ca65a8ff4fb43f88d3d0cb3bd93591fe5842fd0106601aa6f1",
-    "mediaSourceSha256": "dc775291679fd69e554e28ee4cbab457f4e79080fb58542d92a83bc0004cac82",
+    "embeddingSourceSha256": "0ecdc89d1e3591febbd1e2eab1acc85af10ddb00fa96077f52a627adff3fa28b",
+    "mediaSourceSha256": "a9cb79f88d235350ce857460b322cc5501839b886dbff2e8af9d55f5700f92eb",
     "rustcVersion": "rustc 1.97.1 (8bab26f4f 2026-07-14)",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.6.1",
+  "packageVersion": "0.6.2",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "e5785b1c0df60222bf5be92d24ecefeab921712d"
+  "sourceCommit": "7dad1c9a3501ac00c361f8247da8d4f655d9952e"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hraness/sponge",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "A local library for captured articles and verified offline assets.",
   "repository": {
     "type": "git",