@hraness/sponge 0.2.1 → 0.2.2

package/dist/src/indexing/index.d.ts

@@ -23,7 +23,7 @@ export declare class IndexError extends Error {
     code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index';
     constructor(code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index');
 }
-/** Unicode-safe chunks leave room for the fixed document prompt within 8192 UTF-8 bytes. */
+/** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
 export declare function chunkDocument(text: string): string[];
 /** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */
 export declare function createGeneration(db: DatabaseSync, options: {

package/dist/src/indexing/index.js

@@ -1,5 +1,5 @@
 import { createHash, randomUUID } from 'node:crypto';
-import { spaceFingerprint, encodeEmbeddingRequest } from '../inference/embedding/index.js';
+import { spaceFingerprint, encodeEmbeddingRequest, embeddingTextBytes } from '../inference/embedding/index.js';
 export class IndexError extends Error {
     code;
     constructor(code) {
@@ -21,17 +21,25 @@ catch (e) {
 } }
 function epoch(db) { return Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch); }
 function revisions(db) { return db.prepare('SELECT id AS itemId,current_revision AS revisionId FROM items WHERE deleted_at IS NULL AND current_revision IS NOT NULL ORDER BY id').all(); }
-/** Unicode-safe chunks leave room for the fixed document prompt within 8192 UTF-8 bytes. */
-export function chunkDocument(text) { const chunks = []; let part = '', size = 0; for (const char of text) {
+const chunkBytes = embeddingTextBytes - Buffer.byteLength('title: none | text: ');
+/** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
+export function chunkDocument(text) { const chunks = []; let part = '', size = 0, cut = -1, cutSize = 0; for (const char of text) {
     const bytes = Buffer.byteLength(char);
-    if (size + bytes > 7600) {
-        if (part.trim())
-            chunks.push(part);
-        part = '';
-        size = 0;
+    if (size + bytes > chunkBytes) {
+        const at = cut > 0 && cutSize * 2 >= chunkBytes ? cut : part.length;
+        const head = part.slice(0, at);
+        if (head.trim())
+            chunks.push(head);
+        part = part.slice(at);
+        size = Buffer.byteLength(part);
+        cut = -1;
     }
     part += char;
     size += bytes;
+    if (/\s/u.test(char)) {
+        cut = part.length;
+        cutSize = size;
+    }
 } if (part.trim())
     chunks.push(part); return chunks; }
 function snapshot(db) {
@@ -41,7 +49,7 @@ function snapshot(db) {
         for (const doc of docs) {
             const chunks = chunkDocument(String(doc.markdown));
             for (let ordinal = 0; ordinal < chunks.length; ordinal++) {
-                const text = chunks[ordinal], derivationHash = hash(JSON.stringify(['sponge-chunk-v1', String(doc.id), ordinal, text]));
+                const text = chunks[ordinal], derivationHash = hash(JSON.stringify(['sponge-chunk-v2', String(doc.id), ordinal, text]));
                 inputs.push({ ...r, sourceId: JSON.stringify([r.revisionId, 'document', String(doc.id), ordinal]), kind: 'document', representationId: String(doc.id), ordinal, derivationHash });
             }
         }
@@ -73,7 +81,7 @@ export function pendingInputs(db, id, limit = 16) { if (!Number.isSafeInteger(li
     fail('cancelled-index'); if (row.state !== 'building')
     return []; const done = new Set(db.prepare('SELECT source_id FROM vectors WHERE generation_id=? AND input_hash IS NOT NULL').all(id).map(r => String(r.source_id))); return coverage.inputs.filter(i => !done.has(i.sourceId)).slice(0, limit).map(input => { if (input.kind === 'image')
     return input; const doc = db.prepare('SELECT markdown FROM representations WHERE id=? AND revision_id=?').get(input.representationId, input.revisionId); if (!doc || typeof doc.markdown !== 'string')
-    return fail(); const text = chunkDocument(doc.markdown)[input.ordinal]; if (text === undefined || hash(JSON.stringify(['sponge-chunk-v1', input.representationId, input.ordinal, text])) !== input.derivationHash)
+    return fail(); const text = chunkDocument(doc.markdown)[input.ordinal]; if (text === undefined || hash(JSON.stringify(['sponge-chunk-v2', input.representationId, input.ordinal, text])) !== input.derivationHash)
     return fail(); return { ...input, text }; }); }
 export function writeBatch(db, id, fingerprint, batch) {
     if (!batch.length || batch.length > 256)

package/dist/src/inference/embedding/index.d.ts

@@ -9,7 +9,10 @@ export declare const embeddingIdentity: {
     readonly prompt: "search-result-query-title-none-v1";
     readonly protocol: 1;
     readonly backend: "cpu-2threads-f32";
+    readonly textBudget: "utf8-1016-v1";
 };
+/** The model's largest text signature is 1024 tokens; byte-fallback tokenization never yields more tokens than UTF-8 bytes. */
+export declare const embeddingTextBytes = 1016;
 export type EmbeddingInput = {
     kind: 'query' | 'document';
     text: string;

package/dist/src/inference/embedding/index.js

@@ -3,7 +3,9 @@ import { constants } from 'node:fs';
 import { open, mkdtemp, chmod, writeFile, rm, realpath } from 'node:fs/promises';
 import { dirname, join, isAbsolute } from 'node:path';
 import { runOwned } from '../../acquisition/process.js';
-export const embeddingIdentity = { modelSha256: '92dcbea108899e5d6e30d919b0744f90d9967e80c67a4ab5503ac16d54f62eb0', librarySha256: '9526db9ee35e7eb4a5a58ac1e53330032f22a13890bd4ee01818fb380a57a362', runtime: 'LiteRT-LM@0.18.0', dimensions: 768, normalization: 'unit-l2', preprocessing: 'sips-max512-png-litert-v1', prompt: 'search-result-query-title-none-v1', protocol: 1, backend: 'cpu-2threads-f32' };
+export const embeddingIdentity = { modelSha256: '92dcbea108899e5d6e30d919b0744f90d9967e80c67a4ab5503ac16d54f62eb0', librarySha256: '9526db9ee35e7eb4a5a58ac1e53330032f22a13890bd4ee01818fb380a57a362', runtime: 'LiteRT-LM@0.18.0', dimensions: 768, normalization: 'unit-l2', preprocessing: 'sips-max512-png-litert-v1', prompt: 'search-result-query-title-none-v1', protocol: 1, backend: 'cpu-2threads-f32', textBudget: 'utf8-1016-v1' };
+/** The model's largest text signature is 1024 tokens; byte-fallback tokenization never yields more tokens than UTF-8 bytes. */
+export const embeddingTextBytes = 1016;
 export class EmbeddingError extends Error {
     code;
     constructor(code) {
@@ -42,7 +44,7 @@ export function encodeEmbeddingRequest(inputs) {
                 fail();
             payload = Buffer.from((input.kind === 'query' ? 'task: search result | query: ' : 'title: none | text: ') + input.text);
             kind = 1;
-            if (payload.length > 8192)
+            if (payload.length > embeddingTextBytes)
                 fail('limit-exceeded');
         }
         const header = Buffer.alloc(5);

package/native/runtime-manifest.json

@@ -27,14 +27,14 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "ff8c83a1963c3d1e1a63b03279ae4fa9b7c5d914c8ec82115f886a9b23b9eb7d",
+    "asrSourceSha256": "2f8ad0eec7e46e259cba34dd3a37b09a673b553f60a63a37e54c91138db621bc",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "41b8381f87a32b2e9ab2fa24fcde9043463eae2117aa4b7cd758537dbb169d06",
+    "embeddingSourceSha256": "cbeef73ab8e45c4352444ff0c7c74f3850f3a249ca54f86c8ed89461013ecc70",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.2.1",
+  "packageVersion": "0.2.2",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "16156a5bb4b73318da65dbf950d4c3d9f6a4aba9"
+  "sourceCommit": "3e5d95f573c57ed0e3e2a2d2c496d96258bb68dd"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hraness/sponge",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "A local library for captured articles and verified offline assets.",
   "repository": {
     "type": "git",