@stacksjs/ai 0.75.337 → 0.75.341

package/dist/documents.d.ts

@@ -0,0 +1,5 @@
+import type { SearchDocument, SearchResult } from './search';
+/** Split private text before persistence, without a provider request or hidden truncation. */
+export declare function chunkDocument(document: SearchDocument, options?: { size?: number, overlap?: number }): SearchDocument[];
+/** BM25 retrieval over an already authorized corpus. Callers own tenant filtering. */
+export declare function searchDocuments(query: string, documents: SearchDocument[], options?: { topK?: number, maxCharacters?: number }): SearchResult[];

package/dist/documents.js

@@ -0,0 +1 @@
+export function chunkDocument(document,options={}){const{size=1600,overlap=160}=options;if(!Number.isSafeInteger(size)||size<64||!Number.isSafeInteger(overlap)||overlap<0||overlap>=size)throw TypeError("Document size must be at least 64 and overlap must be smaller than size");const chunks=[];for(let start=0;start<document.content.length;){let end=Math.min(start+size,document.content.length);if(end<document.content.length){const boundary=document.content.lastIndexOf(" ",end);if(boundary>start+size/2)end=boundary;const code=document.content.charCodeAt(end-1);if(code>=55296&&code<=56319)end--}const content=document.content.slice(start,end).trim();if(content)chunks.push({id:`${document.id}:${chunks.length}`,content,metadata:{...document.metadata,sourceId:document.id,start,end}});if(end===document.content.length)break;start=Math.max(start+1,end-overlap);const code=document.content.charCodeAt(start);if(code>=56320&&code<=57343)start++}return chunks}function tokens(text){return text.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}]+/gu)??[]}export function searchDocuments(query,documents,options={}){const{topK=5,maxCharacters=12000}=options;if(!Number.isSafeInteger(topK)||topK<1||!Number.isSafeInteger(maxCharacters)||maxCharacters<1)throw TypeError("Retrieval limits must be positive integers");if(!documents.length)return[];const terms=[...new Set(tokens(query))].slice(0,128);if(!terms.length)return[];const corpus=documents.map((document)=>{const words=tokens(document.content),frequencies=new Map;for(const word of words)frequencies.set(word,(frequencies.get(word)??0)+1);return{document,length:words.length,frequencies}}),average=corpus.reduce((sum,entry)=>sum+entry.length,0)/corpus.length||1,counts=new Map(terms.map((term)=>[term,corpus.filter((entry)=>entry.frequencies.has(term)).length])),scored=corpus.map((entry)=>{let score=0;for(const term of terms){const frequency=entry.frequencies.get(term)??0;if(!frequency)continue;const count=counts.get(term),inverse=Math.log(1+(corpus.length-count+0.5)/(count+0.5));score+=inverse*frequency*2.2/(frequency+1.2*(0.25+0.75*entry.length/average))}return{document:entry.document,score}}).filter((entry)=>entry.score>0).sort((a,b)=>b.score-a.score||a.document.id.localeCompare(b.document.id)),results=[];let remaining=maxCharacters;for(const entry of scored){if(entry.document.content.length>remaining)continue;results.push({...entry,rank:results.length+1});remaining-=entry.document.content.length;if(results.length===topK)break}return results}

package/dist/index.d.ts

@@ -21,6 +21,7 @@ export * from './image';
 export * from './speech';
 // Semantic search, embeddings & RAG
 export * from './search';
+export * from './documents';
 // Personalization, sentiment & classification
 export * from './personalization';
 // Model Context Protocol (MCP) client

package/dist/index.js

@@ -1 +1 @@
-export*from"./types";export*from"./drivers";export*from"./client";export*from"./agents";export*from"./buddy";export*from"./context";export*from"./text";export*from"./image";export*from"./speech";export*from"./search";export*from"./personalization";export*from"./mcp";export*from"./mcp-server";export*from"./models";export*from"./utils/client-bedrock";export*from"./utils/client-bedrock-runtime";export{buildMessageWithImages,normalizeMessagesForProvider}from"./utils/vision";export{assistantTurn,toolResultsTurn}from"./utils/tools";export{fetchWithRetry}from"./utils/retry";export{clearUsageReporters,listUsageReporters,onUsage,recordUsage}from"./utils/usage";export{estimateMessageTokens,estimateTokens,sanitizePrompt}from"./utils/tokens";export*from"./utils/model-access";
+export*from"./types";export*from"./drivers";export*from"./client";export*from"./agents";export*from"./buddy";export*from"./context";export*from"./text";export*from"./image";export*from"./speech";export*from"./search";export*from"./documents";export*from"./personalization";export*from"./mcp";export*from"./mcp-server";export*from"./models";export*from"./utils/client-bedrock";export*from"./utils/client-bedrock-runtime";export{buildMessageWithImages,normalizeMessagesForProvider}from"./utils/vision";export{assistantTurn,toolResultsTurn}from"./utils/tools";export{fetchWithRetry}from"./utils/retry";export{clearUsageReporters,listUsageReporters,onUsage,recordUsage}from"./utils/usage";export{estimateMessageTokens,estimateTokens,sanitizePrompt}from"./utils/tokens";export*from"./utils/model-access";

package/package.json

@@ -2,7 +2,7 @@
   "name": "@stacksjs/ai",
   "type": "module",
   "sideEffects": false,
-  "version": "0.75.337",
+  "version": "0.75.341",
   "description": "Stacks Artificial Intelligence.",
   "author": "Chris Breuer",
   "contributors": [
@@ -65,10 +65,10 @@
     "prepublishOnly": "bun run build"
   },
   "dependencies": {
-    "@stacksjs/cli": "0.75.337",
-    "@stacksjs/config": "0.75.337",
-    "@stacksjs/router": "0.75.337",
-    "@stacksjs/storage": "0.75.337",
+    "@stacksjs/cli": "0.75.341",
+    "@stacksjs/config": "0.75.341",
+    "@stacksjs/router": "0.75.341",
+    "@stacksjs/storage": "0.75.341",
     "@stacksjs/ts-cloud": "^0.16.69"
   },
   "peerDependencies": {