@staticn0va/wigolo 0.6.5 → 1.0.0

package/dist/research/brief.js

@@ -9,7 +9,7 @@ const MIN_PHRASE_LEN = 4;
 // shape to produce the final report without needing to re-read raw sources.
 export async function buildResearchBrief(question, sources, subQueries, perSourceCharCap, totalSourcesCharCap, queryType = 'general', comparisonEntities = []) {
     const fetched = sources.filter((s) => s.fetched && s.markdown_content.length > 0);
-    // Highlights reuse the FlashRank-or-paragraph scorer so briefs align with
+    // Highlights reuse the ONNX-reranker-or-paragraph scorer so briefs align with
     // whatever format='highlights' produces for single-query searches.
     const searchItems = fetched.map((s) => ({
         title: s.title,

package/dist/research/brief.js.map

@@ -1 +1 @@
-{"version":3,"file":"brief.js","sourceRoot":"","sources":["../../src/research/brief.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,uEAAuE;AACvE,0EAA0E;AAC1E,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,OAAyB,EACzB,UAAoB,EACpB,gBAAwB,EACxB,mBAA2B,EAC3B,YAAuB,SAAS,EAChC,qBAA+B,EAAE;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAElF,0EAA0E;IAC1E,mEAAmE;IACnE,MAAM,WAAW,GAAuB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1D,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,OAAO,EAAE,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACzC,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;QACpC,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;IAEtF,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,eAAe,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,SAAS,KAAK,YAAY,IAAI,kBAAkB,CAAC,MAAM,IAAI,CAAC;QAC7E,CAAC,CAAC,sBAAsB,CAAC,kBAAkB,EAAE,OAAO,CAAC;QACrD,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,MAAM;QACN,UAAU;QACV,YAAY,EAAE,WAAW;QACzB,mBAAmB,EAAE,gBAAgB;QACrC,sBAAsB,EAAE,mBAAmB;QAC3C,QAAQ,EAAE;YACR,QAAQ,EAAE;gBACR,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrC,gBAAgB,EAAE,eAAe;aAClC;YACD,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,IAAI;SACL;QACD,UAAU,EAAE,SAAS;KACtB,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,4DAA4D;AAC5D,SAAS,WAAW,CAAC,UAAoB,EAAE,OAAyB;IAClE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,MAAM,GAAG,OAAO;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AAC7C,CAAC;AAED,4EAA4E;AAC5E,qEAAqE;AACrE,SAAS,gBAAgB,CAAC,OAAyB;IACjD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,yBAAyB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,mBAAmB;YAChD,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,GAAG;YACzD,CAAC,CAAC,KAAK,CAAC;QACV,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAyB;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,yDAAyD;IACzD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEjD,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,OAAO;aAClB,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;aAC5B,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,cAAc,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACxC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAE1B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAC7D,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,SAAS,EAAE,CAAC;QAChD,IAAI,aAAa,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,MAAM;gBACf,cAAc,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE;gBACzC,UAAU,EAAE,aAAa,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAC7E,OAAO,sBAAsB,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAsB;IACpD,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,wEAAwE;QACxE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAClE,IAAI,YAAY,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAElE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,UAAoB,EAAE,OAAyB;IACjE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpF,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,2CAA2C;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE;aAC9B,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;aAC5B,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,cAAc,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEjC,wDAAwD;QACxD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAEtC,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,CAAC,0BAA0B,KAAK,GAAG,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAAkB,EAClB,OAAyB;IAEzB,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAErF,oDAAoD;IACpD,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;QAC5E,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO;QAChE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IAErE,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QAE3C,wCAAwC;QACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,MAAM,WAAW,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,GAAG,KAAK,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC7B,2CAA2C;YAC3C,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;YACtF,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,EAAE,CAAC;YACf,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,yBAAyB,CAAC,QAAgB;IACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS;QAC5B,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,SAAS;QAC5E,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,MAAM,CAAC,IAAc;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS;IAC9D,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACjE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IAC/D,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACjE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IACpE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAChE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CACzC,CAAC,CAAC"}
+{"version":3,"file":"brief.js","sourceRoot":"","sources":["../../src/research/brief.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,uEAAuE;AACvE,0EAA0E;AAC1E,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,OAAyB,EACzB,UAAoB,EACpB,gBAAwB,EACxB,mBAA2B,EAC3B,YAAuB,SAAS,EAChC,qBAA+B,EAAE;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAElF,8EAA8E;IAC9E,mEAAmE;IACnE,MAAM,WAAW,GAAuB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1D,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,OAAO,EAAE,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACzC,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;QACpC,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;IAEtF,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,eAAe,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,SAAS,KAAK,YAAY,IAAI,kBAAkB,CAAC,MAAM,IAAI,CAAC;QAC7E,CAAC,CAAC,sBAAsB,CAAC,kBAAkB,EAAE,OAAO,CAAC;QACrD,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,MAAM;QACN,UAAU;QACV,YAAY,EAAE,WAAW;QACzB,mBAAmB,EAAE,gBAAgB;QACrC,sBAAsB,EAAE,mBAAmB;QAC3C,QAAQ,EAAE;YACR,QAAQ,EAAE;gBACR,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrC,gBAAgB,EAAE,eAAe;aAClC;YACD,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,IAAI;SACL;QACD,UAAU,EAAE,SAAS;KACtB,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,4DAA4D;AAC5D,SAAS,WAAW,CAAC,UAAoB,EAAE,OAAyB;IAClE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,MAAM,GAAG,OAAO;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AAC7C,CAAC;AAED,4EAA4E;AAC5E,qEAAqE;AACrE,SAAS,gBAAgB,CAAC,OAAyB;IACjD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,yBAAyB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,mBAAmB;YAChD,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,GAAG;YACzD,CAAC,CAAC,KAAK,CAAC;QACV,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAyB;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,yDAAyD;IACzD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEjD,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,OAAO;aAClB,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;aAC5B,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,cAAc,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACxC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAE1B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAC7D,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,SAAS,EAAE,CAAC;QAChD,IAAI,aAAa,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,MAAM;gBACf,cAAc,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE;gBACzC,UAAU,EAAE,aAAa,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAC7E,OAAO,sBAAsB,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAsB;IACpD,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,wEAAwE;QACxE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAClE,IAAI,YAAY,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAElE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,UAAoB,EAAE,OAAyB;IACjE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpF,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,2CAA2C;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE;aAC9B,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;aAC5B,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,cAAc,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEjC,wDAAwD;QACxD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAEtC,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,CAAC,0BAA0B,KAAK,GAAG,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAAkB,EAClB,OAAyB;IAEzB,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAErF,oDAAoD;IACpD,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;QAC5E,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO;QAChE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IAErE,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QAE3C,wCAAwC;QACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,MAAM,WAAW,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,GAAG,KAAK,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC7B,2CAA2C;YAC3C,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;YACtF,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,EAAE,CAAC;YACf,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,yBAAyB,CAAC,QAAgB;IACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS;QAC5B,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,SAAS;QAC5E,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,MAAM,CAAC,IAAc;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS;IAC9D,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACjE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IAC/D,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACjE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IACpE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAChE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CACzC,CAAC,CAAC"}

package/dist/search/evidence.d.ts

@@ -0,0 +1,25 @@
+import type { Citation, EvidenceItem, SearchInput, SearchOutput, SearchResultItem, SourceSpan } from '../types.js';
+export interface BuildEvidenceOptions {
+    maxTokensOut?: number;
+    maxItems?: number;
+}
+export declare function buildEvidenceFromMarkdown(query: string, title: string, url: string, markdown: string, opts?: BuildEvidenceOptions): Promise<EvidenceItem[]>;
+export declare function applyAggregateMarkdownBudget<T>(items: T[], getBody: (item: T) => string, setBody: (item: T, body: string) => void, opts: {
+    maxTokensOut?: number;
+    maxChars?: number;
+}): void;
+export declare function applyTokenBudget(items: EvidenceItem[], maxTokensOut: number): EvidenceItem[];
+export declare function stableCitationId(url: string, start: number): string;
+export declare function buildEvidenceItem(input: {
+    title: string;
+    url: string;
+    sectionHeading: string | null;
+    excerpt: string;
+    score: number;
+    sourceSpan: SourceSpan;
+}): EvidenceItem;
+export declare function applyEvidenceDefault(input: SearchInput, output: SearchOutput, results: SearchResultItem[], query: string): Promise<void>;
+export declare function buildCitationsFromEvidence(results: SearchResultItem[], evidence: EvidenceItem[], baseCitations: Citation[]): Citation[];
+export declare function renderCitationsXml(citations: Citation[]): string;
+export declare function escapeXml(s: string): string;
+//# sourceMappingURL=evidence.d.ts.map

package/dist/search/evidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/search/evidence.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EAER,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,UAAU,EACX,MAAM,aAAa,CAAC;AAYrB,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAMD,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,oBAAyB,GAC9B,OAAO,CAAC,YAAY,EAAE,CAAC,CA+CzB;AAMD,wBAAgB,4BAA4B,CAAC,CAAC,EAC5C,KAAK,EAAE,CAAC,EAAE,EACV,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC5B,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,EACxC,IAAI,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACjD,IAAI,CAoBN;AAKD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,YAAY,EAAE,MAAM,GAAG,YAAY,EAAE,CAa5F;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnE;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,UAAU,CAAC;CACxB,GAAG,YAAY,CAUf;AAED,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,gBAAgB,EAAE,EAC3B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC,CAsFf;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,QAAQ,EAAE,YAAY,EAAE,EACxB,aAAa,EAAE,QAAQ,EAAE,GACxB,QAAQ,EAAE,CAkCZ;AAED,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,CAQhE;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAO3C"}

package/dist/search/evidence.js

@@ -0,0 +1,260 @@
+import { createHash } from 'node:crypto';
+import { extractHighlights } from './highlights.js';
+import { countTokens, truncateByTokens } from './tokens.js';
+import { applyOutputBudget } from './truncate.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('search');
+const DEFAULT_MAX_TOKENS_OUT = 4000;
+const MAX_EVIDENCE_PASSAGES = 20;
+const TRUNCATION_MARKER = '[... content truncated]';
+// Build evidence items from a single page's markdown. Used by per-page tools
+// (fetch, crawl pages, find_similar results, agent/research sources). The
+// returned list is already truncated to fit `maxTokensOut` if provided; pass
+// `maxItems` to cap how many highlights are projected.
+export async function buildEvidenceFromMarkdown(query, title, url, markdown, opts = {}) {
+    if (!markdown)
+        return [];
+    const maxItems = opts.maxItems ?? 1;
+    const synthetic = [{
+            title,
+            url,
+            snippet: '',
+            markdown_content: markdown,
+            relevance_score: 1,
+        }];
+    let result;
+    try {
+        result = await extractHighlights(query, synthetic, Math.max(maxItems, 1));
+    }
+    catch (err) {
+        log.debug('buildEvidenceFromMarkdown: extractHighlights failed', { error: String(err) });
+        return [];
+    }
+    const ranked = result.highlights
+        .slice()
+        .sort((a, b) => b.relevance_score - a.relevance_score)
+        .slice(0, maxItems);
+    const out = [];
+    let used = 0;
+    const budget = opts.maxTokensOut;
+    for (const h of ranked) {
+        let excerpt = h.text;
+        if (budget !== undefined) {
+            const remaining = budget - used;
+            if (remaining <= 0)
+                break;
+            excerpt = truncateByTokens(h.text, remaining);
+            if (!excerpt || excerpt.trim() === TRUNCATION_MARKER)
+                break;
+        }
+        const span = h.source_span ?? { start: 0, end: excerpt.length };
+        out.push(buildEvidenceItem({
+            title: h.source_title || title,
+            url: h.source_url || url,
+            sectionHeading: h.section_heading ?? null,
+            excerpt,
+            score: h.relevance_score,
+            sourceSpan: span,
+        }));
+        if (budget !== undefined)
+            used += countTokens(excerpt);
+    }
+    return out;
+}
+// Walk items in order, capping each item's body text against a shared token
+// budget. Bodies past the budget are cleared (set to ''). Used by all
+// multi-item tools (search markdown_content, find_similar, crawl, research,
+// agent) so per-tool max_tokens_out is an aggregate cap, not per-item.
+export function applyAggregateMarkdownBudget(items, getBody, setBody, opts) {
+    const budget = opts.maxTokensOut;
+    let used = 0;
+    for (const item of items) {
+        const body = getBody(item);
+        if (!body)
+            continue;
+        if (budget !== undefined) {
+            const remaining = budget - used;
+            if (remaining <= 0) {
+                setBody(item, '');
+                continue;
+            }
+            const trimmed = applyOutputBudget(body, { maxTokensOut: remaining, maxChars: opts.maxChars });
+            setBody(item, trimmed);
+            used += countTokens(trimmed);
+        }
+        else {
+            const trimmed = applyOutputBudget(body, { maxChars: opts.maxChars });
+            setBody(item, trimmed);
+        }
+    }
+}
+// Apply an aggregate token budget across an already-built list of evidence
+// items, truncating excerpts in order until the budget is exhausted. Items
+// past the budget are dropped.
+export function applyTokenBudget(items, maxTokensOut) {
+    if (maxTokensOut <= 0)
+        return [];
+    const out = [];
+    let used = 0;
+    for (const item of items) {
+        const remaining = maxTokensOut - used;
+        if (remaining <= 0)
+            break;
+        const excerpt = truncateByTokens(item.excerpt, remaining);
+        if (!excerpt)
+            break;
+        out.push({ ...item, excerpt });
+        used += countTokens(excerpt);
+    }
+    return out;
+}
+export function stableCitationId(url, start) {
+    return createHash('sha1').update(`${url}#${start}`).digest('hex').slice(0, 12);
+}
+export function buildEvidenceItem(input) {
+    return {
+        title: input.title,
+        url: input.url,
+        section_heading: input.sectionHeading,
+        excerpt: input.excerpt,
+        score: input.score,
+        citation_id: stableCitationId(input.url, input.sourceSpan.start),
+        source_span: input.sourceSpan,
+    };
+}
+export async function applyEvidenceDefault(input, output, results, query) {
+    if (results.length === 0)
+        return;
+    const includeFullMarkdown = input.include_full_markdown ?? false;
+    const citationFormat = input.citation_format ?? 'numbered';
+    const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
+    let highlightsResult;
+    try {
+        highlightsResult = await extractHighlights(query, results, MAX_EVIDENCE_PASSAGES);
+    }
+    catch (err) {
+        log.debug('evidence extraction failed', { error: String(err) });
+        const msg = 'evidence extraction failed; results returned without highlights';
+        output.warning = output.warning ? `${output.warning}; ${msg}` : msg;
+        highlightsResult = { highlights: [], citations: [], reranker_used: false };
+    }
+    const ranked = highlightsResult.highlights
+        .slice()
+        .sort((a, b) => b.relevance_score - a.relevance_score);
+    // When the caller sets max_tokens_out explicitly, evidence shares the budget
+    // with citations/results metadata. Reserve room for the structural overhead
+    // so the total stringified output stays under the cap.
+    // NOTE: this relies on JSON.stringify dropping `undefined` keys, and on
+    // applyEvidenceDefault running before any post-evidence mutation that grows
+    // the skeleton (e.g. output.warning); reserve overhead first, mutate later.
+    let evidenceBudget = maxTokensOut;
+    if (input.max_tokens_out !== undefined) {
+        const skeleton = { ...output, citations: undefined, evidence: undefined, citations_xml: undefined };
+        const skeletonTokens = countTokens(JSON.stringify(skeleton));
+        const resultsTokens = countTokens(JSON.stringify(results));
+        const overhead = skeletonTokens + resultsTokens;
+        evidenceBudget = Math.max(0, maxTokensOut - overhead);
+    }
+    const evidence = [];
+    let usedTokens = 0;
+    for (const h of ranked) {
+        if (usedTokens >= evidenceBudget)
+            break;
+        const remaining = evidenceBudget - usedTokens;
+        const excerpt = truncateByTokens(h.text, remaining);
+        if (!excerpt)
+            continue;
+        const span = h.source_span ?? { start: 0, end: excerpt.length };
+        const item = buildEvidenceItem({
+            title: h.source_title,
+            url: h.source_url,
+            sectionHeading: h.section_heading ?? null,
+            excerpt,
+            score: h.relevance_score,
+            sourceSpan: span,
+        });
+        evidence.push(item);
+        usedTokens += countTokens(excerpt);
+    }
+    if (evidence.length > 0) {
+        output.evidence = evidence;
+    }
+    const citations = buildCitationsFromEvidence(results, evidence, highlightsResult.citations);
+    if (citationFormat === 'numbered' || citationFormat === 'json') {
+        if (citations.length > 0)
+            output.citations = citations;
+    }
+    else if (citationFormat === 'anthropic_tags') {
+        if (citations.length > 0) {
+            output.citations = citations;
+            output.citations_xml = renderCitationsXml(citations);
+        }
+    }
+    // Terminal mutation: applyEvidenceDefault is the last step before return.
+    if (!includeFullMarkdown) {
+        for (const r of results) {
+            if (r.markdown_content !== undefined)
+                r.markdown_content = undefined;
+        }
+    }
+    else if (input.max_tokens_out !== undefined) {
+        // Aggregate cap across all results in score order — sum of markdown_content
+        // tokens stays under max_tokens_out; bodies past the budget are dropped.
+        applyAggregateMarkdownBudget(results, (r) => (typeof r.markdown_content === 'string' ? r.markdown_content : ''), (r, body) => { r.markdown_content = body; }, { maxTokensOut: input.max_tokens_out });
+    }
+}
+export function buildCitationsFromEvidence(results, evidence, baseCitations) {
+    // Pick the primary citation_id per source: the first evidence item for that URL
+    // (highest score after sort). Sources whose evidence was budget-cut have no
+    // citation_id — consumers can interpret missing id as "source-level citation,
+    // no specific passage."
+    const primaryByUrl = new Map();
+    for (const ev of evidence) {
+        if (!primaryByUrl.has(ev.url))
+            primaryByUrl.set(ev.url, ev.citation_id);
+    }
+    const baseByUrl = new Map();
+    for (const c of baseCitations)
+        baseByUrl.set(c.url, c);
+    const out = [];
+    for (let i = 0; i < results.length; i++) {
+        const r = results[i];
+        const base = baseByUrl.get(r.url);
+        const citation = base
+            ? { ...base }
+            : {
+                index: i + 1,
+                url: r.url,
+                title: r.title,
+                snippet: r.snippet ?? '',
+            };
+        const primary = primaryByUrl.get(r.url);
+        if (primary !== undefined) {
+            citation.citation_id = primary;
+        }
+        else {
+            // No surviving evidence passage for this source — leave citation_id absent.
+            delete citation.citation_id;
+        }
+        out.push(citation);
+    }
+    return out;
+}
+export function renderCitationsXml(citations) {
+    return citations
+        .map((c) => {
+        const id = c.citation_id ?? stableCitationId(c.url, 0);
+        const inner = escapeXml(`${c.title}\n${c.url}\n${c.snippet}`);
+        return `<source id="${id}">${inner}</source>`;
+    })
+        .join('\n');
+}
+export function escapeXml(s) {
+    return s
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+//# sourceMappingURL=evidence.js.map

package/dist/search/evidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../src/search/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAUzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,iBAAiB,GAAG,yBAAyB,CAAC;AAOpD,6EAA6E;AAC7E,0EAA0E;AAC1E,6EAA6E;AAC7E,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,KAAa,EACb,KAAa,EACb,GAAW,EACX,QAAgB,EAChB,OAA6B,EAAE;IAE/B,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;IACpC,MAAM,SAAS,GAAuB,CAAC;YACrC,KAAK;YACL,GAAG;YACH,OAAO,EAAE,EAAE;YACX,gBAAgB,EAAE,QAAQ;YAC1B,eAAe,EAAE,CAAC;SACnB,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qDAAqD,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU;SAC7B,KAAK,EAAE;SACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC;SACrD,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAEtB,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC;QACrB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC;YAChC,IAAI,SAAS,IAAI,CAAC;gBAAE,MAAM;YAC1B,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC9C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,iBAAiB;gBAAE,MAAM;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAChE,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACzB,KAAK,EAAE,CAAC,CAAC,YAAY,IAAI,KAAK;YAC9B,GAAG,EAAE,CAAC,CAAC,UAAU,IAAI,GAAG;YACxB,cAAc,EAAE,CAAC,CAAC,eAAe,IAAI,IAAI;YACzC,OAAO;YACP,KAAK,EAAE,CAAC,CAAC,eAAe;YACxB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC,CAAC;QACJ,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4EAA4E;AAC5E,sEAAsE;AACtE,4EAA4E;AAC5E,uEAAuE;AACvE,MAAM,UAAU,4BAA4B,CAC1C,KAAU,EACV,OAA4B,EAC5B,OAAwC,EACxC,IAAkD;IAElD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;IACjC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC;YAChC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;gBACnB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClB,SAAS;YACX,CAAC;YACD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9F,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACvB,IAAI,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,2EAA2E;AAC3E,+BAA+B;AAC/B,MAAM,UAAU,gBAAgB,CAAC,KAAqB,EAAE,YAAoB;IAC1E,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,YAAY,GAAG,IAAI,CAAC;QACtC,IAAI,SAAS,IAAI,CAAC;YAAE,MAAM;QAC1B,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO;YAAE,MAAM;QACpB,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/B,IAAI,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,KAAa;IACzD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAOjC;IACC,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,eAAe,EAAE,KAAK,CAAC,cAAc;QACrC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,WAAW,EAAE,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;QAChE,WAAW,EAAE,KAAK,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,KAAkB,EAClB,MAAoB,EACpB,OAA2B,EAC3B,KAAa;IAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEjC,MAAM,mBAAmB,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC;IACjE,MAAM,cAAc,GAAmB,KAAK,CAAC,eAAe,IAAI,UAAU,CAAC;IAC3E,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,IAAI,sBAAsB,CAAC;IAEpE,IAAI,gBAAgB,CAAC;IACrB,IAAI,CAAC;QACH,gBAAgB,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,qBAAqB,CAAC,CAAC;IACpF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,iEAAiE,CAAC;QAC9E,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACpE,gBAAgB,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;IAC7E,CAAC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,UAAU;SACvC,KAAK,EAAE;SACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;IAEzD,6EAA6E;IAC7E,4EAA4E;IAC5E,uDAAuD;IACvD,wEAAwE;IACxE,4EAA4E;IAC5E,4EAA4E;IAC5E,IAAI,cAAc,GAAG,YAAY,CAAC;IAClC,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAiB,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;QAClH,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,cAAc,GAAG,aAAa,CAAC;QAChD,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,UAAU,IAAI,cAAc;YAAE,MAAM;QACxC,MAAM,SAAS,GAAG,cAAc,GAAG,UAAU,CAAC;QAC9C,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAChE,MAAM,IAAI,GAAG,iBAAiB,CAAC;YAC7B,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,GAAG,EAAE,CAAC,CAAC,UAAU;YACjB,cAAc,EAAE,CAAC,CAAC,eAAe,IAAI,IAAI;YACzC,OAAO;YACP,KAAK,EAAE,CAAC,CAAC,eAAe;YACxB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QACH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,UAAU,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAE5F,IAAI,cAAc,KAAK,UAAU,IAAI,cAAc,KAAK,MAAM,EAAE,CAAC;QAC/D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IACzD,CAAC;SAAM,IAAI,cAAc,KAAK,gBAAgB,EAAE,CAAC;QAC/C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;YAC7B,MAAM,CAAC,aAAa,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,CAAC,gBAAgB,KAAK,SAAS;gBAAE,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC;QACvE,CAAC;IACH,CAAC;SAAM,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QAC9C,4EAA4E;QAC5E,yEAAyE;QACzE,4BAA4B,CAC1B,OAAO,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,gBAAgB,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,EACzE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,CAAC,EAC3C,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,EAAE,CACvC,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,OAA2B,EAC3B,QAAwB,EACxB,aAAyB;IAEzB,gFAAgF;IAChF,4EAA4E;IAC5E,8EAA8E;IAC9E,wBAAwB;IACxB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC;YAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,aAAa;QAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAa,IAAI;YAC7B,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE;YACb,CAAC,CAAC;gBACE,KAAK,EAAE,CAAC,GAAG,CAAC;gBACZ,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;aACzB,CAAC;QACN,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,QAAQ,CAAC,WAAW,GAAG,OAAO,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,4EAA4E;YAC5E,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC9B,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,SAAqB;IACtD,OAAO,SAAS;SACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,OAAO,eAAe,EAAE,KAAK,KAAK,WAAW,CAAC;IAChD,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,CAAS;IACjC,OAAO,CAAC;SACL,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC"}

package/dist/search/highlights.d.ts

@@ -2,9 +2,18 @@ import type { SearchResultItem, Citation, Highlight } from '../types.js';
 export interface HighlightSynthesisResult {
     highlights: Highlight[];
     citations: Citation[];
-    flashrank_used: boolean;
+    reranker_used: boolean;
 }
-export declare function splitIntoPassages(markdown: string): string[];
+export interface Passage {
+    text: string;
+    charStart: number;
+    charEnd: number;
+}
+export declare function splitIntoPassages(markdown: string): Passage[];
+export interface AnnotatedPassage extends Passage {
+    sectionHeading: string | null;
+}
+export declare function mapPassageHeadings(markdown: string, passages: Passage[]): AnnotatedPassage[];
 export declare function extractHighlights(query: string, results: SearchResultItem[], maxHighlights?: number): Promise<HighlightSynthesisResult>;
 export declare function fallbackHighlights(results: SearchResultItem[], maxHighlights: number): Highlight[];
 //# sourceMappingURL=highlights.d.ts.map

package/dist/search/highlights.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"highlights.d.ts","sourceRoot":"","sources":["../../src/search/highlights.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUzE,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,cAAc,EAAE,OAAO,CAAC;CACzB;AAYD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAW5D;AAKD,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,gBAAgB,EAAE,EAC3B,aAAa,GAAE,MAA+B,GAC7C,OAAO,CAAC,wBAAwB,CAAC,CA+DnC;AAKD,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,aAAa,EAAE,MAAM,GACpB,SAAS,EAAE,CAgBb"}
+{"version":3,"file":"highlights.d.ts","sourceRoot":"","sources":["../../src/search/highlights.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAYzE,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAwBD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,EAAE,CA6B7D;AAOD,MAAM,WAAW,gBAAiB,SAAQ,OAAO;IAC/C,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAKD,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,OAAO,EAAE,GAClB,gBAAgB,EAAE,CAapB;AAKD,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,gBAAgB,EAAE,EAC3B,aAAa,GAAE,MAA+B,GAC7C,OAAO,CAAC,wBAAwB,CAAC,CAmEnC;AAKD,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,aAAa,EAAE,MAAM,GACpB,SAAS,EAAE,CAkCb"}

package/dist/search/highlights.js

@@ -1,28 +1,89 @@
-import { flashRankRerank, isFlashRankAvailable } from './flashrank.js';
+import { onnxRerank } from './reranker/onnx.js';
+import { getConfig } from '../config.js';
 import { createLogger } from '../logger.js';
+import { parseHeadings, lineStartCharOffsets } from '../extraction/markdown.js';
 const log = createLogger('search');
 const MAX_PASSAGE_LENGTH = 500;
 const MIN_PASSAGE_LENGTH = 50;
 const DEFAULT_MAX_HIGHLIGHTS = 10;
-// Split a single source's markdown into candidate passages for scoring.
-// Filters out headings, table rows, code fences, and short fragments so
-// that scored passages are readable prose.
+function shouldKeep(trimmed) {
+    if (trimmed.length < MIN_PASSAGE_LENGTH)
+        return false;
+    if (trimmed.startsWith('#'))
+        return false;
+    if (trimmed.startsWith('|'))
+        return false;
+    if (trimmed.startsWith('```'))
+        return false;
+    if (trimmed.startsWith('- ') && trimmed.length <= 120)
+        return false;
+    return true;
+}
+// Walk the source markdown block-by-block (separated by blank lines) tracking
+// char offsets so each surviving passage carries an accurate {charStart,
+// charEnd} range pointing back into the original markdown.
 export function splitIntoPassages(markdown) {
     if (!markdown)
         return [];
-    return markdown
-        .split(/\n\n+/)
-        .map((p) => p.trim())
-        .filter((p) => p.length >= MIN_PASSAGE_LENGTH)
-        .filter((p) => !p.startsWith('#'))
-        .filter((p) => !p.startsWith('|'))
-        .filter((p) => !p.startsWith('```'))
-        .filter((p) => !p.startsWith('- ') || p.length > 120)
-        .map((p) => (p.length > MAX_PASSAGE_LENGTH ? p.slice(0, MAX_PASSAGE_LENGTH) : p));
+    const out = [];
+    const re = /\n\n+/g;
+    let blockStart = 0;
+    let m;
+    const consider = (rawStart, rawEnd) => {
+        // raw block is markdown.slice(rawStart, rawEnd); compute trimmed range.
+        const raw = markdown.slice(rawStart, rawEnd);
+        if (!raw)
+            return;
+        let leading = 0;
+        while (leading < raw.length && /\s/.test(raw[leading]))
+            leading++;
+        let trailing = raw.length;
+        while (trailing > leading && /\s/.test(raw[trailing - 1]))
+            trailing--;
+        if (trailing <= leading)
+            return;
+        const trimmedStart = rawStart + leading;
+        const trimmedEnd = rawStart + trailing;
+        const trimmed = markdown.slice(trimmedStart, trimmedEnd);
+        if (!shouldKeep(trimmed))
+            return;
+        const text = trimmed.length > MAX_PASSAGE_LENGTH ? trimmed.slice(0, MAX_PASSAGE_LENGTH) : trimmed;
+        const charEnd = trimmedStart + text.length;
+        out.push({ text, charStart: trimmedStart, charEnd });
+    };
+    while ((m = re.exec(markdown)) !== null) {
+        consider(blockStart, m.index);
+        blockStart = m.index + m[0].length;
+    }
+    consider(blockStart, markdown.length);
+    return out;
 }
-// Score passages across all results and return the top N, FlashRank-first
-// with a graceful first-paragraph fallback when the Python binding is
-// unavailable. Each Highlight carries a source_index suitable for citing.
+// Internal helper preserved for callers that only need the text strings.
+function splitIntoPassageStrings(markdown) {
+    return splitIntoPassages(markdown).map((p) => p.text);
+}
+// Annotate each passage with the nearest preceding markdown heading. Uses
+// `parseHeadings` and a char-offset prefix sum so the lookup is O(passages
+// * headings) without re-parsing markdown for every passage.
+export function mapPassageHeadings(markdown, passages) {
+    const lines = markdown.split('\n');
+    const headings = parseHeadings(lines);
+    const offsets = lineStartCharOffsets(lines);
+    const headingOffsets = headings.map((h) => ({ text: h.text, charStart: offsets[h.lineIndex] }));
+    return passages.map((p) => {
+        let nearest = null;
+        for (const h of headingOffsets) {
+            if (h.charStart <= p.charStart)
+                nearest = h.text;
+            else
+                break;
+        }
+        return { ...p, sectionHeading: nearest };
+    });
+}
+// Score passages across all results and return the top N using the in-process
+// ONNX reranker, with a graceful first-paragraph fallback when reranking is
+// disabled or fails. Each Highlight carries a source_index suitable for citing.
 export async function extractHighlights(query, results, maxHighlights = DEFAULT_MAX_HIGHLIGHTS) {
     const citations = [];
     const candidates = [];
@@ -36,66 +97,88 @@ export async function extractHighlights(query, results, maxHighlights = DEFAULT_
         });
         const source = r.markdown_content ?? r.snippet ?? '';
         const passages = splitIntoPassages(source);
-        for (const text of passages) {
+        const annotated = mapPassageHeadings(source, passages);
+        for (const p of annotated) {
             candidates.push({
-                text,
+                text: p.text,
                 sourceIndex: i + 1,
                 sourceUrl: r.url,
                 sourceTitle: r.title,
+                charStart: p.charStart,
+                charEnd: p.charEnd,
+                sectionHeading: p.sectionHeading,
             });
         }
     }
     if (candidates.length === 0) {
-        // No passages survived the min-length filter (common with snippets-only
-        // results). Fall back to snippet-level highlights so host LLMs still get
-        // structured evidence rather than an empty array.
         return {
             highlights: fallbackHighlights(results, maxHighlights),
             citations,
-            flashrank_used: false,
+            reranker_used: false,
         };
     }
-    const available = await isFlashRankAvailable();
-    if (available) {
-        const scored = await flashRankRerank(query, candidates.map((c, idx) => ({ text: c.text, index: idx })));
-        if (scored && scored.length > 0) {
-            const ranked = scored
-                .slice()
-                .sort((a, b) => b.score - a.score)
-                .slice(0, maxHighlights);
-            const highlights = ranked.map((r) => {
-                const cand = candidates[r.index];
-                return {
-                    text: cand.text,
-                    source_index: cand.sourceIndex,
-                    relevance_score: r.score,
-                    source_url: cand.sourceUrl,
-                    source_title: cand.sourceTitle,
-                };
-            });
-            return { highlights, citations, flashrank_used: true };
+    const cfg = getConfig();
+    if (cfg.reranker === 'onnx') {
+        try {
+            const scored = await onnxRerank(query, candidates.map((c) => ({ text: c.text })), { modelId: cfg.rerankerModel });
+            if (scored.length > 0) {
+                const ranked = scored.slice(0, maxHighlights);
+                const highlights = ranked.map((s) => {
+                    const cand = candidates[s.index];
+                    return {
+                        text: cand.text,
+                        source_index: cand.sourceIndex,
+                        relevance_score: s.score,
+                        source_url: cand.sourceUrl,
+                        source_title: cand.sourceTitle,
+                        section_heading: cand.sectionHeading,
+                        source_span: { start: cand.charStart, end: cand.charEnd },
+                    };
+                });
+                return { highlights, citations, reranker_used: true };
+            }
+        }
+        catch (err) {
+            log.debug('onnx reranker failed, using fallback passages', { error: String(err) });
         }
-        log.debug('flashrank returned null, using fallback passages');
     }
-    return { highlights: fallbackHighlights(results, maxHighlights), citations, flashrank_used: false };
+    return { highlights: fallbackHighlights(results, maxHighlights), citations, reranker_used: false };
 }
-// Fallback when FlashRank is unavailable: take the first substantive paragraph
+// Fallback when the ONNX reranker is unavailable: take the first substantive paragraph
 // from each source (ordered by engine relevance). Preserves citation indices
 // so host LLMs can still cite [N] correctly.
 export function fallbackHighlights(results, maxHighlights) {
     const out = [];
     for (let i = 0; i < results.length && out.length < maxHighlights; i++) {
         const r = results[i];
-        const source = r.markdown_content ?? r.snippet ?? '';
-        const firstPara = splitIntoPassages(source)[0] ?? r.snippet ?? '';
-        if (!firstPara)
+        const source = r.markdown_content ?? '';
+        const passages = source ? splitIntoPassages(source) : [];
+        if (passages.length > 0) {
+            const annotated = mapPassageHeadings(source, [passages[0]])[0];
+            const text = annotated.text.slice(0, MAX_PASSAGE_LENGTH);
+            out.push({
+                text,
+                source_index: i + 1,
+                relevance_score: r.relevance_score,
+                source_url: r.url,
+                source_title: r.title,
+                section_heading: annotated.sectionHeading,
+                source_span: { start: annotated.charStart, end: annotated.charStart + text.length },
+            });
+            continue;
+        }
+        const snippet = r.snippet ?? '';
+        if (!snippet)
             continue;
+        const text = snippet.slice(0, MAX_PASSAGE_LENGTH);
         out.push({
-            text: firstPara.slice(0, MAX_PASSAGE_LENGTH),
+            text,
             source_index: i + 1,
             relevance_score: r.relevance_score,
             source_url: r.url,
             source_title: r.title,
+            section_heading: null,
+            source_span: { start: 0, end: text.length },
         });
     }
     return out;