circle-ir-ai 2.30.0 → 2.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +194 -0
- package/dist/agents/mastra/steps.d.ts +3 -3
- package/dist/agents/mastra/swarm.d.ts.map +1 -1
- package/dist/agents/mastra/swarm.js +277 -238
- package/dist/agents/mastra/swarm.js.map +1 -1
- package/dist/agents/mastra/workflow.d.ts.map +1 -1
- package/dist/agents/mastra/workflow.js +8 -0
- package/dist/agents/mastra/workflow.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/llm/ax-client.d.ts.map +1 -1
- package/dist/llm/ax-client.js +26 -2
- package/dist/llm/ax-client.js.map +1 -1
- package/dist/llm/call-limiter.d.ts +125 -0
- package/dist/llm/call-limiter.d.ts.map +1 -0
- package/dist/llm/call-limiter.js +218 -0
- package/dist/llm/call-limiter.js.map +1 -0
- package/dist/llm/call-logger.d.ts +14 -0
- package/dist/llm/call-logger.d.ts.map +1 -1
- package/dist/llm/call-logger.js.map +1 -1
- package/dist/llm/file-splitter.d.ts +129 -0
- package/dist/llm/file-splitter.d.ts.map +1 -0
- package/dist/llm/file-splitter.js +161 -0
- package/dist/llm/file-splitter.js.map +1 -0
- package/dist/llm/index.d.ts +1 -0
- package/dist/llm/index.d.ts.map +1 -1
- package/dist/llm/index.js +2 -0
- package/dist/llm/index.js.map +1 -1
- package/dist/llm/token-estimator.d.ts +36 -0
- package/dist/llm/token-estimator.d.ts.map +1 -0
- package/dist/llm/token-estimator.js +48 -0
- package/dist/llm/token-estimator.js.map +1 -0
- package/dist/llm/verification.d.ts +11 -0
- package/dist/llm/verification.d.ts.map +1 -1
- package/dist/llm/verification.js +39 -3
- package/dist/llm/verification.js.map +1 -1
- package/package.json +2 -2
|
@@ -97,6 +97,20 @@ export interface LLMCallEntry {
|
|
|
97
97
|
* `errorCategory` for `r2Error` rows.
|
|
98
98
|
*/
|
|
99
99
|
r2Op?: 'get' | 'set' | 'has' | 'delete';
|
|
100
|
+
/**
|
|
101
|
+
* cognium-ai#160 (Sprint L4a) — set on synthetic "no-call" entries
|
|
102
|
+
* emitted when `LLM_MAX_CALLS_PER_SCAN` or `LLM_MAX_CALLS_PER_FILE`
|
|
103
|
+
* caused the call-limiter to reject a call. Mirrors the
|
|
104
|
+
* `skippedByPrefilter` / `verifyCacheHit` / `discoveryCacheHit`
|
|
105
|
+
* patterns so `analyze-llm-log.ts` can surface cap-driven skip rates.
|
|
106
|
+
*/
|
|
107
|
+
skippedByCap?: boolean;
|
|
108
|
+
/**
|
|
109
|
+
* cognium-ai#160 — which cap tripped. `'scan'` = scan-wide cap
|
|
110
|
+
* (`LLM_MAX_CALLS_PER_SCAN`); `'file'` = per-file cap
|
|
111
|
+
* (`LLM_MAX_CALLS_PER_FILE`). Present iff `skippedByCap === true`.
|
|
112
|
+
*/
|
|
113
|
+
skippedCapReason?: 'scan' | 'file';
|
|
100
114
|
}
|
|
101
115
|
/**
|
|
102
116
|
* cognium-ai#163 B3 — categorical bucket for an R2 error. Kept separate
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"call-logger.d.ts","sourceRoot":"","sources":["../../src/llm/call-logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GACxB,gBAAgB,GAChB,aAAa,GACb,SAAS,GACT,YAAY,GACZ,cAAc,GACd,kBAAkB,GAClB,wBAAwB,CAAC;AAE7B,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,iFAAiF;IACjF,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,gBAAgB,GAAG,eAAe,CAAC;IACnD,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1E;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,IAAI,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"call-logger.d.ts","sourceRoot":"","sources":["../../src/llm/call-logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GACxB,gBAAgB,GAChB,aAAa,GACb,SAAS,GACT,YAAY,GACZ,cAAc,GACd,kBAAkB,GAClB,wBAAwB,CAAC;AAE7B,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,iFAAiF;IACjF,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,gBAAgB,GAAG,eAAe,CAAC;IACnD,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1E;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,IAAI,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;IACxC;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACvB,gBAAgB,GAChB,eAAe,GACf,WAAW,GACX,eAAe,GACf,WAAW,GACX,cAAc,GACd,eAAe,CAAC;AAMpB,cAAM,aAAa;IACjB,OAAO,CAAC,IAAI,CAAuB;IACnC,OAAO,CAAC,MAAM,CAAS;IACvB;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAAqB;IAExC,oEAAoE;IACpE,eAAe,IAAI,IAAI;IAMvB,OAAO,CAAC,IAAI;IAwBZ,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,GAAG,IAAI;IAW1C;;;;;OAKG;IACH,OAAO,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAA;KAAE,GAAG,IAAI;IAa5F;;;;OAIG;IACH,iBAAiB,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAA;KAAE,GAAG,IAAI;IActH;;;;OAIG;IACH,oBAAoB,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAA;KAAE,GAAG,IAAI;IAczH;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE;QACf,EAAE,EAAE,WAAW,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QACtC,QAAQ,EAAE,eAAe,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,IAAI;IAiCR;;;;OAIG;IACH,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GAAG,IAAI;CAkB3D;AAuBD;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CA6B/D;AAED,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAO7C;AAED,eAAO,MAAM,aAAa,eAAsB,CAAC;AAEjD;;;GAGG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAChC,IAAI,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,EAChF,IAAI,EAAE,MAAM,OAAO,CAAC;IAAE,KAAK,EAAE,CAAC,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,GAC7E,OAAO,CAAC,CAAC,CAAC,CA4BZ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"call-logger.js","sourceRoot":"","sources":["../../src/llm/call-logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"call-logger.js","sourceRoot":"","sources":["../../src/llm/call-logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAiHzB,MAAM,OAAO,GAAG,eAAe,CAAC;AAChC,MAAM,YAAY,GAAG,UAAU,CAAC;AAChC,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,aAAa;IACT,IAAI,GAAkB,IAAI,CAAC;IAC3B,MAAM,GAAG,KAAK,CAAC;IACvB;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAExC,oEAAoE;IACpE,eAAe;QACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,IAAI;QACV,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,2EAA2E;QAC3E,IAAI,CAAC;YACH,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBAC1C,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,SAAS,EAAE,mBAAmB;gBAC9B,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,IAAI;gBACb,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI;aAC5B,CAAC,GAAG,IAAI,CAAC,CAAC;QACb,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;YAClD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC;IAED,GAAG,CAAC,KAA+B;QACjC,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;QAC7F,IAAI,CAAC;YACH,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,IAA6E;QACnF,IAAI,CAAC,GAAG,CAAC;YACP,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAA6F;QAC7G,IAAI,CAAC,GAAG,CAAC;YACP,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,IAAI;YACb,cAAc,EAAE,IAAI;YACpB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,IAA6F;QAChH,IAAI,CAAC,GAAG,CAAC;YACP,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,IAAI;YACb,iBAAiB,EAAE,IAAI;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAIV;QACC,MAAM,SAAS,GAAG,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,qEAAqE;YACrE,0DAA0D;YAC1D,IAAI,CAAC;gBACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,IAAI,CAAC,EAAE,WAAW,IAAI,CAAC,QAAQ,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CACpF,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,iDAAiD;YACnD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gEAAgE;YAChE,kDAAkD;YAClD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,GAAG,CAAC;YACP,SAAS,EAAE,MAAM,IAAI,CAAC,EAAE,EAAE;YAC1B,KAAK,EAAE,OAAO;YACd,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,EAAE;YACb,aAAa,EAAE,IAAI,CAAC,QAAQ;YAC5B,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,UAAU,CAAC,KAAwC;QACjD,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,sDAAsD;QACtD,MAAM,OAAO,GAAsC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,WAAW;YACpH,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE;YAC5E,CAAC,CAAC,KAAK,CAAC;QACV,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;YAClG,IAAI,CAAC;gBACH,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrC,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU;YACZ,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,GAAG,EAAE,CAAC;YACtC,UAAU,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;CACF;AAED,yEAAyE;AACzE,SAAS,sBAAsB,CAAC,CAAoC;IAClE,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;IACtB,MAAM,GAAG,GAAG,GAAG;QACb,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,GAAG,CAAC,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5E,CAAC,CAAC,UAAU,CAAC;IACf,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,WAAW;QAC3C,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,WAAW,EAAE;QACzC,CAAC,CAAC,WAAW,CAAC;IAChB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO,cAAc,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,UAAU,KAAK,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,EAAE,CAAC;AACtG,CAAC;AAED,6DAA6D;AAC7D,SAAS,UAAU,CAAC,CAAS,EAAE,GAAG,GAAG,GAAG;IACtC,OAAO,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAY;IAC5C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,eAAe,CAAC;IAC5D,MAAM,CAAC,GAAG,GAKT,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,EAAE,cAAc,CAAC;IAE3C,IAAI,IAAI,KAAK,cAAc;QAAE,OAAO,gBAAgB,CAAC;IACrD,IAAI,IAAI,KAAK,cAAc,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,eAAe,CAAC;IACtE,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,UAAU,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACtF,IACE,IAAI,KAAK,WAAW;WACjB,IAAI,KAAK,cAAc;WACvB,IAAI,KAAK,YAAY;WACrB,IAAI,KAAK,WAAW;WACpB,IAAI,KAAK,cAAc,EAC1B,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;QACjF,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,cAAc,CAAC;IACjE,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,GAAG;QAAE,OAAO;IAC9C,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,UAAU;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;AAEjD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAgF,EAChF,IAA8E;IAE9E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;QAC1D,aAAa,CAAC,GAAG,CAAC;YAChB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,aAAa;YACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,OAAO,EAAE,IAAI;YACb,UAAU;SACX,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,aAAa,CAAC,GAAG,CAAC;YAChB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sliding-window file splitter with overlap-aware finding merge
|
|
3
|
+
* (cognium-ai#166 L3).
|
|
4
|
+
*
|
|
5
|
+
* Problem: files larger than the LLM's context window can't be sent
|
|
6
|
+
* single-shot to enrichment/verification. Truncation drops sinks past
|
|
7
|
+
* the cut. Windowed verification (`buildWindowedCode`) helps when we
|
|
8
|
+
* already know source/sink centers, but for the discovery / enrichment
|
|
9
|
+
* pass we don't — we need to feed the entire file to the LLM in
|
|
10
|
+
* multiple chunks and merge the results.
|
|
11
|
+
*
|
|
12
|
+
* Design: line-boundary sliding window. Each chunk is a contiguous
|
|
13
|
+
* range of lines that fits under the byte budget, with a configurable
|
|
14
|
+
* overlap between adjacent chunks so a taint flow that spans a chunk
|
|
15
|
+
* boundary can still be discovered. Findings from all chunks are
|
|
16
|
+
* merged with a dedup key that normalizes line numbers back to the
|
|
17
|
+
* original file, so a finding discovered in the overlap region isn't
|
|
18
|
+
* double-counted.
|
|
19
|
+
*
|
|
20
|
+
* Env vars (read at module load):
|
|
21
|
+
* CP_CHUNK_OVERLAP_RATIO — float, default 0.1 (10% overlap)
|
|
22
|
+
* CP_VERIFY_MAX_CHUNKS_PER_FILE — int, default 4
|
|
23
|
+
*
|
|
24
|
+
* Cap rationale: 4 chunks × 40KB budget = 160KB effective analysis
|
|
25
|
+
* ceiling per file. Files exceeding the cap emit a WARN with the
|
|
26
|
+
* `chunks-skipped` count so operators know to bump the cap or exclude
|
|
27
|
+
* the file. Prevents verifier cost explosion on 50K-line generated code.
|
|
28
|
+
*/
|
|
29
|
+
/** Default overlap fraction — 10% of chunk size. */
|
|
30
|
+
export declare const CHUNK_OVERLAP_RATIO_DEFAULT = 0.1;
|
|
31
|
+
/** Default maximum chunk count per file — 4 chunks. */
|
|
32
|
+
export declare const VERIFY_MAX_CHUNKS_PER_FILE_DEFAULT = 4;
|
|
33
|
+
export declare const CP_CHUNK_OVERLAP_RATIO: number;
|
|
34
|
+
export declare const CP_VERIFY_MAX_CHUNKS_PER_FILE: number;
|
|
35
|
+
/**
|
|
36
|
+
* One chunk of a split file. `startLine` and `endLine` are 1-based,
|
|
37
|
+
* inclusive, and point back to the ORIGINAL file — this is what
|
|
38
|
+
* downstream code uses to normalize finding line numbers.
|
|
39
|
+
*/
|
|
40
|
+
export interface FileChunk {
|
|
41
|
+
/** Chunk index (0-based) in the emission order. */
|
|
42
|
+
index: number;
|
|
43
|
+
/** 1-based start line in the original file (inclusive). */
|
|
44
|
+
startLine: number;
|
|
45
|
+
/** 1-based end line in the original file (inclusive). */
|
|
46
|
+
endLine: number;
|
|
47
|
+
/** The chunk's source text (a contiguous slice of the original code). */
|
|
48
|
+
code: string;
|
|
49
|
+
/** Estimated token count for `code`. */
|
|
50
|
+
estimatedTokens: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Result of `splitFileForVerification`. Small files pass through as a
|
|
54
|
+
* single chunk. Oversized files may be truncated to `maxChunks`, in
|
|
55
|
+
* which case `skippedLines` counts the unprocessed tail.
|
|
56
|
+
*/
|
|
57
|
+
export interface SplitResult {
|
|
58
|
+
chunks: FileChunk[];
|
|
59
|
+
totalLines: number;
|
|
60
|
+
/** Lines skipped because the file exceeded the maxChunks cap. */
|
|
61
|
+
skippedLines: number;
|
|
62
|
+
/** Whether the chunk cap fired. */
|
|
63
|
+
capped: boolean;
|
|
64
|
+
}
|
|
65
|
+
export interface SplitOptions {
|
|
66
|
+
/**
|
|
67
|
+
* Target byte budget per chunk. Defaults are conservative; caller
|
|
68
|
+
* typically passes `MAX_CODE_CONTEXT_LENGTH` from ax-client.
|
|
69
|
+
* Must be > 0.
|
|
70
|
+
*/
|
|
71
|
+
maxBytesPerChunk: number;
|
|
72
|
+
/**
|
|
73
|
+
* Overlap ratio (0..1). Adjacent chunks share this fraction of
|
|
74
|
+
* lines. Default: `CP_CHUNK_OVERLAP_RATIO`.
|
|
75
|
+
*/
|
|
76
|
+
overlapRatio?: number;
|
|
77
|
+
/**
|
|
78
|
+
* Max chunk count per file. Files exceeding this many chunks get
|
|
79
|
+
* their tail dropped with a WARN. Default: `CP_VERIFY_MAX_CHUNKS_PER_FILE`.
|
|
80
|
+
*/
|
|
81
|
+
maxChunks?: number;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Split `code` into overlapping line-boundary chunks that each fit
|
|
85
|
+
* under `maxBytesPerChunk`. Small files pass through as a single
|
|
86
|
+
* chunk. Files that would exceed `maxChunks` have their tail dropped
|
|
87
|
+
* with a WARN log.
|
|
88
|
+
*
|
|
89
|
+
* The line-boundary discipline is deliberate: language parsers assume
|
|
90
|
+
* complete lines, and having chunk boundaries in the middle of a
|
|
91
|
+
* statement would break downstream Tree-sitter parses if the caller
|
|
92
|
+
* chose to re-analyze the chunk.
|
|
93
|
+
*/
|
|
94
|
+
export declare function splitFileForVerification(code: string, opts: SplitOptions): SplitResult;
|
|
95
|
+
/**
|
|
96
|
+
* Finding shape recognised by `mergeChunkedFindings`. Kept
|
|
97
|
+
* intentionally minimal — the merge cares only about the dedup key
|
|
98
|
+
* fields. Callers can pass richer finding types; only the listed
|
|
99
|
+
* fields are read.
|
|
100
|
+
*/
|
|
101
|
+
export interface ChunkedFinding {
|
|
102
|
+
/** CWE identifier (e.g. `CWE-79`). */
|
|
103
|
+
cwe: string;
|
|
104
|
+
/** 1-based line in the ORIGINAL file. */
|
|
105
|
+
sinkLine: number;
|
|
106
|
+
/** 1-based line in the ORIGINAL file. */
|
|
107
|
+
sourceLine?: number;
|
|
108
|
+
/** Optional snippet or dedup discriminator. */
|
|
109
|
+
snippet?: string;
|
|
110
|
+
/**
|
|
111
|
+
* Which chunk this finding came from. Used purely for provenance in
|
|
112
|
+
* the merged output; the dedup key is `(cwe, sinkLine, sourceLine,
|
|
113
|
+
* snippetHash)` and does NOT include the chunk index.
|
|
114
|
+
*/
|
|
115
|
+
chunkIndex?: number;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Merge findings emitted across multiple chunks into a single deduped
|
|
119
|
+
* list. Dedup key: `(cwe, sinkLine, sourceLine, snippetHash)`. The
|
|
120
|
+
* caller is responsible for having already rebased finding line
|
|
121
|
+
* numbers to the ORIGINAL file — the splitter's `startLine` /
|
|
122
|
+
* `endLine` are the mapping the caller should use.
|
|
123
|
+
*
|
|
124
|
+
* When two findings collide on the dedup key, the first-seen
|
|
125
|
+
* (lowest-`chunkIndex`) wins. Chunks in overlap regions therefore
|
|
126
|
+
* don't inflate the finding count.
|
|
127
|
+
*/
|
|
128
|
+
export declare function mergeChunkedFindings<F extends ChunkedFinding>(findings: F[]): F[];
|
|
129
|
+
//# sourceMappingURL=file-splitter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-splitter.d.ts","sourceRoot":"","sources":["../../src/llm/file-splitter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAkBH,oDAAoD;AACpD,eAAO,MAAM,2BAA2B,MAAM,CAAC;AAE/C,uDAAuD;AACvD,eAAO,MAAM,kCAAkC,IAAI,CAAC;AAEpD,eAAO,MAAM,sBAAsB,EAAE,MACkD,CAAC;AAExF,eAAO,MAAM,6BAA6B,EAAE,MACuD,CAAC;AAEpG;;;;GAIG;AACH,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,YAAY,GACjB,WAAW,CA6Eb;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,cAAc,EAC3D,QAAQ,EAAE,CAAC,EAAE,GACZ,CAAC,EAAE,CAUL"}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sliding-window file splitter with overlap-aware finding merge
|
|
3
|
+
* (cognium-ai#166 L3).
|
|
4
|
+
*
|
|
5
|
+
* Problem: files larger than the LLM's context window can't be sent
|
|
6
|
+
* single-shot to enrichment/verification. Truncation drops sinks past
|
|
7
|
+
* the cut. Windowed verification (`buildWindowedCode`) helps when we
|
|
8
|
+
* already know source/sink centers, but for the discovery / enrichment
|
|
9
|
+
* pass we don't — we need to feed the entire file to the LLM in
|
|
10
|
+
* multiple chunks and merge the results.
|
|
11
|
+
*
|
|
12
|
+
* Design: line-boundary sliding window. Each chunk is a contiguous
|
|
13
|
+
* range of lines that fits under the byte budget, with a configurable
|
|
14
|
+
* overlap between adjacent chunks so a taint flow that spans a chunk
|
|
15
|
+
* boundary can still be discovered. Findings from all chunks are
|
|
16
|
+
* merged with a dedup key that normalizes line numbers back to the
|
|
17
|
+
* original file, so a finding discovered in the overlap region isn't
|
|
18
|
+
* double-counted.
|
|
19
|
+
*
|
|
20
|
+
* Env vars (read at module load):
|
|
21
|
+
* CP_CHUNK_OVERLAP_RATIO — float, default 0.1 (10% overlap)
|
|
22
|
+
* CP_VERIFY_MAX_CHUNKS_PER_FILE — int, default 4
|
|
23
|
+
*
|
|
24
|
+
* Cap rationale: 4 chunks × 40KB budget = 160KB effective analysis
|
|
25
|
+
* ceiling per file. Files exceeding the cap emit a WARN with the
|
|
26
|
+
* `chunks-skipped` count so operators know to bump the cap or exclude
|
|
27
|
+
* the file. Prevents verifier cost explosion on 50K-line generated code.
|
|
28
|
+
*/
|
|
29
|
+
import { estimateTokens } from './token-estimator.js';
|
|
30
|
+
function parsePositiveFloat(raw) {
|
|
31
|
+
if (!raw)
|
|
32
|
+
return null;
|
|
33
|
+
const n = parseFloat(raw);
|
|
34
|
+
if (Number.isFinite(n) && n > 0)
|
|
35
|
+
return n;
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
function parsePositiveInt(raw) {
|
|
39
|
+
if (!raw)
|
|
40
|
+
return null;
|
|
41
|
+
const n = parseInt(raw, 10);
|
|
42
|
+
if (Number.isFinite(n) && n > 0)
|
|
43
|
+
return n;
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
/** Default overlap fraction — 10% of chunk size. */
|
|
47
|
+
export const CHUNK_OVERLAP_RATIO_DEFAULT = 0.1;
|
|
48
|
+
/** Default maximum chunk count per file — 4 chunks. */
|
|
49
|
+
export const VERIFY_MAX_CHUNKS_PER_FILE_DEFAULT = 4;
|
|
50
|
+
export const CP_CHUNK_OVERLAP_RATIO = parsePositiveFloat(process.env.CP_CHUNK_OVERLAP_RATIO) ?? CHUNK_OVERLAP_RATIO_DEFAULT;
|
|
51
|
+
export const CP_VERIFY_MAX_CHUNKS_PER_FILE = parsePositiveInt(process.env.CP_VERIFY_MAX_CHUNKS_PER_FILE) ?? VERIFY_MAX_CHUNKS_PER_FILE_DEFAULT;
|
|
52
|
+
/**
|
|
53
|
+
* Split `code` into overlapping line-boundary chunks that each fit
|
|
54
|
+
* under `maxBytesPerChunk`. Small files pass through as a single
|
|
55
|
+
* chunk. Files that would exceed `maxChunks` have their tail dropped
|
|
56
|
+
* with a WARN log.
|
|
57
|
+
*
|
|
58
|
+
* The line-boundary discipline is deliberate: language parsers assume
|
|
59
|
+
* complete lines, and having chunk boundaries in the middle of a
|
|
60
|
+
* statement would break downstream Tree-sitter parses if the caller
|
|
61
|
+
* chose to re-analyze the chunk.
|
|
62
|
+
*/
|
|
63
|
+
export function splitFileForVerification(code, opts) {
|
|
64
|
+
const maxBytes = opts.maxBytesPerChunk;
|
|
65
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
|
|
66
|
+
throw new Error(`splitFileForVerification: maxBytesPerChunk must be > 0, got ${maxBytes}`);
|
|
67
|
+
}
|
|
68
|
+
const overlap = opts.overlapRatio ?? CP_CHUNK_OVERLAP_RATIO;
|
|
69
|
+
const maxChunks = opts.maxChunks ?? CP_VERIFY_MAX_CHUNKS_PER_FILE;
|
|
70
|
+
const lines = code.split(/\r?\n/);
|
|
71
|
+
const totalLines = lines.length;
|
|
72
|
+
// Small file → single chunk pass-through.
|
|
73
|
+
if (code.length <= maxBytes) {
|
|
74
|
+
return {
|
|
75
|
+
chunks: [
|
|
76
|
+
{
|
|
77
|
+
index: 0,
|
|
78
|
+
startLine: 1,
|
|
79
|
+
endLine: totalLines,
|
|
80
|
+
code,
|
|
81
|
+
estimatedTokens: estimateTokens(code),
|
|
82
|
+
},
|
|
83
|
+
],
|
|
84
|
+
totalLines,
|
|
85
|
+
skippedLines: 0,
|
|
86
|
+
capped: false,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
// Estimate line-count that fits under budget. We use the mean line
|
|
90
|
+
// length as a rough sizing heuristic; the actual per-chunk build
|
|
91
|
+
// stops as soon as the accumulated byte count would exceed budget.
|
|
92
|
+
const avgLineLen = Math.max(1, code.length / totalLines);
|
|
93
|
+
const targetLinesPerChunk = Math.max(1, Math.floor(maxBytes / avgLineLen));
|
|
94
|
+
const overlapLines = Math.max(0, Math.floor(targetLinesPerChunk * overlap));
|
|
95
|
+
const chunks = [];
|
|
96
|
+
let cursor = 0; // 0-based line index
|
|
97
|
+
while (cursor < totalLines && chunks.length < maxChunks) {
|
|
98
|
+
// Greedily accumulate lines until adding one more would blow budget.
|
|
99
|
+
let byteCount = 0;
|
|
100
|
+
let endIndex = cursor;
|
|
101
|
+
while (endIndex < totalLines) {
|
|
102
|
+
const lineLen = lines[endIndex].length + 1; // +1 for the newline
|
|
103
|
+
if (byteCount + lineLen > maxBytes && endIndex > cursor)
|
|
104
|
+
break;
|
|
105
|
+
byteCount += lineLen;
|
|
106
|
+
endIndex++;
|
|
107
|
+
}
|
|
108
|
+
// endIndex is now exclusive; guarantee at least one line per chunk.
|
|
109
|
+
if (endIndex === cursor)
|
|
110
|
+
endIndex = cursor + 1;
|
|
111
|
+
const chunkCode = lines.slice(cursor, endIndex).join('\n');
|
|
112
|
+
chunks.push({
|
|
113
|
+
index: chunks.length,
|
|
114
|
+
startLine: cursor + 1,
|
|
115
|
+
endLine: endIndex,
|
|
116
|
+
code: chunkCode,
|
|
117
|
+
estimatedTokens: estimateTokens(chunkCode),
|
|
118
|
+
});
|
|
119
|
+
// Advance cursor by (chunk-length − overlap), never zero or negative.
|
|
120
|
+
const advance = Math.max(1, (endIndex - cursor) - overlapLines);
|
|
121
|
+
cursor += advance;
|
|
122
|
+
}
|
|
123
|
+
const capped = cursor < totalLines;
|
|
124
|
+
const skippedLines = capped ? totalLines - cursor : 0;
|
|
125
|
+
if (capped) {
|
|
126
|
+
console.error(` ⚠ file-splitter: file exceeded maxChunks=${maxChunks}, ` +
|
|
127
|
+
`skipped ${skippedLines}/${totalLines} lines. Bump CP_VERIFY_MAX_CHUNKS_PER_FILE or exclude the file. cognium-ai#166`);
|
|
128
|
+
}
|
|
129
|
+
return { chunks, totalLines, skippedLines, capped };
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Merge findings emitted across multiple chunks into a single deduped
|
|
133
|
+
* list. Dedup key: `(cwe, sinkLine, sourceLine, snippetHash)`. The
|
|
134
|
+
* caller is responsible for having already rebased finding line
|
|
135
|
+
* numbers to the ORIGINAL file — the splitter's `startLine` /
|
|
136
|
+
* `endLine` are the mapping the caller should use.
|
|
137
|
+
*
|
|
138
|
+
* When two findings collide on the dedup key, the first-seen
|
|
139
|
+
* (lowest-`chunkIndex`) wins. Chunks in overlap regions therefore
|
|
140
|
+
* don't inflate the finding count.
|
|
141
|
+
*/
|
|
142
|
+
export function mergeChunkedFindings(findings) {
|
|
143
|
+
const seen = new Map();
|
|
144
|
+
for (const f of findings) {
|
|
145
|
+
const snippetKey = f.snippet ? hashSnippet(f.snippet) : '';
|
|
146
|
+
const key = `${f.cwe}|${f.sinkLine}|${f.sourceLine ?? '-'}|${snippetKey}`;
|
|
147
|
+
if (!seen.has(key)) {
|
|
148
|
+
seen.set(key, f);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return Array.from(seen.values());
|
|
152
|
+
}
|
|
153
|
+
/** Cheap non-cryptographic hash for snippet dedup. */
|
|
154
|
+
function hashSnippet(s) {
|
|
155
|
+
let h = 0;
|
|
156
|
+
for (let i = 0; i < s.length; i++) {
|
|
157
|
+
h = ((h << 5) - h + s.charCodeAt(i)) | 0;
|
|
158
|
+
}
|
|
159
|
+
return h.toString(36);
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=file-splitter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-splitter.js","sourceRoot":"","sources":["../../src/llm/file-splitter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,SAAS,kBAAkB,CAAC,GAAuB;IACjD,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAuB;IAC/C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oDAAoD;AACpD,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC;AAE/C,uDAAuD;AACvD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC;AAEpD,MAAM,CAAC,MAAM,sBAAsB,GACjC,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,2BAA2B,CAAC;AAExF,MAAM,CAAC,MAAM,6BAA6B,GACxC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,IAAI,kCAAkC,CAAC;AAqDpG;;;;;;;;;;GAUG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,IAAkB;IAElB,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,+DAA+D,QAAQ,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,IAAI,sBAAsB,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,6BAA6B,CAAC;IAElE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAEhC,0CAA0C;IAC1C,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,UAAU;oBACnB,IAAI;oBACJ,eAAe,EAAE,cAAc,CAAC,IAAI,CAAC;iBACtC;aACF;YACD,UAAU;YACV,YAAY,EAAE,CAAC;YACf,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,mEAAmE;IACnE,iEAAiE;IACjE,mEAAmE;IACnE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;IACzD,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC;IAC3E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC;IAE5E,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB;IAErC,OAAO,MAAM,GAAG,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QACxD,qEAAqE;QACrE,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,QAAQ,GAAG,MAAM,CAAC;QACtB,OAAO,QAAQ,GAAG,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB;YACjE,IAAI,SAAS,GAAG,OAAO,GAAG,QAAQ,IAAI,QAAQ,GAAG,MAAM;gBAAE,MAAM;YAC/D,SAAS,IAAI,OAAO,CAAC;YACrB,QAAQ,EAAE,CAAC;QACb,CAAC;QACD,oEAAoE;QACpE,IAAI,QAAQ,KAAK,MAAM;YAAE,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,SAAS,EAAE,MAAM,GAAG,CAAC;YACrB,OAAO,EAAE,QAAQ;YACjB,IAAI,EAAE,SAAS;YACf,eAAe,EAAE,cAAc,CAAC,SAAS,CAAC;SAC3C,CAAC,CAAC;QAEH,sEAAsE;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC;QAChE,MAAM,IAAI,OAAO,CAAC;IACpB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;IACnC,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CACX,8CAA8C,SAAS,IAAI;YACzD,WAAW,YAAY,IAAI,UAAU,gFAAgF,CACxH,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;AACtD,CAAC;AAyBD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,oBAAoB,CAClC,QAAa;IAEb,MAAM,IAAI,GAAG,IAAI,GAAG,EAAa,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;QAC1E,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,sDAAsD;AACtD,SAAS,WAAW,CAAC,CAAS;IAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC"}
|
package/dist/llm/index.d.ts
CHANGED
|
@@ -9,5 +9,6 @@ export { EnrichmentEngine, getEnrichmentEngine, enrichType, type EnrichmentResul
|
|
|
9
9
|
export { VerificationEngine, getVerificationEngine, verifyTaintPath, verifyTaintPathsBatch, verifyVulnerability, resetVerifierDedupCache, getVerifierDedupStats, VERIFY_MAX_PAIRS_PER_CHUNK, type VerificationResult, type VerificationInput, type VerificationVerdict, type Exploitability, type BatchVerificationInput, type BatchVerificationResult, } from './verification.js';
|
|
10
10
|
export { DiscoveryEngine, getDiscoveryEngine, discoverVulnerabilities, discoveryResultsToFindings, type DiscoveryResult, type DiscoveredVulnerability, type DiscoveryOptions, } from './discovery.js';
|
|
11
11
|
export { getLanguageContext, generateSourceDiscoveryPrompt, generateSinkDiscoveryPrompt, generateRoleClassificationPrompt, getCWEGuidance, type LanguageContext, type SourcePattern, type SinkPattern, } from './language-context.js';
|
|
12
|
+
export { checkAndCountCall, resetLLMCallLimiter, markSwarmStart, markSwarmEnd, clearSwarmFlagForTesting, getLLMCallStats, LLM_CONCURRENCY, LLM_MAX_CALLS_PER_SCAN, LLM_MAX_CALLS_PER_FILE, } from './call-limiter.js';
|
|
12
13
|
export { sanitizeCodeForPrompt, sanitizeListForPrompt, sanitizeObjectForPrompt, wrapUserCode, formatSystemPrompt, formatCodeBlock, getModelPromptConfig, detectModelFamily, isInputSafe, logInjectionAttempt, type ModelFamily, type ModelPromptConfig, } from './prompt-security.js';
|
|
13
14
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/llm/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,KAAK,SAAS,EACd,KAAK,WAAW,GACjB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,WAAW,EACX,cAAc,EAEd,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,EAC3B,wBAAwB,EACxB,sBAAsB,EACtB,8BAA8B,EAC9B,4BAA4B,GAC7B,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,UAAU,EACV,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,0BAA0B,EAC1B,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,GAC7B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,gBAAgB,GACtB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,2BAA2B,EAC3B,gCAAgC,EAChC,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EACnB,KAAK,WAAW,EAChB,KAAK,iBAAiB,GACvB,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,KAAK,SAAS,EACd,KAAK,WAAW,GACjB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,WAAW,EACX,cAAc,EAEd,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,EAC3B,wBAAwB,EACxB,sBAAsB,EACtB,8BAA8B,EAC9B,4BAA4B,GAC7B,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,UAAU,EACV,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,0BAA0B,EAC1B,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,GAC7B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,gBAAgB,GACtB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,2BAA2B,EAC3B,gCAAgC,EAChC,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,eAAe,EACf,eAAe,EACf,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EACnB,KAAK,WAAW,EAChB,KAAK,iBAAiB,GACvB,MAAM,sBAAsB,CAAC"}
|
package/dist/llm/index.js
CHANGED
|
@@ -17,6 +17,8 @@ export { VerificationEngine, getVerificationEngine, verifyTaintPath, verifyTaint
|
|
|
17
17
|
export { DiscoveryEngine, getDiscoveryEngine, discoverVulnerabilities, discoveryResultsToFindings, } from './discovery.js';
|
|
18
18
|
// Language-Aware Prompts (for multi-language support)
|
|
19
19
|
export { getLanguageContext, generateSourceDiscoveryPrompt, generateSinkDiscoveryPrompt, generateRoleClassificationPrompt, getCWEGuidance, } from './language-context.js';
|
|
20
|
+
// Call limiter (cognium-ai#159 + #160)
|
|
21
|
+
export { checkAndCountCall, resetLLMCallLimiter, markSwarmStart, markSwarmEnd, clearSwarmFlagForTesting, getLLMCallStats, LLM_CONCURRENCY, LLM_MAX_CALLS_PER_SCAN, LLM_MAX_CALLS_PER_FILE, } from './call-limiter.js';
|
|
20
22
|
// Prompt Security (for robustness against prompt injection)
|
|
21
23
|
export { sanitizeCodeForPrompt, sanitizeListForPrompt, sanitizeObjectForPrompt, wrapUserCode, formatSystemPrompt, formatCodeBlock, getModelPromptConfig, detectModelFamily, isInputSafe, logInjectionAttempt, } from './prompt-security.js';
|
|
22
24
|
//# sourceMappingURL=index.js.map
|
package/dist/llm/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,gBAAgB;AAChB,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,GAGzB,MAAM,aAAa,CAAC;AAErB,uDAAuD;AACvD,OAAO,EACL,WAAW,EACX,cAAc;AACd,aAAa;AACb,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,EAC3B,wBAAwB,EACxB,sBAAsB,EACtB,8BAA8B,EAC9B,4BAA4B,GAC7B,MAAM,gBAAgB,CAAC;AAExB,uBAAuB;AACvB,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,UAAU,GAMX,MAAM,iBAAiB,CAAC;AAEzB,yBAAyB;AACzB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,0BAA0B,GAO3B,MAAM,mBAAmB,CAAC;AAE3B,uDAAuD;AACvD,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,uBAAuB,EACvB,0BAA0B,GAI3B,MAAM,gBAAgB,CAAC;AAExB,sDAAsD;AACtD,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,2BAA2B,EAC3B,gCAAgC,EAChC,cAAc,GAIf,MAAM,uBAAuB,CAAC;AAE/B,4DAA4D;AAC5D,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,GAGpB,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,gBAAgB;AAChB,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,GAGzB,MAAM,aAAa,CAAC;AAErB,uDAAuD;AACvD,OAAO,EACL,WAAW,EACX,cAAc;AACd,aAAa;AACb,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,EAC3B,wBAAwB,EACxB,sBAAsB,EACtB,8BAA8B,EAC9B,4BAA4B,GAC7B,MAAM,gBAAgB,CAAC;AAExB,uBAAuB;AACvB,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,UAAU,GAMX,MAAM,iBAAiB,CAAC;AAEzB,yBAAyB;AACzB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,0BAA0B,GAO3B,MAAM,mBAAmB,CAAC;AAE3B,uDAAuD;AACvD,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,uBAAuB,EACvB,0BAA0B,GAI3B,MAAM,gBAAgB,CAAC;AAExB,sDAAsD;AACtD,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,2BAA2B,EAC3B,gCAAgC,EAChC,cAAc,GAIf,MAAM,uBAAuB,CAAC;AAE/B,uCAAuC;AACvC,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,eAAe,EACf,eAAe,EACf,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAE3B,4DAA4D;AAC5D,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,GAGpB,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token estimator for verifier budget-packing (cognium-ai#165 L3).
|
|
3
|
+
*
|
|
4
|
+
* Rough `chars * TOKENS_PER_CHAR` heuristic — no per-provider tokenizer
|
|
5
|
+
* download. Sufficient for budget-packing decisions where being within
|
|
6
|
+
* ±15% is acceptable and being under-budget (over-estimate tokens) is
|
|
7
|
+
* strictly safer than being over-budget.
|
|
8
|
+
*
|
|
9
|
+
* The default `0.25` (≈ 4 chars per token) matches empirical measurements
|
|
10
|
+
* for source code across GPT / Claude / Llama tokenizers within a few
|
|
11
|
+
* percent. Operators who care about a specific provider can tune it via
|
|
12
|
+
* `CP_TOKENS_PER_CHAR` at process start.
|
|
13
|
+
*
|
|
14
|
+
* Rationale for NOT using `js-tiktoken`:
|
|
15
|
+
* - +1MB install, per-provider drift (only ships GPT vocabs)
|
|
16
|
+
* - Calibration debt when Anthropic / Google ship new tokenizers
|
|
17
|
+
* - Budget-packing decisions don't need ±1% accuracy
|
|
18
|
+
*
|
|
19
|
+
* Env vars (read at module load):
|
|
20
|
+
* CP_TOKENS_PER_CHAR — float, default 0.25
|
|
21
|
+
*/
|
|
22
|
+
/** Default: 0.25 tokens/char (≈ 4 chars/token). See file header. */
|
|
23
|
+
export declare const TOKENS_PER_CHAR_DEFAULT = 0.25;
|
|
24
|
+
/**
|
|
25
|
+
* Effective tokens-per-char ratio for this process. Reads
|
|
26
|
+
* `CP_TOKENS_PER_CHAR` at module load; falls back to
|
|
27
|
+
* `TOKENS_PER_CHAR_DEFAULT`.
|
|
28
|
+
*/
|
|
29
|
+
export declare const TOKENS_PER_CHAR: number;
|
|
30
|
+
/**
|
|
31
|
+
* Estimate token count for a text blob. Returns 0 for empty / undefined.
|
|
32
|
+
* Deliberately linear-time (no tokenization); constant-factor cheap
|
|
33
|
+
* enough to call inside packing loops.
|
|
34
|
+
*/
|
|
35
|
+
export declare function estimateTokens(text: string | undefined | null): number;
|
|
36
|
+
//# sourceMappingURL=token-estimator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-estimator.d.ts","sourceRoot":"","sources":["../../src/llm/token-estimator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,oEAAoE;AACpE,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAE5C;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,MACiD,CAAC;AAEhF;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CAGtE"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token estimator for verifier budget-packing (cognium-ai#165 L3).
|
|
3
|
+
*
|
|
4
|
+
* Rough `chars * TOKENS_PER_CHAR` heuristic — no per-provider tokenizer
|
|
5
|
+
* download. Sufficient for budget-packing decisions where being within
|
|
6
|
+
* ±15% is acceptable and being under-budget (over-estimate tokens) is
|
|
7
|
+
* strictly safer than being over-budget.
|
|
8
|
+
*
|
|
9
|
+
* The default `0.25` (≈ 4 chars per token) matches empirical measurements
|
|
10
|
+
* for source code across GPT / Claude / Llama tokenizers within a few
|
|
11
|
+
* percent. Operators who care about a specific provider can tune it via
|
|
12
|
+
* `CP_TOKENS_PER_CHAR` at process start.
|
|
13
|
+
*
|
|
14
|
+
* Rationale for NOT using `js-tiktoken`:
|
|
15
|
+
* - +1MB install, per-provider drift (only ships GPT vocabs)
|
|
16
|
+
* - Calibration debt when Anthropic / Google ship new tokenizers
|
|
17
|
+
* - Budget-packing decisions don't need ±1% accuracy
|
|
18
|
+
*
|
|
19
|
+
* Env vars (read at module load):
|
|
20
|
+
* CP_TOKENS_PER_CHAR — float, default 0.25
|
|
21
|
+
*/
|
|
22
|
+
function parsePositiveFloat(raw) {
|
|
23
|
+
if (!raw)
|
|
24
|
+
return null;
|
|
25
|
+
const n = parseFloat(raw);
|
|
26
|
+
if (Number.isFinite(n) && n > 0)
|
|
27
|
+
return n;
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
/** Default: 0.25 tokens/char (≈ 4 chars/token). See file header. */
|
|
31
|
+
export const TOKENS_PER_CHAR_DEFAULT = 0.25;
|
|
32
|
+
/**
|
|
33
|
+
* Effective tokens-per-char ratio for this process. Reads
|
|
34
|
+
* `CP_TOKENS_PER_CHAR` at module load; falls back to
|
|
35
|
+
* `TOKENS_PER_CHAR_DEFAULT`.
|
|
36
|
+
*/
|
|
37
|
+
export const TOKENS_PER_CHAR = parsePositiveFloat(process.env.CP_TOKENS_PER_CHAR) ?? TOKENS_PER_CHAR_DEFAULT;
|
|
38
|
+
/**
|
|
39
|
+
* Estimate token count for a text blob. Returns 0 for empty / undefined.
|
|
40
|
+
* Deliberately linear-time (no tokenization); constant-factor cheap
|
|
41
|
+
* enough to call inside packing loops.
|
|
42
|
+
*/
|
|
43
|
+
export function estimateTokens(text) {
|
|
44
|
+
if (!text)
|
|
45
|
+
return 0;
|
|
46
|
+
return Math.ceil(text.length * TOKENS_PER_CHAR);
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=token-estimator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-estimator.js","sourceRoot":"","sources":["../../src/llm/token-estimator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,SAAS,kBAAkB,CAAC,GAAuB;IACjD,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAC1B,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,uBAAuB,CAAC;AAEhF;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAA+B;IAC5D,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;AAClD,CAAC"}
|
|
@@ -9,6 +9,17 @@ import { type LLMConfig } from './config.js';
|
|
|
9
9
|
export declare const VERIFY_WINDOW_LINES: number;
|
|
10
10
|
export declare const VERIFY_MAX_CHUNKS: number;
|
|
11
11
|
export declare const VERIFY_MAX_PAIRS_PER_CHUNK: number;
|
|
12
|
+
/**
|
|
13
|
+
* Token budget per verifier LLM call (cognium-ai#165 L3). Complementary
|
|
14
|
+
* to the `MAX_PAIRS_PER_CHUNK` cap: whichever fires first triggers a
|
|
15
|
+
* partition. Default `12000` is safe for 16K-context models with
|
|
16
|
+
* headroom for schema + system prompt + response.
|
|
17
|
+
*
|
|
18
|
+
* Overridable via `CP_VERIFY_TOKEN_BUDGET` (positive integer). Set to a
|
|
19
|
+
* very large value (e.g. `1000000`) to effectively disable and preserve
|
|
20
|
+
* pre-L3 behavior.
|
|
21
|
+
*/
|
|
22
|
+
export declare const VERIFY_TOKEN_BUDGET: number;
|
|
12
23
|
/**
|
|
13
24
|
* Detect minified / bundled single-line content. #102.
|
|
14
25
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verification.d.ts","sourceRoot":"","sources":["../../src/llm/verification.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"verification.d.ts","sourceRoot":"","sources":["../../src/llm/verification.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAoClE,eAAO,MAAM,mBAAmB,QAC6B,CAAC;AAE9D,eAAO,MAAM,iBAAiB,QAC4B,CAAC;AAE3D,eAAO,MAAM,0BAA0B,QAI9B,CAAC;AAEV;;;;;;;;;GASG;AACH,eAAO,MAAM,mBAAmB,QACgC,CAAC;AAEjE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAWvD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,WAAW,GAAE,MAA4B,GACxC,MAAM,CAmCR;AAMD,MAAM,MAAM,mBAAmB,GAAG,eAAe,GAAG,gBAAgB,GAAG,WAAW,CAAC;AACnF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAEhE,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,cAAc,CAAC;IAG/B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IAGzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAEhC,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IAGF,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IAGF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IAGtB,UAAU,EAAE,OAAO,CAAC;IACpB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAqID,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;IACH,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,KAAK,CAAC;QACX,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,mBAAmB,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,cAAc,CAAC;KAChC,CAAC,CAAC;IACH,OAAO,EAAE;QACP,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAGZ;gBAEU,MAAM,CAAC,EAAE,WAAW;IAKhC;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;YAWrD,UAAU;IAuCxB;;;OAGG;IACG,WAAW,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAiBlF,OAAO,CAAC,UAAU;YAMJ,eAAe;IAkG7B;;OAEG;YACW,gBAAgB;IA+B9B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,eAAe;IA6BvB;;;;;;;;;;;;;;;OAeG;YACW,qBAAqB;IA2DnC;;;;;;;;OAQG;IACH,OAAO,CAAC,uBAAuB;IAW/B;;;;OAIG;YACW,kBAAkB;IAqEhC;;OAEG;IACH,OAAO,CAAC,WAAW;IAsBnB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAW7B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAOhC;AAMD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,kBAAkB,CAE9E;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAE3F;AAoDD,kFAAkF;AAClF,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C;AAED,kFAAkF;AAClF,wBAAgB,qBAAqB,IAAI,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAK5E;AAED;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,sBAAsB,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,uBAAuB,CAAC,CA4E3I;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,kBAAkB,CAAC,CAoB7B"}
|
package/dist/llm/verification.js
CHANGED
|
@@ -10,6 +10,7 @@ import { getDefaultLLMConfig } from './config.js';
|
|
|
10
10
|
import { withLLMContext } from './debug-context.js';
|
|
11
11
|
import { getVerifyCache, computeVerifyCacheKey } from '../cache/verify-cache.js';
|
|
12
12
|
import { llmCallLogger } from './call-logger.js';
|
|
13
|
+
import { estimateTokens } from './token-estimator.js';
|
|
13
14
|
// ============================================================================
|
|
14
15
|
// #86 — Windowed verification configuration
|
|
15
16
|
// ============================================================================
|
|
@@ -43,6 +44,17 @@ function firstPositiveInt(...raws) {
|
|
|
43
44
|
export const VERIFY_WINDOW_LINES = firstPositiveInt(process.env.LLM_VERIFY_WINDOW_LINES) ?? 80;
|
|
44
45
|
export const VERIFY_MAX_CHUNKS = firstPositiveInt(process.env.LLM_VERIFY_MAX_CHUNKS) ?? 4;
|
|
45
46
|
export const VERIFY_MAX_PAIRS_PER_CHUNK = firstPositiveInt(process.env.CP_VERIFICATION_CHUNK_SIZE, process.env.LLM_MAX_PAIRS_PER_CHUNK) ?? 20;
|
|
47
|
+
/**
|
|
48
|
+
* Token budget per verifier LLM call (cognium-ai#165 L3). Complementary
|
|
49
|
+
* to the `MAX_PAIRS_PER_CHUNK` cap: whichever fires first triggers a
|
|
50
|
+
* partition. Default `12000` is safe for 16K-context models with
|
|
51
|
+
* headroom for schema + system prompt + response.
|
|
52
|
+
*
|
|
53
|
+
* Overridable via `CP_VERIFY_TOKEN_BUDGET` (positive integer). Set to a
|
|
54
|
+
* very large value (e.g. `1000000`) to effectively disable and preserve
|
|
55
|
+
* pre-L3 behavior.
|
|
56
|
+
*/
|
|
57
|
+
export const VERIFY_TOKEN_BUDGET = firstPositiveInt(process.env.CP_VERIFY_TOKEN_BUDGET) ?? 12_000;
|
|
46
58
|
/**
|
|
47
59
|
* Detect minified / bundled single-line content. #102.
|
|
48
60
|
*
|
|
@@ -331,12 +343,36 @@ export class VerificationEngine {
|
|
|
331
343
|
// latency + risk of context-window pressure.
|
|
332
344
|
const MAX_PAIRS_PER_CHUNK = VERIFY_MAX_PAIRS_PER_CHUNK;
|
|
333
345
|
const totalPairs = input.sources.length * input.sinks.length;
|
|
334
|
-
//
|
|
335
|
-
|
|
346
|
+
// cognium-ai#165 L3 — token budget check runs alongside pair-count
|
|
347
|
+
// check. Estimator is deliberately conservative (chars/4 default);
|
|
348
|
+
// errs on over-count so we split earlier rather than blow the
|
|
349
|
+
// context window. Prompt scaffold constant (~400 tokens for the
|
|
350
|
+
// BATCH_VERIFICATION_PROMPT template + system message) is added
|
|
351
|
+
// to the code + sources + sinks estimate.
|
|
352
|
+
const PROMPT_SCAFFOLD_TOKENS = 400;
|
|
353
|
+
const estimatedTokens = estimateTokens(input.code) +
|
|
354
|
+
estimateTokens(input.sources.map((s) => `${s.line}${s.type}${s.variable ?? ''}`).join('')) +
|
|
355
|
+
estimateTokens(input.sinks.map((s) => `${s.line}${s.type}${s.cwe}${s.method ?? ''}`).join('')) +
|
|
356
|
+
PROMPT_SCAFFOLD_TOKENS;
|
|
357
|
+
// If small enough by both metrics, process in one call.
|
|
358
|
+
if (totalPairs <= MAX_PAIRS_PER_CHUNK && estimatedTokens <= VERIFY_TOKEN_BUDGET) {
|
|
336
359
|
return this.verifyBatchChunk(input, startTime, modelUsed);
|
|
337
360
|
}
|
|
361
|
+
// Compute the effective per-chunk pair budget: whichever of
|
|
362
|
+
// MAX_PAIRS_PER_CHUNK or the token-derived cap is smaller.
|
|
363
|
+
// Token-derived cap: pair-count-equivalent under the token budget.
|
|
364
|
+
let effectiveMaxPairs = MAX_PAIRS_PER_CHUNK;
|
|
365
|
+
if (estimatedTokens > VERIFY_TOKEN_BUDGET && totalPairs > 0) {
|
|
366
|
+
// Ratio of budget to estimate → fraction of pairs that fit under budget.
|
|
367
|
+
// Guard against 0-token estimates (empty code) → don't shrink pair count.
|
|
368
|
+
const budgetRatio = VERIFY_TOKEN_BUDGET / Math.max(1, estimatedTokens);
|
|
369
|
+
const tokenDerivedCap = Math.max(1, Math.floor(totalPairs * budgetRatio));
|
|
370
|
+
effectiveMaxPairs = Math.min(MAX_PAIRS_PER_CHUNK, tokenDerivedCap);
|
|
371
|
+
console.error(` ↻ verification: token-budget split ${estimatedTokens} → ${VERIFY_TOKEN_BUDGET} tokens ` +
|
|
372
|
+
`(pair cap ${MAX_PAIRS_PER_CHUNK} → ${effectiveMaxPairs}). cognium-ai#165`);
|
|
373
|
+
}
|
|
338
374
|
// Otherwise, chunk the sources and process in parallel
|
|
339
|
-
const chunkSize = Math.max(1, Math.floor(
|
|
375
|
+
const chunkSize = Math.max(1, Math.floor(effectiveMaxPairs / Math.max(1, input.sinks.length)));
|
|
340
376
|
const sourceChunks = [];
|
|
341
377
|
for (let i = 0; i < input.sources.length; i += chunkSize) {
|
|
342
378
|
sourceChunks.push(input.sources.slice(i, i + chunkSize));
|