@adonis-agora/agent 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/NOTICE +31 -0
- package/README.md +5 -5
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +1 -0
- package/dist/configure.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/rag/index.d.ts +2 -1
- package/dist/src/rag/index.d.ts.map +1 -1
- package/dist/src/rag/index.js +2 -1
- package/dist/src/rag/index.js.map +1 -1
- package/dist/src/rag/openai-embeddings.d.ts +20 -2
- package/dist/src/rag/openai-embeddings.d.ts.map +1 -1
- package/dist/src/rag/openai-embeddings.js +107 -42
- package/dist/src/rag/openai-embeddings.js.map +1 -1
- package/dist/src/rag/pg-vector-store.d.ts +27 -0
- package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
- package/dist/src/rag/pg-vector-store.js +2 -2
- package/dist/src/rag/pg-vector-store.js.map +1 -1
- package/dist/src/rag/tree/build.d.ts +96 -0
- package/dist/src/rag/tree/build.d.ts.map +1 -0
- package/dist/src/rag/tree/build.js +659 -0
- package/dist/src/rag/tree/build.js.map +1 -0
- package/dist/src/rag/tree/headings.d.ts +29 -0
- package/dist/src/rag/tree/headings.d.ts.map +1 -0
- package/dist/src/rag/tree/headings.js +227 -0
- package/dist/src/rag/tree/headings.js.map +1 -0
- package/dist/src/rag/tree/index-document.d.ts +33 -0
- package/dist/src/rag/tree/index-document.d.ts.map +1 -0
- package/dist/src/rag/tree/index-document.js +36 -0
- package/dist/src/rag/tree/index-document.js.map +1 -0
- package/dist/src/rag/tree/index.d.ts +13 -0
- package/dist/src/rag/tree/index.d.ts.map +1 -0
- package/dist/src/rag/tree/index.js +12 -0
- package/dist/src/rag/tree/index.js.map +1 -0
- package/dist/src/rag/tree/keyword-tree-llm.d.ts +15 -0
- package/dist/src/rag/tree/keyword-tree-llm.d.ts.map +1 -0
- package/dist/src/rag/tree/keyword-tree-llm.js +105 -0
- package/dist/src/rag/tree/keyword-tree-llm.js.map +1 -0
- package/dist/src/rag/tree/llm.d.ts +179 -0
- package/dist/src/rag/tree/llm.d.ts.map +1 -0
- package/dist/src/rag/tree/llm.js +264 -0
- package/dist/src/rag/tree/llm.js.map +1 -0
- package/dist/src/rag/tree/navigate-tool.d.ts +58 -0
- package/dist/src/rag/tree/navigate-tool.d.ts.map +1 -0
- package/dist/src/rag/tree/navigate-tool.js +95 -0
- package/dist/src/rag/tree/navigate-tool.js.map +1 -0
- package/dist/src/rag/tree/navigate.d.ts +137 -0
- package/dist/src/rag/tree/navigate.d.ts.map +1 -0
- package/dist/src/rag/tree/navigate.js +404 -0
- package/dist/src/rag/tree/navigate.js.map +1 -0
- package/dist/src/rag/tree/pg-tree-store.d.ts +45 -0
- package/dist/src/rag/tree/pg-tree-store.d.ts.map +1 -0
- package/dist/src/rag/tree/pg-tree-store.js +205 -0
- package/dist/src/rag/tree/pg-tree-store.js.map +1 -0
- package/dist/src/rag/tree/store.d.ts +57 -0
- package/dist/src/rag/tree/store.d.ts.map +1 -0
- package/dist/src/rag/tree/store.js +66 -0
- package/dist/src/rag/tree/store.js.map +1 -0
- package/dist/src/rag/tree/two-stage.d.ts +54 -0
- package/dist/src/rag/tree/two-stage.d.ts.map +1 -0
- package/dist/src/rag/tree/two-stage.js +92 -0
- package/dist/src/rag/tree/two-stage.js.map +1 -0
- package/dist/src/rag/tree/types.d.ts +135 -0
- package/dist/src/rag/tree/types.d.ts.map +1 -0
- package/dist/src/rag/tree/types.js +21 -0
- package/dist/src/rag/tree/types.js.map +1 -0
- package/dist/stubs/database/migrations/create_agent_rag_trees.stub +40 -0
- package/package.json +2 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../../../src/rag/tree/build.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACnF,OAAO,EACL,WAAW,EACX,cAAc,EACd,UAAU,EAEV,uBAAuB,GAExB,MAAM,UAAU,CAAC;AAQlB,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAiEtC,MAAM,cAAc,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,cAAc,EAAE,OAAO;IACvB,eAAe,EAAE,MAAM;IACvB,SAAS,EAAE,OAAO;IAClB,aAAa,EAAE,MAAM;CACb,CAAC;AAEX,MAAM,gBAAgB,GACpB,uFAAuF,CAAC;AAC1F,MAAM,cAAc,GAClB,mLAAmL,CAAC;AAEtL,mFAAmF;AACnF,MAAM,UAAU,OAAO,CAAC,KAAwB;IAK9C,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACpD,KAAK;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9D,CAAC,CAAC,CAAC;QACJ,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM;YACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CACvC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE;gBACnB,CAAC,CAAC;oBACE;wBACE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE;wBAC3B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;wBACnD,IAAI;wBACJ,WAAW,EAAE,IAAI;qBAClB;iBACF;gBACH,CAAC,CAAC,EAAE,CACP;YACH,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,OAAO;KACd,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAwB,EAAE,KAAiB;IAClE,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACtB,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACnD,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC;YACrE,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;gBACb,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAClF,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI;YACJ,WAAW,EAAE,KAAK;SACnB,CAAC,CAAC;IACL,CAAC;IACD,gFAAgF;IAChF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;AAClD,CAAC;AAUD,8FAA8F;AAC9F,SAAS,YAAY,CAAC,QAA2B,EAAE,SAAiB;IAClE,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAyC,EAAE,CAAC;IACvD,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAClC,6EAA6E;QAC7E,IAAI,GAAG,GAAG,SAAS,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YAC1D,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAChE,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAc;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,OAAO,CAAC,IAAI;YACvB,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;YACpC,QAAQ,EAAE,EAAE;SACb,CAAC;QACF,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClF,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IACH,yDAAyD;IACzD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QAC/C,KAAK,CAAC,OAAO,CAAC;YACZ,KAAK,EAAE,cAAc;YACrB,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC;YAC5B,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,iGAAiG;AACjG,SAAS,SAAS,CAAC,IAAY,EAAE,OAA4B;IAC3D,MAAM,IAAI,GAAG,IAAI;SACd,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;SACpC,IAAI,CACH,CAAC,SAAS,EAAE,EAAE,CACZ,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAC7D,CAAC;IACJ,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACrF,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB,EAAE,KAAa,EAAE,GAAW;IAC/D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IACxB,IAAI,KAAK,EAAE,IAAI,KAAK,SAAS,IAAI,IAAI,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9F,CAAC;IACD,OAAO,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;AACrF,CAAC;AAED,sGAAsG;AACtG,SAAS,UAAU,CACjB,KAAiB,EACjB,KAAa,EACb,GAAW,EACX,SAAiB,EACjB,WAAmB,EACnB,UAAkB,EAClB,OAA4B;IAE5B,IAAI,KAAK,GAAgB,EAAE,CAAC;IAC5B,KAAK,IAAI,IAAI,GAAG,KAAK,EAAE,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,SAAS,EAAE,CAAC;QACtD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;YACjE,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;IACL,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,CAAC,MAAM,GAAG,WAAW,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACxD,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;QACzE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,GAAG,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;YAC7C,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,GAAG,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,KAAK,GAAG,OAAO,CAAC;QAChB,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2FAA2F;AAC3F,SAAS,gBAAgB,CAAC,KAAkB,EAAE,IAAI,GAAa,EAAE;IAC/D,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QACxC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,KAAK,CACZ,KAAkB,EAClB,KAAiB,EACjB,KAAa,EACb,OAMC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAChC,IACE,KAAK,IAAI,OAAO,CAAC,QAAQ;YACzB,CAAC,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAC7D,CAAC;YACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7E,OAAO;gBACL,GAAG,IAAI;gBACP,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,QAAQ,EAAE,EAAE;aACb,CAAC;QACJ,CAAC;QACD,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzB,IACE,QAAQ,CAAC,MAAM,KAAK,CAAC;YACrB,IAAI,KAAK,SAAS;YAClB,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS;YACjC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAC7B,CAAC;YACD,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YACtF,QAAQ,GAAG,UAAU,CACnB,KAAK,EACL,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,OAAO,EACZ,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,GAAG,KAAK,EACxB,OAAO,CAAC,OAAO,CAChB,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;IAC3E,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,IACE,QAAQ,KAAK,SAAS;YACtB,QAAQ,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC9B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC1B,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS;YACrC,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EACjC,CAAC;YACD,QAAQ,CAAC,QAAQ,GAAG;gBAClB,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK;gBACV,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;aACzB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,QAAQ,CAAC,CAAC,SAAS,CAAC,KAAkB;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,EAAE,IAAI,EAAE,CAAC;QACf,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,KAA0B,EAAE,KAAa,EAAE,GAAW;IAC7E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,KAAK,IAAI,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACnF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,KAA0B,EAAE,IAAsB;IAClE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC/F,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,IAAI,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC/E,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACX,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,KAAwB,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAChE,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;YAChB,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;IACtB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAW;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9C,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACjE,CAAC;AAED,mGAAmG;AACnG,KAAK,UAAU,WAAW,CACxB,GAAgB,EAChB,KAAiB,EACjB,OAAkD;IAElD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAe,EAAE,CAAC;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,GAAG,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,IAAI,MAAM,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,kBAAkB,OAAO,CAAC,MAAM,oDAAoD,OAAO,CAAC,QAAQ,GAAG,CACxG,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClC,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI;SACjF,CAAC,CAAC,CAAC;QACJ,MAAM,IAAI,GAAG,KAAK;aACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,GAAG,CAAC;aACxE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,MAAM,GAAG;;;;EAIjB,IAAI;;8EAEwE,CAAC;QAC3E,MAAM,KAAK,GAAG,cAAc,CAC1B,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,KAAK,CAAC,CACrF,CAAC;QACF,MAAM,QAAQ,GAAG,KAAK,EAAE,QAAQ,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,KAAK,IAAI,KAAK,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAChD,SAAS;YACX,CAAC;YACD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;YAChE,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1D,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,CAAC,KAAK,CAAC,IAAI,EAAE;gBACb,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrB,EAAE,GAAG,KAAK;gBACV,EAAE,GAAG,IAAI,EACT,CAAC;gBACD,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,QAAQ,KAAK,SAAS,IAAI,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACjD,SAAS;YACX,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC3B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClF,IAAI,EAAE,EAAE;gBACR,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8DAA8D;AAC9D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAmB,EACnB,KAAa,EACb,IAA4C;IAE5C,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC5F,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;YACrB,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAM,EAAE,KAAK,CAAC,CAAC;QACxD,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAwB,EACxB,OAAiC;IAEjC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;IACtD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC;IAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,UAAU,CAAC;IAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,UAAU,CAAC;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,UAAU,CAAC;IACxD,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC;IAC9C,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,GAAG,CAAC;IACvD,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC;IAE7D,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,UAAU,CAC5B,IAAI,CAAC,SAAS,CAAC;QACb,CAAC,EAAE,CAAC;QACJ,IAAI;QACJ,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;QAC9B,QAAQ,EACN,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,IAAI,IAAI;YACrB,OAAO,CAAC,KAAK,IAAI,IAAI;YACrB,OAAO,CAAC,IAAI,IAAI,IAAI;SACrB,CAAC,IAAI,IAAI;QACZ,KAAK,EAAE,UAAU;QACjB,QAAQ;QACR,SAAS;QACT,YAAY;QACZ,WAAW;QACX,WAAW;QACX,MAAM;QACN,SAAS;QACT,QAAQ;QACR,YAAY,EAAE,YAAY,IAAI,UAAU;QACxC,eAAe;QACf,iBAAiB;KAClB,CAAC,CACH,CAAC;IACF,IAAI,OAAO,CAAC,QAAQ,EAAE,WAAW,KAAK,WAAW,EAAE,CAAC;QAClD,6EAA6E;QAC7E,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QACzD,OAAO;YACL,IAAI,EAAE;gBACJ,GAAG,QAAQ;gBACX,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;YACD,KAAK;YACL,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,WAAW,CAC5B,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAC1D,EAAE,GAAG,cAAc,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EACxC,OAAO,CAAC,MAAM,CACf,CAAC;IACF,IAAI,cAAkC,CAAC;IAEvC,kBAAkB;IAClB,IAAI,SAAS,GAAwB,MAAM,CAAC;IAC5C,IAAI,QAAQ,GAAsB,EAAE,CAAC;IACrC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;QAC/C,SAAS,GAAG,SAAS,CAAC;QACtB,QAAQ,GAAG,OAAO,CAAC;IACrB,CAAC;SAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1D,SAAS,GAAG,UAAU,CAAC;QACvB,QAAQ,GAAG,QAAQ,CAAC;IACtB,CAAC;SAAM,IAAI,MAAM,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,cAAc,CAC1B,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAC9B,MAAM,KAAK,UAAU,CACtB,CAAC;QACF,IAAI,KAAK,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YAChC,SAAS,GAAG,UAAU,CAAC;YACvB,QAAQ,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,IAAI,SAAS,KAAK,MAAM,IAAI,YAAY,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3E,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE;gBAC7C,WAAW,EAAE,OAAO,CAAC,oBAAoB,IAAI,MAAM;gBACnD,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC;aACvD,CAAC,CAAC;YACH,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC7C,SAAS,GAAG,KAAK,CAAC;gBAClB,QAAQ,GAAG,KAAK,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,+BAA+B,CAAC;YACnD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc;gBACZ,KAAK,YAAY,uBAAuB;oBACtC,CAAC,CAAC,4BAA4B,KAAK,CAAC,OAAO,EAAE;oBAC7C,CAAC,CAAC,2BAA4B,KAAe,CAAC,OAAO,EAAE,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GACT,SAAS,KAAK,MAAM;QAClB,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAClB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC;QACrF,CAAC,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE;QACpC,QAAQ;QACR,YAAY;QACZ,SAAS;QACT,WAAW;QACX,OAAO;KACR,CAAC,CAAC;IAEH,iCAAiC;IACjC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,QAAQ,GAAG,CAAC,KAAkB,EAAsB,EAAE,CAC1D,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjB,OAAO,IAAI,CAAC,CAAC;QACb,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC;QAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC;QAC1C,OAAO;YACL,EAAE;YACF,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,WAAW,EAAE,UAAU,CACrB,GAAG,IAAI,CAAC,KAAK,SAAS,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CACrF;YACD,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;SAClC,CAAC;IACJ,CAAC,CAAC,CAAC;IACL,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE/B,2FAA2F;IAC3F,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,IACE,IAAI,CAAC,OAAO,KAAK,SAAS;gBAC1B,CAAC,IAAI,CAAC,aAAa,KAAK,KAAK,IAAI,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,EACjE,CAAC;gBACD,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAuB,EAAE,CAAC;IAC5C,KAAK,IAAI,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,GAAI,CAAC;QAChD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAsB,CAAC;QAC/C,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;YACtB,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC;YAC9B,eAAe,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,eAAe,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChC,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QAC5E,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;aACzF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;aAC5B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,6BAA6B,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,YAAY;;iBAE9E,IAAI,CAAC,KAAK;EACzB,WAAW,CAAC,CAAC,CAAC,iBAAiB,WAAW,IAAI,CAAC,CAAC,CAAC,EAAE;;EAEnD,OAAO;;6BAEoB,CAAC;QAC1B,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAC7B,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EACvD,cAAc,EACd,MAAM,EACN,GAAG,CACJ,CAAC;gBACF,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,OAAO,IAAI,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBACzE,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;oBACnB,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBACtC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;oBAC3B,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,uBAAuB,EAAE,CAAC;oBAC7C,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;gBACD,cAAc,KAAK,mBAAoB,KAAe,CAAC,OAAO,EAAE,CAAC;YACnE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,YAAY,EAAE,CAAC;QACjB,cAAc,KAAK,uDAAuD,CAAC;IAC7E,CAAC;IAED,IAAI,WAA+B,CAAC;IACpC,MAAM,qBAAqB,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9D,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACjE,CAAC,CAAC,CAAC;IACJ,IAAI,QAAQ,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,wEAAwE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE;;EAE9H,qBAAqB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;iCAEhF,CAAC;QAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,WAAW,GAAG,QAAQ,CACpB,MAAM,MAAM,CAAC,IAAI,CACf;oBACE,IAAI,EAAE,UAAU;oBAChB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC5D,OAAO,EAAE,qBAAqB;iBAC/B,EACD,cAAc,EACd,MAAM,EACN,GAAG,CACJ,EACD,KAAK,CACN,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,cAAc,KAAK,uBAAwB,KAAe,CAAC,OAAO,EAAE,CAAC;YACvE,CAAC;QACH,CAAC;IACH,CAAC;IACD,IACE,WAAW,KAAK,SAAS;QACzB,OAAO,CAAC,QAAQ,EAAE,WAAW,KAAK,SAAS;QAC3C,eAAe,GAAG,CAAC,EACnB,CAAC;QACD,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;IAC7C,CAAC;IAED,MAAM,IAAI,GAAiB;QACzB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,OAAO,EAAE,CAAC;QACV,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,SAAS;QACT,WAAW;QACX,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,KAAK;QACL,KAAK,EAAE;YACL,QAAQ,EAAE,MAAM,CAAC,KAAK;YACtB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;YAChC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,eAAe,EAAE,MAAM,CAAC,SAAS;YACjC,eAAe;SAChB;QACD,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAClC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAC3C,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic structure detection — the cheap path that runs before any LLM. Finds headings in
|
|
3
|
+
* page text: markdown headers, and the numbered/keyword headings long structured documents use
|
|
4
|
+
* (`PART 52`, `Subpart 9.1`, `Item 7.`, `52.236-5 Material and Workmanship.`, `3.2 Scope`).
|
|
5
|
+
*/
|
|
6
|
+
/** A heading found in (or declared for) the document, before nesting. */
|
|
7
|
+
export interface DetectedHeading {
|
|
8
|
+
title: string;
|
|
9
|
+
level: number;
|
|
10
|
+
unit: number;
|
|
11
|
+
/** The heading is the first text of its unit, so the previous section ends in the unit before. */
|
|
12
|
+
atUnitStart: boolean;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Detect headings in each unit's text. `markdownOnly` restricts detection to markdown headers (for
|
|
16
|
+
* text you know is markdown, where a numbered line is more likely a list item).
|
|
17
|
+
*
|
|
18
|
+
* Besides running headers (below), it drops **table-of-contents listings**: five or more
|
|
19
|
+
* consecutive keyword/numbered heading lines with no body text between them are a contents page
|
|
20
|
+
* (a regulation part opens with one), not sections — only a leading top-level heading of the run
|
|
21
|
+
* (`PART 2—…`) and its last line (the first real heading, which body text follows) are kept. Markdown headers are explicit and never dropped this way.
|
|
22
|
+
*/
|
|
23
|
+
export declare function detectHeadings(texts: readonly string[], markdownOnly?: boolean): DetectedHeading[];
|
|
24
|
+
/**
|
|
25
|
+
* Lines that repeat across many units (running headers/footers, page furniture), digits ignored so
|
|
26
|
+
* `Page 3 of 40` and `Page 4 of 40` count as one. Used to pick meaningful lines for group titles.
|
|
27
|
+
*/
|
|
28
|
+
export declare function runningLines(texts: readonly string[]): Set<string>;
|
|
29
|
+
//# sourceMappingURL=headings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"headings.d.ts","sourceRoot":"","sources":["../../../../src/rag/tree/headings.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,kGAAkG;IAClG,WAAW,EAAE,OAAO,CAAC;CACtB;AAkDD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,YAAY,UAAQ,GAAG,eAAe,EAAE,CAgHhG;AAuCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAclE"}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic structure detection — the cheap path that runs before any LLM. Finds headings in
|
|
3
|
+
* page text: markdown headers, and the numbered/keyword headings long structured documents use
|
|
4
|
+
* (`PART 52`, `Subpart 9.1`, `Item 7.`, `52.236-5 Material and Workmanship.`, `3.2 Scope`).
|
|
5
|
+
*/
|
|
6
|
+
const MARKDOWN = /^(#{1,6})\s+(.+?)\s*#*\s*$/;
|
|
7
|
+
/** Keyword headings, with the level each keyword usually sits at. */
|
|
8
|
+
const KEYWORD_LEVEL = {
|
|
9
|
+
part: 1,
|
|
10
|
+
chapter: 1,
|
|
11
|
+
title: 1,
|
|
12
|
+
appendix: 1,
|
|
13
|
+
annex: 1,
|
|
14
|
+
exhibit: 1,
|
|
15
|
+
schedule: 1,
|
|
16
|
+
subpart: 2,
|
|
17
|
+
article: 2,
|
|
18
|
+
item: 2,
|
|
19
|
+
section: 3,
|
|
20
|
+
};
|
|
21
|
+
const KEYWORD = /^(PART|Part|CHAPTER|Chapter|TITLE|APPENDIX|Appendix|ANNEX|Annex|EXHIBIT|Exhibit|SCHEDULE|Schedule|SUBPART|Subpart|ARTICLE|Article|ITEM|Item|SECTION|Section)\s+([0-9]{1,4}[A-Z]?(?:[.\-–][0-9A-Z]{1,4}){0,3}|[IVXLC]{1,6}|[A-Z])\b\.?(?:\s*[—–:\-.]\s*|\s+)?(.{0,110})$/;
|
|
22
|
+
/** `3.2 Scope`, `52.236–5 Material and Workmanship.`, `1.1.4 Definitions` — dotted numbering only. */
|
|
23
|
+
const NUMBERED = /^(\d{1,3}(?:\.\d{1,4}){1,4})([-–]\d{1,4})?\.?\s+([A-Z[][^\n]{1,100})$/;
|
|
24
|
+
function clean(title) {
|
|
25
|
+
return title
|
|
26
|
+
.replace(/\s+/g, ' ')
|
|
27
|
+
.replace(/[\s.:—–-]+$/, '')
|
|
28
|
+
.trim();
|
|
29
|
+
}
|
|
30
|
+
/** Does this line look like prose or a table row rather than a heading? */
|
|
31
|
+
function looksLikeBody(rest) {
|
|
32
|
+
return /(\d\)?|[%$])\s*$/.test(rest) || rest.split(' ').length > 16;
|
|
33
|
+
}
|
|
34
|
+
/** Words a heading title rarely ends with — a line ending in one is wrapped body text. */
|
|
35
|
+
const DANGLING = new Set('a an the of to in for and or as by with on at from that which is are be than under'.split(' '));
|
|
36
|
+
function dangling(title) {
|
|
37
|
+
if (/[,;(]$/.test(title)) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
const last = title.split(/\s+/).pop()?.toLowerCase() ?? '';
|
|
41
|
+
return DANGLING.has(last);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Detect headings in each unit's text. `markdownOnly` restricts detection to markdown headers (for
|
|
45
|
+
* text you know is markdown, where a numbered line is more likely a list item).
|
|
46
|
+
*
|
|
47
|
+
* Besides running headers (below), it drops **table-of-contents listings**: five or more
|
|
48
|
+
* consecutive keyword/numbered heading lines with no body text between them are a contents page
|
|
49
|
+
* (a regulation part opens with one), not sections — only a leading top-level heading of the run
|
|
50
|
+
* (`PART 2—…`) and its last line (the first real heading, which body text follows) are kept. Markdown headers are explicit and never dropped this way.
|
|
51
|
+
*/
|
|
52
|
+
export function detectHeadings(texts, markdownOnly = false) {
|
|
53
|
+
const headings = [];
|
|
54
|
+
let inFence = false;
|
|
55
|
+
/** The current run of consecutive non-markdown heading lines, as indexes into `headings`. */
|
|
56
|
+
let run = [];
|
|
57
|
+
const dropped = new Set();
|
|
58
|
+
const closeRun = () => {
|
|
59
|
+
if (run.length >= 5) {
|
|
60
|
+
run.forEach((entry, position) => {
|
|
61
|
+
const heading = headings[entry.index];
|
|
62
|
+
// The last line of the run is followed by body text: that one is the real heading the
|
|
63
|
+
// contents page was listing ahead of.
|
|
64
|
+
const keep = position === run.length - 1 ||
|
|
65
|
+
(position === 0 && entry.keyword && heading !== undefined && heading.level === 1);
|
|
66
|
+
if (!keep) {
|
|
67
|
+
dropped.add(entry.index);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
run = [];
|
|
72
|
+
};
|
|
73
|
+
texts.forEach((text, unit) => {
|
|
74
|
+
let seenText = false;
|
|
75
|
+
const lines = text.split('\n').map((raw) => raw.trim());
|
|
76
|
+
for (let at = 0; at < lines.length; at++) {
|
|
77
|
+
let line = lines[at] ?? '';
|
|
78
|
+
if (line.startsWith('```')) {
|
|
79
|
+
inFence = !inFence;
|
|
80
|
+
seenText = true;
|
|
81
|
+
closeRun();
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (line === '') {
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (inFence || line.length > 140) {
|
|
88
|
+
seenText = true;
|
|
89
|
+
closeRun();
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const atUnitStart = !seenText;
|
|
93
|
+
seenText = true;
|
|
94
|
+
const markdown = MARKDOWN.exec(line);
|
|
95
|
+
if (markdown?.[1] !== undefined && markdown[2] !== undefined) {
|
|
96
|
+
closeRun();
|
|
97
|
+
const title = clean(markdown[2].replace(/\*\*/g, ''));
|
|
98
|
+
if (title !== '') {
|
|
99
|
+
headings.push({ title, level: markdown[1].length, unit, atUnitStart });
|
|
100
|
+
}
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (markdownOnly) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const candidate = KEYWORD.test(line) || NUMBERED.test(line);
|
|
107
|
+
if (candidate) {
|
|
108
|
+
// A heading hyphenated over lines (`52.236–3 Site Investigation and Condi-` / `tions
|
|
109
|
+
// Affecting the Work.`): join up to two continuation lines.
|
|
110
|
+
for (let joined = 0; joined < 2; joined++) {
|
|
111
|
+
const next = lines[at + 1];
|
|
112
|
+
if (!/[a-z]-$/.test(line) || next === undefined || next === '' || next.length > 80) {
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
line = `${line.slice(0, -1)}${next}`;
|
|
116
|
+
at += 1;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const keyword = KEYWORD.exec(line);
|
|
120
|
+
if (keyword?.[1] !== undefined && keyword[2] !== undefined) {
|
|
121
|
+
const rest = keyword[3] ?? '';
|
|
122
|
+
const allCaps = keyword[1] === keyword[1].toUpperCase();
|
|
123
|
+
if (!looksLikeBody(rest) &&
|
|
124
|
+
!/^[a-z,;)]/.test(rest) &&
|
|
125
|
+
(rest !== '' || allCaps) &&
|
|
126
|
+
!dangling(clean(line))) {
|
|
127
|
+
const level = KEYWORD_LEVEL[keyword[1].toLowerCase()] ?? 2;
|
|
128
|
+
run.push({ index: headings.length, keyword: true });
|
|
129
|
+
headings.push({ title: clean(line), level, unit, atUnitStart });
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const numbered = NUMBERED.exec(line);
|
|
134
|
+
if (numbered?.[1] !== undefined &&
|
|
135
|
+
numbered[3] !== undefined &&
|
|
136
|
+
!looksLikeBody(numbered[3]) &&
|
|
137
|
+
!dangling(clean(numbered[3]))) {
|
|
138
|
+
// Below the keyword levels: `52.236` is 3, `52.236–5` is 4, `1.1.4` is 4.
|
|
139
|
+
const level = Math.min(6, numbered[1].split('.').length + 1 + (numbered[2] ? 1 : 0));
|
|
140
|
+
run.push({ index: headings.length, keyword: false });
|
|
141
|
+
headings.push({ title: clean(line), level, unit, atUnitStart });
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
// A body line ends a run of heading lines; a heading-shaped line that was rejected (a contents
|
|
145
|
+
// entry cut mid-title), a short Title Case line (its wrapped second line) or page furniture
|
|
146
|
+
// does not.
|
|
147
|
+
if (!candidate && /[A-Za-z]{3,}/.test(line) && !isPageFurniture(line) && !isTitleCase(line)) {
|
|
148
|
+
closeRun();
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
closeRun();
|
|
153
|
+
return dropContentsDuplicates(dropRunningHeaders(headings.filter((_, index) => !dropped.has(index)), texts.length));
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* A numbered title (`19.800 General`, `Subpart 19.8—…`) names one section; seen twice, the earlier
|
|
157
|
+
* one is a contents entry pointing at the later — keep the later. Titles without a number
|
|
158
|
+
* ("General", "Definitions") legitimately repeat and are all kept.
|
|
159
|
+
*/
|
|
160
|
+
function dropContentsDuplicates(headings) {
|
|
161
|
+
const last = new Map();
|
|
162
|
+
headings.forEach((heading, index) => {
|
|
163
|
+
if (/\d/.test(heading.title)) {
|
|
164
|
+
last.set(heading.title.toLowerCase(), index);
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
return headings.filter((heading, index) => {
|
|
168
|
+
const key = heading.title.toLowerCase();
|
|
169
|
+
return !last.has(key) || last.get(key) === index;
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
/** A short line whose longer words are mostly capitalized — a title fragment, not prose. */
|
|
173
|
+
function isTitleCase(line) {
|
|
174
|
+
if (line.length > 70 || /\.\s*$/.test(line)) {
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
const words = line.split(/\s+/).filter((word) => /^[A-Za-z]{4,}/.test(word));
|
|
178
|
+
return (words.length > 0 && words.filter((word) => /^[A-Z]/.test(word)).length / words.length >= 0.6);
|
|
179
|
+
}
|
|
180
|
+
/** A page number, a running citation line (`48 CFR Ch. 1 (10–1–24 Edition)`) — neither body nor heading. */
|
|
181
|
+
function isPageFurniture(line) {
|
|
182
|
+
return (line.length < 50 &&
|
|
183
|
+
/^[\d\s\-–—.()]*(\d+\s*CFR|Federal Acquisition Regulation|Page \d+)/i.test(line));
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Lines that repeat across many units (running headers/footers, page furniture), digits ignored so
|
|
187
|
+
* `Page 3 of 40` and `Page 4 of 40` count as one. Used to pick meaningful lines for group titles.
|
|
188
|
+
*/
|
|
189
|
+
export function runningLines(texts) {
|
|
190
|
+
const counts = new Map();
|
|
191
|
+
for (const text of texts) {
|
|
192
|
+
const seen = new Set();
|
|
193
|
+
for (const raw of text.split('\n')) {
|
|
194
|
+
const key = raw.trim().replace(/\d+/g, '#').toLowerCase();
|
|
195
|
+
if (key && !seen.has(key)) {
|
|
196
|
+
seen.add(key);
|
|
197
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const threshold = Math.max(3, Math.ceil(texts.length * 0.2));
|
|
202
|
+
return new Set([...counts].filter(([, count]) => count >= threshold).map(([key]) => key));
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Remove running headers/footers: the same title repeated on many units ("PART 52—SOLICITATION
|
|
206
|
+
* PROVISIONS" printed atop every page) is page furniture, not structure. Keeps the first occurrence.
|
|
207
|
+
*/
|
|
208
|
+
function dropRunningHeaders(headings, unitCount) {
|
|
209
|
+
const counts = new Map();
|
|
210
|
+
for (const heading of headings) {
|
|
211
|
+
const key = heading.title.toLowerCase();
|
|
212
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
213
|
+
}
|
|
214
|
+
const seen = new Set();
|
|
215
|
+
const threshold = Math.max(3, Math.ceil(unitCount * 0.2));
|
|
216
|
+
return headings.filter((heading) => {
|
|
217
|
+
const key = heading.title.toLowerCase();
|
|
218
|
+
if ((counts.get(key) ?? 0) >= threshold) {
|
|
219
|
+
if (seen.has(key)) {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
seen.add(key);
|
|
224
|
+
return true;
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
//# sourceMappingURL=headings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"headings.js","sourceRoot":"","sources":["../../../../src/rag/tree/headings.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH,MAAM,QAAQ,GAAG,4BAA4B,CAAC;AAE9C,qEAAqE;AACrE,MAAM,aAAa,GAA2B;IAC5C,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,OAAO,EAAE,CAAC;IACV,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC;IACV,OAAO,EAAE,CAAC;IACV,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC;CACX,CAAC;AAEF,MAAM,OAAO,GACX,yQAAyQ,CAAC;AAE5Q,sGAAsG;AACtG,MAAM,QAAQ,GAAG,uEAAuE,CAAC;AAEzF,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,KAAK;SACT,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,2EAA2E;AAC3E,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC;AACtE,CAAC;AAED,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,oFAAoF,CAAC,KAAK,CAAC,GAAG,CAAC,CAChG,CAAC;AAEF,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3D,OAAO,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,KAAwB,EAAE,YAAY,GAAG,KAAK;IAC3E,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,6FAA6F;IAC7F,IAAI,GAAG,GAA0C,EAAE,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACtC,sFAAsF;gBACtF,sCAAsC;gBACtC,MAAM,IAAI,GACR,QAAQ,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC;oBAC3B,CAAC,QAAQ,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QACD,GAAG,GAAG,EAAE,CAAC;IACX,CAAC,CAAC;IACF,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;QAC3B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,GAAG,CAAC,OAAO,CAAC;gBACnB,QAAQ,GAAG,IAAI,CAAC;gBAChB,QAAQ,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;YACD,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YACD,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACjC,QAAQ,GAAG,IAAI,CAAC;gBAChB,QAAQ,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;YACD,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC;YAC9B,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC7D,QAAQ,EAAE,CAAC;gBACX,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;gBACtD,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;oBACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;gBACzE,CAAC;gBACD,SAAS;YACX,CAAC;YACD,IAAI,YAAY,EAAE,CAAC;gBACjB,SAAS;YACX,CAAC;YACD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,SAAS,EAAE,CAAC;gBACd,qFAAqF;gBACrF,4DAA4D;gBAC5D,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;oBAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC3B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;wBACnF,MAAM;oBACR,CAAC;oBACD,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC;oBACrC,EAAE,IAAI,CAAC,CAAC;gBACV,CAAC;YACH,CAAC;YACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACxD,IACE,CAAC,aAAa,CAAC,IAAI,CAAC;oBACpB,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;oBACvB,CAAC,IAAI,KAAK,EAAE,IAAI,OAAO,CAAC;oBACxB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EACtB,CAAC;oBACD,MAAM,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;oBAC3D,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;oBAChE,SAAS;gBACX,CAAC;YACH,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrC,IACE,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS;gBAC3B,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS;gBACzB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC3B,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAC7B,CAAC;gBACD,0EAA0E;gBAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACrF,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChE,SAAS;YACX,CAAC;YACD,+FAA+F;YAC/F,4FAA4F;YAC5F,YAAY;YACZ,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5F,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,QAAQ,EAAE,CAAC;IACX,OAAO,sBAAsB,CAC3B,kBAAkB,CAChB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAClD,KAAK,CAAC,MAAM,CACb,CACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,QAA2B;IACzD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAClC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,4FAA4F;AAC5F,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7E,OAAO,CACL,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAC7F,CAAC;AACJ,CAAC;AAED,4GAA4G;AAC5G,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,CACL,IAAI,CAAC,MAAM,GAAG,EAAE;QAChB,qEAAqE,CAAC,IAAI,CAAC,IAAI,CAAC,CACjF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,KAAwB;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAC1D,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;IAC7D,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,QAA2B,EAAE,SAAiB;IACxE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;QACjC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type BuildDocumentTreeOptions } from './build.js';
|
|
2
|
+
import type { DocumentTreeStore } from './store.js';
|
|
3
|
+
import type { DocumentTree, DocumentTreeInput } from './types.js';
|
|
4
|
+
export interface IndexDocumentTreeOptions extends Omit<BuildDocumentTreeOptions, 'previous'> {
|
|
5
|
+
store: DocumentTreeStore;
|
|
6
|
+
/**
|
|
7
|
+
* Only documents with at least this many units (pages, or sections) get a tree; a shorter one is
|
|
8
|
+
* skipped and any tree it had is removed. Default 20 — tree navigation pays off on long
|
|
9
|
+
* structured documents and costs latency and LLM spend for nothing on short notes.
|
|
10
|
+
*/
|
|
11
|
+
minUnits?: number;
|
|
12
|
+
/** …or at least this many characters, whichever is met first. Default 60 000 (~15k tokens). */
|
|
13
|
+
minChars?: number;
|
|
14
|
+
}
|
|
15
|
+
export type IndexDocumentTreeResult = {
|
|
16
|
+
status: 'skipped';
|
|
17
|
+
reason: string;
|
|
18
|
+
} | {
|
|
19
|
+
status: 'unchanged';
|
|
20
|
+
tree: DocumentTree;
|
|
21
|
+
} | {
|
|
22
|
+
status: 'built';
|
|
23
|
+
tree: DocumentTree;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* Build (or rebuild) a document's tree and store it — the call to make at upload/ingestion time,
|
|
27
|
+
* next to chunking and embedding. Short documents are skipped (see `minUnits`/`minChars`); an
|
|
28
|
+
* unchanged document costs one store read and no LLM call; a changed one reuses the summaries of
|
|
29
|
+
* its unchanged sections. Never throws for a document the LLM cannot structure: the tree falls back
|
|
30
|
+
* to page groups and `tree.stats.fallbackReason` says why. Store errors do propagate.
|
|
31
|
+
*/
|
|
32
|
+
export declare function indexDocumentTree(input: DocumentTreeInput, options: IndexDocumentTreeOptions): Promise<IndexDocumentTreeResult>;
|
|
33
|
+
//# sourceMappingURL=index-document.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-document.d.ts","sourceRoot":"","sources":["../../../../src/rag/tree/index-document.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,wBAAwB,EAA8B,MAAM,YAAY,CAAC;AACvF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAElE,MAAM,WAAW,wBAAyB,SAAQ,IAAI,CAAC,wBAAwB,EAAE,UAAU,CAAC;IAC1F,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,uBAAuB,GAC/B;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,YAAY,CAAA;CAAE,GAC3C;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,YAAY,CAAA;CAAE,CAAC;AAE5C;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,iBAAiB,EACxB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CA2BlC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { buildDocumentTree, toUnits } from './build.js';
|
|
2
|
+
/**
|
|
3
|
+
* Build (or rebuild) a document's tree and store it — the call to make at upload/ingestion time,
|
|
4
|
+
* next to chunking and embedding. Short documents are skipped (see `minUnits`/`minChars`); an
|
|
5
|
+
* unchanged document costs one store read and no LLM call; a changed one reuses the summaries of
|
|
6
|
+
* its unchanged sections. Never throws for a document the LLM cannot structure: the tree falls back
|
|
7
|
+
* to page groups and `tree.stats.fallbackReason` says why. Store errors do propagate.
|
|
8
|
+
*/
|
|
9
|
+
export async function indexDocumentTree(input, options) {
|
|
10
|
+
const { store, minUnits = 20, minChars = 60_000, ...build } = options;
|
|
11
|
+
const { units } = toUnits(input);
|
|
12
|
+
const chars = units.reduce((total, unit) => total + unit.text.length, 0);
|
|
13
|
+
if (units.length < minUnits && chars < minChars) {
|
|
14
|
+
await store.remove(options.documentId);
|
|
15
|
+
return {
|
|
16
|
+
status: 'skipped',
|
|
17
|
+
reason: `${units.length} units / ${chars} chars is below the threshold (${minUnits} units or ${minChars} chars)`,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
const previous = await store.get(options.documentId);
|
|
21
|
+
const built = await buildDocumentTree(input, {
|
|
22
|
+
...build,
|
|
23
|
+
...(previous !== undefined ? { previous } : {}),
|
|
24
|
+
});
|
|
25
|
+
if (built.unchanged) {
|
|
26
|
+
const metadataMoved = JSON.stringify(previous?.metadata ?? null) !== JSON.stringify(built.tree.metadata ?? null) ||
|
|
27
|
+
previous?.source !== built.tree.source;
|
|
28
|
+
if (metadataMoved) {
|
|
29
|
+
await store.put(built.tree, built.units);
|
|
30
|
+
}
|
|
31
|
+
return { status: 'unchanged', tree: built.tree };
|
|
32
|
+
}
|
|
33
|
+
await store.put(built.tree, built.units);
|
|
34
|
+
return { status: 'built', tree: built.tree };
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=index-document.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-document.js","sourceRoot":"","sources":["../../../../src/rag/tree/index-document.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiC,iBAAiB,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAqBvF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAwB,EACxB,OAAiC;IAEjC,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,EAAE,EAAE,QAAQ,GAAG,MAAM,EAAE,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACzE,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACvC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,YAAY,KAAK,kCAAkC,QAAQ,aAAa,QAAQ,SAAS;SACjH,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE;QAC3C,GAAG,KAAK;QACR,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChD,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,aAAa,GACjB,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;YAC1F,QAAQ,EAAE,MAAM,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QACzC,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;IACnD,CAAC;IACD,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;AAC/C,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { type BuildDocumentTreeOptions, type BuiltDocumentTree, buildDocumentTree, spanText, } from './build.js';
|
|
2
|
+
export { type DetectedHeading, detectHeadings } from './headings.js';
|
|
3
|
+
export { type IndexDocumentTreeOptions, type IndexDocumentTreeResult, indexDocumentTree, } from './index-document.js';
|
|
4
|
+
export { keywordTreeLlm } from './keyword-tree-llm.js';
|
|
5
|
+
export { cachedTreeLlm, estimateTokens, type OpenAiChatTreeLlmOptions, openAiChatTreeLlm, parseJsonReply, stableHash, type TreeBudget, TreeBudgetExceededError, type TreeLlm, type TreeLlmCache, type TreeLlmCandidate, type TreeLlmRequest, type TreeLlmResponse, type TreeLlmTask, treeLlmFromModelProvider, } from './llm.js';
|
|
6
|
+
export { type DocumentNavigation, type NavigatedNode, type NavigateOptions, type NavigationResult, type NavigationStep, TreeNavigationRetriever, type TreeNavigationRetrieverOptions, } from './navigate.js';
|
|
7
|
+
export { createNavigateDocumentTool, type NavigateDocumentInput, type NavigateDocumentOutput, type NavigateDocumentToolOptions, } from './navigate-tool.js';
|
|
8
|
+
export { PgDocumentTreeStore, type PgDocumentTreeStoreOptions } from './pg-tree-store.js';
|
|
9
|
+
export { type DocumentTreeHeader, type DocumentTreeStore, MemoryDocumentTreeStore, } from './store.js';
|
|
10
|
+
export { TwoStageRetriever, type TwoStageRetrieverOptions } from './two-stage.js';
|
|
11
|
+
export type { DocumentTree, DocumentTreeInput, DocumentTreeNode, TreeBuildStats, TreeHeading, TreeSection, TreeStructureSource, TreeSummarySource, TreeUnit, } from './types.js';
|
|
12
|
+
export { indexTree, walkTree } from './types.js';
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/rag/tree/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,wBAAwB,EAC7B,KAAK,iBAAiB,EACtB,iBAAiB,EACjB,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,KAAK,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EACL,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC5B,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,cAAc,EACd,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,KAAK,UAAU,EACf,uBAAuB,EACvB,KAAK,OAAO,EACZ,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,wBAAwB,GACzB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,uBAAuB,EACvB,KAAK,8BAA8B,GACpC,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,0BAA0B,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,GACjC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC1F,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,uBAAuB,GACxB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,iBAAiB,EAAE,KAAK,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAClF,YAAY,EACV,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,WAAW,EACX,WAAW,EACX,mBAAmB,EACnB,iBAAiB,EACjB,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { buildDocumentTree, spanText, } from './build.js';
|
|
2
|
+
export { detectHeadings } from './headings.js';
|
|
3
|
+
export { indexDocumentTree, } from './index-document.js';
|
|
4
|
+
export { keywordTreeLlm } from './keyword-tree-llm.js';
|
|
5
|
+
export { cachedTreeLlm, estimateTokens, openAiChatTreeLlm, parseJsonReply, stableHash, TreeBudgetExceededError, treeLlmFromModelProvider, } from './llm.js';
|
|
6
|
+
export { TreeNavigationRetriever, } from './navigate.js';
|
|
7
|
+
export { createNavigateDocumentTool, } from './navigate-tool.js';
|
|
8
|
+
export { PgDocumentTreeStore } from './pg-tree-store.js';
|
|
9
|
+
export { MemoryDocumentTreeStore, } from './store.js';
|
|
10
|
+
export { TwoStageRetriever } from './two-stage.js';
|
|
11
|
+
export { indexTree, walkTree } from './types.js';
|
|
12
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/rag/tree/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,iBAAiB,EACjB,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EAAwB,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAGL,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,cAAc,EAEd,iBAAiB,EACjB,cAAc,EACd,UAAU,EAEV,uBAAuB,EAOvB,wBAAwB,GACzB,MAAM,UAAU,CAAC;AAClB,OAAO,EAML,uBAAuB,GAExB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,0BAA0B,GAI3B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAmC,MAAM,oBAAoB,CAAC;AAC1F,OAAO,EAGL,uBAAuB,GACxB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,iBAAiB,EAAiC,MAAM,gBAAgB,CAAC;AAYlF,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { TreeLlm } from './llm.js';
|
|
2
|
+
/**
|
|
3
|
+
* A deterministic stand-in for the tree LLM — no model, no network — for tests and offline
|
|
4
|
+
* development. It answers from the structured {@link TreeLlmRequest.task}, never the prose:
|
|
5
|
+
*
|
|
6
|
+
* - `navigate` / `select`: ranks candidates by how many of the question's terms their title and
|
|
7
|
+
* summary share, keeps those sharing any (deepest first on ties in single-pass mode); in a beam,
|
|
8
|
+
* candidates with subsections are expanded, the rest read;
|
|
9
|
+
* - `summarize`: the section's first sentence; `describe`: the top-level titles;
|
|
10
|
+
* - `structure`: lines in ALL CAPS (or markdown headers) become level-1 sections.
|
|
11
|
+
*
|
|
12
|
+
* Wrap it to count calls, inject failures or slow replies when testing budgets.
|
|
13
|
+
*/
|
|
14
|
+
export declare function keywordTreeLlm(): TreeLlm;
|
|
15
|
+
//# sourceMappingURL=keyword-tree-llm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keyword-tree-llm.d.ts","sourceRoot":"","sources":["../../../../src/rag/tree/keyword-tree-llm.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAmC,MAAM,UAAU,CAAC;AA0BzE;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,IAAI,OAAO,CAgFxC"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
const STOP = new Set('the and for are was were with that this from which what when where who whom how does did has have had not but its into under over than then there their about any all can may must shall should would could will our your they them his her per each other such only also more most'.split(' '));
|
|
2
|
+
function terms(text) {
|
|
3
|
+
return new Set((text.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}.-]*[\p{L}\p{N}]|[\p{L}\p{N}]{3,}/gu) ?? []).filter((term) => term.length >= 3 && !STOP.has(term)));
|
|
4
|
+
}
|
|
5
|
+
function overlap(query, text) {
|
|
6
|
+
let score = 0;
|
|
7
|
+
for (const term of terms(text)) {
|
|
8
|
+
if (query.has(term)) {
|
|
9
|
+
score += 1;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return score;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* A deterministic stand-in for the tree LLM — no model, no network — for tests and offline
|
|
16
|
+
* development. It answers from the structured {@link TreeLlmRequest.task}, never the prose:
|
|
17
|
+
*
|
|
18
|
+
* - `navigate` / `select`: ranks candidates by how many of the question's terms their title and
|
|
19
|
+
* summary share, keeps those sharing any (deepest first on ties in single-pass mode); in a beam,
|
|
20
|
+
* candidates with subsections are expanded, the rest read;
|
|
21
|
+
* - `summarize`: the section's first sentence; `describe`: the top-level titles;
|
|
22
|
+
* - `structure`: lines in ALL CAPS (or markdown headers) become level-1 sections.
|
|
23
|
+
*
|
|
24
|
+
* Wrap it to count calls, inject failures or slow replies when testing budgets.
|
|
25
|
+
*/
|
|
26
|
+
export function keywordTreeLlm() {
|
|
27
|
+
return async (request) => {
|
|
28
|
+
const { task } = request;
|
|
29
|
+
let text;
|
|
30
|
+
switch (task.kind) {
|
|
31
|
+
case 'navigate': {
|
|
32
|
+
const query = terms(task.query);
|
|
33
|
+
const ranked = task.candidates
|
|
34
|
+
.map((candidate, order) => ({
|
|
35
|
+
candidate,
|
|
36
|
+
order,
|
|
37
|
+
score: overlap(query, `${candidate.title} ${candidate.summary ?? ''}`),
|
|
38
|
+
}))
|
|
39
|
+
.filter((entry) => entry.score > 0)
|
|
40
|
+
.sort((a, b) => b.score - a.score ||
|
|
41
|
+
(b.candidate.depth ?? 0) - (a.candidate.depth ?? 0) ||
|
|
42
|
+
a.order - b.order)
|
|
43
|
+
.slice(0, task.maxNodes)
|
|
44
|
+
.map((entry) => entry.candidate);
|
|
45
|
+
text =
|
|
46
|
+
task.mode === 'single-pass'
|
|
47
|
+
? JSON.stringify({ thinking: 'keyword overlap', read: ranked.map((c) => c.id) })
|
|
48
|
+
: JSON.stringify({
|
|
49
|
+
thinking: 'keyword overlap',
|
|
50
|
+
expand: ranked.filter((c) => c.hasChildren).map((c) => c.id),
|
|
51
|
+
read: ranked.filter((c) => !c.hasChildren).map((c) => c.id),
|
|
52
|
+
});
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
case 'select': {
|
|
56
|
+
const query = terms(task.query);
|
|
57
|
+
const ranked = task.documents
|
|
58
|
+
.map((document) => ({
|
|
59
|
+
id: document.id,
|
|
60
|
+
score: overlap(query, `${document.title ?? ''} ${document.description ?? ''}`),
|
|
61
|
+
}))
|
|
62
|
+
.filter((entry) => entry.score > 0)
|
|
63
|
+
.sort((a, b) => b.score - a.score)
|
|
64
|
+
.slice(0, task.maxDocuments)
|
|
65
|
+
.map((entry) => entry.id);
|
|
66
|
+
text = JSON.stringify({ thinking: 'keyword overlap', documents: ranked });
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
case 'summarize': {
|
|
70
|
+
const flat = task.text.replace(/\s+/g, ' ').trim();
|
|
71
|
+
const sentence = /^.*?[.!?](\s|$)/.exec(flat)?.[0] ?? flat;
|
|
72
|
+
text = `${task.title}: ${sentence.slice(0, 200).trim()}`;
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
case 'describe':
|
|
76
|
+
text = `${task.title ?? 'Document'} covering ${task.outline.map((entry) => entry.title).join(', ')}.`;
|
|
77
|
+
break;
|
|
78
|
+
case 'structure': {
|
|
79
|
+
const sections = [];
|
|
80
|
+
for (const unit of task.units) {
|
|
81
|
+
for (const line of unit.text.split('\n')) {
|
|
82
|
+
const trimmed = line.trim();
|
|
83
|
+
const markdown = /^(#{1,6})\s+(.+)$/.exec(trimmed);
|
|
84
|
+
if (markdown?.[1] !== undefined && markdown[2] !== undefined) {
|
|
85
|
+
sections.push({ title: markdown[2], level: markdown[1].length, unit: unit.unit });
|
|
86
|
+
}
|
|
87
|
+
else if (/^[A-Z][A-Z0-9 ,.'&-]{3,80}$/.test(trimmed) && /[A-Z]{3}/.test(trimmed)) {
|
|
88
|
+
sections.push({ title: trimmed, level: 1, unit: unit.unit });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
text = JSON.stringify({ sections });
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
text,
|
|
98
|
+
usage: {
|
|
99
|
+
inputTokens: Math.ceil(request.prompt.length / 4),
|
|
100
|
+
outputTokens: Math.ceil(text.length / 4),
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=keyword-tree-llm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keyword-tree-llm.js","sourceRoot":"","sources":["../../../../src/rag/tree/keyword-tree-llm.ts"],"names":[],"mappings":"AAEA,MAAM,IAAI,GAAG,IAAI,GAAG,CAClB,qRAAqR,CAAC,KAAK,CACzR,GAAG,CACJ,CACF,CAAC;AAEF,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,GAAG,CACZ,CACE,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,4DAA4D,CAAC,IAAI,EAAE,CAC7F,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CACxD,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,KAAkB,EAAE,IAAY;IAC/C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,KAAK,EAAE,OAAuB,EAA4B,EAAE;QACjE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;QACzB,IAAI,IAAY,CAAC;QACjB,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,KAAK,UAAU,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU;qBAC3B,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1B,SAAS;oBACT,KAAK;oBACL,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;iBACvE,CAAC,CAAC;qBACF,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;qBAClC,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;oBACjB,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,CAAC;oBACnD,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CACpB;qBACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC;qBACvB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACnC,IAAI;oBACF,IAAI,CAAC,IAAI,KAAK,aAAa;wBACzB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;wBAChF,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;4BACb,QAAQ,EAAE,iBAAiB;4BAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BAC5D,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC5D,CAAC,CAAC;gBACT,MAAM;YACR,CAAC;YACD,KAAK,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS;qBAC1B,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAClB,EAAE,EAAE,QAAQ,CAAC,EAAE;oBACf,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,QAAQ,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;iBAC/E,CAAC,CAAC;qBACF,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;qBAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;qBACjC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC;qBAC3B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC5B,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC1E,MAAM;YACR,CAAC;YACD,KAAK,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACnD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;gBAC3D,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzD,MAAM;YACR,CAAC;YACD,KAAK,UAAU;gBACb,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBACtG,MAAM;YACR,KAAK,WAAW,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAqD,EAAE,CAAC;gBACtE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;wBACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;wBAC5B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACnD,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;4BAC7D,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;wBACpF,CAAC;6BAAM,IAAI,6BAA6B,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;4BACnF,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;wBAC/D,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACpC,MAAM;YACR,CAAC;QACH,CAAC;QACD,OAAO;YACL,IAAI;YACJ,KAAK,EAAE;gBACL,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;gBACjD,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aACzC;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|