@apart-tech/intelligence-core 1.11.5 → 1.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/ability.d.ts +4 -4
- package/dist/auth/ability.d.ts.map +1 -1
- package/dist/auth/ability.js +17 -11
- package/dist/auth/ability.js.map +1 -1
- package/dist/auth/ability.test.js +25 -12
- package/dist/auth/ability.test.js.map +1 -1
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -1
- package/dist/config/index.js.map +1 -1
- package/dist/db/tenant.d.ts.map +1 -1
- package/dist/db/tenant.js +8 -0
- package/dist/db/tenant.js.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/services/__tests__/chunk-service.test.d.ts +2 -0
- package/dist/services/__tests__/chunk-service.test.d.ts.map +1 -0
- package/dist/services/__tests__/chunk-service.test.js +111 -0
- package/dist/services/__tests__/chunk-service.test.js.map +1 -0
- package/dist/services/__tests__/chunker.test.d.ts +2 -0
- package/dist/services/__tests__/chunker.test.d.ts.map +1 -0
- package/dist/services/__tests__/chunker.test.js +113 -0
- package/dist/services/__tests__/chunker.test.js.map +1 -0
- package/dist/services/__tests__/node-service.test.d.ts +2 -0
- package/dist/services/__tests__/node-service.test.d.ts.map +1 -0
- package/dist/services/__tests__/node-service.test.js +207 -0
- package/dist/services/__tests__/node-service.test.js.map +1 -0
- package/dist/services/__tests__/pii-detector-service.test.js +51 -0
- package/dist/services/__tests__/pii-detector-service.test.js.map +1 -1
- package/dist/services/__tests__/pii-encryption-service.test.js +37 -0
- package/dist/services/__tests__/pii-encryption-service.test.js.map +1 -1
- package/dist/services/__tests__/search-service.test.d.ts +2 -0
- package/dist/services/__tests__/search-service.test.d.ts.map +1 -0
- package/dist/services/__tests__/search-service.test.js +163 -0
- package/dist/services/__tests__/search-service.test.js.map +1 -0
- package/dist/services/backfill-chunks.d.ts +30 -0
- package/dist/services/backfill-chunks.d.ts.map +1 -0
- package/dist/services/backfill-chunks.js +55 -0
- package/dist/services/backfill-chunks.js.map +1 -0
- package/dist/services/chunk-service.d.ts +45 -0
- package/dist/services/chunk-service.d.ts.map +1 -0
- package/dist/services/chunk-service.js +111 -0
- package/dist/services/chunk-service.js.map +1 -0
- package/dist/services/chunker.d.ts +32 -0
- package/dist/services/chunker.d.ts.map +1 -0
- package/dist/services/chunker.js +289 -0
- package/dist/services/chunker.js.map +1 -0
- package/dist/services/context-service.d.ts +3 -1
- package/dist/services/context-service.d.ts.map +1 -1
- package/dist/services/context-service.js +17 -1
- package/dist/services/context-service.js.map +1 -1
- package/dist/services/node-service.d.ts +12 -1
- package/dist/services/node-service.d.ts.map +1 -1
- package/dist/services/node-service.js +54 -11
- package/dist/services/node-service.js.map +1 -1
- package/dist/services/pii-detector-service.d.ts +1 -0
- package/dist/services/pii-detector-service.d.ts.map +1 -1
- package/dist/services/pii-detector-service.js +95 -2
- package/dist/services/pii-detector-service.js.map +1 -1
- package/dist/services/pii-encryption-service.d.ts +10 -0
- package/dist/services/pii-encryption-service.d.ts.map +1 -1
- package/dist/services/pii-encryption-service.js +32 -0
- package/dist/services/pii-encryption-service.js.map +1 -1
- package/dist/services/search-service.d.ts +30 -1
- package/dist/services/search-service.d.ts.map +1 -1
- package/dist/services/search-service.js +262 -45
- package/dist/services/search-service.js.map +1 -1
- package/dist/services/tag-service.d.ts +78 -0
- package/dist/services/tag-service.d.ts.map +1 -0
- package/dist/services/tag-service.js +639 -0
- package/dist/services/tag-service.js.map +1 -0
- package/dist/services/tag-service.test.d.ts +2 -0
- package/dist/services/tag-service.test.d.ts.map +1 -0
- package/dist/services/tag-service.test.js +448 -0
- package/dist/services/tag-service.test.js.map +1 -0
- package/dist/services/workspace-service.d.ts +2 -0
- package/dist/services/workspace-service.d.ts.map +1 -1
- package/dist/services/workspace-service.js +7 -1
- package/dist/services/workspace-service.js.map +1 -1
- package/dist/types/index.d.ts +67 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +2 -2
- package/prisma/schema.prisma +180 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pii-encryption-service.test.js","sourceRoot":"","sources":["../../../src/services/__tests__/pii-encryption-service.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;IAEzC,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,aAAa;aACpB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,0CAA0C,EAAE;gBAC7E,MAAM;gBACN,IAAI,EAAE,SAAS;gBACf,eAAe,EAAE,CAAC,OAAO,CAAC;aAC3B,CAAC,CAAC;YAEH,qCAAqC;YACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,0BAA0B;YAC1B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAC3D,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,QAAQ,GAAG,iDAAiD,CAAC;YACnE,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC7E,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9D,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,IAAI,GAAG,aAAa,CAAC;YAC3B,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAC9D,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC7D,gEAAgE;YAChE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CACtC;gBACE,KAAK,EAAE,0BAA0B;gBACjC,OAAO,EAAE,iCAAiC;gBAC1C,QAAQ,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;aACzC,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAC5B,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,KAAK,GAAG;gBACZ,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;aAC3B,CAAC;YACF,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE9E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,QAAQ,GAAG;gBACf,KAAK,EAAE,0BAA0B;gBACjC,OAAO,EAAE,qBAAqB;gBAC9B,QAAQ,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE;aACnC,CAAC;YAEF,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG;gBACX,EAAE,EAAE,MAAM;gBACV,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,EAAE;aACnC,CAAC;YAEF,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAC3D,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,kEAAkE,CAAC;YACtF,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,CAAC,KAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,mEAAmE;YACnE,MAAM,CAAC,KAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"pii-encryption-service.test.js","sourceRoot":"","sources":["../../../src/services/__tests__/pii-encryption-service.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;IAEzC,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,aAAa;aACpB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,iCAAiC,EAAE;gBACpE,MAAM;gBACN,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,0CAA0C,EAAE;gBAC7E,MAAM;gBACN,IAAI,EAAE,SAAS;gBACf,eAAe,EAAE,CAAC,OAAO,CAAC;aAC3B,CAAC,CAAC;YAEH,qCAAqC;YACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,0BAA0B;YAC1B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAC3D,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,QAAQ,GAAG,iDAAiD,CAAC;YACnE,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC7E,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9D,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,IAAI,GAAG,aAAa,CAAC;YAC3B,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAC9D,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC7D,gEAAgE;YAChE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CACtC;gBACE,KAAK,EAAE,0BAA0B;gBACjC,OAAO,EAAE,iCAAiC;gBAC1C,QAAQ,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;aACzC,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAC5B,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,KAAK,GAAG;gBACZ,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;aAC3B,CAAC;YACF,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE9E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,QAAQ,GAAG;gBACf,KAAK,EAAE,0BAA0B;gBACjC,OAAO,EAAE,qBAAqB;gBAC9B,QAAQ,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE;aACnC,CAAC;YAEF,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG;gBACX,EAAE,EAAE,MAAM;gBACV,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,EAAE;aACnC,CAAC;YAEF,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;YACnE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,yCAAyC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACtF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,wCAAwC,CAAC,CAAC;YAC1E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACpC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAC3D,MAAM;gBACN,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,kEAAkE,CAAC;YACtF,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,CAAC,KAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,mEAAmE;YACnE,MAAM,CAAC,KAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-service.test.d.ts","sourceRoot":"","sources":["../../../src/services/__tests__/search-service.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
2
|
+
import { SearchService } from "../search-service.js";
|
|
3
|
+
import { PiiDetectorService } from "../pii-detector-service.js";
|
|
4
|
+
import { PiiEncryptionService } from "../pii-encryption-service.js";
|
|
5
|
+
function createMockDb() {
|
|
6
|
+
return {
|
|
7
|
+
$queryRaw: vi.fn().mockResolvedValue([]),
|
|
8
|
+
$executeRaw: vi.fn().mockResolvedValue(1),
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
function createMockEmbeddings() {
|
|
12
|
+
return {
|
|
13
|
+
embed: vi.fn(async () => [0.1, 0.2, 0.3]),
|
|
14
|
+
embedBatch: vi.fn(async (texts) => texts.map(() => [0.1, 0.2, 0.3])),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
const testConfig = {
|
|
18
|
+
database: { url: "postgresql://localhost/test" },
|
|
19
|
+
embeddings: { provider: "mock", model: "mock", dimensions: 3 },
|
|
20
|
+
search: {
|
|
21
|
+
defaultLimit: 10,
|
|
22
|
+
includeDrafts: true,
|
|
23
|
+
semanticWeight: 1,
|
|
24
|
+
keywordWeight: 1,
|
|
25
|
+
substringWeight: 0.5,
|
|
26
|
+
trigramWeight: 0.3,
|
|
27
|
+
trigramThreshold: 0.3,
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
describe("SearchService", () => {
|
|
31
|
+
const detector = new PiiDetectorService();
|
|
32
|
+
const piiEncryption = new PiiEncryptionService(detector);
|
|
33
|
+
const tenantCtx = { organizationId: "org-1" };
|
|
34
|
+
describe("PII stripping in search", () => {
|
|
35
|
+
let db;
|
|
36
|
+
let embeddings;
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
db = createMockDb();
|
|
39
|
+
embeddings = createMockEmbeddings();
|
|
40
|
+
});
|
|
41
|
+
it("strips email from query before embed() in encrypt mode", async () => {
|
|
42
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx, piiEncryption, { mode: "encrypt", orgKey: Buffer.alloc(32) });
|
|
43
|
+
await service.search({ query: "find john@example.com" });
|
|
44
|
+
expect(embeddings.embed).toHaveBeenCalledWith(expect.stringContaining("[EMAIL]"), expect.anything());
|
|
45
|
+
expect(embeddings.embed).toHaveBeenCalledWith(expect.not.stringContaining("john@example.com"), expect.anything());
|
|
46
|
+
});
|
|
47
|
+
it("does not strip when mode is disabled", async () => {
|
|
48
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx, piiEncryption, { mode: "disabled" });
|
|
49
|
+
await service.search({ query: "find john@example.com" });
|
|
50
|
+
expect(embeddings.embed).toHaveBeenCalledWith(expect.stringContaining("john@example.com"), expect.anything());
|
|
51
|
+
});
|
|
52
|
+
it("logs PII query to pii_query_logs on PII detection", async () => {
|
|
53
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx, piiEncryption, { mode: "encrypt", orgKey: Buffer.alloc(32) });
|
|
54
|
+
await service.search({ query: "find john@example.com" });
|
|
55
|
+
// Allow time for fire-and-forget
|
|
56
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
57
|
+
expect(db.$executeRaw).toHaveBeenCalled();
|
|
58
|
+
// Verify audit log INSERT was called (look for pii_query_logs in the SQL)
|
|
59
|
+
const execCalls = db.$executeRaw.mock.calls;
|
|
60
|
+
const auditCall = execCalls.find((call) => {
|
|
61
|
+
const tpl = call[0];
|
|
62
|
+
return tpl.raw.join("").includes("pii_query_logs");
|
|
63
|
+
});
|
|
64
|
+
expect(auditCall).toBeDefined();
|
|
65
|
+
});
|
|
66
|
+
it("does not log audit when query has no PII", async () => {
|
|
67
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx, piiEncryption, { mode: "encrypt", orgKey: Buffer.alloc(32) });
|
|
68
|
+
await service.search({ query: "normal search query" });
|
|
69
|
+
// Allow time for fire-and-forget
|
|
70
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
71
|
+
const execCalls = db.$executeRaw.mock.calls;
|
|
72
|
+
const auditCall = execCalls.find((call) => {
|
|
73
|
+
const tpl = call[0];
|
|
74
|
+
return tpl.raw.join("").includes("pii_query_logs");
|
|
75
|
+
});
|
|
76
|
+
expect(auditCall).toBeUndefined();
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
describe("chunk-level search and snippet propagation", () => {
|
|
80
|
+
let db;
|
|
81
|
+
let embeddings;
|
|
82
|
+
beforeEach(() => {
|
|
83
|
+
db = createMockDb();
|
|
84
|
+
embeddings = createMockEmbeddings();
|
|
85
|
+
});
|
|
86
|
+
it("runs chunk semantic search in parallel with node-level search", async () => {
|
|
87
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx);
|
|
88
|
+
await service.search({ query: "test query" });
|
|
89
|
+
// embed() should be called twice: once for chunk search, once for node fallback
|
|
90
|
+
expect(embeddings.embed).toHaveBeenCalledTimes(2);
|
|
91
|
+
});
|
|
92
|
+
it("returns snippet from chunk semantic results", async () => {
|
|
93
|
+
// Mock chunk search to return results with snippets
|
|
94
|
+
const mockChunkResult = {
|
|
95
|
+
id: "node-1",
|
|
96
|
+
type: "note",
|
|
97
|
+
title: "Test",
|
|
98
|
+
content: "Full content",
|
|
99
|
+
metadata: {},
|
|
100
|
+
status: "approved",
|
|
101
|
+
created_by: "test",
|
|
102
|
+
created_at: new Date(),
|
|
103
|
+
updated_at: new Date(),
|
|
104
|
+
version: 1,
|
|
105
|
+
domain_id: null,
|
|
106
|
+
organization_id: "org-1",
|
|
107
|
+
has_pii: false,
|
|
108
|
+
embedding_pii_mode: null,
|
|
109
|
+
similarity: 0.95,
|
|
110
|
+
chunk_content: "Relevant chunk text",
|
|
111
|
+
chunk_index: 2,
|
|
112
|
+
};
|
|
113
|
+
db.$queryRaw = vi.fn().mockResolvedValue([mockChunkResult]);
|
|
114
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx);
|
|
115
|
+
const results = await service.search({ query: "test query" });
|
|
116
|
+
// The result should carry the snippet
|
|
117
|
+
const nodeResult = results.find((r) => r.node.id === "node-1");
|
|
118
|
+
if (nodeResult) {
|
|
119
|
+
expect(nodeResult.snippet).toBe("Relevant chunk text");
|
|
120
|
+
expect(nodeResult.snippetChunkIndex).toBe(2);
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
it("preserves snippet through RRF when merging results", async () => {
|
|
124
|
+
// Test the RRF function directly by exercising the search method
|
|
125
|
+
// with all queries returning the same node — chunk query with snippet, keyword without
|
|
126
|
+
const mockNode = {
|
|
127
|
+
id: "node-1",
|
|
128
|
+
type: "note",
|
|
129
|
+
title: "Test",
|
|
130
|
+
content: "Full content",
|
|
131
|
+
metadata: {},
|
|
132
|
+
status: "approved",
|
|
133
|
+
created_by: "test",
|
|
134
|
+
created_at: new Date(),
|
|
135
|
+
updated_at: new Date(),
|
|
136
|
+
version: 1,
|
|
137
|
+
domain_id: null,
|
|
138
|
+
organization_id: "org-1",
|
|
139
|
+
has_pii: false,
|
|
140
|
+
embedding_pii_mode: null,
|
|
141
|
+
};
|
|
142
|
+
// All $queryRaw calls return the same node with chunk data
|
|
143
|
+
// (the chunk search result includes chunk_content and chunk_index)
|
|
144
|
+
db.$queryRaw = vi.fn().mockResolvedValue([{
|
|
145
|
+
...mockNode,
|
|
146
|
+
similarity: 0.95,
|
|
147
|
+
chunk_content: "Best chunk",
|
|
148
|
+
chunk_index: 1,
|
|
149
|
+
rank: 0.8,
|
|
150
|
+
match_score: 1.0,
|
|
151
|
+
trgm_score: 0.5,
|
|
152
|
+
}]);
|
|
153
|
+
const service = new SearchService(db, embeddings, testConfig, tenantCtx);
|
|
154
|
+
const results = await service.search({ query: "test query" });
|
|
155
|
+
// Node should appear in results (from multiple lists → hybrid)
|
|
156
|
+
const result = results.find((r) => r.node.id === "node-1");
|
|
157
|
+
expect(result).toBeDefined();
|
|
158
|
+
// Snippet should be preserved from the chunk semantic result
|
|
159
|
+
expect(result.snippet).toBe("Best chunk");
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
//# sourceMappingURL=search-service.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-service.test.js","sourceRoot":"","sources":["../../../src/services/__tests__/search-service.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAIpE,SAAS,YAAY;IACnB,OAAO;QACL,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACxC,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;KACf,CAAC;AAC/B,CAAC;AAED,SAAS,oBAAoB;IAC3B,OAAO;QACL,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QACzC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;KAC/E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,GAAgB;IAC9B,QAAQ,EAAE,EAAE,GAAG,EAAE,6BAA6B,EAAE;IAChD,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE;IAC9D,MAAM,EAAE;QACN,YAAY,EAAE,EAAE;QAChB,aAAa,EAAE,IAAI;QACnB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,GAAG;QACpB,aAAa,EAAE,GAAG;QAClB,gBAAgB,EAAE,GAAG;KACtB;CACa,CAAC;AAEjB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,MAAM,QAAQ,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAC1C,MAAM,aAAa,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;IAE9C,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,IAAI,EAAmC,CAAC;QACxC,IAAI,UAAmD,CAAC;QAExD,UAAU,CAAC,GAAG,EAAE;YACd,EAAE,GAAG,YAAY,EAAE,CAAC;YACpB,UAAU,GAAG,oBAAoB,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,OAAO,GAAG,IAAI,aAAa,CAC/B,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EACrC,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAC7D,CAAC;YAEF,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAEzD,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAClC,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YACF,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,EAC/C,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,OAAO,GAAG,IAAI,aAAa,CAC/B,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EACrC,aAAa,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CACpC,CAAC;YAEF,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAEzD,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,EAC3C,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,OAAO,GAAG,IAAI,aAAa,CAC/B,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EACrC,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAC7D,CAAC;YAEF,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAEzD,iCAAiC;YACjC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YAE5C,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,gBAAgB,EAAE,CAAC;YAC1C,0EAA0E;YAC1E,MAAM,SAAS,GAAI,EAAE,CAAC,WAAwC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC1E,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;gBACxC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAA+B,CAAC;gBAClD,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,OAAO,GAAG,IAAI,aAAa,CAC/B,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EACrC,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAC7D,CAAC;YAEF,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC,CAAC;YAEvD,iCAAiC;YACjC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YAE5C,MAAM,SAAS,GAAI,EAAE,CAAC,WAAwC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC1E,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;gBACxC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAA+B,CAAC;gBAClD,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4CAA4C,EAAE,GAAG,EAAE;QAC1D,IAAI,EAAmC,CAAC;QACxC,IAAI,UAAmD,CAAC;QAExD,UAAU,CAAC,GAAG,EAAE;YACd,EAAE,GAAG,YAAY,EAAE,CAAC;YACpB,UAAU,GAAG,oBAAoB,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAEzE,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;YAE9C,gFAAgF;YAChF,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,oDAAoD;YACpD,MAAM,eAAe,GAAG;gBACtB,EAAE,EAAE,QAAQ;gBACZ,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,cAAc;gBACvB,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,UAAU;gBAClB,UAAU,EAAE,MAAM;gBAClB,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,IAAI;gBACf,eAAe,EAAE,OAAO;gBACxB,OAAO,EAAE,KAAK;gBACd,kBAAkB,EAAE,IAAI;gBACxB,UAAU,EAAE,IAAI;gBAChB,aAAa,EAAE,qBAAqB;gBACpC,WAAW,EAAE,CAAC;aACf,CAAC;YACF,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;YAE5D,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YACzE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;YAE9D,sCAAsC;YACtC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;YAC/D,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBACvD,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,iEAAiE;YACjE,uFAAuF;YACvF,MAAM,QAAQ,GAAG;gBACf,EAAE,EAAE,QAAQ;gBACZ,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,cAAc;gBACvB,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,UAAU;gBAClB,UAAU,EAAE,MAAM;gBAClB,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,UAAU,EAAE,IAAI,IAAI,EAAE;gBACtB,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,IAAI;gBACf,eAAe,EAAE,OAAO;gBACxB,OAAO,EAAE,KAAK;gBACd,kBAAkB,EAAE,IAAI;aACzB,CAAC;YAEF,2DAA2D;YAC3D,mEAAmE;YACnE,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC;oBACxC,GAAG,QAAQ;oBACX,UAAU,EAAE,IAAI;oBAChB,aAAa,EAAE,YAAY;oBAC3B,WAAW,EAAE,CAAC;oBACd,IAAI,EAAE,GAAG;oBACT,WAAW,EAAE,GAAG;oBAChB,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YACzE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;YAE9D,+DAA+D;YAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;YAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,6DAA6D;YAC7D,MAAM,CAAC,MAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PrismaClient } from "@prisma/client";
|
|
2
|
+
import type { EmbeddingProvider } from "../types/index.js";
|
|
3
|
+
import type { TenantContext } from "../db/tenant.js";
|
|
4
|
+
import type { PiiEncryptionService } from "./pii-encryption-service.js";
|
|
5
|
+
import type { PiiEncryptionOptions } from "./pii-encryption-service.js";
|
|
6
|
+
export interface BackfillProgress {
|
|
7
|
+
completed: number;
|
|
8
|
+
total: number;
|
|
9
|
+
errors: number;
|
|
10
|
+
}
|
|
11
|
+
export interface BackfillOptions {
|
|
12
|
+
/** Skip nodes that already have chunks (default: true) */
|
|
13
|
+
skipExisting?: boolean;
|
|
14
|
+
/** Batch size for cursor pagination (default: 50) */
|
|
15
|
+
batchSize?: number;
|
|
16
|
+
/** AbortSignal for cancellation */
|
|
17
|
+
signal?: AbortSignal;
|
|
18
|
+
/** Progress callback */
|
|
19
|
+
onProgress?: (progress: BackfillProgress) => void;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Backfill chunks for all existing nodes that don't have them.
|
|
23
|
+
* Cursor-based pagination, per-node error handling (logs + continues).
|
|
24
|
+
*/
|
|
25
|
+
export declare function backfillChunks(db: PrismaClient, embeddings: EmbeddingProvider, tenantCtx: TenantContext, piiEncryption?: PiiEncryptionService, piiOptions?: PiiEncryptionOptions, options?: BackfillOptions): Promise<{
|
|
26
|
+
processed: number;
|
|
27
|
+
skipped: number;
|
|
28
|
+
errors: number;
|
|
29
|
+
}>;
|
|
30
|
+
//# sourceMappingURL=backfill-chunks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill-chunks.d.ts","sourceRoot":"","sources":["../../src/services/backfill-chunks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,CAAC;CACnD;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,YAAY,EAChB,UAAU,EAAE,iBAAiB,EAC7B,SAAS,EAAE,aAAa,EACxB,aAAa,CAAC,EAAE,oBAAoB,EACpC,UAAU,CAAC,EAAE,oBAAoB,EACjC,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAsDjE"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { ChunkService } from "./chunk-service.js";
|
|
2
|
+
/**
|
|
3
|
+
* Backfill chunks for all existing nodes that don't have them.
|
|
4
|
+
* Cursor-based pagination, per-node error handling (logs + continues).
|
|
5
|
+
*/
|
|
6
|
+
export async function backfillChunks(db, embeddings, tenantCtx, piiEncryption, piiOptions, options) {
|
|
7
|
+
const skipExisting = options?.skipExisting ?? true;
|
|
8
|
+
const batchSize = options?.batchSize ?? 50;
|
|
9
|
+
const chunkService = new ChunkService(db, embeddings, tenantCtx, piiEncryption, piiOptions);
|
|
10
|
+
let processed = 0;
|
|
11
|
+
let skipped = 0;
|
|
12
|
+
let errors = 0;
|
|
13
|
+
let cursor;
|
|
14
|
+
// Count total for progress reporting
|
|
15
|
+
const totalCount = await db.node.count();
|
|
16
|
+
while (true) {
|
|
17
|
+
if (options?.signal?.aborted)
|
|
18
|
+
break;
|
|
19
|
+
const nodes = await db.node.findMany({
|
|
20
|
+
take: batchSize,
|
|
21
|
+
...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
|
|
22
|
+
orderBy: { id: "asc" },
|
|
23
|
+
select: { id: true, title: true, content: true },
|
|
24
|
+
});
|
|
25
|
+
if (nodes.length === 0)
|
|
26
|
+
break;
|
|
27
|
+
cursor = nodes[nodes.length - 1].id;
|
|
28
|
+
for (const node of nodes) {
|
|
29
|
+
if (options?.signal?.aborted)
|
|
30
|
+
break;
|
|
31
|
+
if (skipExisting) {
|
|
32
|
+
const existingChunks = await chunkService.getByNodeId(node.id);
|
|
33
|
+
if (existingChunks.length > 0) {
|
|
34
|
+
skipped++;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
await chunkService.chunkAndEmbed(node.id, node.title, node.content);
|
|
40
|
+
processed++;
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
errors++;
|
|
44
|
+
console.error(`[backfill-chunks] Failed to chunk node ${node.id}:`, err);
|
|
45
|
+
}
|
|
46
|
+
options?.onProgress?.({
|
|
47
|
+
completed: processed + skipped + errors,
|
|
48
|
+
total: totalCount,
|
|
49
|
+
errors,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return { processed, skipped, errors };
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=backfill-chunks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill-chunks.js","sourceRoot":"","sources":["../../src/services/backfill-chunks.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAmBlD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,EAAgB,EAChB,UAA6B,EAC7B,SAAwB,EACxB,aAAoC,EACpC,UAAiC,EACjC,OAAyB;IAEzB,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,IAAI,CAAC;IACnD,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAE5F,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAA0B,CAAC;IAE/B,qCAAqC;IACrC,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO;YAAE,MAAM;QAEpC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;YACnC,IAAI,EAAE,SAAS;YACf,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,OAAO,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;YACtB,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;SACjD,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAC9B,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO;gBAAE,MAAM;YAEpC,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC/D,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,OAAO,EAAE,CAAC;oBACV,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBACpE,SAAS,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;gBACT,OAAO,CAAC,KAAK,CAAC,0CAA0C,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC3E,CAAC;YAED,OAAO,EAAE,UAAU,EAAE,CAAC;gBACpB,SAAS,EAAE,SAAS,GAAG,OAAO,GAAG,MAAM;gBACvC,KAAK,EAAE,UAAU;gBACjB,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACxC,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { PrismaClient } from "@prisma/client";
|
|
2
|
+
import type { EmbeddingProvider } from "../types/index.js";
|
|
3
|
+
import { type TenantContext } from "../db/tenant.js";
|
|
4
|
+
import type { PiiEncryptionService } from "./pii-encryption-service.js";
|
|
5
|
+
import type { PiiEncryptionOptions } from "./pii-encryption-service.js";
|
|
6
|
+
export declare class ChunkService {
|
|
7
|
+
private db;
|
|
8
|
+
private embeddings;
|
|
9
|
+
private piiEncryption?;
|
|
10
|
+
private piiOptions?;
|
|
11
|
+
private tenantCtx;
|
|
12
|
+
constructor(db: PrismaClient, embeddings: EmbeddingProvider, tenantCtx?: TenantContext, piiEncryption?: PiiEncryptionService | undefined, piiOptions?: PiiEncryptionOptions | undefined);
|
|
13
|
+
/**
|
|
14
|
+
* Chunk a node's content, embed each chunk, and store in node_chunks.
|
|
15
|
+
* Replaces any existing chunks for the node (atomic replace).
|
|
16
|
+
*/
|
|
17
|
+
chunkAndEmbed(nodeId: string, title: string, content: string): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Delete all chunks for a node.
|
|
20
|
+
*/
|
|
21
|
+
deleteByNodeId(nodeId: string): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Get all chunks for a node, ordered by chunk_index.
|
|
24
|
+
*/
|
|
25
|
+
getByNodeId(nodeId: string): Promise<Array<{
|
|
26
|
+
id: string;
|
|
27
|
+
nodeId: string;
|
|
28
|
+
chunkIndex: number;
|
|
29
|
+
content: string;
|
|
30
|
+
tokenCount: number;
|
|
31
|
+
headingContext: string | null;
|
|
32
|
+
startOffset: number;
|
|
33
|
+
endOffset: number;
|
|
34
|
+
overlapPrev: number;
|
|
35
|
+
embeddingPiiMode: string | null;
|
|
36
|
+
}>>;
|
|
37
|
+
/**
|
|
38
|
+
* Insert chunks with embeddings via raw SQL.
|
|
39
|
+
* Uses gen_random_uuid() in VALUES (Accelerate proxy gotcha).
|
|
40
|
+
*/
|
|
41
|
+
private insertChunksBatch;
|
|
42
|
+
private stripForEmbedding;
|
|
43
|
+
private resolvePiiMode;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=chunk-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-service.d.ts","sourceRoot":"","sources":["../../src/services/chunk-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAqC,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,qBAAa,YAAY;IAIrB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,UAAU;IAElB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;IAPrB,OAAO,CAAC,SAAS,CAAgB;gBAGvB,EAAE,EAAE,YAAY,EAChB,UAAU,EAAE,iBAAiB,EACrC,SAAS,CAAC,EAAE,aAAa,EACjB,aAAa,CAAC,EAAE,oBAAoB,YAAA,EACpC,UAAU,CAAC,EAAE,oBAAoB,YAAA;IAK3C;;;OAGG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlF;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASnD;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/C,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;QAC9B,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;KACjC,CAAC,CAAC;IAgBH;;;OAGG;YACW,iBAAiB;IA6C/B,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,cAAc;CAIvB"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { tenantWhere, SINGLE_TENANT_ORG_ID } from "../db/tenant.js";
|
|
2
|
+
import { chunkText } from "./chunker.js";
|
|
3
|
+
export class ChunkService {
|
|
4
|
+
db;
|
|
5
|
+
embeddings;
|
|
6
|
+
piiEncryption;
|
|
7
|
+
piiOptions;
|
|
8
|
+
tenantCtx;
|
|
9
|
+
constructor(db, embeddings, tenantCtx, piiEncryption, piiOptions) {
|
|
10
|
+
this.db = db;
|
|
11
|
+
this.embeddings = embeddings;
|
|
12
|
+
this.piiEncryption = piiEncryption;
|
|
13
|
+
this.piiOptions = piiOptions;
|
|
14
|
+
this.tenantCtx = tenantCtx ?? { organizationId: SINGLE_TENANT_ORG_ID };
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Chunk a node's content, embed each chunk, and store in node_chunks.
|
|
18
|
+
* Replaces any existing chunks for the node (atomic replace).
|
|
19
|
+
*/
|
|
20
|
+
async chunkAndEmbed(nodeId, title, content) {
|
|
21
|
+
// Strip PII before chunking to prevent PII reaching the provider
|
|
22
|
+
const strippedText = this.stripForEmbedding(`${title}\n\n${content}`);
|
|
23
|
+
const chunks = chunkText(strippedText);
|
|
24
|
+
if (chunks.length === 0)
|
|
25
|
+
return;
|
|
26
|
+
// Prepend heading context to each chunk for embedding
|
|
27
|
+
const textsForEmbedding = chunks.map((c) => c.headingContext ? `${c.headingContext}: ${c.content}` : c.content);
|
|
28
|
+
const embeddings = await this.embeddings.embedBatch(textsForEmbedding, {
|
|
29
|
+
inputType: "document",
|
|
30
|
+
});
|
|
31
|
+
// Atomic replace: delete existing, then insert new
|
|
32
|
+
await this.deleteByNodeId(nodeId);
|
|
33
|
+
await this.insertChunksBatch(nodeId, chunks, embeddings);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Delete all chunks for a node.
|
|
37
|
+
*/
|
|
38
|
+
async deleteByNodeId(nodeId) {
|
|
39
|
+
const orgFilter = tenantWhere(this.tenantCtx);
|
|
40
|
+
await this.db.$executeRaw `
|
|
41
|
+
DELETE FROM node_chunks
|
|
42
|
+
WHERE node_id = ${nodeId}::uuid
|
|
43
|
+
AND ${orgFilter}
|
|
44
|
+
`;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Get all chunks for a node, ordered by chunk_index.
|
|
48
|
+
*/
|
|
49
|
+
async getByNodeId(nodeId) {
|
|
50
|
+
const orgFilter = tenantWhere(this.tenantCtx);
|
|
51
|
+
return this.db.$queryRaw `
|
|
52
|
+
SELECT id, node_id as "nodeId", chunk_index as "chunkIndex",
|
|
53
|
+
content, token_count as "tokenCount",
|
|
54
|
+
heading_context as "headingContext",
|
|
55
|
+
start_offset as "startOffset", end_offset as "endOffset",
|
|
56
|
+
overlap_prev as "overlapPrev",
|
|
57
|
+
embedding_pii_mode as "embeddingPiiMode"
|
|
58
|
+
FROM node_chunks
|
|
59
|
+
WHERE node_id = ${nodeId}::uuid
|
|
60
|
+
AND ${orgFilter}
|
|
61
|
+
ORDER BY chunk_index ASC
|
|
62
|
+
`;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Insert chunks with embeddings via raw SQL.
|
|
66
|
+
* Uses gen_random_uuid() in VALUES (Accelerate proxy gotcha).
|
|
67
|
+
*/
|
|
68
|
+
async insertChunksBatch(nodeId, chunks, embeddings) {
|
|
69
|
+
if (chunks.length === 0)
|
|
70
|
+
return;
|
|
71
|
+
const piiMode = this.resolvePiiMode();
|
|
72
|
+
const orgId = this.tenantCtx.organizationId;
|
|
73
|
+
const values = [];
|
|
74
|
+
const params = [];
|
|
75
|
+
let paramIdx = 1;
|
|
76
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
77
|
+
const chunk = chunks[i];
|
|
78
|
+
const embeddingStr = `[${embeddings[i].join(",")}]`;
|
|
79
|
+
values.push(`(gen_random_uuid(), $${paramIdx}::uuid, $${paramIdx + 1}, $${paramIdx + 2}, $${paramIdx + 3}, $${paramIdx + 4}, $${paramIdx + 5}, $${paramIdx + 6}, $${paramIdx + 7}, $${paramIdx + 8}::vector, $${paramIdx + 9}, $${paramIdx + 10}::uuid, NOW())`);
|
|
80
|
+
params.push(nodeId, // node_id
|
|
81
|
+
chunk.chunkIndex, // chunk_index
|
|
82
|
+
chunk.content, // content
|
|
83
|
+
chunk.tokenCount, // token_count
|
|
84
|
+
chunk.headingContext, // heading_context
|
|
85
|
+
chunk.startOffset, // start_offset
|
|
86
|
+
chunk.endOffset, // end_offset
|
|
87
|
+
chunk.overlapPrev, // overlap_prev
|
|
88
|
+
embeddingStr, // embedding
|
|
89
|
+
piiMode, // embedding_pii_mode
|
|
90
|
+
orgId);
|
|
91
|
+
paramIdx += 11;
|
|
92
|
+
}
|
|
93
|
+
const sql = `
|
|
94
|
+
INSERT INTO node_chunks (id, node_id, chunk_index, content, token_count, heading_context, start_offset, end_offset, overlap_prev, embedding, embedding_pii_mode, organization_id, created_at)
|
|
95
|
+
VALUES ${values.join(", ")}
|
|
96
|
+
`;
|
|
97
|
+
await this.db.$queryRawUnsafe(sql, ...params);
|
|
98
|
+
}
|
|
99
|
+
stripForEmbedding(text) {
|
|
100
|
+
if (!this.piiEncryption || !this.piiOptions || this.piiOptions.mode === "disabled") {
|
|
101
|
+
return text;
|
|
102
|
+
}
|
|
103
|
+
return this.piiEncryption.stripPii(text, this.piiOptions.allowedPiiTypes).text;
|
|
104
|
+
}
|
|
105
|
+
resolvePiiMode() {
|
|
106
|
+
if (!this.piiOptions || this.piiOptions.mode === "disabled")
|
|
107
|
+
return null;
|
|
108
|
+
return this.piiOptions.mode;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=chunk-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-service.js","sourceRoot":"","sources":["../../src/services/chunk-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAsB,MAAM,iBAAiB,CAAC;AAGxF,OAAO,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE1D,MAAM,OAAO,YAAY;IAIb;IACA;IAEA;IACA;IAPF,SAAS,CAAgB;IAEjC,YACU,EAAgB,EAChB,UAA6B,EACrC,SAAyB,EACjB,aAAoC,EACpC,UAAiC;QAJjC,OAAE,GAAF,EAAE,CAAc;QAChB,eAAU,GAAV,UAAU,CAAmB;QAE7B,kBAAa,GAAb,aAAa,CAAuB;QACpC,eAAU,GAAV,UAAU,CAAuB;QAEzC,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,EAAE,cAAc,EAAE,oBAAoB,EAAE,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,KAAa,EAAE,OAAe;QAChE,iEAAiE;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,KAAK,OAAO,OAAO,EAAE,CAAC,CAAC;QACtE,MAAM,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;QAEvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,sDAAsD;QACtD,MAAM,iBAAiB,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACzC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CACnE,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrE,SAAS,EAAE,UAAU;SACtB,CAAC,CAAC;QAEH,mDAAmD;QACnD,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;wBAEL,MAAM;cAChB,SAAS;KAClB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,MAAc;QAY9B,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,CAAA;;;;;;;;wBAQJ,MAAM;cAChB,SAAS;;KAElB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAC7B,MAAc,EACd,MAAoB,EACpB,UAAsB;QAEtB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC;QAE5C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,YAAY,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAEpD,MAAM,CAAC,IAAI,CACT,wBAAwB,QAAQ,YAAY,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,CAAC,cAAc,QAAQ,GAAG,CAAC,MAAM,QAAQ,GAAG,EAAE,gBAAgB,CACpP,CAAC;YACF,MAAM,CAAC,IAAI,CACT,MAAM,EAAY,UAAU;YAC5B,KAAK,CAAC,UAAU,EAAE,cAAc;YAChC,KAAK,CAAC,OAAO,EAAK,UAAU;YAC5B,KAAK,CAAC,UAAU,EAAE,cAAc;YAChC,KAAK,CAAC,cAAc,EAAE,kBAAkB;YACxC,KAAK,CAAC,WAAW,EAAE,eAAe;YAClC,KAAK,CAAC,SAAS,EAAI,aAAa;YAChC,KAAK,CAAC,WAAW,EAAE,eAAe;YAClC,YAAY,EAAO,YAAY;YAC/B,OAAO,EAAY,qBAAqB;YACxC,KAAK,CACN,CAAC;YACF,QAAQ,IAAI,EAAE,CAAC;QACjB,CAAC;QAED,MAAM,GAAG,GAAG;;eAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;KAC3B,CAAC;QAEF,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC;IAChD,CAAC;IAEO,iBAAiB,CAAC,IAAY;QACpC,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACnF,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC;IACjF,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC;QACzE,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure text splitting function for content chunking.
|
|
3
|
+
* No I/O — takes text, returns chunk descriptors.
|
|
4
|
+
*/
|
|
5
|
+
export interface ChunkInput {
|
|
6
|
+
content: string;
|
|
7
|
+
chunkIndex: number;
|
|
8
|
+
tokenCount: number;
|
|
9
|
+
headingContext: string | null;
|
|
10
|
+
startOffset: number;
|
|
11
|
+
endOffset: number;
|
|
12
|
+
overlapPrev: number;
|
|
13
|
+
}
|
|
14
|
+
/** Target chunk size in estimated tokens */
|
|
15
|
+
export declare const TARGET_CHUNK_TOKENS = 400;
|
|
16
|
+
/** Hard cap — never exceed this */
|
|
17
|
+
export declare const MAX_CHUNK_TOKENS = 600;
|
|
18
|
+
/** Rough token estimate: ~4 chars per token */
|
|
19
|
+
export declare function estimateTokens(text: string): number;
|
|
20
|
+
/**
|
|
21
|
+
* Main entry point: split text into chunks suitable for embedding.
|
|
22
|
+
*
|
|
23
|
+
* Algorithm:
|
|
24
|
+
* 1. Parse markdown headings into sections
|
|
25
|
+
* 2. Within each section, split into paragraphs
|
|
26
|
+
* 3. Accumulate paragraphs into chunks up to ~TARGET_CHUNK_TOKENS
|
|
27
|
+
* 4. Structural boundaries (headings, paragraph breaks) → overlap = 0
|
|
28
|
+
* 5. Forced mid-paragraph splits → overlap = trailing sentences
|
|
29
|
+
* 6. Code blocks are atomic — never split within
|
|
30
|
+
*/
|
|
31
|
+
export declare function chunkText(text: string): ChunkInput[];
|
|
32
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/services/chunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,4CAA4C;AAC5C,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,mCAAmC;AACnC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAsND;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,CAgGpD"}
|