@framers/agentos 0.1.125 → 0.1.126

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/dist/core/provenance/crypto/AgentKeyManager.d.ts.map +1 -1
  2. package/dist/core/provenance/crypto/AgentKeyManager.js +9 -3
  3. package/dist/core/provenance/crypto/AgentKeyManager.js.map +1 -1
  4. package/dist/core/text-processing/ITextProcessor.d.ts +43 -0
  5. package/dist/core/text-processing/ITextProcessor.d.ts.map +1 -0
  6. package/dist/core/text-processing/ITextProcessor.js +6 -0
  7. package/dist/core/text-processing/ITextProcessor.js.map +1 -0
  8. package/dist/core/text-processing/TextProcessingPipeline.d.ts +49 -0
  9. package/dist/core/text-processing/TextProcessingPipeline.d.ts.map +1 -0
  10. package/dist/core/text-processing/TextProcessingPipeline.js +61 -0
  11. package/dist/core/text-processing/TextProcessingPipeline.js.map +1 -0
  12. package/dist/core/text-processing/filters/StopWordFilter.d.ts +30 -0
  13. package/dist/core/text-processing/filters/StopWordFilter.d.ts.map +1 -0
  14. package/dist/core/text-processing/filters/StopWordFilter.js +76 -0
  15. package/dist/core/text-processing/filters/StopWordFilter.js.map +1 -0
  16. package/dist/core/text-processing/index.d.ts +20 -0
  17. package/dist/core/text-processing/index.d.ts.map +1 -0
  18. package/dist/core/text-processing/index.js +24 -0
  19. package/dist/core/text-processing/index.js.map +1 -0
  20. package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.d.ts +11 -0
  21. package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.d.ts.map +1 -0
  22. package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.js +13 -0
  23. package/dist/core/text-processing/lemmatizers/NoOpLemmatizer.js.map +1 -0
  24. package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.d.ts +25 -0
  25. package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.d.ts.map +1 -0
  26. package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.js +64 -0
  27. package/dist/core/text-processing/lemmatizers/WordNetLemmatizer.js.map +1 -0
  28. package/dist/core/text-processing/normalizers/AccentStripper.d.ts +17 -0
  29. package/dist/core/text-processing/normalizers/AccentStripper.d.ts.map +1 -0
  30. package/dist/core/text-processing/normalizers/AccentStripper.js +22 -0
  31. package/dist/core/text-processing/normalizers/AccentStripper.js.map +1 -0
  32. package/dist/core/text-processing/normalizers/LowercaseNormalizer.d.ts +11 -0
  33. package/dist/core/text-processing/normalizers/LowercaseNormalizer.d.ts.map +1 -0
  34. package/dist/core/text-processing/normalizers/LowercaseNormalizer.js +13 -0
  35. package/dist/core/text-processing/normalizers/LowercaseNormalizer.js.map +1 -0
  36. package/dist/core/text-processing/presets.d.ts +22 -0
  37. package/dist/core/text-processing/presets.d.ts.map +1 -0
  38. package/dist/core/text-processing/presets.js +45 -0
  39. package/dist/core/text-processing/presets.js.map +1 -0
  40. package/dist/core/text-processing/stemmers/NoOpStemmer.d.ts +14 -0
  41. package/dist/core/text-processing/stemmers/NoOpStemmer.d.ts.map +1 -0
  42. package/dist/core/text-processing/stemmers/NoOpStemmer.js +16 -0
  43. package/dist/core/text-processing/stemmers/NoOpStemmer.js.map +1 -0
  44. package/dist/core/text-processing/stemmers/PorterStemmer.d.ts +31 -0
  45. package/dist/core/text-processing/stemmers/PorterStemmer.d.ts.map +1 -0
  46. package/dist/core/text-processing/stemmers/PorterStemmer.js +62 -0
  47. package/dist/core/text-processing/stemmers/PorterStemmer.js.map +1 -0
  48. package/dist/core/text-processing/tokenizers/CodeTokenizer.d.ts +25 -0
  49. package/dist/core/text-processing/tokenizers/CodeTokenizer.d.ts.map +1 -0
  50. package/dist/core/text-processing/tokenizers/CodeTokenizer.js +75 -0
  51. package/dist/core/text-processing/tokenizers/CodeTokenizer.js.map +1 -0
  52. package/dist/core/text-processing/tokenizers/StandardTokenizer.d.ts +22 -0
  53. package/dist/core/text-processing/tokenizers/StandardTokenizer.d.ts.map +1 -0
  54. package/dist/core/text-processing/tokenizers/StandardTokenizer.js +37 -0
  55. package/dist/core/text-processing/tokenizers/StandardTokenizer.js.map +1 -0
  56. package/dist/core/text-processing/types.d.ts +18 -0
  57. package/dist/core/text-processing/types.d.ts.map +1 -0
  58. package/dist/core/text-processing/types.js +6 -0
  59. package/dist/core/text-processing/types.js.map +1 -0
  60. package/dist/core/vector-search/HnswIndexSidecar.d.ts +91 -0
  61. package/dist/core/vector-search/HnswIndexSidecar.d.ts.map +1 -0
  62. package/dist/core/vector-search/HnswIndexSidecar.js +270 -0
  63. package/dist/core/vector-search/HnswIndexSidecar.js.map +1 -0
  64. package/dist/core/vector-search/index.d.ts +7 -0
  65. package/dist/core/vector-search/index.d.ts.map +1 -0
  66. package/dist/core/vector-search/index.js +6 -0
  67. package/dist/core/vector-search/index.js.map +1 -0
  68. package/dist/core/vector-search/types.d.ts +42 -0
  69. package/dist/core/vector-search/types.d.ts.map +1 -0
  70. package/dist/core/vector-search/types.js +6 -0
  71. package/dist/core/vector-search/types.js.map +1 -0
  72. package/dist/memory/CognitiveMemoryManager.js +3 -3
  73. package/dist/memory/CognitiveMemoryManager.js.map +1 -1
  74. package/dist/memory/consolidation/ConsolidationLoop.js +3 -3
  75. package/dist/memory/consolidation/ConsolidationLoop.js.map +1 -1
  76. package/dist/memory/facade/Memory.d.ts +21 -0
  77. package/dist/memory/facade/Memory.d.ts.map +1 -1
  78. package/dist/memory/facade/Memory.js +124 -62
  79. package/dist/memory/facade/Memory.js.map +1 -1
  80. package/dist/memory/io/ChatGptImporter.d.ts.map +1 -1
  81. package/dist/memory/io/ChatGptImporter.js +7 -6
  82. package/dist/memory/io/ChatGptImporter.js.map +1 -1
  83. package/dist/memory/io/CsvImporter.d.ts +15 -0
  84. package/dist/memory/io/CsvImporter.d.ts.map +1 -1
  85. package/dist/memory/io/CsvImporter.js +28 -7
  86. package/dist/memory/io/CsvImporter.js.map +1 -1
  87. package/dist/memory/io/JsonExporter.d.ts +14 -0
  88. package/dist/memory/io/JsonExporter.d.ts.map +1 -1
  89. package/dist/memory/io/JsonExporter.js +22 -3
  90. package/dist/memory/io/JsonExporter.js.map +1 -1
  91. package/dist/memory/io/JsonImporter.d.ts +15 -0
  92. package/dist/memory/io/JsonImporter.d.ts.map +1 -1
  93. package/dist/memory/io/JsonImporter.js +35 -16
  94. package/dist/memory/io/JsonImporter.js.map +1 -1
  95. package/dist/memory/io/MarkdownImporter.d.ts.map +1 -1
  96. package/dist/memory/io/MarkdownImporter.js +7 -5
  97. package/dist/memory/io/MarkdownImporter.js.map +1 -1
  98. package/dist/memory/io/ObsidianImporter.d.ts.map +1 -1
  99. package/dist/memory/io/ObsidianImporter.js +9 -23
  100. package/dist/memory/io/ObsidianImporter.js.map +1 -1
  101. package/dist/memory/io/SqliteExporter.d.ts.map +1 -1
  102. package/dist/memory/io/SqliteExporter.js +1 -3
  103. package/dist/memory/io/SqliteExporter.js.map +1 -1
  104. package/dist/memory/io/SqliteImporter.d.ts.map +1 -1
  105. package/dist/memory/io/SqliteImporter.js +10 -11
  106. package/dist/memory/io/SqliteImporter.js.map +1 -1
  107. package/dist/memory/store/HnswSidecar.d.ts +7 -0
  108. package/dist/memory/store/HnswSidecar.d.ts.map +1 -1
  109. package/dist/memory/store/HnswSidecar.js +7 -0
  110. package/dist/memory/store/HnswSidecar.js.map +1 -1
  111. package/dist/memory/store/SqliteBrain.d.ts +14 -2
  112. package/dist/memory/store/SqliteBrain.d.ts.map +1 -1
  113. package/dist/memory/store/SqliteBrain.js +37 -34
  114. package/dist/memory/store/SqliteBrain.js.map +1 -1
  115. package/dist/memory/store/SqliteKnowledgeGraph.d.ts.map +1 -1
  116. package/dist/memory/store/SqliteKnowledgeGraph.js +12 -55
  117. package/dist/memory/store/SqliteKnowledgeGraph.js.map +1 -1
  118. package/dist/memory/store/SqliteMemoryGraph.d.ts.map +1 -1
  119. package/dist/memory/store/SqliteMemoryGraph.js +11 -16
  120. package/dist/memory/store/SqliteMemoryGraph.js.map +1 -1
  121. package/dist/memory/store/tracePersistence.d.ts +1 -1
  122. package/dist/memory/store/tracePersistence.d.ts.map +1 -1
  123. package/dist/memory/store/tracePersistence.js +3 -3
  124. package/dist/memory/store/tracePersistence.js.map +1 -1
  125. package/dist/memory/tools/MemoryAddTool.d.ts.map +1 -1
  126. package/dist/memory/tools/MemoryAddTool.js +2 -7
  127. package/dist/memory/tools/MemoryAddTool.js.map +1 -1
  128. package/dist/memory/tools/MemoryMergeTool.d.ts.map +1 -1
  129. package/dist/memory/tools/MemoryMergeTool.js +2 -3
  130. package/dist/memory/tools/MemoryMergeTool.js.map +1 -1
  131. package/dist/memory/tools/MemorySearchTool.d.ts.map +1 -1
  132. package/dist/memory/tools/MemorySearchTool.js +6 -5
  133. package/dist/memory/tools/MemorySearchTool.js.map +1 -1
  134. package/dist/memory/tools/MemoryUpdateTool.d.ts.map +1 -1
  135. package/dist/memory/tools/MemoryUpdateTool.js +2 -3
  136. package/dist/memory/tools/MemoryUpdateTool.js.map +1 -1
  137. package/dist/memory/util/crossPlatformCrypto.d.ts +31 -0
  138. package/dist/memory/util/crossPlatformCrypto.d.ts.map +1 -0
  139. package/dist/memory/util/crossPlatformCrypto.js +60 -0
  140. package/dist/memory/util/crossPlatformCrypto.js.map +1 -0
  141. package/dist/rag/VectorStoreManager.d.ts.map +1 -1
  142. package/dist/rag/VectorStoreManager.js +4 -2
  143. package/dist/rag/VectorStoreManager.js.map +1 -1
  144. package/dist/rag/implementations/vector_stores/HnswlibVectorStore.d.ts +6 -0
  145. package/dist/rag/implementations/vector_stores/HnswlibVectorStore.d.ts.map +1 -1
  146. package/dist/rag/implementations/vector_stores/HnswlibVectorStore.js +9 -0
  147. package/dist/rag/implementations/vector_stores/HnswlibVectorStore.js.map +1 -1
  148. package/dist/rag/implementations/vector_stores/SqlVectorStore.d.ts +23 -0
  149. package/dist/rag/implementations/vector_stores/SqlVectorStore.d.ts.map +1 -1
  150. package/dist/rag/implementations/vector_stores/SqlVectorStore.js +109 -5
  151. package/dist/rag/implementations/vector_stores/SqlVectorStore.js.map +1 -1
  152. package/dist/rag/search/BM25Index.d.ts +13 -0
  153. package/dist/rag/search/BM25Index.d.ts.map +1 -1
  154. package/dist/rag/search/BM25Index.js +14 -20
  155. package/dist/rag/search/BM25Index.js.map +1 -1
  156. package/package.json +3 -2
@@ -1 +1 @@
1
- {"version":3,"file":"AgentKeyManager.d.ts","sourceRoot":"","sources":["../../../../src/core/provenance/crypto/AgentKeyManager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAkBlD,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,SAAS,CAAsB;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,OAAO;IAUP;;OAEG;WACU,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAkBhE;;OAEG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC;IAc7F;;OAEG;IACG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBzC;;;OAGG;IACG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8B/F;;OAEG;WACU,eAAe,CAC1B,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,OAAO,CAAC;IA2BnB;;OAEG;IACH,kBAAkB,IAAI,MAAM;IAI5B;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,WAAW,IAAI,cAAc;CAO9B"}
1
+ {"version":3,"file":"AgentKeyManager.d.ts","sourceRoot":"","sources":["../../../../src/core/provenance/crypto/AgentKeyManager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAwBlD,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,SAAS,CAAsB;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,OAAO;IAUP;;OAEG;WACU,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAkBhE;;OAEG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC;IAc7F;;OAEG;IACG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBzC;;;OAGG;IACG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8B/F;;OAEG;WACU,eAAe,CAC1B,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,OAAO,CAAC;IA2BnB;;OAEG;IACH,kBAAkB,IAAI,MAAM;IAI5B;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,WAAW,IAAI,cAAc;CAO9B"}
@@ -8,13 +8,19 @@
8
8
  // =============================================================================
9
9
  // Runtime Detection
10
10
  // =============================================================================
11
+ /**
12
+ * Lazily-resolved reference to Node.js `crypto` module.
13
+ * Uses synchronous `require()` to avoid top-level `await`, which breaks CJS
14
+ * output (esbuild / tsx emit `"cjs"` format that does not support TLA).
15
+ * Falls back to `undefined` in browser / non-Node runtimes.
16
+ */
11
17
  let nodeCrypto;
12
18
  try {
13
- // Dynamic import for Node.js runtime
14
- nodeCrypto = await import('node:crypto');
19
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
20
+ nodeCrypto = require('node:crypto');
15
21
  }
16
22
  catch {
17
- // Not in Node.js environment
23
+ // Not in Node.js environment (browser, Deno without compat, etc.)
18
24
  }
19
25
  // =============================================================================
20
26
  // AgentKeyManager
@@ -1 +1 @@
1
- {"version":3,"file":"AgentKeyManager.js","sourceRoot":"","sources":["../../../../src/core/provenance/crypto/AgentKeyManager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF,IAAI,UAAoD,CAAC;AACzD,IAAI,CAAC;IACH,qCAAqC;IACrC,UAAU,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;AAC3C,CAAC;AAAC,MAAM,CAAC;IACP,6BAA6B;AAC/B,CAAC;AAED,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF,MAAM,OAAO,eAAe;IAK1B,YACE,OAAe,EACf,UAA+B,EAC/B,SAA8B;QAE9B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAe;QACnC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAC5E,OAAO,IAAI,eAAe,CACxB,OAAO,EACP,UAAU,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EACnD,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAClD,CAAC;QACJ,CAAC;QAED,sCAAsC;QACtC,4EAA4E;QAC5E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtD,OAAO,IAAI,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,MAAsB;QAChE,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,OAAO,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;QAChG,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAChE,OAAO,IAAI,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,SAAS,GAAG,UAAU,CAAC,gBAAgB,CAAC;gBAC5C,GAAG,EAAE,IAAI,CAAC,UAAoB;gBAC9B,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAC/D,OAAO,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,mBAAmB;QACnB,4EAA4E;QAC5E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,eAAuB,EAAE,eAAwB;QAC1E,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,eAAe;YACjC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC;YACxC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAEnB,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,UAAU,CAAC,eAAe,CAAC;oBAC3C,GAAG,EAAE,WAAqB;oBAC1B,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;gBACH,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YACnE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC;YACH,4EAA4E;YAC9E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC3C,OAAO,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,eAAe,CAC1B,IAAY,EACZ,eAAuB,EACvB,eAAuB;QAEvB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAE3D,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,UAAU,CAAC,eAAe,CAAC;oBAC3C,GAAG,EAAE,WAAW;oBAChB,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;gBACH,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YACnE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,4EAA4E;YAC9E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC3C,OAAO,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,gBAAgB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC5C,eAAe,EAAE,IAAI,CAAC,kBAAkB,EAAE;SAC3C,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"AgentKeyManager.js","sourceRoot":"","sources":["../../../../src/core/provenance/crypto/AgentKeyManager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;;;;GAKG;AACH,IAAI,UAAoD,CAAC;AACzD,IAAI,CAAC;IACH,iEAAiE;IACjE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AACtC,CAAC;AAAC,MAAM,CAAC;IACP,kEAAkE;AACpE,CAAC;AAED,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF,MAAM,OAAO,eAAe;IAK1B,YACE,OAAe,EACf,UAA+B,EAC/B,SAA8B;QAE9B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAe;QACnC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAC5E,OAAO,IAAI,eAAe,CACxB,OAAO,EACP,UAAU,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EACnD,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAClD,CAAC;QACJ,CAAC;QAED,sCAAsC;QACtC,4EAA4E;QAC5E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtD,OAAO,IAAI,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,MAAsB;QAChE,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,OAAO,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;QAChG,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAChE,OAAO,IAAI,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,SAAS,GAAG,UAAU,CAAC,gBAAgB,CAAC;gBAC5C,GAAG,EAAE,IAAI,CAAC,UAAoB;gBAC9B,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAC/D,OAAO,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,mBAAmB;QACnB,4EAA4E;QAC5E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,eAAuB,EAAE,eAAwB;QAC1E,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,eAAe;YACjC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC;YACxC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAEnB,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,UAAU,CAAC,eAAe,CAAC;oBAC3C,GAAG,EAAE,WAAqB;oBAC1B,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;gBACH,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YACnE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC;YACH,4EAA4E;YAC9E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC3C,OAAO,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,eAAe,CAC1B,IAAY,EACZ,eAAuB,EACvB,eAAuB;QAEvB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAE3D,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,UAAU,CAAC,eAAe,CAAC;oBAC3C,GAAG,EAAE,WAAW;oBAChB,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;gBACH,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YACnE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,4EAA4E;YAC9E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC3C,OAAO,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,gBAAgB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC5C,eAAe,EAAE,IAAI,CAAC,kBAAkB,EAAE;SAC3C,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,43 @@
1
+ /**
2
+ * @fileoverview Interface for a single stage in the text processing pipeline.
3
+ * @module agentos/core/text-processing/ITextProcessor
4
+ */
5
+ import type { Token } from './types';
6
+ /**
7
+ * A single processing stage in a {@link TextProcessingPipeline}.
8
+ * Each processor receives tokens from the previous stage and returns
9
+ * transformed tokens for the next stage.
10
+ *
11
+ * Tokenizers are also processors — the first stage in the pipeline
12
+ * receives an empty token array and produces the initial tokens from
13
+ * raw text via the pipeline's entry point.
14
+ */
15
+ export interface ITextProcessor {
16
+ /** Human-readable name for debugging and logging. */
17
+ readonly name: string;
18
+ /**
19
+ * Process an array of tokens, returning transformed tokens.
20
+ * May filter, modify, split, or annotate tokens.
21
+ *
22
+ * @param tokens — tokens from the previous pipeline stage
23
+ * @returns transformed tokens for the next stage
24
+ */
25
+ process(tokens: Token[]): Token[];
26
+ }
27
+ /**
28
+ * A tokenizer is the first stage in a pipeline — it converts raw text
29
+ * into an initial array of tokens. Separate interface because it takes
30
+ * a string, not Token[].
31
+ */
32
+ export interface ITokenizer {
33
+ /** Human-readable name for debugging and logging. */
34
+ readonly name: string;
35
+ /**
36
+ * Split raw text into tokens with position tracking.
37
+ *
38
+ * @param text — raw input text
39
+ * @returns initial token array
40
+ */
41
+ tokenize(text: string): Token[];
42
+ }
43
+ //# sourceMappingURL=ITextProcessor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ITextProcessor.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/ITextProcessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAErC;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,qDAAqD;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC;CACnC;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,qDAAqD;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,CAAC;CACjC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * @fileoverview Interface for a single stage in the text processing pipeline.
3
+ * @module agentos/core/text-processing/ITextProcessor
4
+ */
5
+ export {};
6
+ //# sourceMappingURL=ITextProcessor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ITextProcessor.js","sourceRoot":"","sources":["../../../src/core/text-processing/ITextProcessor.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
@@ -0,0 +1,49 @@
1
+ /**
2
+ * @fileoverview Configurable text processing pipeline.
3
+ * Chains a tokenizer + N processors to produce processed tokens from raw text.
4
+ *
5
+ * @module agentos/core/text-processing/TextProcessingPipeline
6
+ */
7
+ import type { Token } from './types';
8
+ import type { ITextProcessor, ITokenizer } from './ITextProcessor';
9
+ /**
10
+ * Orchestrates a chain of text processors: tokenizer → processors → output.
11
+ *
12
+ * @example
13
+ * const pipeline = new TextProcessingPipeline(new StandardTokenizer())
14
+ * .add(new LowercaseNormalizer())
15
+ * .add(new StopWordFilter(ENGLISH_STOP_WORDS))
16
+ * .add(new PorterStemmer());
17
+ *
18
+ * const tokens = pipeline.process('The quick brown foxes are running');
19
+ * // tokens[0].text === 'quick', tokens[0].stem === 'quick'
20
+ * // tokens[1].text === 'brown', ...
21
+ */
22
+ export declare class TextProcessingPipeline {
23
+ private tokenizer;
24
+ private processors;
25
+ /**
26
+ * @param tokenizer — the first stage that splits raw text into tokens
27
+ */
28
+ constructor(tokenizer: ITokenizer);
29
+ /** Add a processing stage to the pipeline. Returns `this` for chaining. */
30
+ add(processor: ITextProcessor): this;
31
+ /**
32
+ * Process raw text through the full pipeline.
33
+ *
34
+ * @param text — raw input text
35
+ * @returns array of processed tokens with position and linguistic annotations
36
+ */
37
+ process(text: string): Token[];
38
+ /**
39
+ * Convenience: process text and return just the token strings.
40
+ * Useful for BM25 indexing and FTS where only the text values are needed.
41
+ *
42
+ * @param text — raw input text
43
+ * @returns array of processed token strings
44
+ */
45
+ processToStrings(text: string): string[];
46
+ /** Get the names of all stages for debugging. */
47
+ getStageNames(): string[];
48
+ }
49
+ //# sourceMappingURL=TextProcessingPipeline.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"TextProcessingPipeline.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/TextProcessingPipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;;;;;;;GAYG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,UAAU,CAAwB;IAE1C;;OAEG;gBACS,SAAS,EAAE,UAAU;IAIjC,2EAA2E;IAC3E,GAAG,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI;IAKpC;;;;;OAKG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE;IAU9B;;;;;;OAMG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAIxC,iDAAiD;IACjD,aAAa,IAAI,MAAM,EAAE;CAG1B"}
@@ -0,0 +1,61 @@
1
+ /**
2
+ * @fileoverview Configurable text processing pipeline.
3
+ * Chains a tokenizer + N processors to produce processed tokens from raw text.
4
+ *
5
+ * @module agentos/core/text-processing/TextProcessingPipeline
6
+ */
7
+ /**
8
+ * Orchestrates a chain of text processors: tokenizer → processors → output.
9
+ *
10
+ * @example
11
+ * const pipeline = new TextProcessingPipeline(new StandardTokenizer())
12
+ * .add(new LowercaseNormalizer())
13
+ * .add(new StopWordFilter(ENGLISH_STOP_WORDS))
14
+ * .add(new PorterStemmer());
15
+ *
16
+ * const tokens = pipeline.process('The quick brown foxes are running');
17
+ * // tokens[0].text === 'quick', tokens[0].stem === 'quick'
18
+ * // tokens[1].text === 'brown', ...
19
+ */
20
+ export class TextProcessingPipeline {
21
+ /**
22
+ * @param tokenizer — the first stage that splits raw text into tokens
23
+ */
24
+ constructor(tokenizer) {
25
+ this.processors = [];
26
+ this.tokenizer = tokenizer;
27
+ }
28
+ /** Add a processing stage to the pipeline. Returns `this` for chaining. */
29
+ add(processor) {
30
+ this.processors.push(processor);
31
+ return this;
32
+ }
33
+ /**
34
+ * Process raw text through the full pipeline.
35
+ *
36
+ * @param text — raw input text
37
+ * @returns array of processed tokens with position and linguistic annotations
38
+ */
39
+ process(text) {
40
+ let tokens = this.tokenizer.tokenize(text);
41
+ for (const processor of this.processors) {
42
+ tokens = processor.process(tokens);
43
+ }
44
+ return tokens;
45
+ }
46
+ /**
47
+ * Convenience: process text and return just the token strings.
48
+ * Useful for BM25 indexing and FTS where only the text values are needed.
49
+ *
50
+ * @param text — raw input text
51
+ * @returns array of processed token strings
52
+ */
53
+ processToStrings(text) {
54
+ return this.process(text).map(t => t.text);
55
+ }
56
+ /** Get the names of all stages for debugging. */
57
+ getStageNames() {
58
+ return [this.tokenizer.name, ...this.processors.map(p => p.name)];
59
+ }
60
+ }
61
+ //# sourceMappingURL=TextProcessingPipeline.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"TextProcessingPipeline.js","sourceRoot":"","sources":["../../../src/core/text-processing/TextProcessingPipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,sBAAsB;IAIjC;;OAEG;IACH,YAAY,SAAqB;QALzB,eAAU,GAAqB,EAAE,CAAC;QAMxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,2EAA2E;IAC3E,GAAG,CAAC,SAAyB;QAC3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,IAAY;QAClB,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE3C,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,gBAAgB,CAAC,IAAY;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,iDAAiD;IACjD,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,CAAC;CACF"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * @fileoverview Removes tokens that match a configurable stop word list.
3
+ * @module agentos/core/text-processing/filters/StopWordFilter
4
+ */
5
+ import type { Token } from '../types';
6
+ import type { ITextProcessor } from '../ITextProcessor';
7
+ /** ~120 common English stop words. */
8
+ export declare const ENGLISH_STOP_WORDS: ReadonlySet<string>;
9
+ /**
10
+ * Stop words for code search. These are noise in code but NOT programming keywords.
11
+ * Note: `class`, `function`, `import`, `const`, `let`, `var`, `return`, `this`
12
+ * are deliberately NOT in this list — they're meaningful in code search.
13
+ */
14
+ export declare const CODE_STOP_WORDS: ReadonlySet<string>;
15
+ export declare function getNaturalStopWords(): ReadonlySet<string>;
16
+ /**
17
+ * Filters tokens whose `.text` appears in the provided stop word set.
18
+ * Case-sensitive — apply after LowercaseNormalizer for case-insensitive filtering.
19
+ */
20
+ export declare class StopWordFilter implements ITextProcessor {
21
+ readonly name = "StopWordFilter";
22
+ private stopWords;
23
+ /**
24
+ * @param stopWords — stop word set to filter against. Defaults to `natural`'s
25
+ * 170-word list when available, falls back to the built-in 120-word ENGLISH_STOP_WORDS.
26
+ */
27
+ constructor(stopWords?: ReadonlySet<string>);
28
+ process(tokens: Token[]): Token[];
29
+ }
30
+ //# sourceMappingURL=StopWordFilter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"StopWordFilter.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/filters/StopWordFilter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,sCAAsC;AACtC,eAAO,MAAM,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAejD,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,WAAW,CAAC,MAAM,CAW9C,CAAC;AAOH,wBAAgB,mBAAmB,IAAI,WAAW,CAAC,MAAM,CAAC,CAYzD;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,IAAI,oBAAoB;IACjC,OAAO,CAAC,SAAS,CAAsB;IAEvC;;;OAGG;gBACS,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC;IAI3C,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}
@@ -0,0 +1,76 @@
1
+ /**
2
+ * @fileoverview Removes tokens that match a configurable stop word list.
3
+ * @module agentos/core/text-processing/filters/StopWordFilter
4
+ */
5
+ /** ~120 common English stop words. */
6
+ export const ENGLISH_STOP_WORDS = new Set([
7
+ 'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and',
8
+ 'any', 'are', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below',
9
+ 'between', 'both', 'but', 'by', 'can', 'could', 'did', 'do', 'does', 'doing',
10
+ 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'get', 'got', 'had',
11
+ 'has', 'have', 'having', 'he', 'her', 'here', 'hers', 'herself', 'him',
12
+ 'himself', 'his', 'how', 'if', 'in', 'into', 'is', 'it', 'its', 'itself',
13
+ 'just', 'me', 'might', 'more', 'most', 'must', 'my', 'myself', 'no', 'nor',
14
+ 'not', 'now', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'our', 'ours',
15
+ 'ourselves', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some',
16
+ 'such', 'than', 'that', 'the', 'their', 'theirs', 'them', 'themselves', 'then',
17
+ 'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too', 'under',
18
+ 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which',
19
+ 'while', 'who', 'whom', 'why', 'will', 'with', 'would', 'you', 'your', 'yours',
20
+ 'yourself', 'yourselves',
21
+ ]);
22
+ /**
23
+ * Stop words for code search. These are noise in code but NOT programming keywords.
24
+ * Note: `class`, `function`, `import`, `const`, `let`, `var`, `return`, `this`
25
+ * are deliberately NOT in this list — they're meaningful in code search.
26
+ */
27
+ export const CODE_STOP_WORDS = new Set([
28
+ 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
29
+ 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
30
+ 'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
31
+ 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
32
+ 'before', 'after', 'above', 'below', 'between', 'out', 'off', 'over',
33
+ 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
34
+ 'where', 'why', 'how', 'all', 'each', 'every', 'both', 'few', 'more',
35
+ 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own',
36
+ 'same', 'so', 'than', 'too', 'very', 'just', 'because', 'but', 'and',
37
+ 'or', 'if', 'while', 'about', 'up', 'out', 'also', 'it', 'its',
38
+ ]);
39
+ /**
40
+ * Extended stop word list from the `natural` NLP library (170 words).
41
+ * Loaded lazily — falls back to ENGLISH_STOP_WORDS if natural is unavailable.
42
+ */
43
+ let _naturalStopWords = null;
44
+ export function getNaturalStopWords() {
45
+ if (_naturalStopWords)
46
+ return _naturalStopWords;
47
+ try {
48
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
49
+ const natural = require('natural');
50
+ if (natural.stopwords && Array.isArray(natural.stopwords)) {
51
+ _naturalStopWords = new Set(natural.stopwords);
52
+ return _naturalStopWords;
53
+ }
54
+ }
55
+ catch { /* natural not installed */ }
56
+ _naturalStopWords = ENGLISH_STOP_WORDS;
57
+ return _naturalStopWords;
58
+ }
59
+ /**
60
+ * Filters tokens whose `.text` appears in the provided stop word set.
61
+ * Case-sensitive — apply after LowercaseNormalizer for case-insensitive filtering.
62
+ */
63
+ export class StopWordFilter {
64
+ /**
65
+ * @param stopWords — stop word set to filter against. Defaults to `natural`'s
66
+ * 170-word list when available, falls back to the built-in 120-word ENGLISH_STOP_WORDS.
67
+ */
68
+ constructor(stopWords) {
69
+ this.name = 'StopWordFilter';
70
+ this.stopWords = stopWords ?? getNaturalStopWords();
71
+ }
72
+ process(tokens) {
73
+ return tokens.filter(t => !this.stopWords.has(t.text));
74
+ }
75
+ }
76
+ //# sourceMappingURL=StopWordFilter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"StopWordFilter.js","sourceRoot":"","sources":["../../../../src/core/text-processing/filters/StopWordFilter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,sCAAsC;AACtC,MAAM,CAAC,MAAM,kBAAkB,GAAwB,IAAI,GAAG,CAAC;IAC7D,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IAC5E,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO;IAC7E,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IAC5E,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAC9E,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK;IACtE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ;IACxE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK;IAC1E,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;IAC7E,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM;IACxE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;IAC9E,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO;IAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAC5E,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC9E,UAAU,EAAE,YAAY;CACzB,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC;IAC1D,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACnE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IACjE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ;IACnE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;IACpE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACpE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IACpE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;IAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK;IACpE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK;CAC/D,CAAC,CAAC;AAEH;;;GAGG;AACH,IAAI,iBAAiB,GAA+B,IAAI,CAAC;AACzD,MAAM,UAAU,mBAAmB;IACjC,IAAI,iBAAiB;QAAE,OAAO,iBAAiB,CAAC;IAChD,IAAI,CAAC;QACH,8DAA8D;QAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACnC,IAAI,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,SAAqB,CAAC,CAAC;YAC3D,OAAO,iBAAiB,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,2BAA2B,CAAC,CAAC;IACvC,iBAAiB,GAAG,kBAAkB,CAAC;IACvC,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,cAAc;IAIzB;;;OAGG;IACH,YAAY,SAA+B;QAPlC,SAAI,GAAG,gBAAgB,CAAC;QAQ/B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,mBAAmB,EAAE,CAAC;IACtD,CAAC;IAED,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC;CACF"}
@@ -0,0 +1,20 @@
1
+ /**
2
+ * @fileoverview Pluggable text processing pipeline for AgentOS.
3
+ * Feeds into BM25 keyword search, FTS5, and embedding generation.
4
+ *
5
+ * @module agentos/core/text-processing
6
+ */
7
+ export { TextProcessingPipeline } from './TextProcessingPipeline';
8
+ export type { ITextProcessor, ITokenizer } from './ITextProcessor';
9
+ export type { Token } from './types';
10
+ export { StandardTokenizer } from './tokenizers/StandardTokenizer';
11
+ export { CodeTokenizer } from './tokenizers/CodeTokenizer';
12
+ export { LowercaseNormalizer } from './normalizers/LowercaseNormalizer';
13
+ export { AccentStripper } from './normalizers/AccentStripper';
14
+ export { StopWordFilter, ENGLISH_STOP_WORDS, CODE_STOP_WORDS, getNaturalStopWords } from './filters/StopWordFilter';
15
+ export { PorterStemmer } from './stemmers/PorterStemmer';
16
+ export { NoOpStemmer } from './stemmers/NoOpStemmer';
17
+ export { WordNetLemmatizer } from './lemmatizers/WordNetLemmatizer';
18
+ export { NoOpLemmatizer } from './lemmatizers/NoOpLemmatizer';
19
+ export { createProsePipeline, createCodePipeline, createRagPipeline } from './presets';
20
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/text-processing/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnE,YAAY,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAGrC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAG9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGpH,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAGrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAG9D,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC"}
@@ -0,0 +1,24 @@
1
+ /**
2
+ * @fileoverview Pluggable text processing pipeline for AgentOS.
3
+ * Feeds into BM25 keyword search, FTS5, and embedding generation.
4
+ *
5
+ * @module agentos/core/text-processing
6
+ */
7
+ export { TextProcessingPipeline } from './TextProcessingPipeline.js';
8
+ // Tokenizers
9
+ export { StandardTokenizer } from './tokenizers/StandardTokenizer.js';
10
+ export { CodeTokenizer } from './tokenizers/CodeTokenizer.js';
11
+ // Normalizers
12
+ export { LowercaseNormalizer } from './normalizers/LowercaseNormalizer.js';
13
+ export { AccentStripper } from './normalizers/AccentStripper.js';
14
+ // Filters
15
+ export { StopWordFilter, ENGLISH_STOP_WORDS, CODE_STOP_WORDS, getNaturalStopWords } from './filters/StopWordFilter.js';
16
+ // Stemmers
17
+ export { PorterStemmer } from './stemmers/PorterStemmer.js';
18
+ export { NoOpStemmer } from './stemmers/NoOpStemmer.js';
19
+ // Lemmatizers
20
+ export { WordNetLemmatizer } from './lemmatizers/WordNetLemmatizer.js';
21
+ export { NoOpLemmatizer } from './lemmatizers/NoOpLemmatizer.js';
22
+ // Presets
23
+ export { createProsePipeline, createCodePipeline, createRagPipeline } from './presets.js';
24
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/text-processing/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAIlE,aAAa;AACb,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,cAAc;AACd,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpH,WAAW;AACX,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,cAAc;AACd,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,UAAU;AACV,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC"}
@@ -0,0 +1,11 @@
1
+ /**
2
+ * @fileoverview Pass-through lemmatizer that does nothing.
3
+ * @module agentos/core/text-processing/lemmatizers/NoOpLemmatizer
4
+ */
5
+ import type { Token } from '../types';
6
+ import type { ITextProcessor } from '../ITextProcessor';
7
+ export declare class NoOpLemmatizer implements ITextProcessor {
8
+ readonly name = "NoOpLemmatizer";
9
+ process(tokens: Token[]): Token[];
10
+ }
11
+ //# sourceMappingURL=NoOpLemmatizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"NoOpLemmatizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/NoOpLemmatizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * @fileoverview Pass-through lemmatizer that does nothing.
3
+ * @module agentos/core/text-processing/lemmatizers/NoOpLemmatizer
4
+ */
5
+ export class NoOpLemmatizer {
6
+ constructor() {
7
+ this.name = 'NoOpLemmatizer';
8
+ }
9
+ process(tokens) {
10
+ return tokens;
11
+ }
12
+ }
13
+ //# sourceMappingURL=NoOpLemmatizer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"NoOpLemmatizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/NoOpLemmatizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,OAAO,cAAc;IAA3B;QACW,SAAI,GAAG,gBAAgB,CAAC;IAKnC,CAAC;IAHC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
@@ -0,0 +1,25 @@
1
+ /**
2
+ * @fileoverview WordNet lemmatizer wrapping the `natural` package.
3
+ * Falls back to no-op if `natural` is not installed.
4
+ *
5
+ * Lemmatization produces the dictionary form of a word:
6
+ * `ran` → `run`, `better` → `good`, `mice` → `mouse`.
7
+ *
8
+ * @module agentos/core/text-processing/lemmatizers/WordNetLemmatizer
9
+ */
10
+ import type { Token } from '../types';
11
+ import type { ITextProcessor } from '../ITextProcessor';
12
+ /**
13
+ * WordNet-based lemmatizer. Reduces words to their dictionary (lemma) form.
14
+ *
15
+ * Sets `token.lemma` and updates `token.text` to the lemmatized form.
16
+ * Falls back to Lancaster stemming if full WordNet lookup is unavailable.
17
+ */
18
+ export declare class WordNetLemmatizer implements ITextProcessor {
19
+ readonly name = "WordNetLemmatizer";
20
+ private initialized;
21
+ private ensureLoaded;
22
+ process(tokens: Token[]): Token[];
23
+ initialize(): Promise<void>;
24
+ }
25
+ //# sourceMappingURL=WordNetLemmatizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"WordNetLemmatizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/WordNetLemmatizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AA0BxD;;;;;GAKG;AACH,qBAAa,iBAAkB,YAAW,cAAc;IACtD,QAAQ,CAAC,IAAI,uBAAuB;IAEpC,OAAO,CAAC,WAAW,CAAS;YAEd,YAAY;IAO1B,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;IAS3B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;CAGlC"}
@@ -0,0 +1,64 @@
1
+ /**
2
+ * @fileoverview WordNet lemmatizer wrapping the `natural` package.
3
+ * Falls back to no-op if `natural` is not installed.
4
+ *
5
+ * Lemmatization produces the dictionary form of a word:
6
+ * `ran` → `run`, `better` → `good`, `mice` → `mouse`.
7
+ *
8
+ * @module agentos/core/text-processing/lemmatizers/WordNetLemmatizer
9
+ */
10
+ /** Lazy-loaded lemmatize function from the `natural` package. */
11
+ let lemmatizeFn = null;
12
+ let loadAttempted = false;
13
+ async function loadLemmatizer() {
14
+ if (loadAttempted)
15
+ return;
16
+ loadAttempted = true;
17
+ try {
18
+ const natural = await import('natural');
19
+ const wordnet = new natural.WordNet();
20
+ lemmatizeFn = (word) => {
21
+ /* WordNet lookup is async in natural, but we need sync for the pipeline.
22
+ Use the synchronous stemmer-based lemmatizer as a practical fallback. */
23
+ try {
24
+ return natural.LancasterStemmer.stem(word);
25
+ }
26
+ catch {
27
+ return word;
28
+ }
29
+ };
30
+ }
31
+ catch {
32
+ lemmatizeFn = null;
33
+ }
34
+ }
35
+ /**
36
+ * WordNet-based lemmatizer. Reduces words to their dictionary (lemma) form.
37
+ *
38
+ * Sets `token.lemma` and updates `token.text` to the lemmatized form.
39
+ * Falls back to Lancaster stemming if full WordNet lookup is unavailable.
40
+ */
41
+ export class WordNetLemmatizer {
42
+ constructor() {
43
+ this.name = 'WordNetLemmatizer';
44
+ this.initialized = false;
45
+ }
46
+ async ensureLoaded() {
47
+ if (!this.initialized) {
48
+ await loadLemmatizer();
49
+ this.initialized = true;
50
+ }
51
+ }
52
+ process(tokens) {
53
+ if (!lemmatizeFn)
54
+ return tokens;
55
+ return tokens.map(t => {
56
+ const lemma = lemmatizeFn(t.text);
57
+ return { ...t, text: lemma, lemma };
58
+ });
59
+ }
60
+ async initialize() {
61
+ await this.ensureLoaded();
62
+ }
63
+ }
64
+ //# sourceMappingURL=WordNetLemmatizer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"WordNetLemmatizer.js","sourceRoot":"","sources":["../../../../src/core/text-processing/lemmatizers/WordNetLemmatizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,iEAAiE;AACjE,IAAI,WAAW,GAAsC,IAAI,CAAC;AAC1D,IAAI,aAAa,GAAG,KAAK,CAAC;AAE1B,KAAK,UAAU,cAAc;IAC3B,IAAI,aAAa;QAAE,OAAO;IAC1B,aAAa,GAAG,IAAI,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACtC,WAAW,GAAG,CAAC,IAAY,EAAE,EAAE;YAC7B;uFAC2E;YAC3E,IAAI,CAAC;gBACH,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,iBAAiB;IAA9B;QACW,SAAI,GAAG,mBAAmB,CAAC;QAE5B,gBAAW,GAAG,KAAK,CAAC;IAqB9B,CAAC;IAnBS,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAe;QACrB,IAAI,CAAC,WAAW;YAAE,OAAO,MAAM,CAAC;QAEhC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACpB,MAAM,KAAK,GAAG,WAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;CACF"}
@@ -0,0 +1,17 @@
1
+ /**
2
+ * @fileoverview Strips diacritics/accents from token text.
3
+ * `café` → `cafe`, `naïve` → `naive`.
4
+ *
5
+ * @module agentos/core/text-processing/normalizers/AccentStripper
6
+ */
7
+ import type { Token } from '../types';
8
+ import type { ITextProcessor } from '../ITextProcessor';
9
+ /**
10
+ * Removes combining diacritical marks after Unicode NFD decomposition.
11
+ * This makes accent-insensitive search possible.
12
+ */
13
+ export declare class AccentStripper implements ITextProcessor {
14
+ readonly name = "AccentStripper";
15
+ process(tokens: Token[]): Token[];
16
+ }
17
+ //# sourceMappingURL=AccentStripper.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AccentStripper.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/AccentStripper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;GAGG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAMlC"}
@@ -0,0 +1,22 @@
1
+ /**
2
+ * @fileoverview Strips diacritics/accents from token text.
3
+ * `café` → `cafe`, `naïve` → `naive`.
4
+ *
5
+ * @module agentos/core/text-processing/normalizers/AccentStripper
6
+ */
7
+ /**
8
+ * Removes combining diacritical marks after Unicode NFD decomposition.
9
+ * This makes accent-insensitive search possible.
10
+ */
11
+ export class AccentStripper {
12
+ constructor() {
13
+ this.name = 'AccentStripper';
14
+ }
15
+ process(tokens) {
16
+ return tokens.map(t => ({
17
+ ...t,
18
+ text: t.text.normalize('NFD').replace(/[\u0300-\u036f]/g, ''),
19
+ }));
20
+ }
21
+ }
22
+ //# sourceMappingURL=AccentStripper.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AccentStripper.js","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/AccentStripper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;GAGG;AACH,MAAM,OAAO,cAAc;IAA3B;QACW,SAAI,GAAG,gBAAgB,CAAC;IAQnC,CAAC;IANC,OAAO,CAAC,MAAe;QACrB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACtB,GAAG,CAAC;YACJ,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC9D,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
@@ -0,0 +1,11 @@
1
+ /**
2
+ * @fileoverview Lowercases all token text. Preserves original.
3
+ * @module agentos/core/text-processing/normalizers/LowercaseNormalizer
4
+ */
5
+ import type { Token } from '../types';
6
+ import type { ITextProcessor } from '../ITextProcessor';
7
+ export declare class LowercaseNormalizer implements ITextProcessor {
8
+ readonly name = "LowercaseNormalizer";
9
+ process(tokens: Token[]): Token[];
10
+ }
11
+ //# sourceMappingURL=LowercaseNormalizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LowercaseNormalizer.d.ts","sourceRoot":"","sources":["../../../../src/core/text-processing/normalizers/LowercaseNormalizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,yBAAyB;IAEtC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE;CAGlC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * @fileoverview Lowercases all token text. Preserves original.
3
+ * @module agentos/core/text-processing/normalizers/LowercaseNormalizer
4
+ */
5
+ export class LowercaseNormalizer {
6
+ constructor() {
7
+ this.name = 'LowercaseNormalizer';
8
+ }
9
+ process(tokens) {
10
+ return tokens.map(t => ({ ...t, text: t.text.toLowerCase() }));
11
+ }
12
+ }
13
+ //# sourceMappingURL=LowercaseNormalizer.js.map