@lossless-claude/lcm 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/.claude-plugin/commands/lcm-compact.md +31 -0
  2. package/.claude-plugin/commands/lcm-curate.md +31 -0
  3. package/.claude-plugin/commands/lcm-diagnose.md +29 -0
  4. package/.claude-plugin/commands/lcm-doctor.md +23 -0
  5. package/.claude-plugin/commands/lcm-dogfood.md +101 -0
  6. package/.claude-plugin/commands/lcm-import.md +48 -0
  7. package/.claude-plugin/commands/lcm-promote.md +29 -0
  8. package/.claude-plugin/commands/lcm-sensitive.md +55 -0
  9. package/.claude-plugin/commands/lcm-stats.md +19 -0
  10. package/.claude-plugin/commands/lcm-status.md +27 -0
  11. package/.claude-plugin/hooks/README.md +47 -0
  12. package/.claude-plugin/marketplace.json +20 -0
  13. package/.claude-plugin/mcp.mjs +12 -0
  14. package/.claude-plugin/plugin.json +46 -0
  15. package/.claude-plugin/skills/lcm-context/SKILL.md +107 -0
  16. package/.claude-plugin/skills/lcm-dogfood/SKILL.md +102 -0
  17. package/.claude-plugin/skills/lcm-dogfood/references/checks.md +239 -0
  18. package/.claude-plugin/skills/lcm-dogfood/references/known-issues.md +11 -0
  19. package/.claude-plugin/skills/lcm-dogfood/scripts/db-integrity.js +40 -0
  20. package/.claude-plugin/skills/lcm-dogfood/scripts/prompt-search-test.js +35 -0
  21. package/.claude-plugin/skills/lcm-e2e/SKILL.md +61 -0
  22. package/.claude-plugin/skills/lcm-e2e/checklist.md +367 -0
  23. package/.claude-plugin/skills/lossless-claude-upgrade/SKILL.md +47 -0
  24. package/LICENSE +21 -0
  25. package/README.md +231 -0
  26. package/dist/bin/lcm.d.ts +2 -0
  27. package/dist/bin/lcm.js +461 -0
  28. package/dist/bin/lcm.js.map +1 -0
  29. package/dist/installer/dry-run-deps.d.ts +23 -0
  30. package/dist/installer/dry-run-deps.js +66 -0
  31. package/dist/installer/dry-run-deps.js.map +1 -0
  32. package/dist/installer/install.d.ts +39 -0
  33. package/dist/installer/install.js +236 -0
  34. package/dist/installer/install.js.map +1 -0
  35. package/dist/installer/uninstall.d.ts +11 -0
  36. package/dist/installer/uninstall.js +80 -0
  37. package/dist/installer/uninstall.js.map +1 -0
  38. package/dist/src/batch-compact.d.ts +16 -0
  39. package/dist/src/batch-compact.js +121 -0
  40. package/dist/src/batch-compact.js.map +1 -0
  41. package/dist/src/compaction.d.ts +198 -0
  42. package/dist/src/compaction.js +964 -0
  43. package/dist/src/compaction.js.map +1 -0
  44. package/dist/src/connectors/constants.d.ts +5 -0
  45. package/dist/src/connectors/constants.js +6 -0
  46. package/dist/src/connectors/constants.js.map +1 -0
  47. package/dist/src/connectors/installer.d.ts +16 -0
  48. package/dist/src/connectors/installer.js +200 -0
  49. package/dist/src/connectors/installer.js.map +1 -0
  50. package/dist/src/connectors/registry.d.ts +4 -0
  51. package/dist/src/connectors/registry.js +264 -0
  52. package/dist/src/connectors/registry.js.map +1 -0
  53. package/dist/src/connectors/template-service.d.ts +5 -0
  54. package/dist/src/connectors/template-service.js +54 -0
  55. package/dist/src/connectors/template-service.js.map +1 -0
  56. package/dist/src/connectors/templates/base.md +1 -0
  57. package/dist/src/connectors/templates/mcp-base.md +1 -0
  58. package/dist/src/connectors/templates/sections/command-reference.md +15 -0
  59. package/dist/src/connectors/templates/sections/mcp-workflow.md +18 -0
  60. package/dist/src/connectors/templates/sections/workflow.md +29 -0
  61. package/dist/src/connectors/templates/skill/SKILL.md +74 -0
  62. package/dist/src/connectors/types.d.ts +19 -0
  63. package/dist/src/connectors/types.js +10 -0
  64. package/dist/src/connectors/types.js.map +1 -0
  65. package/dist/src/daemon/client.d.ts +9 -0
  66. package/dist/src/daemon/client.js +28 -0
  67. package/dist/src/daemon/client.js.map +1 -0
  68. package/dist/src/daemon/config.d.ts +48 -0
  69. package/dist/src/daemon/config.js +67 -0
  70. package/dist/src/daemon/config.js.map +1 -0
  71. package/dist/src/daemon/lifecycle.d.ts +19 -0
  72. package/dist/src/daemon/lifecycle.js +102 -0
  73. package/dist/src/daemon/lifecycle.js.map +1 -0
  74. package/dist/src/daemon/orientation.d.ts +1 -0
  75. package/dist/src/daemon/orientation.js +9 -0
  76. package/dist/src/daemon/orientation.js.map +1 -0
  77. package/dist/src/daemon/project-queue.d.ts +1 -0
  78. package/dist/src/daemon/project-queue.js +17 -0
  79. package/dist/src/daemon/project-queue.js.map +1 -0
  80. package/dist/src/daemon/project.d.ts +7 -0
  81. package/dist/src/daemon/project.js +25 -0
  82. package/dist/src/daemon/project.js.map +1 -0
  83. package/dist/src/daemon/proxy-manager.d.ts +21 -0
  84. package/dist/src/daemon/proxy-manager.js +205 -0
  85. package/dist/src/daemon/proxy-manager.js.map +1 -0
  86. package/dist/src/daemon/routes/compact.d.ts +13 -0
  87. package/dist/src/daemon/routes/compact.js +195 -0
  88. package/dist/src/daemon/routes/compact.js.map +1 -0
  89. package/dist/src/daemon/routes/describe.d.ts +3 -0
  90. package/dist/src/daemon/routes/describe.js +39 -0
  91. package/dist/src/daemon/routes/describe.js.map +1 -0
  92. package/dist/src/daemon/routes/expand.d.ts +3 -0
  93. package/dist/src/daemon/routes/expand.js +41 -0
  94. package/dist/src/daemon/routes/expand.js.map +1 -0
  95. package/dist/src/daemon/routes/grep.d.ts +3 -0
  96. package/dist/src/daemon/routes/grep.js +43 -0
  97. package/dist/src/daemon/routes/grep.js.map +1 -0
  98. package/dist/src/daemon/routes/ingest.d.ts +3 -0
  99. package/dist/src/daemon/routes/ingest.js +101 -0
  100. package/dist/src/daemon/routes/ingest.js.map +1 -0
  101. package/dist/src/daemon/routes/promote.d.ts +4 -0
  102. package/dist/src/daemon/routes/promote.js +104 -0
  103. package/dist/src/daemon/routes/promote.js.map +1 -0
  104. package/dist/src/daemon/routes/prompt-search.d.ts +3 -0
  105. package/dist/src/daemon/routes/prompt-search.js +65 -0
  106. package/dist/src/daemon/routes/prompt-search.js.map +1 -0
  107. package/dist/src/daemon/routes/recent.d.ts +3 -0
  108. package/dist/src/daemon/routes/recent.js +37 -0
  109. package/dist/src/daemon/routes/recent.js.map +1 -0
  110. package/dist/src/daemon/routes/restore.d.ts +3 -0
  111. package/dist/src/daemon/routes/restore.js +120 -0
  112. package/dist/src/daemon/routes/restore.js.map +1 -0
  113. package/dist/src/daemon/routes/search.d.ts +2 -0
  114. package/dist/src/daemon/routes/search.js +66 -0
  115. package/dist/src/daemon/routes/search.js.map +1 -0
  116. package/dist/src/daemon/routes/status.d.ts +3 -0
  117. package/dist/src/daemon/routes/status.js +80 -0
  118. package/dist/src/daemon/routes/status.js.map +1 -0
  119. package/dist/src/daemon/routes/store.d.ts +2 -0
  120. package/dist/src/daemon/routes/store.js +46 -0
  121. package/dist/src/daemon/routes/store.js.map +1 -0
  122. package/dist/src/daemon/server.d.ts +19 -0
  123. package/dist/src/daemon/server.js +183 -0
  124. package/dist/src/daemon/server.js.map +1 -0
  125. package/dist/src/daemon/summarizer.d.ts +11 -0
  126. package/dist/src/daemon/summarizer.js +51 -0
  127. package/dist/src/daemon/summarizer.js.map +1 -0
  128. package/dist/src/db/config.d.ts +31 -0
  129. package/dist/src/db/config.js +83 -0
  130. package/dist/src/db/config.js.map +1 -0
  131. package/dist/src/db/connection.d.ts +3 -0
  132. package/dist/src/db/connection.js +62 -0
  133. package/dist/src/db/connection.js.map +1 -0
  134. package/dist/src/db/features.d.ts +11 -0
  135. package/dist/src/db/features.js +36 -0
  136. package/dist/src/db/features.js.map +1 -0
  137. package/dist/src/db/migration.d.ts +4 -0
  138. package/dist/src/db/migration.js +499 -0
  139. package/dist/src/db/migration.js.map +1 -0
  140. package/dist/src/db/promoted.d.ts +47 -0
  141. package/dist/src/db/promoted.js +96 -0
  142. package/dist/src/db/promoted.js.map +1 -0
  143. package/dist/src/db/redaction-stats.d.ts +6 -0
  144. package/dist/src/db/redaction-stats.js +16 -0
  145. package/dist/src/db/redaction-stats.js.map +1 -0
  146. package/dist/src/diagnose.d.ts +39 -0
  147. package/dist/src/diagnose.js +432 -0
  148. package/dist/src/diagnose.js.map +1 -0
  149. package/dist/src/doctor/doctor.d.ts +4 -0
  150. package/dist/src/doctor/doctor.js +378 -0
  151. package/dist/src/doctor/doctor.js.map +1 -0
  152. package/dist/src/doctor/types.d.ts +24 -0
  153. package/dist/src/doctor/types.js +2 -0
  154. package/dist/src/doctor/types.js.map +1 -0
  155. package/dist/src/expansion.d.ts +100 -0
  156. package/dist/src/expansion.js +268 -0
  157. package/dist/src/expansion.js.map +1 -0
  158. package/dist/src/hooks/auto-heal.d.ts +12 -0
  159. package/dist/src/hooks/auto-heal.js +49 -0
  160. package/dist/src/hooks/auto-heal.js.map +1 -0
  161. package/dist/src/hooks/compact.d.ts +5 -0
  162. package/dist/src/hooks/compact.js +22 -0
  163. package/dist/src/hooks/compact.js.map +1 -0
  164. package/dist/src/hooks/dispatch.d.ts +7 -0
  165. package/dist/src/hooks/dispatch.js +36 -0
  166. package/dist/src/hooks/dispatch.js.map +1 -0
  167. package/dist/src/hooks/probe-precompact.d.ts +2 -0
  168. package/dist/src/hooks/probe-precompact.js +17 -0
  169. package/dist/src/hooks/probe-precompact.js.map +1 -0
  170. package/dist/src/hooks/probe-sessionstart.d.ts +2 -0
  171. package/dist/src/hooks/probe-sessionstart.js +18 -0
  172. package/dist/src/hooks/probe-sessionstart.js.map +1 -0
  173. package/dist/src/hooks/restore.d.ts +5 -0
  174. package/dist/src/hooks/restore.js +19 -0
  175. package/dist/src/hooks/restore.js.map +1 -0
  176. package/dist/src/hooks/session-end.d.ts +16 -0
  177. package/dist/src/hooks/session-end.js +73 -0
  178. package/dist/src/hooks/session-end.js.map +1 -0
  179. package/dist/src/hooks/user-prompt.d.ts +5 -0
  180. package/dist/src/hooks/user-prompt.js +31 -0
  181. package/dist/src/hooks/user-prompt.js.map +1 -0
  182. package/dist/src/import.d.ts +24 -0
  183. package/dist/src/import.js +119 -0
  184. package/dist/src/import.js.map +1 -0
  185. package/dist/src/large-files.d.ts +28 -0
  186. package/dist/src/large-files.js +413 -0
  187. package/dist/src/large-files.js.map +1 -0
  188. package/dist/src/llm/anthropic.d.ts +9 -0
  189. package/dist/src/llm/anthropic.js +54 -0
  190. package/dist/src/llm/anthropic.js.map +1 -0
  191. package/dist/src/llm/claude-process.d.ts +2 -0
  192. package/dist/src/llm/claude-process.js +55 -0
  193. package/dist/src/llm/claude-process.js.map +1 -0
  194. package/dist/src/llm/codex-process.d.ts +15 -0
  195. package/dist/src/llm/codex-process.js +142 -0
  196. package/dist/src/llm/codex-process.js.map +1 -0
  197. package/dist/src/llm/mock-summarizer.d.ts +9 -0
  198. package/dist/src/llm/mock-summarizer.js +17 -0
  199. package/dist/src/llm/mock-summarizer.js.map +1 -0
  200. package/dist/src/llm/openai.d.ts +10 -0
  201. package/dist/src/llm/openai.js +52 -0
  202. package/dist/src/llm/openai.js.map +1 -0
  203. package/dist/src/llm/types.d.ts +6 -0
  204. package/dist/src/llm/types.js +2 -0
  205. package/dist/src/llm/types.js.map +1 -0
  206. package/dist/src/mcp/server.d.ts +9 -0
  207. package/dist/src/mcp/server.js +128 -0
  208. package/dist/src/mcp/server.js.map +1 -0
  209. package/dist/src/mcp/tools/lcm-describe.d.ts +14 -0
  210. package/dist/src/mcp/tools/lcm-describe.js +12 -0
  211. package/dist/src/mcp/tools/lcm-describe.js.map +1 -0
  212. package/dist/src/mcp/tools/lcm-doctor.d.ts +8 -0
  213. package/dist/src/mcp/tools/lcm-doctor.js +9 -0
  214. package/dist/src/mcp/tools/lcm-doctor.js.map +1 -0
  215. package/dist/src/mcp/tools/lcm-expand.d.ts +18 -0
  216. package/dist/src/mcp/tools/lcm-expand.js +13 -0
  217. package/dist/src/mcp/tools/lcm-expand.js.map +1 -0
  218. package/dist/src/mcp/tools/lcm-grep.d.ts +27 -0
  219. package/dist/src/mcp/tools/lcm-grep.js +15 -0
  220. package/dist/src/mcp/tools/lcm-grep.js.map +1 -0
  221. package/dist/src/mcp/tools/lcm-search.d.ts +33 -0
  222. package/dist/src/mcp/tools/lcm-search.js +15 -0
  223. package/dist/src/mcp/tools/lcm-search.js.map +1 -0
  224. package/dist/src/mcp/tools/lcm-stats.d.ts +14 -0
  225. package/dist/src/mcp/tools/lcm-stats.js +11 -0
  226. package/dist/src/mcp/tools/lcm-stats.js.map +1 -0
  227. package/dist/src/mcp/tools/lcm-store.d.ts +26 -0
  228. package/dist/src/mcp/tools/lcm-store.js +22 -0
  229. package/dist/src/mcp/tools/lcm-store.js.map +1 -0
  230. package/dist/src/memory/index.d.ts +22 -0
  231. package/dist/src/memory/index.js +21 -0
  232. package/dist/src/memory/index.js.map +1 -0
  233. package/dist/src/promotion/dedup.d.ts +19 -0
  234. package/dist/src/promotion/dedup.js +42 -0
  235. package/dist/src/promotion/dedup.js.map +1 -0
  236. package/dist/src/promotion/detector.d.ts +15 -0
  237. package/dist/src/promotion/detector.js +31 -0
  238. package/dist/src/promotion/detector.js.map +1 -0
  239. package/dist/src/prompts/condensed-d1.yaml +38 -0
  240. package/dist/src/prompts/condensed-d2.yaml +32 -0
  241. package/dist/src/prompts/condensed-d3plus.yaml +32 -0
  242. package/dist/src/prompts/leaf-aggressive.yaml +34 -0
  243. package/dist/src/prompts/leaf-normal.yaml +34 -0
  244. package/dist/src/prompts/loader.d.ts +9 -0
  245. package/dist/src/prompts/loader.js +37 -0
  246. package/dist/src/prompts/loader.js.map +1 -0
  247. package/dist/src/prompts/promoted-merge.yaml +18 -0
  248. package/dist/src/prompts/system.yaml +5 -0
  249. package/dist/src/retrieval.d.ts +122 -0
  250. package/dist/src/retrieval.js +214 -0
  251. package/dist/src/retrieval.js.map +1 -0
  252. package/dist/src/scrub.d.ts +46 -0
  253. package/dist/src/scrub.js +184 -0
  254. package/dist/src/scrub.js.map +1 -0
  255. package/dist/src/sensitive.d.ts +4 -0
  256. package/dist/src/sensitive.js +258 -0
  257. package/dist/src/sensitive.js.map +1 -0
  258. package/dist/src/stats.d.ts +34 -0
  259. package/dist/src/stats.js +260 -0
  260. package/dist/src/stats.js.map +1 -0
  261. package/dist/src/store/conversation-store.d.ts +115 -0
  262. package/dist/src/store/conversation-store.js +447 -0
  263. package/dist/src/store/conversation-store.js.map +1 -0
  264. package/dist/src/store/fts5-sanitize.d.ts +23 -0
  265. package/dist/src/store/fts5-sanitize.js +30 -0
  266. package/dist/src/store/fts5-sanitize.js.map +1 -0
  267. package/dist/src/store/full-text-fallback.d.ts +16 -0
  268. package/dist/src/store/full-text-fallback.js +60 -0
  269. package/dist/src/store/full-text-fallback.js.map +1 -0
  270. package/dist/src/store/index.d.ts +4 -0
  271. package/dist/src/store/index.js +3 -0
  272. package/dist/src/store/index.js.map +1 -0
  273. package/dist/src/store/summary-store.d.ts +118 -0
  274. package/dist/src/store/summary-store.js +570 -0
  275. package/dist/src/store/summary-store.js.map +1 -0
  276. package/dist/src/summarize.d.ts +59 -0
  277. package/dist/src/summarize.js +619 -0
  278. package/dist/src/summarize.js.map +1 -0
  279. package/dist/src/transcript.d.ts +7 -0
  280. package/dist/src/transcript.js +51 -0
  281. package/dist/src/transcript.js.map +1 -0
  282. package/dist/src/types.d.ts +116 -0
  283. package/dist/src/types.js +8 -0
  284. package/dist/src/types.js.map +1 -0
  285. package/docs/agent-tools.md +187 -0
  286. package/docs/architecture.md +224 -0
  287. package/docs/configuration.md +168 -0
  288. package/docs/fts5.md +161 -0
  289. package/docs/hook-protocol.md +41 -0
  290. package/docs/privacy.md +101 -0
  291. package/mcp.mjs +17 -0
  292. package/package.json +79 -0
@@ -0,0 +1,5 @@
1
+ name: system
2
+ description: System prompt for the LCM summarization engine
3
+ variables: []
4
+ template: |
5
+ You are a context-compaction summarization engine. Follow user instructions exactly and return plain text summary content only.
@@ -0,0 +1,122 @@
1
+ import type { ConversationStore, MessageSearchResult } from "./store/conversation-store.js";
2
+ import type { SummaryStore, SummarySearchResult } from "./store/summary-store.js";
3
+ export interface DescribeResult {
4
+ id: string;
5
+ type: "summary" | "file";
6
+ /** Summary-specific fields */
7
+ summary?: {
8
+ conversationId: number;
9
+ kind: "leaf" | "condensed";
10
+ content: string;
11
+ depth: number;
12
+ tokenCount: number;
13
+ descendantCount: number;
14
+ descendantTokenCount: number;
15
+ sourceMessageTokenCount: number;
16
+ fileIds: string[];
17
+ parentIds: string[];
18
+ childIds: string[];
19
+ messageIds: number[];
20
+ earliestAt: Date | null;
21
+ latestAt: Date | null;
22
+ subtree: Array<{
23
+ summaryId: string;
24
+ parentSummaryId: string | null;
25
+ depthFromRoot: number;
26
+ kind: "leaf" | "condensed";
27
+ depth: number;
28
+ tokenCount: number;
29
+ descendantCount: number;
30
+ descendantTokenCount: number;
31
+ sourceMessageTokenCount: number;
32
+ earliestAt: Date | null;
33
+ latestAt: Date | null;
34
+ childCount: number;
35
+ path: string;
36
+ }>;
37
+ createdAt: Date;
38
+ };
39
+ /** File-specific fields */
40
+ file?: {
41
+ conversationId: number;
42
+ fileName: string | null;
43
+ mimeType: string | null;
44
+ byteSize: number | null;
45
+ storageUri: string;
46
+ explorationSummary: string | null;
47
+ createdAt: Date;
48
+ };
49
+ }
50
+ export interface GrepInput {
51
+ query: string;
52
+ mode: "regex" | "full_text";
53
+ scope: "messages" | "summaries" | "both";
54
+ conversationId?: number;
55
+ since?: Date;
56
+ before?: Date;
57
+ limit?: number;
58
+ }
59
+ export interface GrepResult {
60
+ messages: MessageSearchResult[];
61
+ summaries: SummarySearchResult[];
62
+ totalMatches: number;
63
+ }
64
+ export interface ExpandInput {
65
+ summaryId: string;
66
+ /** Max traversal depth (default 1) */
67
+ depth?: number;
68
+ /** Include raw source messages at leaf level */
69
+ includeMessages?: boolean;
70
+ /** Max tokens to return before truncating */
71
+ tokenCap?: number;
72
+ }
73
+ export interface ExpandResult {
74
+ /** Child summaries found */
75
+ children: Array<{
76
+ summaryId: string;
77
+ kind: "leaf" | "condensed";
78
+ content: string;
79
+ tokenCount: number;
80
+ }>;
81
+ /** Source messages (only if includeMessages=true and hitting leaf summaries) */
82
+ messages: Array<{
83
+ messageId: number;
84
+ role: string;
85
+ content: string;
86
+ tokenCount: number;
87
+ }>;
88
+ /** Total estimated tokens in result */
89
+ estimatedTokens: number;
90
+ /** Whether result was truncated due to tokenCap */
91
+ truncated: boolean;
92
+ }
93
+ export declare class RetrievalEngine {
94
+ private conversationStore;
95
+ private summaryStore;
96
+ constructor(conversationStore: ConversationStore, summaryStore: SummaryStore);
97
+ /**
98
+ * Describe an LCM item by ID.
99
+ *
100
+ * - IDs starting with "sum_" are looked up as summaries (with lineage).
101
+ * - IDs starting with "file_" are looked up as large files.
102
+ * - Returns null if the item is not found.
103
+ */
104
+ describe(id: string): Promise<DescribeResult | null>;
105
+ private describeSummary;
106
+ private describeFile;
107
+ /**
108
+ * Search compacted history using regex or full-text search.
109
+ *
110
+ * Depending on `scope`, searches messages, summaries, or both (in parallel).
111
+ */
112
+ grep(input: GrepInput): Promise<GrepResult>;
113
+ /**
114
+ * Expand a summary to its children and/or source messages.
115
+ *
116
+ * - Condensed summaries: returns child summaries, recursing up to `depth`.
117
+ * - Leaf summaries with `includeMessages`: fetches the source messages.
118
+ * - Respects `tokenCap` and sets `truncated` when the cap is exceeded.
119
+ */
120
+ expand(input: ExpandInput): Promise<ExpandResult>;
121
+ private expandRecursive;
122
+ }
@@ -0,0 +1,214 @@
1
+ // ── Helpers ──────────────────────────────────────────────────────────────────
2
+ /** Rough token estimate: ~4 chars per token. */
3
+ function estimateTokens(content) {
4
+ return Math.ceil(content.length / 4);
5
+ }
6
+ // ── RetrievalEngine ──────────────────────────────────────────────────────────
7
+ export class RetrievalEngine {
8
+ conversationStore;
9
+ summaryStore;
10
+ constructor(conversationStore, summaryStore) {
11
+ this.conversationStore = conversationStore;
12
+ this.summaryStore = summaryStore;
13
+ }
14
+ // ── describe ─────────────────────────────────────────────────────────────
15
+ /**
16
+ * Describe an LCM item by ID.
17
+ *
18
+ * - IDs starting with "sum_" are looked up as summaries (with lineage).
19
+ * - IDs starting with "file_" are looked up as large files.
20
+ * - Returns null if the item is not found.
21
+ */
22
+ async describe(id) {
23
+ if (id.startsWith("sum_")) {
24
+ return this.describeSummary(id);
25
+ }
26
+ if (id.startsWith("file_")) {
27
+ return this.describeFile(id);
28
+ }
29
+ return null;
30
+ }
31
+ async describeSummary(id) {
32
+ const summary = await this.summaryStore.getSummary(id);
33
+ if (!summary) {
34
+ return null;
35
+ }
36
+ // Fetch lineage in parallel
37
+ const [parents, children, messageIds, subtree] = await Promise.all([
38
+ this.summaryStore.getSummaryParents(id),
39
+ this.summaryStore.getSummaryChildren(id),
40
+ this.summaryStore.getSummaryMessages(id),
41
+ this.summaryStore.getSummarySubtree(id),
42
+ ]);
43
+ return {
44
+ id,
45
+ type: "summary",
46
+ summary: {
47
+ conversationId: summary.conversationId,
48
+ kind: summary.kind,
49
+ content: summary.content,
50
+ depth: summary.depth,
51
+ tokenCount: summary.tokenCount,
52
+ descendantCount: summary.descendantCount,
53
+ descendantTokenCount: summary.descendantTokenCount,
54
+ sourceMessageTokenCount: summary.sourceMessageTokenCount,
55
+ fileIds: summary.fileIds,
56
+ parentIds: parents.map((p) => p.summaryId),
57
+ childIds: children.map((c) => c.summaryId),
58
+ messageIds,
59
+ earliestAt: summary.earliestAt,
60
+ latestAt: summary.latestAt,
61
+ subtree: subtree.map((node) => ({
62
+ summaryId: node.summaryId,
63
+ parentSummaryId: node.parentSummaryId,
64
+ depthFromRoot: node.depthFromRoot,
65
+ kind: node.kind,
66
+ depth: node.depth,
67
+ tokenCount: node.tokenCount,
68
+ descendantCount: node.descendantCount,
69
+ descendantTokenCount: node.descendantTokenCount,
70
+ sourceMessageTokenCount: node.sourceMessageTokenCount,
71
+ earliestAt: node.earliestAt,
72
+ latestAt: node.latestAt,
73
+ childCount: node.childCount,
74
+ path: node.path,
75
+ })),
76
+ createdAt: summary.createdAt,
77
+ },
78
+ };
79
+ }
80
+ async describeFile(id) {
81
+ const file = await this.summaryStore.getLargeFile(id);
82
+ if (!file) {
83
+ return null;
84
+ }
85
+ return {
86
+ id,
87
+ type: "file",
88
+ file: {
89
+ conversationId: file.conversationId,
90
+ fileName: file.fileName,
91
+ mimeType: file.mimeType,
92
+ byteSize: file.byteSize,
93
+ storageUri: file.storageUri,
94
+ explorationSummary: file.explorationSummary,
95
+ createdAt: file.createdAt,
96
+ },
97
+ };
98
+ }
99
+ // ── grep ─────────────────────────────────────────────────────────────────
100
+ /**
101
+ * Search compacted history using regex or full-text search.
102
+ *
103
+ * Depending on `scope`, searches messages, summaries, or both (in parallel).
104
+ */
105
+ async grep(input) {
106
+ const { query, mode, scope, conversationId, since, before, limit } = input;
107
+ const searchInput = { query, mode, conversationId, since, before, limit };
108
+ let messages = [];
109
+ let summaries = [];
110
+ if (scope === "messages") {
111
+ messages = await this.conversationStore.searchMessages(searchInput);
112
+ }
113
+ else if (scope === "summaries") {
114
+ summaries = await this.summaryStore.searchSummaries(searchInput);
115
+ }
116
+ else {
117
+ // scope === "both" — run in parallel
118
+ [messages, summaries] = await Promise.all([
119
+ this.conversationStore.searchMessages(searchInput),
120
+ this.summaryStore.searchSummaries(searchInput),
121
+ ]);
122
+ }
123
+ messages.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
124
+ summaries.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
125
+ return {
126
+ messages,
127
+ summaries,
128
+ totalMatches: messages.length + summaries.length,
129
+ };
130
+ }
131
+ // ── expand ───────────────────────────────────────────────────────────────
132
+ /**
133
+ * Expand a summary to its children and/or source messages.
134
+ *
135
+ * - Condensed summaries: returns child summaries, recursing up to `depth`.
136
+ * - Leaf summaries with `includeMessages`: fetches the source messages.
137
+ * - Respects `tokenCap` and sets `truncated` when the cap is exceeded.
138
+ */
139
+ async expand(input) {
140
+ const depth = input.depth ?? 1;
141
+ const includeMessages = input.includeMessages ?? false;
142
+ const tokenCap = input.tokenCap ?? Infinity;
143
+ const result = {
144
+ children: [],
145
+ messages: [],
146
+ estimatedTokens: 0,
147
+ truncated: false,
148
+ };
149
+ await this.expandRecursive(input.summaryId, depth, includeMessages, tokenCap, result);
150
+ return result;
151
+ }
152
+ async expandRecursive(summaryId, depth, includeMessages, tokenCap, result) {
153
+ if (depth <= 0) {
154
+ return;
155
+ }
156
+ if (result.truncated) {
157
+ return;
158
+ }
159
+ const summary = await this.summaryStore.getSummary(summaryId);
160
+ if (!summary) {
161
+ return;
162
+ }
163
+ if (summary.kind === "condensed") {
164
+ const children = await this.summaryStore.getSummaryChildren(summaryId);
165
+ for (const child of children) {
166
+ if (result.truncated) {
167
+ break;
168
+ }
169
+ // Check if adding this child would exceed the token cap
170
+ if (result.estimatedTokens + child.tokenCount > tokenCap) {
171
+ result.truncated = true;
172
+ break;
173
+ }
174
+ result.children.push({
175
+ summaryId: child.summaryId,
176
+ kind: child.kind,
177
+ content: child.content,
178
+ tokenCount: child.tokenCount,
179
+ });
180
+ result.estimatedTokens += child.tokenCount;
181
+ // Recurse into children if depth allows
182
+ if (depth > 1) {
183
+ await this.expandRecursive(child.summaryId, depth - 1, includeMessages, tokenCap, result);
184
+ }
185
+ }
186
+ }
187
+ else if (summary.kind === "leaf" && includeMessages) {
188
+ // Leaf summary — fetch source messages
189
+ const messageIds = await this.summaryStore.getSummaryMessages(summaryId);
190
+ for (const msgId of messageIds) {
191
+ if (result.truncated) {
192
+ break;
193
+ }
194
+ const msg = await this.conversationStore.getMessageById(msgId);
195
+ if (!msg) {
196
+ continue;
197
+ }
198
+ const tokenCount = msg.tokenCount || estimateTokens(msg.content);
199
+ if (result.estimatedTokens + tokenCount > tokenCap) {
200
+ result.truncated = true;
201
+ break;
202
+ }
203
+ result.messages.push({
204
+ messageId: msg.messageId,
205
+ role: msg.role,
206
+ content: msg.content,
207
+ tokenCount,
208
+ });
209
+ result.estimatedTokens += tokenCount;
210
+ }
211
+ }
212
+ }
213
+ }
214
+ //# sourceMappingURL=retrieval.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retrieval.js","sourceRoot":"","sources":["../../src/retrieval.ts"],"names":[],"mappings":"AA6GA,gFAAgF;AAEhF,gDAAgD;AAChD,SAAS,cAAc,CAAC,OAAe;IACrC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,gFAAgF;AAEhF,MAAM,OAAO,eAAe;IAEhB;IACA;IAFV,YACU,iBAAoC,EACpC,YAA0B;QAD1B,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,iBAAY,GAAZ,YAAY,CAAc;IACjC,CAAC;IAEJ,4EAA4E;IAE5E;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,EAAU;QACvB,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,EAAU;QACtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QACd,CAAC;QAED,4BAA4B;QAC5B,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACjE,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS;YACf,OAAO,EAAE;gBACP,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;gBAClD,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1C,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1C,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;iBAChB,CAAC,CAAC;gBACH,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,EAAU;QACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE;gBACJ,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;SACF,CAAC;IACJ,CAAC;IAED,4EAA4E;IAE5E;;;;OAIG;IACH,KAAK,CAAC,IAAI,CAAC,KAAgB;QACzB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;QAE3E,MAAM,WAAW,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAE1E,IAAI,QAAQ,GAA0B,EAAE,CAAC;QACzC,IAAI,SAAS,GAA0B,EAAE,CAAC;QAE1C,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QACtE,CAAC;aAAM,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YACjC,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,qCAAqC;YACrC,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACxC,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,WAAW,CAAC;gBAClD,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,WAAW,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAExE,OAAO;YACL,QAAQ;YACR,SAAS;YACT,YAAY,EAAE,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;SACjD,CAAC;IACJ,CAAC;IAED,4EAA4E;IAE5E;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,CAAC,KAAkB;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;QAC/B,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAE5C,MAAM,MAAM,GAAiB;YAC3B,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,EAAE;YACZ,eAAe,EAAE,CAAC;YAClB,SAAS,EAAE,KAAK;SACjB,CAAC;QAEF,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEtF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAAiB,EACjB,KAAa,EACb,eAAwB,EACxB,QAAgB,EAChB,MAAoB;QAEpB,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAEvE,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;oBACrB,MAAM;gBACR,CAAC;gBAED,wDAAwD;gBACxD,IAAI,MAAM,CAAC,eAAe,GAAG,KAAK,CAAC,UAAU,GAAG,QAAQ,EAAE,CAAC;oBACzD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM;gBACR,CAAC;gBAED,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACnB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;iBAC7B,CAAC,CAAC;gBACH,MAAM,CAAC,eAAe,IAAI,KAAK,CAAC,UAAU,CAAC;gBAE3C,wCAAwC;gBACxC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACd,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC5F,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,eAAe,EAAE,CAAC;YACtD,uCAAuC;YACvC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAEzE,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;oBACrB,MAAM;gBACR,CAAC;gBAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC/D,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,SAAS;gBACX,CAAC;gBAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAEjE,IAAI,MAAM,CAAC,eAAe,GAAG,UAAU,GAAG,QAAQ,EAAE,CAAC;oBACnD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM;gBACR,CAAC;gBAED,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACnB,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,UAAU;iBACX,CAAC,CAAC;gBACH,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,46 @@
1
+ export declare const BUILT_IN_PATTERNS: string[];
2
+ export interface ScrubCounts {
3
+ text: string;
4
+ builtIn: number;
5
+ global: number;
6
+ project: number;
7
+ }
8
+ export declare class ScrubEngine {
9
+ private readonly spanningPatterns;
10
+ private readonly tokenPatterns;
11
+ /** Original index (into the combined [builtIn, global, project] array) for each spanning pattern. */
12
+ private readonly _spanningOrigIdx;
13
+ /** Original index for each token pattern. */
14
+ private readonly _tokenOrigIdx;
15
+ /** Number of global patterns (for category accounting). */
16
+ private readonly _globalPatternCount;
17
+ readonly invalidPatterns: string[];
18
+ constructor(globalPatterns: string[], projectPatterns: string[]);
19
+ /**
20
+ * Redact all matching patterns in text, returning the scrubbed text along
21
+ * with per-category counts of how many redactions were made.
22
+ *
23
+ * Strategy:
24
+ * - "Spanning" patterns (those that can match across whitespace) are applied
25
+ * to the full text via a multi-range merge to avoid one pattern consuming
26
+ * another's matches.
27
+ * - "Token" patterns (no whitespace/dot in source) are applied token-by-token
28
+ * so that greedy `.*`-style patterns in one token don't eat adjacent tokens.
29
+ */
30
+ scrubWithCounts(text: string): ScrubCounts;
31
+ /**
32
+ * Redact all matching patterns in text, replacing matches with [REDACTED].
33
+ *
34
+ * Strategy:
35
+ * - "Spanning" patterns (those that can match across whitespace) are applied
36
+ * to the full text via a multi-range merge to avoid one pattern consuming
37
+ * another's matches.
38
+ * - "Token" patterns (no whitespace/dot in source) are applied token-by-token
39
+ * so that greedy `.*`-style patterns in one token don't eat adjacent tokens.
40
+ */
41
+ scrub(text: string): string;
42
+ /** Parse a sensitive-patterns.txt file. Returns empty array if file is absent. */
43
+ static loadProjectPatterns(filePath: string): Promise<string[]>;
44
+ /** Build a ScrubEngine for a given project directory. */
45
+ static forProject(globalPatterns: string[], projectDir: string): Promise<ScrubEngine>;
46
+ }
@@ -0,0 +1,184 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ export const BUILT_IN_PATTERNS = [
4
+ "sk-[A-Za-z0-9]{20,}",
5
+ "sk-ant-[A-Za-z0-9\\-]{40,}",
6
+ "ghp_[A-Za-z0-9]{36}",
7
+ "AKIA[0-9A-Z]{16}",
8
+ "-----BEGIN .* KEY-----",
9
+ "Bearer [A-Za-z0-9\\-._~+/]+=*",
10
+ "[Pp]assword\\s*[:=]\\s*\\S+",
11
+ ];
12
+ /**
13
+ * Returns true if a regex pattern source can match across whitespace boundaries.
14
+ * Patterns containing a literal space, \s, or a dot (which matches space) are
15
+ * considered "spanning" and will be applied to the full text rather than
16
+ * token-by-token.
17
+ */
18
+ function isSpanningPattern(source) {
19
+ // Check for literal space or the escape sequence \s — unambiguous spanning intent.
20
+ // Use string includes (not regex) so we detect the two-char sequence \s, not whitespace chars.
21
+ if (source.includes(" ") || source.includes("\\s"))
22
+ return true;
23
+ // Check for unescaped `.` which can match spaces
24
+ // Walk the source and look for `.` not preceded by `\`
25
+ for (let i = 0; i < source.length; i++) {
26
+ if (source[i] === "\\") {
27
+ i++; // skip escaped char
28
+ continue;
29
+ }
30
+ if (source[i] === ".")
31
+ return true;
32
+ }
33
+ return false;
34
+ }
35
+ export class ScrubEngine {
36
+ spanningPatterns = [];
37
+ tokenPatterns = [];
38
+ /** Original index (into the combined [builtIn, global, project] array) for each spanning pattern. */
39
+ _spanningOrigIdx = [];
40
+ /** Original index for each token pattern. */
41
+ _tokenOrigIdx = [];
42
+ /** Number of global patterns (for category accounting). */
43
+ _globalPatternCount;
44
+ invalidPatterns = [];
45
+ constructor(globalPatterns, projectPatterns) {
46
+ this._globalPatternCount = globalPatterns.length;
47
+ const all = [...BUILT_IN_PATTERNS, ...globalPatterns, ...projectPatterns];
48
+ for (let i = 0; i < all.length; i++) {
49
+ const source = all[i];
50
+ try {
51
+ const regex = new RegExp(source, "g");
52
+ if (isSpanningPattern(source)) {
53
+ this.spanningPatterns.push({ source, regex });
54
+ this._spanningOrigIdx.push(i);
55
+ }
56
+ else {
57
+ this.tokenPatterns.push({ source, regex });
58
+ this._tokenOrigIdx.push(i);
59
+ }
60
+ }
61
+ catch {
62
+ this.invalidPatterns.push(source);
63
+ }
64
+ }
65
+ }
66
+ /**
67
+ * Redact all matching patterns in text, returning the scrubbed text along
68
+ * with per-category counts of how many redactions were made.
69
+ *
70
+ * Strategy:
71
+ * - "Spanning" patterns (those that can match across whitespace) are applied
72
+ * to the full text via a multi-range merge to avoid one pattern consuming
73
+ * another's matches.
74
+ * - "Token" patterns (no whitespace/dot in source) are applied token-by-token
75
+ * so that greedy `.*`-style patterns in one token don't eat adjacent tokens.
76
+ */
77
+ scrubWithCounts(text) {
78
+ const builtInCount = BUILT_IN_PATTERNS.length;
79
+ const globalCount = this._globalPatternCount;
80
+ const taggedRanges = [];
81
+ for (let pi = 0; pi < this.spanningPatterns.length; pi++) {
82
+ const { regex } = this.spanningPatterns[pi];
83
+ regex.lastIndex = 0;
84
+ let m;
85
+ while ((m = regex.exec(text)) !== null) {
86
+ taggedRanges.push({ range: [m.index, m.index + m[0].length], idx: this._spanningOrigIdx[pi] });
87
+ if (m[0].length === 0)
88
+ regex.lastIndex++;
89
+ }
90
+ }
91
+ // Step 2: apply token patterns per whitespace-separated segment
92
+ const segments = text.split(/(\s+)/);
93
+ let offset = 0;
94
+ for (const seg of segments) {
95
+ if (!/^\s+$/.test(seg) && this.tokenPatterns.length > 0) {
96
+ for (let pi = 0; pi < this.tokenPatterns.length; pi++) {
97
+ const { regex } = this.tokenPatterns[pi];
98
+ regex.lastIndex = 0;
99
+ let m;
100
+ while ((m = regex.exec(seg)) !== null) {
101
+ taggedRanges.push({ range: [offset + m.index, offset + m.index + m[0].length], idx: this._tokenOrigIdx[pi] });
102
+ if (m[0].length === 0)
103
+ regex.lastIndex++;
104
+ }
105
+ }
106
+ }
107
+ offset += seg.length;
108
+ }
109
+ if (taggedRanges.length === 0)
110
+ return { text, builtIn: 0, global: 0, project: 0 };
111
+ // Sort by start position
112
+ taggedRanges.sort((a, b) => a.range[0] - b.range[0]);
113
+ // Merge overlapping ranges; when overlaps occur, the lowest original pattern
114
+ // index wins so that built-in > global > project and earlier patterns win.
115
+ const merged = [];
116
+ let cur = taggedRanges[0];
117
+ for (let i = 1; i < taggedRanges.length; i++) {
118
+ const next = taggedRanges[i];
119
+ if (next.range[0] <= cur.range[1]) {
120
+ cur = { range: [cur.range[0], Math.max(cur.range[1], next.range[1])], idx: Math.min(cur.idx, next.idx) };
121
+ }
122
+ else {
123
+ merged.push(cur);
124
+ cur = next;
125
+ }
126
+ }
127
+ merged.push(cur);
128
+ // Count redactions by category
129
+ let builtIn = 0;
130
+ let global = 0;
131
+ let project = 0;
132
+ for (const { idx } of merged) {
133
+ if (idx < builtInCount)
134
+ builtIn++;
135
+ else if (idx < builtInCount + globalCount)
136
+ global++;
137
+ else
138
+ project++;
139
+ }
140
+ // Build result string
141
+ let result = "";
142
+ let pos = 0;
143
+ for (const { range: [s, e] } of merged) {
144
+ result += text.slice(pos, s) + "[REDACTED]";
145
+ pos = e;
146
+ }
147
+ result += text.slice(pos);
148
+ return { text: result, builtIn, global, project };
149
+ }
150
+ /**
151
+ * Redact all matching patterns in text, replacing matches with [REDACTED].
152
+ *
153
+ * Strategy:
154
+ * - "Spanning" patterns (those that can match across whitespace) are applied
155
+ * to the full text via a multi-range merge to avoid one pattern consuming
156
+ * another's matches.
157
+ * - "Token" patterns (no whitespace/dot in source) are applied token-by-token
158
+ * so that greedy `.*`-style patterns in one token don't eat adjacent tokens.
159
+ */
160
+ scrub(text) {
161
+ return this.scrubWithCounts(text).text;
162
+ }
163
+ /** Parse a sensitive-patterns.txt file. Returns empty array if file is absent. */
164
+ static async loadProjectPatterns(filePath) {
165
+ try {
166
+ const content = await readFile(filePath, "utf-8");
167
+ return content
168
+ .split("\n")
169
+ .map((line) => line.trim())
170
+ .filter((line) => line.length > 0 && !line.startsWith("#"));
171
+ }
172
+ catch (err) {
173
+ if (err.code === "ENOENT")
174
+ return [];
175
+ throw err;
176
+ }
177
+ }
178
+ /** Build a ScrubEngine for a given project directory. */
179
+ static async forProject(globalPatterns, projectDir) {
180
+ const projectPatterns = await ScrubEngine.loadProjectPatterns(join(projectDir, "sensitive-patterns.txt"));
181
+ return new ScrubEngine(globalPatterns, projectPatterns);
182
+ }
183
+ }
184
+ //# sourceMappingURL=scrub.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrub.js","sourceRoot":"","sources":["../../src/scrub.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,CAAC,MAAM,iBAAiB,GAAa;IACzC,qBAAqB;IACrB,4BAA4B;IAC5B,qBAAqB;IACrB,kBAAkB;IAClB,wBAAwB;IACxB,+BAA+B;IAC/B,6BAA6B;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,MAAc;IACvC,mFAAmF;IACnF,+FAA+F;IAC/F,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChE,iDAAiD;IACjD,uDAAuD;IACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvB,CAAC,EAAE,CAAC,CAAC,oBAAoB;YACzB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AASD,MAAM,OAAO,WAAW;IACL,gBAAgB,GAA6C,EAAE,CAAC;IAChE,aAAa,GAA6C,EAAE,CAAC;IAC9E,qGAAqG;IACpF,gBAAgB,GAAa,EAAE,CAAC;IACjD,6CAA6C;IAC5B,aAAa,GAAa,EAAE,CAAC;IAC9C,2DAA2D;IAC1C,mBAAmB,CAAS;IACpC,eAAe,GAAa,EAAE,CAAC;IAExC,YAAY,cAAwB,EAAE,eAAyB;QAC7D,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC,MAAM,CAAC;QACjD,MAAM,GAAG,GAAG,CAAC,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,CAAC;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBACtC,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBAC9C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBAC3C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACH,eAAe,CAAC,IAAY;QAC1B,MAAM,YAAY,GAAG,iBAAiB,CAAC,MAAM,CAAC;QAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC;QAI7C,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YACzD,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YAC5C,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;YACpB,IAAI,CAAyB,CAAC;YAC9B,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACvC,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC/F,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;oBAAE,KAAK,CAAC,SAAS,EAAE,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxD,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;oBACtD,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;oBACzC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;oBACpB,IAAI,CAAyB,CAAC;oBAC9B,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;wBACtC,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC9G,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;4BAAE,KAAK,CAAC,SAAS,EAAE,CAAC;oBAC3C,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC;QACvB,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAElF,yBAAyB;QACzB,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAErD,6EAA6E;QAC7E,2EAA2E;QAC3E,MAAM,MAAM,GAAoD,EAAE,CAAC;QACnE,IAAI,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClC,GAAG,GAAG,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3G,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,GAAG,GAAG,IAAI,CAAC;YACb,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEjB,+BAA+B;QAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,MAAM,EAAE,CAAC;YAC7B,IAAI,GAAG,GAAG,YAAY;gBAAE,OAAO,EAAE,CAAC;iBAC7B,IAAI,GAAG,GAAG,YAAY,GAAG,WAAW;gBAAE,MAAM,EAAE,CAAC;;gBAC/C,OAAO,EAAE,CAAC;QACjB,CAAC;QAED,sBAAsB;QACtB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YACvC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC;YAC5C,GAAG,GAAG,CAAC,CAAC;QACV,CAAC;QACD,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACpD,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;IACzC,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,QAAgB;QAC/C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,OAAO;iBACX,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,EAAE,CAAC;YAChE,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,cAAwB,EACxB,UAAkB;QAElB,MAAM,eAAe,GAAG,MAAM,WAAW,CAAC,mBAAmB,CAC3D,IAAI,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAC3C,CAAC;QACF,OAAO,IAAI,WAAW,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;IAC1D,CAAC;CACF"}
@@ -0,0 +1,4 @@
1
+ export declare function handleSensitive(argv: string[], cwd: string, configPath?: string): Promise<{
2
+ exitCode: number;
3
+ stdout: string;
4
+ }>;