@psiclawops/hypermem 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/ARCHITECTURE.md +30 -38
  2. package/README.md +83 -35
  3. package/dist/background-indexer.d.ts +14 -3
  4. package/dist/background-indexer.d.ts.map +1 -1
  5. package/dist/background-indexer.js +126 -18
  6. package/dist/budget-policy.d.ts +22 -0
  7. package/dist/budget-policy.d.ts.map +1 -0
  8. package/dist/budget-policy.js +27 -0
  9. package/dist/cache.d.ts +11 -0
  10. package/dist/cache.d.ts.map +1 -1
  11. package/dist/compositor-utils.d.ts +31 -0
  12. package/dist/compositor-utils.d.ts.map +1 -0
  13. package/dist/compositor-utils.js +47 -0
  14. package/dist/compositor.d.ts +163 -1
  15. package/dist/compositor.d.ts.map +1 -1
  16. package/dist/compositor.js +862 -130
  17. package/dist/content-hash.d.ts +43 -0
  18. package/dist/content-hash.d.ts.map +1 -0
  19. package/dist/content-hash.js +75 -0
  20. package/dist/context-store.d.ts +54 -0
  21. package/dist/context-store.d.ts.map +1 -1
  22. package/dist/context-store.js +102 -0
  23. package/dist/contradiction-audit-store.d.ts +54 -0
  24. package/dist/contradiction-audit-store.d.ts.map +1 -0
  25. package/dist/contradiction-audit-store.js +88 -0
  26. package/dist/contradiction-resolution-policy.d.ts +21 -0
  27. package/dist/contradiction-resolution-policy.d.ts.map +1 -0
  28. package/dist/contradiction-resolution-policy.js +17 -0
  29. package/dist/degradation.d.ts +102 -0
  30. package/dist/degradation.d.ts.map +1 -0
  31. package/dist/degradation.js +141 -0
  32. package/dist/dreaming-promoter.d.ts +38 -0
  33. package/dist/dreaming-promoter.d.ts.map +1 -1
  34. package/dist/dreaming-promoter.js +68 -2
  35. package/dist/index.d.ts +68 -6
  36. package/dist/index.d.ts.map +1 -1
  37. package/dist/index.js +402 -26
  38. package/dist/knowledge-lint.d.ts +2 -0
  39. package/dist/knowledge-lint.d.ts.map +1 -1
  40. package/dist/knowledge-lint.js +40 -1
  41. package/dist/library-schema.d.ts +7 -2
  42. package/dist/library-schema.d.ts.map +1 -1
  43. package/dist/library-schema.js +236 -1
  44. package/dist/message-store.d.ts +64 -1
  45. package/dist/message-store.d.ts.map +1 -1
  46. package/dist/message-store.js +137 -1
  47. package/dist/open-domain.js +1 -1
  48. package/dist/proactive-pass.d.ts +2 -2
  49. package/dist/proactive-pass.d.ts.map +1 -1
  50. package/dist/proactive-pass.js +66 -12
  51. package/dist/replay-recovery.d.ts +29 -0
  52. package/dist/replay-recovery.d.ts.map +1 -0
  53. package/dist/replay-recovery.js +82 -0
  54. package/dist/reranker.d.ts +95 -0
  55. package/dist/reranker.d.ts.map +1 -0
  56. package/dist/reranker.js +308 -0
  57. package/dist/schema.d.ts +1 -1
  58. package/dist/schema.d.ts.map +1 -1
  59. package/dist/schema.js +46 -1
  60. package/dist/session-flusher.d.ts +2 -2
  61. package/dist/session-flusher.d.ts.map +1 -1
  62. package/dist/session-flusher.js +1 -1
  63. package/dist/temporal-store.js +2 -2
  64. package/dist/tool-artifact-store.d.ts +98 -0
  65. package/dist/tool-artifact-store.d.ts.map +1 -0
  66. package/dist/tool-artifact-store.js +244 -0
  67. package/dist/topic-detector.js +2 -2
  68. package/dist/topic-store.d.ts +6 -0
  69. package/dist/topic-store.d.ts.map +1 -1
  70. package/dist/topic-store.js +39 -0
  71. package/dist/types.d.ts +233 -1
  72. package/dist/types.d.ts.map +1 -1
  73. package/dist/vector-store.d.ts +2 -1
  74. package/dist/vector-store.d.ts.map +1 -1
  75. package/dist/vector-store.js +3 -0
  76. package/dist/version.d.ts +10 -10
  77. package/dist/version.d.ts.map +1 -1
  78. package/dist/version.js +10 -10
  79. package/package.json +6 -4
@@ -44,6 +44,44 @@ function getMaxMessageIndex(db, conversationId) {
44
44
  .get(conversationId);
45
45
  return typeof row.max_index === 'number' ? row.max_index : -1;
46
46
  }
47
+ /**
48
+ * Filter candidate message ids down to rows that are safe to delete without
49
+ * violating HyperMem's FK edges.
50
+ *
51
+ * Current blockers:
52
+ * - summary_messages.message_id -> messages.id
53
+ * - messages.parent_id -> messages.id (child rows point at parent rows)
54
+ */
55
+ function getDeletableMessageIds(db, candidateIds) {
56
+ if (candidateIds.length === 0)
57
+ return { deletableIds: [], blockedIds: [] };
58
+ const placeholders = candidateIds.map(() => '?').join(', ');
59
+ const blocked = db
60
+ .prepare(`
61
+ SELECT DISTINCT id
62
+ FROM (
63
+ SELECT sm.message_id AS id
64
+ FROM summary_messages sm
65
+ WHERE sm.message_id IN (${placeholders})
66
+
67
+ UNION
68
+
69
+ SELECT parent.id AS id
70
+ FROM messages child
71
+ JOIN messages parent ON parent.id = child.parent_id
72
+ WHERE child.parent_id IN (${placeholders})
73
+ ) blocked
74
+ `)
75
+ .all(...candidateIds, ...candidateIds);
76
+ if (blocked.length === 0)
77
+ return { deletableIds: candidateIds, blockedIds: [] };
78
+ const blockedIds = blocked.map(row => row.id);
79
+ const blockedSet = new Set(blockedIds);
80
+ return {
81
+ deletableIds: candidateIds.filter(id => !blockedSet.has(id)),
82
+ blockedIds,
83
+ };
84
+ }
47
85
  /**
48
86
  * Decide if a message is noise based on content + is_heartbeat flag.
49
87
  *
@@ -73,7 +111,7 @@ function isNoiseMessage(textContent, isHeartbeat) {
73
111
  * Deletions are wrapped in a single transaction. The FTS5 trigger handles
74
112
  * index cleanup automatically (msg_fts_ad fires on DELETE).
75
113
  */
76
- export function runNoiseSweep(db, conversationId, recentWindowSize = 20) {
114
+ export function runNoiseSweep(db, conversationId, recentWindowSize = 20, maxCandidates = Infinity) {
77
115
  const ZERO = { messagesDeleted: 0, passType: 'noise_sweep' };
78
116
  try {
79
117
  const safeWindow = resolveSafeWindow(recentWindowSize);
@@ -101,11 +139,16 @@ export function runNoiseSweep(db, conversationId, recentWindowSize = 20) {
101
139
  .all(conversationId, cutoff);
102
140
  if (candidates.length === 0)
103
141
  return ZERO;
104
- // Filter to noise messages
105
- const toDelete = candidates.filter(row => isNoiseMessage(row.text_content, row.is_heartbeat));
106
- if (toDelete.length === 0)
142
+ // Filter to noise messages, respecting per-pass candidate cap
143
+ const toDelete = candidates.filter(row => isNoiseMessage(row.text_content, row.is_heartbeat)).slice(0, Number.isFinite(maxCandidates) ? maxCandidates : undefined);
144
+ if (toDelete.length === 0) {
107
145
  return ZERO;
108
- const ids = toDelete.map(r => r.id);
146
+ }
147
+ const candidateIds = toDelete.map(r => r.id);
148
+ const { deletableIds: ids, blockedIds } = getDeletableMessageIds(db, candidateIds);
149
+ if (ids.length === 0) {
150
+ return ZERO;
151
+ }
109
152
  // Delete in a transaction; use chunked IN clauses to avoid
110
153
  // SQLite's SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
111
154
  let totalDeleted = 0;
@@ -126,6 +169,9 @@ export function runNoiseSweep(db, conversationId, recentWindowSize = 20) {
126
169
  db.prepare('ROLLBACK').run();
127
170
  throw innerErr;
128
171
  }
172
+ if (totalDeleted > 0) {
173
+ console.log(`[proactive-pass] Noise sweep conversation=${conversationId} candidates=${candidates.length} noise=${candidateIds.length} deleted=${totalDeleted} skippedReferenced=${blockedIds.length} cutoff=${cutoff}`);
174
+ }
129
175
  return { messagesDeleted: totalDeleted, passType: 'noise_sweep' };
130
176
  }
131
177
  catch (err) {
@@ -150,16 +196,18 @@ export function runNoiseSweep(db, conversationId, recentWindowSize = 20) {
150
196
  *
151
197
  * Mutations are committed in a single transaction.
152
198
  */
153
- export function runToolDecay(db, conversationId, recentWindowSize = 40) {
199
+ export function runToolDecay(db, conversationId, recentWindowSize = 40, maxCandidates = Infinity) {
154
200
  const ZERO = { messagesUpdated: 0, bytesFreed: 0, passType: 'tool_decay' };
155
201
  try {
156
202
  const safeWindow = resolveSafeWindow(recentWindowSize);
157
203
  const maxIndex = getMaxMessageIndex(db, conversationId);
158
- if (maxIndex < 0)
204
+ if (maxIndex < 0) {
159
205
  return ZERO;
206
+ }
160
207
  const cutoff = maxIndex - safeWindow;
161
- if (cutoff <= 0)
208
+ if (cutoff <= 0) {
162
209
  return ZERO;
210
+ }
163
211
  // Fetch messages with large tool_results outside the recent window.
164
212
  const candidates = db
165
213
  .prepare(`
@@ -171,11 +219,13 @@ export function runToolDecay(db, conversationId, recentWindowSize = 40) {
171
219
  AND length(tool_results) > 2000
172
220
  `)
173
221
  .all(conversationId, cutoff);
174
- if (candidates.length === 0)
222
+ if (candidates.length === 0) {
175
223
  return ZERO;
176
- // Build the update list by processing each candidate.
224
+ }
225
+ // Build the update list by processing each candidate, respecting per-pass cap.
226
+ const cappedCandidates = Number.isFinite(maxCandidates) ? candidates.slice(0, maxCandidates) : candidates;
177
227
  const updates = [];
178
- for (const row of candidates) {
228
+ for (const row of cappedCandidates) {
179
229
  let parsed;
180
230
  try {
181
231
  parsed = JSON.parse(row.tool_results);
@@ -207,8 +257,9 @@ export function runToolDecay(db, conversationId, recentWindowSize = 40) {
207
257
  updates.push({ id: row.id, newJson, savedBytes });
208
258
  }
209
259
  }
210
- if (updates.length === 0)
260
+ if (updates.length === 0) {
211
261
  return ZERO;
262
+ }
212
263
  let totalUpdated = 0;
213
264
  let totalBytesFreed = 0;
214
265
  db.prepare('BEGIN').run();
@@ -225,6 +276,9 @@ export function runToolDecay(db, conversationId, recentWindowSize = 40) {
225
276
  db.prepare('ROLLBACK').run();
226
277
  throw innerErr;
227
278
  }
279
+ if (totalUpdated > 0) {
280
+ console.log(`[proactive-pass] Tool decay conversation=${conversationId} candidates=${candidates.length} updated=${totalUpdated} bytesFreed=${totalBytesFreed} cutoff=${cutoff}`);
281
+ }
228
282
  return {
229
283
  messagesUpdated: totalUpdated,
230
284
  bytesFreed: totalBytesFreed,
@@ -0,0 +1,29 @@
1
+ import { type ReplayMarker, type ReplayState } from './degradation.js';
2
+ export interface ReplayRecoveryInputs {
3
+ currentState?: ReplayState | '' | null;
4
+ runtimeTokens: number;
5
+ redisTokens: number;
6
+ effectiveBudget: number;
7
+ }
8
+ export interface ReplayRecoveryDecision {
9
+ active: boolean;
10
+ shouldSkipCacheReplay: boolean;
11
+ trimTargetOverride?: number;
12
+ historyDepthCap?: number;
13
+ emittedMarker?: ReplayMarker;
14
+ emittedText?: string;
15
+ nextState: ReplayState | null;
16
+ }
17
+ export declare const REPLAY_RECOVERY_POLICY: {
18
+ readonly enterPressure: 0.8;
19
+ readonly exitPressure: 0.65;
20
+ readonly redisColdFraction: 0.2;
21
+ readonly enterTrimTarget: 0.2;
22
+ readonly stabilizingTrimTarget: 0.35;
23
+ readonly historyDepthCap: 60;
24
+ readonly redisFloorTokens: 500;
25
+ };
26
+ export declare function isColdRedisReplay(inputs: ReplayRecoveryInputs): boolean;
27
+ export declare function isReplayRecovered(inputs: ReplayRecoveryInputs): boolean;
28
+ export declare function decideReplayRecovery(inputs: ReplayRecoveryInputs): ReplayRecoveryDecision;
29
+ //# sourceMappingURL=replay-recovery.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"replay-recovery.d.ts","sourceRoot":"","sources":["../src/replay-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,OAAO,CAAC;IAChB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,WAAW,GAAG,IAAI,CAAC;CAC/B;AAED,eAAO,MAAM,sBAAsB;;;;;;;;CAQzB,CAAC;AAgBX,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAKvE;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAKvE;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,oBAAoB,GAAG,sBAAsB,CA+DzF"}
@@ -0,0 +1,82 @@
1
+ import { formatReplayMarker, } from './degradation.js';
2
+ export const REPLAY_RECOVERY_POLICY = {
3
+ enterPressure: 0.80,
4
+ exitPressure: 0.65,
5
+ redisColdFraction: 0.20,
6
+ enterTrimTarget: 0.20,
7
+ stabilizingTrimTarget: 0.35,
8
+ historyDepthCap: 60,
9
+ redisFloorTokens: 500,
10
+ };
11
+ function coldRedisThreshold(runtimeTokens) {
12
+ return Math.max(REPLAY_RECOVERY_POLICY.redisFloorTokens, Math.floor(runtimeTokens * REPLAY_RECOVERY_POLICY.redisColdFraction));
13
+ }
14
+ function recoveredRedisThreshold(effectiveBudget) {
15
+ return Math.max(REPLAY_RECOVERY_POLICY.redisFloorTokens, Math.floor(effectiveBudget * REPLAY_RECOVERY_POLICY.redisColdFraction));
16
+ }
17
+ export function isColdRedisReplay(inputs) {
18
+ return (inputs.runtimeTokens > inputs.effectiveBudget * REPLAY_RECOVERY_POLICY.enterPressure &&
19
+ inputs.redisTokens < coldRedisThreshold(inputs.runtimeTokens));
20
+ }
21
+ export function isReplayRecovered(inputs) {
22
+ return (inputs.runtimeTokens <= inputs.effectiveBudget * REPLAY_RECOVERY_POLICY.exitPressure &&
23
+ inputs.redisTokens >= recoveredRedisThreshold(inputs.effectiveBudget));
24
+ }
25
+ export function decideReplayRecovery(inputs) {
26
+ const currentState = inputs.currentState ?? null;
27
+ if (!currentState) {
28
+ if (!isColdRedisReplay(inputs)) {
29
+ return {
30
+ active: false,
31
+ shouldSkipCacheReplay: false,
32
+ nextState: null,
33
+ };
34
+ }
35
+ const emittedMarker = {
36
+ state: 'entering',
37
+ status: 'bounded',
38
+ reason: 'replay_cold_redis',
39
+ summary: 'cold restart, keep the window bounded',
40
+ };
41
+ return {
42
+ active: true,
43
+ shouldSkipCacheReplay: true,
44
+ trimTargetOverride: REPLAY_RECOVERY_POLICY.enterTrimTarget,
45
+ historyDepthCap: REPLAY_RECOVERY_POLICY.historyDepthCap,
46
+ emittedMarker,
47
+ emittedText: formatReplayMarker(emittedMarker),
48
+ nextState: 'stabilizing',
49
+ };
50
+ }
51
+ if (isReplayRecovered(inputs)) {
52
+ const emittedMarker = {
53
+ state: 'exited',
54
+ status: 'bounded',
55
+ reason: 'replay_exited',
56
+ summary: 'stable window restored',
57
+ };
58
+ return {
59
+ active: false,
60
+ shouldSkipCacheReplay: false,
61
+ emittedMarker,
62
+ emittedText: formatReplayMarker(emittedMarker),
63
+ nextState: null,
64
+ };
65
+ }
66
+ const emittedMarker = {
67
+ state: 'stabilizing',
68
+ status: 'bounded',
69
+ reason: 'replay_stabilizing',
70
+ summary: 'replay window stabilizing, keep it bounded',
71
+ };
72
+ return {
73
+ active: true,
74
+ shouldSkipCacheReplay: true,
75
+ trimTargetOverride: REPLAY_RECOVERY_POLICY.stabilizingTrimTarget,
76
+ historyDepthCap: REPLAY_RECOVERY_POLICY.historyDepthCap,
77
+ emittedMarker,
78
+ emittedText: formatReplayMarker(emittedMarker),
79
+ nextState: 'stabilizing',
80
+ };
81
+ }
82
+ //# sourceMappingURL=replay-recovery.js.map
@@ -0,0 +1,95 @@
1
+ /**
2
+ * hypermem Reranker
3
+ *
4
+ * Pluggable reranking interface with circuit-breaker protection and graceful
5
+ * degradation. Callers that receive null fall back to the original document
6
+ * order without disruption.
7
+ *
8
+ * Providers:
9
+ * - ZeroEntropyReranker — https://api.zeroentropy.dev/v1/rerank (zerank-2)
10
+ * - OpenRouterReranker — https://openrouter.ai/api/v1/rerank (cohere/rerank-4-pro)
11
+ * - OllamaReranker — http://localhost:11434/api/chat (yes/no classification)
12
+ *
13
+ * API key resolution order (per provider):
14
+ * zeroentropy: config.zeroEntropyApiKey → ZEROENTROPY_API_KEY env var
15
+ * openrouter: config.openrouterApiKey → OPENROUTER_API_KEY env var
16
+ */
17
+ export interface RerankResult {
18
+ index: number;
19
+ score: number;
20
+ content: string;
21
+ }
22
+ export interface RerankerProvider {
23
+ readonly name: string;
24
+ /**
25
+ * Reranks `documents` by relevance to `query`.
26
+ * Returns null on any failure — callers MUST fall back to original order.
27
+ */
28
+ rerank(query: string, documents: string[], topK?: number): Promise<RerankResult[] | null>;
29
+ }
30
+ export interface RerankerConfig {
31
+ provider: 'zeroentropy' | 'openrouter' | 'local' | 'none';
32
+ minCandidates: number;
33
+ maxDocuments: number;
34
+ topK: number;
35
+ timeoutMs: number;
36
+ zeroEntropyApiKey?: string;
37
+ zeroEntropyModel?: string;
38
+ openrouterApiKey?: string;
39
+ openrouterModel?: string;
40
+ ollamaUrl?: string;
41
+ ollamaModel?: string;
42
+ }
43
+ export declare class ZeroEntropyReranker implements RerankerProvider {
44
+ readonly name = "zeroentropy";
45
+ private readonly circuit;
46
+ private readonly apiKey;
47
+ private readonly model;
48
+ private readonly timeoutMs;
49
+ constructor(apiKey: string, model?: string, timeoutMs?: number);
50
+ rerank(query: string, documents: string[], topK?: number): Promise<RerankResult[] | null>;
51
+ }
52
+ export declare class OpenRouterReranker implements RerankerProvider {
53
+ readonly name = "openrouter";
54
+ private readonly circuit;
55
+ private readonly apiKey;
56
+ private readonly model;
57
+ private readonly timeoutMs;
58
+ constructor(apiKey: string, model?: string, timeoutMs?: number);
59
+ rerank(query: string, documents: string[], topK?: number): Promise<RerankResult[] | null>;
60
+ }
61
+ export declare class OllamaReranker implements RerankerProvider {
62
+ readonly name = "ollama";
63
+ private readonly circuit;
64
+ private readonly baseUrl;
65
+ private readonly model;
66
+ private readonly timeoutMs;
67
+ constructor(baseUrl?: string, model?: string, timeoutMs?: number);
68
+ /**
69
+ * Scores documents sequentially — one chat call per document.
70
+ * The Qwen3-Reranker-0.6B model responds with "yes" (relevant) or "no".
71
+ * Score: yes → 1.0, anything else → 0.0.
72
+ *
73
+ * Sequential iteration is required because Ollama's /api/chat is stateless
74
+ * per-request and running calls in parallel would overload a local GPU.
75
+ * Returns null on the first failure to preserve circuit breaker semantics.
76
+ */
77
+ rerank(query: string, documents: string[], topK?: number): Promise<RerankResult[] | null>;
78
+ }
79
+ /**
80
+ * Creates a RerankerProvider from the supplied config.
81
+ *
82
+ * API key resolution order:
83
+ * zeroentropy: config.zeroEntropyApiKey → ZEROENTROPY_API_KEY env var
84
+ * openrouter: config.openrouterApiKey → OPENROUTER_API_KEY env var
85
+ *
86
+ * Returns null when:
87
+ * - provider is 'none'
88
+ * - provider is 'zeroentropy' and no key found in config or env
89
+ * - provider is 'openrouter' and no key found in config or env
90
+ *
91
+ * 'local' (Ollama) never returns null from the factory — it has no required
92
+ * API key. The provider itself returns null on runtime failure.
93
+ */
94
+ export declare function createReranker(config: RerankerConfig): RerankerProvider | null;
95
+ //# sourceMappingURL=reranker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAgFH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC;CAC3F;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,aAAa,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC;IAC1D,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAID,qBAAa,mBAAoB,YAAW,gBAAgB;IAC1D,QAAQ,CAAC,IAAI,iBAAiB;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,MAAM,EAAE,MAAM,EAAE,KAAK,SAAa,EAAE,SAAS,SAAO;IAM1D,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,IAAI,SAAK,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC;CAqC5F;AAID,qBAAa,kBAAmB,YAAW,gBAAgB;IACzD,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,MAAM,EAAE,MAAM,EAAE,KAAK,SAAwB,EAAE,SAAS,SAAO;IAMrE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,IAAI,SAAK,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC;CAqC5F;AAQD,qBAAa,cAAe,YAAW,gBAAgB;IACrD,QAAQ,CAAC,IAAI,YAAY;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAGjC,OAAO,SAA2B,EAClC,KAAK,SAAuC,EAC5C,SAAS,SAAS;IAOpB;;;;;;;;OAQG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,IAAI,SAAK,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC;CA4C5F;AAID;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,GAAG,IAAI,CAyB9E"}
@@ -0,0 +1,308 @@
1
+ /**
2
+ * hypermem Reranker
3
+ *
4
+ * Pluggable reranking interface with circuit-breaker protection and graceful
5
+ * degradation. Callers that receive null fall back to the original document
6
+ * order without disruption.
7
+ *
8
+ * Providers:
9
+ * - ZeroEntropyReranker — https://api.zeroentropy.dev/v1/rerank (zerank-2)
10
+ * - OpenRouterReranker — https://openrouter.ai/api/v1/rerank (cohere/rerank-4-pro)
11
+ * - OllamaReranker — http://localhost:11434/api/chat (yes/no classification)
12
+ *
13
+ * API key resolution order (per provider):
14
+ * zeroentropy: config.zeroEntropyApiKey → ZEROENTROPY_API_KEY env var
15
+ * openrouter: config.openrouterApiKey → OPENROUTER_API_KEY env var
16
+ */
17
+ // ── Circuit Breaker ───────────────────────────────────────────────────────────
18
+ /** Number of consecutive failures before the circuit opens. */
19
+ const CIRCUIT_FAILURE_THRESHOLD = 3;
20
+ /** How long the circuit stays open after tripping (ms). */
21
+ const CIRCUIT_DISABLE_DURATION_MS = 60_000;
22
+ /**
23
+ * CircuitBreaker protects a provider from repeated calls during sustained
24
+ * failures.
25
+ *
26
+ * State machine:
27
+ * CLOSED — normal operation; failures increment the counter.
28
+ * OPEN — provider disabled; isOpen() returns true for DISABLE_DURATION.
29
+ * HALF-OPEN — one probe allowed after the disable window expires;
30
+ * success resets the counter, failure re-opens the circuit.
31
+ *
32
+ * Transition: CLOSED → OPEN when consecutiveFailures >= FAILURE_THRESHOLD.
33
+ * Transition: OPEN → HALF-OPEN when Date.now() >= disabledUntil.
34
+ * Transition: HALF-OPEN → CLOSED on recordSuccess().
35
+ * Transition: HALF-OPEN → OPEN on recordFailure() (reset window starts fresh).
36
+ */
37
+ class CircuitBreaker {
38
+ /** Consecutive failures since last success. Reset to 0 on any success. */
39
+ consecutiveFailures = 0;
40
+ /**
41
+ * Epoch ms after which the circuit allows a probe.
42
+ * 0 means the circuit is CLOSED (not disabled).
43
+ */
44
+ disabledUntil = 0;
45
+ /** Returns true when the provider should be skipped. */
46
+ isOpen() {
47
+ if (this.disabledUntil === 0)
48
+ return false; // CLOSED
49
+ if (Date.now() >= this.disabledUntil) {
50
+ // Disable window expired — transition to HALF-OPEN: allow one probe.
51
+ // We clear disabledUntil here so the next call goes through.
52
+ // If it fails, recordFailure() re-opens the circuit with a fresh window.
53
+ this.disabledUntil = 0;
54
+ return false;
55
+ }
56
+ return true; // OPEN
57
+ }
58
+ /** Call on a successful provider response. Resets all failure state. */
59
+ recordSuccess() {
60
+ this.consecutiveFailures = 0;
61
+ this.disabledUntil = 0;
62
+ }
63
+ /**
64
+ * Call on any provider failure (non-2xx, timeout, parse error, etc.).
65
+ * After CIRCUIT_FAILURE_THRESHOLD consecutive failures, opens the circuit
66
+ * for CIRCUIT_DISABLE_DURATION_MS milliseconds.
67
+ */
68
+ recordFailure() {
69
+ this.consecutiveFailures += 1;
70
+ if (this.consecutiveFailures >= CIRCUIT_FAILURE_THRESHOLD) {
71
+ // Trip the circuit — disable for the configured window.
72
+ // Reset consecutiveFailures so the probe after the window isn't
73
+ // pre-tripped by the old count.
74
+ this.disabledUntil = Date.now() + CIRCUIT_DISABLE_DURATION_MS;
75
+ this.consecutiveFailures = 0;
76
+ }
77
+ }
78
+ }
79
+ // ── Helpers ───────────────────────────────────────────────────────────────────
80
+ /** Returns an AbortController that fires after `ms` and a cleanup handle. */
81
+ function withTimeout(ms) {
82
+ const controller = new AbortController();
83
+ const id = setTimeout(() => controller.abort(), ms);
84
+ return { controller, clear: () => clearTimeout(id) };
85
+ }
86
+ // ── ZeroEntropyReranker ───────────────────────────────────────────────────────
87
+ export class ZeroEntropyReranker {
88
+ name = 'zeroentropy';
89
+ circuit = new CircuitBreaker();
90
+ apiKey;
91
+ model;
92
+ timeoutMs;
93
+ constructor(apiKey, model = 'zerank-2', timeoutMs = 2000) {
94
+ this.apiKey = apiKey;
95
+ this.model = model;
96
+ this.timeoutMs = timeoutMs;
97
+ }
98
+ async rerank(query, documents, topK = 10) {
99
+ if (this.circuit.isOpen())
100
+ return null;
101
+ const { controller, clear } = withTimeout(this.timeoutMs);
102
+ try {
103
+ const response = await fetch('https://api.zeroentropy.dev/v1/rerank', {
104
+ method: 'POST',
105
+ headers: {
106
+ 'Content-Type': 'application/json',
107
+ Authorization: `Bearer ${this.apiKey}`,
108
+ },
109
+ body: JSON.stringify({ query, documents, model: this.model, top_n: topK }),
110
+ signal: controller.signal,
111
+ });
112
+ clear();
113
+ if (!response.ok) {
114
+ this.circuit.recordFailure();
115
+ return null;
116
+ }
117
+ const data = (await response.json());
118
+ if (!Array.isArray(data.results)) {
119
+ this.circuit.recordFailure();
120
+ return null;
121
+ }
122
+ this.circuit.recordSuccess();
123
+ return data.results.map((r) => {
124
+ const score = r.score ?? r.relevance_score ?? 0;
125
+ const content = resolveDocumentText(r.content, r.document, documents[r.index]);
126
+ return { index: r.index, score, content };
127
+ });
128
+ }
129
+ catch {
130
+ clear();
131
+ this.circuit.recordFailure();
132
+ return null;
133
+ }
134
+ }
135
+ }
136
+ // ── OpenRouterReranker ────────────────────────────────────────────────────────
137
+ export class OpenRouterReranker {
138
+ name = 'openrouter';
139
+ circuit = new CircuitBreaker();
140
+ apiKey;
141
+ model;
142
+ timeoutMs;
143
+ constructor(apiKey, model = 'cohere/rerank-4-pro', timeoutMs = 2000) {
144
+ this.apiKey = apiKey;
145
+ this.model = model;
146
+ this.timeoutMs = timeoutMs;
147
+ }
148
+ async rerank(query, documents, topK = 10) {
149
+ if (this.circuit.isOpen())
150
+ return null;
151
+ const { controller, clear } = withTimeout(this.timeoutMs);
152
+ try {
153
+ const response = await fetch('https://openrouter.ai/api/v1/rerank', {
154
+ method: 'POST',
155
+ headers: {
156
+ 'Content-Type': 'application/json',
157
+ Authorization: `Bearer ${this.apiKey}`,
158
+ },
159
+ body: JSON.stringify({ query, documents, model: this.model, top_n: topK }),
160
+ signal: controller.signal,
161
+ });
162
+ clear();
163
+ if (!response.ok) {
164
+ this.circuit.recordFailure();
165
+ return null;
166
+ }
167
+ const data = (await response.json());
168
+ if (!Array.isArray(data.results)) {
169
+ this.circuit.recordFailure();
170
+ return null;
171
+ }
172
+ this.circuit.recordSuccess();
173
+ return data.results.map((r) => {
174
+ const score = r.relevance_score ?? r.score ?? 0;
175
+ const content = resolveDocumentText(undefined, r.document, documents[r.index]);
176
+ return { index: r.index, score, content };
177
+ });
178
+ }
179
+ catch {
180
+ clear();
181
+ this.circuit.recordFailure();
182
+ return null;
183
+ }
184
+ }
185
+ }
186
+ // ── OllamaReranker ────────────────────────────────────────────────────────────
187
+ const OLLAMA_SYSTEM_PROMPT = 'Judge whether the Document meets the requirements based on the Query and the Instruct provided. ' +
188
+ 'Note that the answer can only be "yes" or "no".';
189
+ export class OllamaReranker {
190
+ name = 'ollama';
191
+ circuit = new CircuitBreaker();
192
+ baseUrl;
193
+ model;
194
+ timeoutMs;
195
+ constructor(baseUrl = 'http://localhost:11434', model = 'dengcao/Qwen3-Reranker-0.6B:Q5_K_M', timeoutMs = 10_000) {
196
+ this.baseUrl = baseUrl.replace(/\/$/, '');
197
+ this.model = model;
198
+ this.timeoutMs = timeoutMs;
199
+ }
200
+ /**
201
+ * Scores documents sequentially — one chat call per document.
202
+ * The Qwen3-Reranker-0.6B model responds with "yes" (relevant) or "no".
203
+ * Score: yes → 1.0, anything else → 0.0.
204
+ *
205
+ * Sequential iteration is required because Ollama's /api/chat is stateless
206
+ * per-request and running calls in parallel would overload a local GPU.
207
+ * Returns null on the first failure to preserve circuit breaker semantics.
208
+ */
209
+ async rerank(query, documents, topK = 10) {
210
+ if (this.circuit.isOpen())
211
+ return null;
212
+ const scored = [];
213
+ for (let i = 0; i < documents.length; i++) {
214
+ const doc = documents[i];
215
+ const userContent = `Instruct: Given a query, retrieve relevant passages that answer the query\n\n` +
216
+ `Query: ${query}\n\nDocument: ${doc}`;
217
+ const { controller, clear } = withTimeout(this.timeoutMs);
218
+ try {
219
+ const response = await fetch(`${this.baseUrl}/api/chat`, {
220
+ method: 'POST',
221
+ headers: { 'Content-Type': 'application/json' },
222
+ body: JSON.stringify({
223
+ model: this.model,
224
+ messages: [
225
+ { role: 'system', content: OLLAMA_SYSTEM_PROMPT },
226
+ { role: 'user', content: userContent },
227
+ ],
228
+ stream: false,
229
+ }),
230
+ signal: controller.signal,
231
+ });
232
+ clear();
233
+ if (!response.ok) {
234
+ this.circuit.recordFailure();
235
+ return null;
236
+ }
237
+ const data = (await response.json());
238
+ const answer = (data.message?.content ?? '').trim().toLowerCase();
239
+ // "yes" prefix → relevant; anything else (including "no") → not relevant
240
+ const score = answer.startsWith('yes') ? 1.0 : 0.0;
241
+ scored.push({ index: i, score, content: doc });
242
+ }
243
+ catch {
244
+ clear();
245
+ this.circuit.recordFailure();
246
+ return null;
247
+ }
248
+ }
249
+ this.circuit.recordSuccess();
250
+ // Sort by score descending and return top-K
251
+ scored.sort((a, b) => b.score - a.score);
252
+ return scored.slice(0, topK);
253
+ }
254
+ }
255
+ // ── Factory ───────────────────────────────────────────────────────────────────
256
+ /**
257
+ * Creates a RerankerProvider from the supplied config.
258
+ *
259
+ * API key resolution order:
260
+ * zeroentropy: config.zeroEntropyApiKey → ZEROENTROPY_API_KEY env var
261
+ * openrouter: config.openrouterApiKey → OPENROUTER_API_KEY env var
262
+ *
263
+ * Returns null when:
264
+ * - provider is 'none'
265
+ * - provider is 'zeroentropy' and no key found in config or env
266
+ * - provider is 'openrouter' and no key found in config or env
267
+ *
268
+ * 'local' (Ollama) never returns null from the factory — it has no required
269
+ * API key. The provider itself returns null on runtime failure.
270
+ */
271
+ export function createReranker(config) {
272
+ switch (config.provider) {
273
+ case 'none':
274
+ return null;
275
+ case 'zeroentropy': {
276
+ const apiKey = config.zeroEntropyApiKey ?? process.env['ZEROENTROPY_API_KEY'];
277
+ if (!apiKey)
278
+ return null;
279
+ return new ZeroEntropyReranker(apiKey, config.zeroEntropyModel ?? 'zerank-2', config.timeoutMs);
280
+ }
281
+ case 'openrouter': {
282
+ const apiKey = config.openrouterApiKey ?? process.env['OPENROUTER_API_KEY'];
283
+ if (!apiKey)
284
+ return null;
285
+ return new OpenRouterReranker(apiKey, config.openrouterModel ?? 'cohere/rerank-4-pro', config.timeoutMs);
286
+ }
287
+ case 'local': {
288
+ return new OllamaReranker(config.ollamaUrl ?? 'http://localhost:11434', config.ollamaModel ?? 'dengcao/Qwen3-Reranker-0.6B:Q5_K_M', config.timeoutMs);
289
+ }
290
+ }
291
+ }
292
+ // ── Private Helpers ───────────────────────────────────────────────────────────
293
+ /**
294
+ * Resolves the document text from a reranking API response.
295
+ * Preference order: explicit `content` string → `document` string →
296
+ * `document.text` string → original document from the input array.
297
+ */
298
+ function resolveDocumentText(content, document, fallback) {
299
+ if (typeof content === 'string')
300
+ return content;
301
+ if (typeof document === 'string')
302
+ return document;
303
+ if (document !== null && typeof document === 'object' && typeof document.text === 'string') {
304
+ return document.text;
305
+ }
306
+ return fallback ?? '';
307
+ }
308
+ //# sourceMappingURL=reranker.js.map
package/dist/schema.d.ts CHANGED
@@ -6,7 +6,7 @@
6
6
  * Contains ONLY conversation data — structured knowledge lives in library.db.
7
7
  */
8
8
  import type { DatabaseSync } from 'node:sqlite';
9
- export declare const LATEST_SCHEMA_VERSION = 8;
9
+ export declare const LATEST_SCHEMA_VERSION = 10;
10
10
  /**
11
11
  * Run migrations on an agent message database.
12
12
  */