@xdarkicex/openclaw-memory-libravdb 1.4.5 → 1.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/HOOK.md +14 -0
  2. package/README.md +32 -2
  3. package/dist/cli.d.ts +39 -0
  4. package/dist/cli.js +208 -0
  5. package/dist/context-engine.d.ts +56 -0
  6. package/dist/context-engine.js +125 -0
  7. package/dist/dream-promotion.d.ts +47 -0
  8. package/dist/dream-promotion.js +363 -0
  9. package/dist/dream-routing.d.ts +6 -0
  10. package/dist/dream-routing.js +31 -0
  11. package/dist/durable-namespace.d.ts +6 -0
  12. package/dist/durable-namespace.js +24 -0
  13. package/dist/grpc-client.d.ts +23 -0
  14. package/dist/grpc-client.js +104 -0
  15. package/dist/index.d.ts +10 -0
  16. package/dist/index.js +40 -0
  17. package/dist/lifecycle-hooks.d.ts +4 -0
  18. package/dist/lifecycle-hooks.js +64 -0
  19. package/dist/markdown-hash.d.ts +3 -0
  20. package/dist/markdown-hash.js +82 -0
  21. package/dist/markdown-ingest.d.ts +43 -0
  22. package/dist/markdown-ingest.js +464 -0
  23. package/dist/memory-provider.d.ts +4 -0
  24. package/dist/memory-provider.js +13 -0
  25. package/dist/memory-runtime.d.ts +118 -0
  26. package/dist/memory-runtime.js +217 -0
  27. package/dist/plugin-runtime.d.ts +28 -0
  28. package/dist/plugin-runtime.js +127 -0
  29. package/dist/proto/intelligence_kernel/v1/kernel.proto +378 -0
  30. package/dist/recall-cache.d.ts +2 -0
  31. package/dist/recall-cache.js +30 -0
  32. package/dist/rpc-protobuf-codecs.d.ts +70 -0
  33. package/dist/rpc-protobuf-codecs.js +77 -0
  34. package/dist/rpc.d.ts +14 -0
  35. package/dist/rpc.js +121 -0
  36. package/dist/sidecar.d.ts +34 -0
  37. package/dist/sidecar.js +535 -0
  38. package/dist/types.d.ts +163 -0
  39. package/dist/types.js +1 -0
  40. package/docs/contributing.md +14 -13
  41. package/docs/install.md +7 -9
  42. package/docs/installation.md +23 -16
  43. package/docs/uninstall.md +1 -1
  44. package/index.js +2 -0
  45. package/openclaw.plugin.json +2 -2
  46. package/package.json +39 -16
  47. package/packaging/README.md +0 -71
  48. package/packaging/homebrew/libravdbd.rb.tmpl +0 -224
  49. package/packaging/launchd/com.xdarkicex.libravdbd.plist +0 -32
  50. package/packaging/systemd/libravdbd.service +0 -12
  51. package/src/cli.ts +0 -299
  52. package/src/comparison-experiments.ts +0 -128
  53. package/src/context-engine.ts +0 -1451
  54. package/src/continuity.ts +0 -93
  55. package/src/dream-promotion.ts +0 -492
  56. package/src/dream-routing.ts +0 -40
  57. package/src/durable-namespace.ts +0 -34
  58. package/src/index.ts +0 -47
  59. package/src/lifecycle-hooks.ts +0 -96
  60. package/src/markdown-hash.ts +0 -104
  61. package/src/markdown-ingest.ts +0 -627
  62. package/src/memory-provider.ts +0 -25
  63. package/src/memory-runtime.ts +0 -283
  64. package/src/openclaw-plugin-sdk.d.ts +0 -59
  65. package/src/plugin-runtime.ts +0 -116
  66. package/src/recall-cache.ts +0 -34
  67. package/src/recall-utils.ts +0 -131
  68. package/src/rpc.ts +0 -84
  69. package/src/scoring.ts +0 -632
  70. package/src/sidecar.ts +0 -486
  71. package/src/temporal.ts +0 -1010
  72. package/src/tokens.ts +0 -52
  73. package/src/types.ts +0 -277
  74. package/tsconfig.json +0 -20
  75. package/tsconfig.tests.json +0 -12
package/src/temporal.ts DELETED
@@ -1,1010 +0,0 @@
1
- import { estimateTokens } from "./tokens.js";
2
- import type { SearchResult } from "./types.js";
3
- import {
4
- createComparisonProfileSummary,
5
- resolveComparisonExperimentConfig,
6
- type ComparisonExperimentConfig,
7
- type ComparisonProfileSummary,
8
- } from "./comparison-experiments.js";
9
-
10
- const TEMPORAL_PATTERN_WEIGHTS: Array<{ label: string; weight: number; patterns: RegExp[] }> = [
11
- {
12
- label: "how many days",
13
- weight: 1.0,
14
- patterns: [/\bhow\s+many\s+days\b/i],
15
- },
16
- {
17
- label: "how long",
18
- weight: 0.9,
19
- patterns: [/\bhow\s+long\b/i],
20
- },
21
- {
22
- label: "before or after",
23
- weight: 0.8,
24
- patterns: [/\bbefore\b/i, /\bafter\b/i],
25
- },
26
- {
27
- label: "since or between",
28
- weight: 0.7,
29
- patterns: [/\bsince\b/i, /\bbetween\b/i],
30
- },
31
- {
32
- label: "first or earlier",
33
- weight: 0.8,
34
- patterns: [/\bfirst\b/i, /\bearlier\b/i, /\bwhich\s+came\s+first\b/i],
35
- },
36
- {
37
- label: "when did",
38
- weight: 0.7,
39
- patterns: [/\bwhen\s+did\b/i],
40
- },
41
- ];
42
-
43
- const TEMPORAL_ANCHOR_PATTERNS: RegExp[] = [
44
- /\b\d{4}-\d{2}-\d{2}\b/g,
45
- /\b\d{1,2}\/\d{1,2}(?:\/\d{2,4})?\b/g,
46
- /\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\s+\d{1,2}(?:st|nd|rd|th)?(?:,\s*\d{4})?\b/gi,
47
- /\b(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/gi,
48
- /\b(?:today|yesterday|tomorrow|last\s+(?:week|month|year|night|saturday|sunday)|next\s+(?:week|month|year|monday|tuesday|wednesday|thursday|friday|saturday|sunday)|mid-?[a-z]+)\b/gi,
49
- /\b\d{1,2}:\d{2}(?:\s?[ap]m)?\b/gi,
50
- /\b(?:19|20)\d{2}\b/g,
51
- /\b\d{10,13}\b/g,
52
- ];
53
-
54
- const TEMPORAL_XI_NORM = 1.5;
55
- const TEMPORAL_XI_THRESHOLD = 0.3;
56
- const TEMPORAL_ANCHOR_NORM = 3;
57
- const TEMPORAL_ANCHOR_CACHE_MAX = 4096;
58
- const temporalAnchorCache = new Map<string, number>();
59
-
60
- const TEMPORAL_SLOT_STOPWORDS = new Set([
61
- "the",
62
- "and",
63
- "for",
64
- "with",
65
- "that",
66
- "this",
67
- "have",
68
- "from",
69
- "your",
70
- "what",
71
- "when",
72
- "where",
73
- "which",
74
- "would",
75
- "could",
76
- "should",
77
- "about",
78
- "into",
79
- "some",
80
- "them",
81
- "they",
82
- "been",
83
- "just",
84
- "want",
85
- "looking",
86
- "look",
87
- "help",
88
- "need",
89
- "recommend",
90
- "suggestions",
91
- "suggest",
92
- "advice",
93
- "think",
94
- "also",
95
- "did",
96
- "does",
97
- "do",
98
- "after",
99
- "before",
100
- "since",
101
- "between",
102
- "first",
103
- "earlier",
104
- "many",
105
- "days",
106
- "long",
107
- "how",
108
- "did",
109
- "take",
110
- "took",
111
- "it",
112
- "me",
113
- "my",
114
- "i",
115
- ]);
116
-
117
- const COMPARISON_GENERIC_SLOT_PREFIXES = new Set([
118
- "trip",
119
- "event",
120
- "device",
121
- "thing",
122
- "item",
123
- "time",
124
- "place",
125
- "activity",
126
- "experience",
127
- "job",
128
- "role",
129
- "project",
130
- "task",
131
- "purchase",
132
- "plan",
133
- "change",
134
- "decision",
135
- "move",
136
- "visit",
137
- "meeting",
138
- "session",
139
- ]);
140
-
141
- const COMPARISON_GENERIC_AFFILIATION_TERMS = new Set([
142
- ...COMPARISON_GENERIC_SLOT_PREFIXES,
143
- "solo",
144
- ]);
145
-
146
- const COMPARISON_FIRST_PERSON_CLAUSE_PATTERNS: RegExp[] = [
147
- /\bi\b/gi,
148
- /\bi'm\b/gi,
149
- /\bi've\b/gi,
150
- ];
151
-
152
- const COMPARISON_PROSPECTIVE_PERSONAL_PATTERNS: RegExp[] = [
153
- /\b(?:i\s+am\s+|i'm\s+)?considering\b/gi,
154
- /\b(?:i\s+am\s+|i'm\s+)?planning\b/gi,
155
- /\bi\s+am\s+thinking\s+about\b/gi,
156
- /\bi'?m\s+thinking\s+about\b/gi,
157
- /\bi\s+have\s+been\s+looking\b/gi,
158
- /\bi'?ve\s+been\s+looking\b/gi,
159
- /\bi\s+am\s+looking\s+(?:at|into)\b/gi,
160
- /\bi'?m\s+looking\s+(?:at|into)\b/gi,
161
- /\bi\s+want\s+to\b/gi,
162
- /\bi\s+would\s+like\s+to\b/gi,
163
- /\bi\s+am\s+hoping\s+to\b/gi,
164
- /\bi'?m\s+hoping\s+to\b/gi,
165
- /\bi\s+am\s+going\s+to\s+(?:visit|go|try|do)\b/gi,
166
- /\bi'?m\s+going\s+to\s+(?:visit|go|try|do)\b/gi,
167
- /\bi\s+am\s+trying\s+to\s+decide\b/gi,
168
- /\bi'?m\s+trying\s+to\s+decide\b/gi,
169
- /\bi\s+am\s+trying\s+to\s+(?:plan|figure)\b/gi,
170
- /\bi'?m\s+trying\s+to\s+(?:plan|figure)\b/gi,
171
- ];
172
-
173
- export interface TemporalQuerySignal {
174
- indicator: number;
175
- active: boolean;
176
- matchedPatterns: string[];
177
- }
178
-
179
- export interface TemporalSelectorGuardDecision {
180
- shouldApply: boolean;
181
- slots: string[];
182
- reason: string;
183
- }
184
-
185
- export interface TemporalRecoveryDebugCandidate {
186
- id: string;
187
- text: string;
188
- selected: boolean;
189
- temporalAnchorDensity: number;
190
- semanticScore: number;
191
- recencyScore: number;
192
- slotCoverage: number;
193
- slotMatches: string[];
194
- finalScore: number;
195
- rationale: string;
196
- comparisonSide?: 0 | 1 | null;
197
- comparisonSlot?: string;
198
- comparisonSlotRecall?: number;
199
- comparisonSlotPrecision?: number;
200
- comparisonSlotSpecificity?: number;
201
- comparisonSlotPositionWeightedRecall?: number;
202
- comparisonSlotPositionWeightedPrecision?: number;
203
- comparisonSlotPositionWeightedSpecificity?: number;
204
- comparisonFirstPersonClauseCount?: number;
205
- comparisonProspectivePersonalVerbCount?: number;
206
- comparisonPlanningDensity?: number;
207
- comparisonPastness?: number;
208
- comparisonSideWitnessScore?: number;
209
- }
210
-
211
- export interface TemporalRecoveryRankingResult {
212
- ranked: SearchResult[];
213
- debug: TemporalRecoveryDebugCandidate[];
214
- temporalQuery: TemporalQuerySignal;
215
- slots: string[];
216
- comparisonCoverageApplied?: boolean;
217
- comparisonCoverageSlots?: string[];
218
- comparisonCoverageMinTokens?: number;
219
- comparisonWitnessIds?: string[];
220
- comparisonProfile?: ComparisonProfileSummary;
221
- }
222
-
223
- let activeComparisonProfile: ComparisonProfileSummary | null = null;
224
- let activeComparisonExperimentConfig: ComparisonExperimentConfig | null = null;
225
-
226
- export function detectTemporalQuerySignal(queryText: string): TemporalQuerySignal {
227
- const matchedPatterns: string[] = [];
228
- let weightedMatches = 0;
229
-
230
- for (const entry of TEMPORAL_PATTERN_WEIGHTS) {
231
- if (entry.patterns.some((pattern) => pattern.test(queryText))) {
232
- matchedPatterns.push(entry.label);
233
- weightedMatches += entry.weight;
234
- }
235
- }
236
-
237
- const indicator = clamp01(weightedMatches / TEMPORAL_XI_NORM);
238
- return {
239
- indicator,
240
- active: indicator >= TEMPORAL_XI_THRESHOLD,
241
- matchedPatterns,
242
- };
243
- }
244
-
245
- export function getTemporalAnchorDensity(docKey: string, text: string): number {
246
- const cacheKey = `${docKey}\n${text}`;
247
- const cached = temporalAnchorCache.get(cacheKey);
248
- if (typeof cached === "number") {
249
- touchTemporalAnchorCache(cacheKey, cached);
250
- return cached;
251
- }
252
-
253
- const uniqueMatches = new Set<string>();
254
- for (const pattern of TEMPORAL_ANCHOR_PATTERNS) {
255
- for (const match of text.matchAll(pattern)) {
256
- const value = match[0]?.trim().toLowerCase();
257
- if (value) {
258
- uniqueMatches.add(value);
259
- }
260
- }
261
- }
262
-
263
- const density = clamp01(uniqueMatches.size / TEMPORAL_ANCHOR_NORM);
264
- touchTemporalAnchorCache(cacheKey, density);
265
- return density;
266
- }
267
-
268
- export function rankTemporalRecoveryCandidates(
269
- items: SearchResult[],
270
- opts: {
271
- queryText: string;
272
- maxSelected?: number;
273
- nowMs?: number;
274
- recencyLambda?: number;
275
- selectionTokenBudget?: number;
276
- },
277
- ): TemporalRecoveryRankingResult {
278
- const comparisonExperiment = resolveComparisonExperimentConfig();
279
- const temporalQuery = detectTemporalQuerySignal(opts.queryText);
280
- const slots = extractTemporalSlots(opts.queryText);
281
- const isComparisonQuery = temporalQuery.matchedPatterns.includes("first or earlier");
282
- const effectiveSlots = isComparisonQuery ? filterComparisonSlots(slots) : slots;
283
- const comparisonSlots = isComparisonQuery ? deriveComparisonSideSlots(effectiveSlots) : [];
284
- const recencyLambda = Math.max(0, opts.recencyLambda ?? 0.00001);
285
- const now = opts.nowMs ?? Date.now();
286
- const maxSelected = Math.max(1, Math.floor(opts.maxSelected ?? 3));
287
- const selectionTokenBudget = Math.max(0, Math.floor(opts.selectionTokenBudget ?? Number.MAX_SAFE_INTEGER));
288
- const comparisonProfile = comparisonExperiment.profilingEnabled && isComparisonQuery
289
- ? createComparisonProfileSummary(comparisonExperiment.ablationMode)
290
- : null;
291
- const previousProfile = activeComparisonProfile;
292
- const previousExperimentConfig = activeComparisonExperimentConfig;
293
- activeComparisonProfile = comparisonProfile;
294
- activeComparisonExperimentConfig = comparisonExperiment;
295
- const totalStart = comparisonProfile ? process.hrtime.bigint() : 0n;
296
-
297
- try {
298
- if (comparisonProfile) {
299
- comparisonProfile.rawCandidateCount = items.length;
300
- }
301
-
302
- const decorateStart = comparisonProfile ? process.hrtime.bigint() : 0n;
303
- const decorated = items.map((item) => {
304
- const semanticScore = clamp01(typeof item.finalScore === "number" ? item.finalScore : item.score ?? 0);
305
- const recencyScore = computeRecencyScore(item, now, recencyLambda);
306
- const temporalAnchorDensity = getTemporalAnchorDensity(
307
- `${typeof item.metadata.collection === "string" ? item.metadata.collection : "unknown"}::${item.id}`,
308
- item.text,
309
- );
310
- const { coverage, matches } = computeSlotCoverage(effectiveSlots, item.text);
311
- const tokenEstimate = estimateTokensWithProfile(item.text);
312
- const comparisonSide = comparisonSlots.length === 2
313
- ? computeComparisonSideAffiliation(comparisonSlots, item.text)
314
- : null;
315
- const comparisonMetrics = comparisonSide !== null
316
- ? computeComparisonSlotSpecificityMetrics(
317
- comparisonSlots[comparisonSide]!,
318
- item.text,
319
- comparisonSlots[1 - comparisonSide],
320
- )
321
- : null;
322
- const comparisonSideWitnessScore = comparisonMetrics
323
- ? comparisonMetrics.sideWitnessScore
324
- : undefined;
325
- const finalScore = clamp01(isComparisonQuery
326
- ? computeComparisonFinalScore({
327
- semanticScore,
328
- recencyScore,
329
- temporalAnchorDensity,
330
- coverage,
331
- comparisonSideWitnessScore,
332
- temporalQueryActive: temporalQuery.active,
333
- })
334
- : (0.40 * semanticScore) +
335
- (0.25 * recencyScore) +
336
- (0.20 * temporalAnchorDensity) +
337
- (0.15 * coverage) +
338
- (temporalQuery.active ? 0.05 : 0));
339
- return {
340
- item,
341
- semanticScore,
342
- recencyScore,
343
- temporalAnchorDensity,
344
- slotCoverage: coverage,
345
- slotMatches: matches,
346
- tokenEstimate,
347
- comparisonSide,
348
- comparisonMetrics,
349
- comparisonSideWitnessScore,
350
- finalScore,
351
- };
352
- });
353
- if (comparisonProfile) {
354
- comparisonProfile.decorateMs += elapsedMs(decorateStart);
355
- comparisonProfile.comparisonCandidateCount = decorated.filter((candidate) => candidate.comparisonSide !== null).length;
356
- comparisonProfile.side0AffiliatedCount = decorated.filter((candidate) => candidate.comparisonSide === 0).length;
357
- comparisonProfile.side1AffiliatedCount = decorated.filter((candidate) => candidate.comparisonSide === 1).length;
358
- }
359
-
360
- const selectedIDs = new Set<string>();
361
- const coveredSlots = new Set<string>();
362
- const selected: SearchResult[] = [];
363
- let comparisonCoverageApplied = false;
364
- let comparisonCoverageMinTokens: number | undefined;
365
- let comparisonWitnessIds: string[] | undefined;
366
-
367
- if (comparisonSlots.length === 2) {
368
- const sideCandidates = comparisonSlots.map((_, sideIndex) => {
369
- const candidates = decorated
370
- .filter((candidate) => candidate.comparisonSide === sideIndex);
371
- recordSort(candidates.length);
372
- return candidates.sort((left, right) => {
373
- const leftWitnessScore = left.comparisonSideWitnessScore ?? Number.NEGATIVE_INFINITY;
374
- const rightWitnessScore = right.comparisonSideWitnessScore ?? Number.NEGATIVE_INFINITY;
375
- if (rightWitnessScore !== leftWitnessScore) {
376
- return rightWitnessScore - leftWitnessScore;
377
- }
378
- if (right.finalScore !== left.finalScore) {
379
- return right.finalScore - left.finalScore;
380
- }
381
- return left.tokenEstimate - right.tokenEstimate;
382
- });
383
- });
384
- const cheapestSideTokenSum = sideCandidates.reduce((sum, candidates) => (
385
- candidates.length > 0 ? sum + candidates.reduce((best, candidate) => Math.min(best, candidate.tokenEstimate), Number.POSITIVE_INFINITY) : sum
386
- ), 0);
387
- if (sideCandidates.every((candidates) => candidates.length > 0) && Number.isFinite(cheapestSideTokenSum)) {
388
- comparisonCoverageMinTokens = cheapestSideTokenSum;
389
- const bestPair = findBestComparisonCoveragePair(sideCandidates[0], sideCandidates[1], selectionTokenBudget);
390
- if (bestPair) {
391
- for (const candidate of bestPair) {
392
- selectedIDs.add(candidate.item.id);
393
- for (const slot of candidate.slotMatches) {
394
- coveredSlots.add(slot);
395
- }
396
- selected.push({
397
- ...candidate.item,
398
- finalScore: candidate.finalScore,
399
- });
400
- }
401
- comparisonCoverageApplied = true;
402
- comparisonWitnessIds = [bestPair[0].item.id, bestPair[1].item.id];
403
- }
404
- }
405
- }
406
-
407
- const greedyStart = comparisonProfile ? process.hrtime.bigint() : 0n;
408
- for (let pass = selected.length; pass < maxSelected; pass += 1) {
409
- let best: (typeof decorated)[number] | null = null;
410
- let bestScore = Number.NEGATIVE_INFINITY;
411
-
412
- for (const candidate of decorated) {
413
- if (selectedIDs.has(candidate.item.id)) {
414
- continue;
415
- }
416
- const marginalCoverage =
417
- candidate.slotMatches.filter((slot) => !coveredSlots.has(slot)).length / Math.max(1, effectiveSlots.length);
418
- const combined = candidate.finalScore + (0.25 * marginalCoverage);
419
- if (combined > bestScore) {
420
- best = candidate;
421
- bestScore = combined;
422
- }
423
- }
424
-
425
- if (!best || bestScore < 0.12) {
426
- break;
427
- }
428
-
429
- selectedIDs.add(best.item.id);
430
- for (const slot of best.slotMatches) {
431
- coveredSlots.add(slot);
432
- }
433
- selected.push({
434
- ...best.item,
435
- finalScore: best.finalScore,
436
- });
437
- }
438
- if (comparisonProfile) {
439
- comparisonProfile.greedyFillMs += elapsedMs(greedyStart);
440
- }
441
-
442
- const remainingCandidates = decorated
443
- .filter((candidate) => !selectedIDs.has(candidate.item.id));
444
- recordSort(remainingCandidates.length);
445
- const remaining = remainingCandidates
446
- .sort((left, right) => right.finalScore - left.finalScore)
447
- .map((candidate) => ({
448
- ...candidate.item,
449
- finalScore: candidate.finalScore,
450
- }));
451
-
452
- const ranked = [...selected, ...remaining];
453
- const debugStart = comparisonProfile ? process.hrtime.bigint() : 0n;
454
- const debugCandidates = [...decorated];
455
- recordSort(debugCandidates.length);
456
- const debug = debugCandidates
457
- .sort((left, right) => right.finalScore - left.finalScore)
458
- .map((candidate) => ({
459
- id: candidate.item.id,
460
- text: candidate.item.text,
461
- selected: selectedIDs.has(candidate.item.id),
462
- temporalAnchorDensity: candidate.temporalAnchorDensity,
463
- semanticScore: candidate.semanticScore,
464
- recencyScore: candidate.recencyScore,
465
- slotCoverage: candidate.slotCoverage,
466
- slotMatches: candidate.slotMatches,
467
- finalScore: candidate.finalScore,
468
- rationale: buildTemporalRecoveryRationale(candidate.slotCoverage, candidate.temporalAnchorDensity, candidate.semanticScore),
469
- comparisonSide: candidate.comparisonSide,
470
- comparisonSlot: candidate.comparisonSide !== null ? comparisonSlots[candidate.comparisonSide] : undefined,
471
- comparisonSlotRecall: candidate.comparisonMetrics?.recall,
472
- comparisonSlotPrecision: candidate.comparisonMetrics?.precision,
473
- comparisonSlotSpecificity: candidate.comparisonMetrics?.specificity,
474
- comparisonSlotPositionWeightedRecall: candidate.comparisonMetrics?.positionWeightedRecall,
475
- comparisonSlotPositionWeightedPrecision: candidate.comparisonMetrics?.positionWeightedPrecision,
476
- comparisonSlotPositionWeightedSpecificity: candidate.comparisonMetrics?.positionWeightedSpecificity,
477
- comparisonFirstPersonClauseCount: candidate.comparisonMetrics?.firstPersonClauseCount,
478
- comparisonProspectivePersonalVerbCount: candidate.comparisonMetrics?.prospectivePersonalVerbCount,
479
- comparisonPlanningDensity: candidate.comparisonMetrics?.planningDensity,
480
- comparisonPastness: candidate.comparisonMetrics?.pastness,
481
- comparisonSideWitnessScore: candidate.comparisonSideWitnessScore,
482
- }));
483
- if (comparisonProfile) {
484
- comparisonProfile.debugBuildMs += elapsedMs(debugStart);
485
- comparisonProfile.rankTotalMs += elapsedMs(totalStart);
486
- }
487
-
488
- return {
489
- ranked,
490
- debug,
491
- temporalQuery,
492
- slots: effectiveSlots,
493
- comparisonCoverageApplied,
494
- comparisonCoverageSlots: comparisonSlots.length === 2 ? comparisonSlots : undefined,
495
- comparisonCoverageMinTokens,
496
- comparisonWitnessIds,
497
- comparisonProfile: comparisonProfile ?? undefined,
498
- };
499
- } finally {
500
- activeComparisonProfile = previousProfile;
501
- activeComparisonExperimentConfig = previousExperimentConfig;
502
- }
503
- }
504
-
505
- export function decideTemporalSelectorGuard(
506
- queryText: string,
507
- temporalQuery: TemporalQuerySignal = detectTemporalQuerySignal(queryText),
508
- ): TemporalSelectorGuardDecision {
509
- const slots = extractTemporalSlots(queryText);
510
- if (!temporalQuery.active) {
511
- return {
512
- shouldApply: false,
513
- slots,
514
- reason: "temporal query gate inactive",
515
- };
516
- }
517
-
518
- const strongCompositionalPattern = temporalQuery.matchedPatterns.some((pattern) =>
519
- pattern === "how many days" ||
520
- pattern === "how long" ||
521
- pattern === "before or after" ||
522
- pattern === "since or between"
523
- );
524
- const strongComparisonPattern = temporalQuery.matchedPatterns.some((pattern) =>
525
- pattern === "first or earlier"
526
- );
527
- if (!strongCompositionalPattern && !strongComparisonPattern) {
528
- return {
529
- shouldApply: false,
530
- slots,
531
- reason: "query lacks strong compositional temporal pattern",
532
- };
533
- }
534
-
535
- if (strongComparisonPattern && !strongCompositionalPattern) {
536
- if (slots.length < 1 || slots.length > 4) {
537
- return {
538
- shouldApply: false,
539
- slots,
540
- reason: "comparison query did not resolve to a sensible number of slots",
541
- };
542
- }
543
- return {
544
- shouldApply: true,
545
- slots,
546
- reason: "comparison query with temporal entity extraction",
547
- };
548
- }
549
-
550
- if (slots.length !== 2) {
551
- return {
552
- shouldApply: false,
553
- slots,
554
- reason: "query did not resolve to exactly two temporal slots",
555
- };
556
- }
557
-
558
- return {
559
- shouldApply: true,
560
- slots,
561
- reason: "strong temporal query with two-slot decomposition",
562
- };
563
- }
564
-
565
- export function resetTemporalCachesForTest(): void {
566
- temporalAnchorCache.clear();
567
- }
568
-
569
- function extractTemporalSlots(text: string): string[] {
570
- const clauses = splitTemporalClauses(text);
571
- const slots = new Set<string>();
572
-
573
- for (const clause of clauses) {
574
- const terms = normalizeTerms(clause)
575
- .filter((term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term));
576
- if (terms.length === 0) {
577
- continue;
578
- }
579
- if (terms.length <= 3) {
580
- slots.add(terms.join(" "));
581
- continue;
582
- }
583
- slots.add(terms.slice(0, 4).join(" "));
584
- slots.add(terms.slice(-4).join(" "));
585
- }
586
-
587
- if (slots.size === 0) {
588
- const fallback = normalizeTerms(text).filter((term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term));
589
- if (fallback.length > 0) {
590
- slots.add(fallback.slice(0, 4).join(" "));
591
- }
592
- }
593
-
594
- return [...slots].slice(0, 4);
595
- }
596
-
597
- function computeSlotCoverage(slots: string[], candidateText: string): { coverage: number; matches: string[] } {
598
- if (slots.length === 0) {
599
- return { coverage: 0, matches: [] };
600
- }
601
-
602
- const candidateTerms = new Set(normalizeTerms(candidateText));
603
- const matches: string[] = [];
604
- let covered = 0;
605
-
606
- for (const slot of slots) {
607
- const slotTerms = normalizeTerms(slot).filter((term) => term.length >= 3);
608
- if (slotTerms.length === 0) {
609
- continue;
610
- }
611
- const overlap = slotTerms.filter((term) => candidateTerms.has(term)).length / slotTerms.length;
612
- if (overlap >= 0.5) {
613
- covered += 1;
614
- matches.push(slot);
615
- }
616
- }
617
-
618
- return {
619
- coverage: covered / Math.max(1, slots.length),
620
- matches,
621
- };
622
- }
623
-
624
- function filterComparisonSlots(slots: string[]): string[] {
625
- const filtered = slots.filter((slot) => {
626
- const terms = normalizeTerms(slot).filter(
627
- (term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term),
628
- );
629
- if (terms.length === 0) {
630
- return false;
631
- }
632
- if (terms.length === 1 && COMPARISON_GENERIC_SLOT_PREFIXES.has(terms[0]!)) {
633
- return false;
634
- }
635
- return true;
636
- });
637
-
638
- return filtered.length >= 1 ? filtered : slots;
639
- }
640
-
641
- function deriveComparisonSideSlots(slots: string[]): string[] {
642
- const specificSlots = slots.filter((slot) => {
643
- const terms = normalizeTerms(slot).filter(
644
- (term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term),
645
- );
646
- if (terms.length === 0) {
647
- return false;
648
- }
649
- return !(terms.length <= 2 && COMPARISON_GENERIC_SLOT_PREFIXES.has(terms[0]!));
650
- });
651
- const usableSlots = specificSlots.length >= 2 ? specificSlots : slots;
652
- return usableSlots.slice(0, 2);
653
- }
654
-
655
- function computeComparisonSideScore(slots: string[], candidateText: string): number {
656
- if (slots.length < 2) {
657
- return 0;
658
- }
659
-
660
- const overlaps = slots
661
- .map((slot) => computeSlotCoverage([slot], candidateText).coverage)
662
- .sort((left, right) => right - left);
663
- const strongest = overlaps[0] ?? 0;
664
- const second = overlaps[1] ?? 0;
665
- const asymmetry = Math.abs(strongest - second);
666
-
667
- return clamp01((0.6 * asymmetry) + (0.4 * strongest));
668
- }
669
-
670
- function computeComparisonSideAffiliation(slots: string[], candidateText: string): 0 | 1 | null {
671
- const start = activeComparisonProfile ? process.hrtime.bigint() : 0n;
672
- try {
673
- if (slots.length < 2) {
674
- return null;
675
- }
676
-
677
- const leftCoverage = activeComparisonExperimentConfig?.disableDiscriminativeAffiliation
678
- ? computeSlotCoverage([slots[0]!], candidateText).coverage
679
- : computeComparisonAffiliationCoverage(slots[0]!, slots[1]!, candidateText);
680
- const rightCoverage = activeComparisonExperimentConfig?.disableDiscriminativeAffiliation
681
- ? computeSlotCoverage([slots[1]!], candidateText).coverage
682
- : computeComparisonAffiliationCoverage(slots[1]!, slots[0]!, candidateText);
683
- if (leftCoverage >= 0.5 && leftCoverage > rightCoverage) {
684
- return 0;
685
- }
686
- if (rightCoverage >= 0.5 && rightCoverage > leftCoverage) {
687
- return 1;
688
- }
689
- return null;
690
- } finally {
691
- if (activeComparisonProfile) {
692
- activeComparisonProfile.sideAffiliationMs += elapsedMs(start);
693
- }
694
- }
695
- }
696
-
697
- function computeComparisonAffiliationCoverage(slot: string, otherSlot: string, candidateText: string): number {
698
- const slotTerms = normalizeTerms(slot).filter(
699
- (term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term),
700
- );
701
- const otherTerms = new Set(normalizeTerms(otherSlot).filter(
702
- (term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term),
703
- ));
704
- const discriminativeTerms = slotTerms.filter((term) => (
705
- !COMPARISON_GENERIC_AFFILIATION_TERMS.has(term) && !otherTerms.has(term)
706
- ));
707
- if (discriminativeTerms.length === 0) {
708
- return 0;
709
- }
710
-
711
- const candidateTerms = new Set(normalizeTerms(candidateText));
712
- const matched = discriminativeTerms.filter((term) => candidateTerms.has(term)).length;
713
- return matched / discriminativeTerms.length;
714
- }
715
-
716
- function computeComparisonSlotSpecificityMetrics(
717
- slot: string,
718
- candidateText: string,
719
- otherSlot?: string,
720
- ): {
721
- recall: number;
722
- precision: number;
723
- specificity: number;
724
- positionWeightedRecall: number;
725
- positionWeightedPrecision: number;
726
- positionWeightedSpecificity: number;
727
- firstPersonClauseCount: number;
728
- prospectivePersonalVerbCount: number;
729
- planningDensity: number;
730
- pastness: number;
731
- sideWitnessScore: number;
732
- } {
733
- const start = activeComparisonProfile ? process.hrtime.bigint() : 0n;
734
- try {
735
- const slotTerms = normalizeContentTerms(slot);
736
- const candidateTerms = normalizeContentTerms(candidateText);
737
- const candidateTermSequence = normalizeContentTermSequence(candidateText);
738
- const firstPositions = buildFirstContentTermPositions(candidateTermSequence);
739
- const matchedTerms = [...slotTerms].filter((term) => candidateTerms.has(term));
740
- const matched = matchedTerms.length;
741
- const positionWeightedMatched = matchedTerms.reduce((sum, term) => {
742
- const position = firstPositions.get(term);
743
- if (typeof position !== "number") {
744
- return sum;
745
- }
746
- const earlyThreshold = candidateTermSequence.length * 0.3;
747
- return sum + (position < earlyThreshold ? 1 : 0.5);
748
- }, 0);
749
- const recall = slotTerms.size > 0 ? matched / slotTerms.size : 0;
750
- const precision = matched / Math.max(5, candidateTerms.size);
751
- const useSpecificity = slotTerms.size >= 2;
752
- const specificity = recall * precision;
753
- const positionWeightedRecall = slotTerms.size > 0 ? positionWeightedMatched / slotTerms.size : 0;
754
- const positionWeightedPrecision = positionWeightedMatched / Math.max(5, candidateTerms.size);
755
- const positionWeightedSpecificity = positionWeightedRecall * positionWeightedPrecision;
756
- const { firstPersonClauseCount, prospectivePersonalVerbCount, planningDensity, pastness } = computePastness(candidateText);
757
- const otherSlotTerms = otherSlot ? normalizeContentTerms(otherSlot) : new Set<string>();
758
- const otherMatched = otherSlotTerms.size > 0
759
- ? [...otherSlotTerms].filter((term) => candidateTerms.has(term)).length
760
- : 0;
761
- const otherSlotRecall = otherSlotTerms.size > 0 ? otherMatched / otherSlotTerms.size : 0;
762
- const purity = clamp01(1 - otherSlotRecall);
763
- const purityMultiplier = activeComparisonExperimentConfig?.disableContaminationPenalty
764
- ? 1
765
- : 0.7 + (0.3 * purity);
766
- const rawWitnessScore = useSpecificity
767
- ? (activeComparisonExperimentConfig?.disableWitnessPositionPastness
768
- ? specificity
769
- : positionWeightedSpecificity * Math.max(0.6, pastness))
770
- : recall;
771
- return {
772
- recall,
773
- precision,
774
- specificity,
775
- positionWeightedRecall,
776
- positionWeightedPrecision,
777
- positionWeightedSpecificity,
778
- firstPersonClauseCount,
779
- prospectivePersonalVerbCount,
780
- planningDensity,
781
- pastness,
782
- sideWitnessScore: clamp01(rawWitnessScore * purityMultiplier),
783
- };
784
- } finally {
785
- if (activeComparisonProfile) {
786
- activeComparisonProfile.specificityMs += elapsedMs(start);
787
- }
788
- }
789
- }
790
-
791
- function findBestComparisonCoveragePair(
792
- leftCandidates: Array<{
793
- item: SearchResult;
794
- finalScore: number;
795
- comparisonSideWitnessScore?: number;
796
- tokenEstimate: number;
797
- slotMatches: string[];
798
- }>,
799
- rightCandidates: Array<{
800
- item: SearchResult;
801
- finalScore: number;
802
- comparisonSideWitnessScore?: number;
803
- tokenEstimate: number;
804
- slotMatches: string[];
805
- }>,
806
- selectionTokenBudget: number,
807
- ): Array<{
808
- item: SearchResult;
809
- finalScore: number;
810
- comparisonSideWitnessScore?: number;
811
- tokenEstimate: number;
812
- slotMatches: string[];
813
- }> | null {
814
- const start = activeComparisonProfile ? process.hrtime.bigint() : 0n;
815
- let bestPair: Array<{
816
- item: SearchResult;
817
- finalScore: number;
818
- comparisonSideWitnessScore?: number;
819
- tokenEstimate: number;
820
- slotMatches: string[];
821
- }> | null = null;
822
- let bestPairScore = Number.NEGATIVE_INFINITY;
823
-
824
- for (const left of leftCandidates) {
825
- for (const right of rightCandidates) {
826
- if (left.item.id === right.item.id) {
827
- continue;
828
- }
829
- const totalTokens = left.tokenEstimate + right.tokenEstimate;
830
- if (totalTokens > selectionTokenBudget) {
831
- continue;
832
- }
833
- const leftWitness = activeComparisonExperimentConfig?.disablePairScoreOnWitness
834
- ? left.finalScore
835
- : (left.comparisonSideWitnessScore ?? left.finalScore);
836
- const rightWitness = activeComparisonExperimentConfig?.disablePairScoreOnWitness
837
- ? right.finalScore
838
- : (right.comparisonSideWitnessScore ?? right.finalScore);
839
- const pairScore = leftWitness + rightWitness;
840
- const currentTokens = bestPair ? bestPair[0]!.tokenEstimate + bestPair[1]!.tokenEstimate : Number.POSITIVE_INFINITY;
841
- if (pairScore > bestPairScore || (pairScore === bestPairScore && totalTokens < currentTokens)) {
842
- bestPair = [left, right];
843
- bestPairScore = pairScore;
844
- }
845
- }
846
- }
847
- if (activeComparisonProfile) {
848
- activeComparisonProfile.pairSelectionMs += elapsedMs(start);
849
- }
850
- return bestPair;
851
- }
852
-
853
- function buildTemporalRecoveryRationale(slotCoverage: number, anchorDensity: number, semanticScore: number): string {
854
- if (slotCoverage >= 0.5 && anchorDensity >= 0.5) {
855
- return "slot coverage and temporal anchors both supported this candidate";
856
- }
857
- if (slotCoverage >= 0.5) {
858
- return "slot coverage lifted this candidate toward the query's subevents";
859
- }
860
- if (anchorDensity >= 0.5) {
861
- return "temporal anchors lifted this candidate toward the query's date logic";
862
- }
863
- if (semanticScore >= 0.6) {
864
- return "semantic similarity kept this candidate in the temporal pool";
865
- }
866
- return "candidate remained in the bounded temporal recovery pool";
867
- }
868
-
869
- function computeRecencyScore(item: SearchResult, now: number, recencyLambda: number): number {
870
- const ts = typeof item.metadata.ts === "number" ? item.metadata.ts : now;
871
- const ageSeconds = Math.max(0, now - ts) / 1000;
872
- return Math.exp(-recencyLambda * ageSeconds);
873
- }
874
-
875
- function splitTemporalClauses(text: string): string[] {
876
- return text
877
- .split(/(?:\bafter\b|\bbefore\b|\bbetween\b|\bor\b|\band\b|\bthen\b|[?.!,;\n]+)/i)
878
- .map((part) => part.trim())
879
- .filter((part) => part.length > 0);
880
- }
881
-
882
- function normalizeTerms(text: string): string[] {
883
- if (activeComparisonProfile) {
884
- activeComparisonProfile.normalizeTermsCalls += 1;
885
- }
886
- return text
887
- .toLowerCase()
888
- .split(/[^a-z0-9_]+/i)
889
- .filter((term) => term.length > 0);
890
- }
891
-
892
- function normalizeContentTerms(text: string): Set<string> {
893
- if (activeComparisonProfile) {
894
- activeComparisonProfile.normalizeContentTermsCalls += 1;
895
- }
896
- return new Set(
897
- normalizeContentTermSequence(text),
898
- );
899
- }
900
-
901
- function normalizeContentTermSequence(text: string): string[] {
902
- if (activeComparisonProfile) {
903
- activeComparisonProfile.normalizeContentTermsCalls += 1;
904
- }
905
- return normalizeTerms(text).filter((term) => term.length >= 3 && !TEMPORAL_SLOT_STOPWORDS.has(term));
906
- }
907
-
908
- function buildFirstContentTermPositions(terms: string[]): Map<string, number> {
909
- const positions = new Map<string, number>();
910
- for (let index = 0; index < terms.length; index += 1) {
911
- const term = terms[index]!;
912
- if (!positions.has(term)) {
913
- positions.set(term, index);
914
- }
915
- }
916
- return positions;
917
- }
918
-
919
- function computeComparisonFinalScore({
920
- semanticScore,
921
- recencyScore,
922
- temporalAnchorDensity,
923
- coverage,
924
- comparisonSideWitnessScore,
925
- temporalQueryActive,
926
- }: {
927
- semanticScore: number;
928
- recencyScore: number;
929
- temporalAnchorDensity: number;
930
- coverage: number;
931
- comparisonSideWitnessScore?: number;
932
- temporalQueryActive: boolean;
933
- }): number {
934
- if (activeComparisonExperimentConfig?.disableComparisonBlend) {
935
- return (0.40 * semanticScore) +
936
- (0.25 * recencyScore) +
937
- (0.20 * temporalAnchorDensity) +
938
- (0.15 * coverage) +
939
- (temporalQueryActive ? 0.05 : 0);
940
- }
941
-
942
- return (0.15 * semanticScore) +
943
- (0.15 * recencyScore) +
944
- (0.15 * coverage) +
945
- (0.55 * (comparisonSideWitnessScore ?? 0));
946
- }
947
-
948
- function estimateTokensWithProfile(text: string): number {
949
- if (activeComparisonProfile) {
950
- activeComparisonProfile.estimateTokensCalls += 1;
951
- }
952
- return estimateTokens(text);
953
- }
954
-
955
- function recordSort(length: number): void {
956
- if (!activeComparisonProfile) {
957
- return;
958
- }
959
- activeComparisonProfile.sortCalls += 1;
960
- activeComparisonProfile.totalSortedLength += length;
961
- }
962
-
963
- function elapsedMs(start: bigint): number {
964
- return Number(process.hrtime.bigint() - start) / 1_000_000;
965
- }
966
-
967
- function computePastness(text: string): {
968
- firstPersonClauseCount: number;
969
- prospectivePersonalVerbCount: number;
970
- planningDensity: number;
971
- pastness: number;
972
- } {
973
- const lower = text.toLowerCase();
974
- const firstPersonClauseCount = Math.max(1, countPatternMatches(lower, COMPARISON_FIRST_PERSON_CLAUSE_PATTERNS));
975
- const prospectivePersonalVerbCount = countPatternMatches(lower, COMPARISON_PROSPECTIVE_PERSONAL_PATTERNS);
976
- const planningDensity = prospectivePersonalVerbCount / firstPersonClauseCount;
977
- return {
978
- firstPersonClauseCount,
979
- prospectivePersonalVerbCount,
980
- planningDensity,
981
- pastness: clamp01(1 - planningDensity),
982
- };
983
- }
984
-
985
- function countPatternMatches(text: string, patterns: RegExp[]): number {
986
- let count = 0;
987
- for (const pattern of patterns) {
988
- for (const _match of text.matchAll(pattern)) {
989
- count += 1;
990
- }
991
- }
992
- return count;
993
- }
994
-
995
- function touchTemporalAnchorCache(cacheKey: string, value: number): void {
996
- if (temporalAnchorCache.has(cacheKey)) {
997
- temporalAnchorCache.delete(cacheKey);
998
- }
999
- temporalAnchorCache.set(cacheKey, value);
1000
- if (temporalAnchorCache.size > TEMPORAL_ANCHOR_CACHE_MAX) {
1001
- const oldestKey = temporalAnchorCache.keys().next().value;
1002
- if (typeof oldestKey === "string") {
1003
- temporalAnchorCache.delete(oldestKey);
1004
- }
1005
- }
1006
- }
1007
-
1008
- function clamp01(value: number): number {
1009
- return Math.min(1, Math.max(0, value));
1010
- }