@oh-my-pi/pi-mnemopi 16.3.6 → 16.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/types/cli.d.ts +1 -1
- package/dist/types/config.d.ts +1 -1
- package/dist/types/core/beam/consolidate.d.ts +8 -2
- package/dist/types/core/beam/helpers.d.ts +4 -4
- package/dist/types/core/beam/index.d.ts +3 -3
- package/dist/types/core/beam/recall.d.ts +21 -1
- package/dist/types/core/beam/store.d.ts +1 -1
- package/dist/types/core/beam/types.d.ts +18 -0
- package/dist/types/core/binary-vectors.d.ts +3 -3
- package/dist/types/core/embeddings.d.ts +3 -3
- package/dist/types/core/episodic-graph.d.ts +1 -1
- package/dist/types/core/extraction.d.ts +28 -1
- package/dist/types/core/index.d.ts +6 -6
- package/dist/types/core/local-llm.d.ts +2 -2
- package/dist/types/core/memory.d.ts +5 -5
- package/dist/types/core/migrations/e6-triplestore-split.d.ts +1 -1
- package/dist/types/core/migrations/index.d.ts +1 -1
- package/dist/types/core/orchestrator.d.ts +2 -2
- package/dist/types/core/polyphonic-recall.d.ts +5 -5
- package/dist/types/core/query-cache.d.ts +1 -1
- package/dist/types/core/shmr.d.ts +1 -1
- package/dist/types/core/temporal-parser.d.ts +1 -1
- package/dist/types/core/triples.d.ts +1 -1
- package/dist/types/core/veracity-consolidation.d.ts +1 -1
- package/dist/types/dr/index.d.ts +1 -1
- package/dist/types/dr/recovery.d.ts +1 -1
- package/dist/types/index.d.ts +6 -6
- package/dist/types/mcp-server.d.ts +1 -1
- package/dist/types/migrations/e6-triplestore-split.d.ts +1 -1
- package/dist/types/migrations/index.d.ts +1 -1
- package/package.json +4 -4
- package/src/core/beam/consolidate.ts +76 -12
- package/src/core/beam/recall.ts +57 -8
- package/src/core/beam/store.ts +5 -5
- package/src/core/beam/types.ts +18 -0
- package/src/core/extraction.ts +184 -55
- package/src/core/memory.ts +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [16.3.7] - 2026-07-05
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Added `RecallOptions.contentPreviewChars` to allow customizing or disabling the content preview cap (default is 500, set to 0 for full content).
|
|
10
|
+
- Added `RecallResult.truncated` and `RecallResult.full_length` properties to easily identify clipped previews without parsing trailing markers.
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- Fixed background LLM fact extraction to preserve specific extractor categories (`instructions`, `preferences`, `timelines`, and `kg` triples) in MEMORIA tables and graph triples instead of flattening them into generic `fact/entity` rows.
|
|
15
|
+
- Improved recall previews and `factLine` context to append a trailing ellipsis (`…`) when content is clipped, preventing mid-word truncation without a marker.
|
|
16
|
+
|
|
5
17
|
## [16.3.5] - 2026-07-04
|
|
6
18
|
|
|
7
19
|
### Fixed
|
package/dist/types/cli.d.ts
CHANGED
package/dist/types/config.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Env, envBool, envDisabled, envFloat, envInt, envOneOf, envOptionalString, envString, envTruthy } from "./util/env";
|
|
1
|
+
import { type Env, envBool, envDisabled, envFloat, envInt, envOneOf, envOptionalString, envString, envTruthy } from "./util/env.js";
|
|
2
2
|
export type { Env };
|
|
3
3
|
export { envBool, envDisabled, envFloat, envInt, envOneOf, envOptionalString, envString, envTruthy };
|
|
4
4
|
export declare const DEFAULT_DATA_DIR: string;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type ExtractedFactCategories } from "../extraction.js";
|
|
2
|
+
import type { BeamMemoryState, BeamStats, JsonValue, MemoriaRetrieveResult, Metadata, SleepResult } from "./types.js";
|
|
2
3
|
type Row = Record<string, unknown>;
|
|
3
4
|
type FactCounts = {
|
|
4
5
|
metric: number;
|
|
@@ -18,7 +19,12 @@ type ConsolidateOptions = {
|
|
|
18
19
|
};
|
|
19
20
|
export declare function consolidateToEpisodic(beam: BeamMemoryState, summary: string, sourceWmIds: readonly string[], source?: string, importance?: number, options?: ConsolidateOptions): string;
|
|
20
21
|
export declare function detectLanguage(_beam: BeamMemoryState, text: string): string;
|
|
21
|
-
|
|
22
|
+
type StoreFactStringOptions = {
|
|
23
|
+
routeHeuristicCategories?: boolean;
|
|
24
|
+
};
|
|
25
|
+
export declare function storeFactStrings(beam: BeamMemoryState, facts: readonly string[], messageIdx?: number, sourceMemoryId?: string | null, importance?: number, options?: StoreFactStringOptions): number;
|
|
26
|
+
/** Store category-preserving LLM extraction output in MEMORIA and KG tables. */
|
|
27
|
+
export declare function storeExtractedFactCategories(beam: BeamMemoryState, extracted: ExtractedFactCategories, messageIdx?: number, sourceMemoryId?: string | null, importance?: number): number;
|
|
22
28
|
export declare function extractAndStoreFacts(beam: BeamMemoryState, content: string, messageIdx?: number, sourceMemoryId?: string | null): FactCounts;
|
|
23
29
|
export declare function memoriaRetrieve(beam: BeamMemoryState, query: string, ability?: string | null, topK?: number): MemoriaRetrieveResult;
|
|
24
30
|
export declare function getEpisodicStats(beam: BeamMemoryState, authorId?: string | null, authorType?: string | null, channelId?: string | null): BeamStats;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { sha256Hex16 } from "../../util/ids";
|
|
3
|
-
import { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, recallTokens } from "../../util/regex";
|
|
4
|
-
import type { BeamMemoryState, Metadata } from "./types";
|
|
2
|
+
import { sha256Hex16 } from "../../util/ids.js";
|
|
3
|
+
import { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, recallTokens } from "../../util/regex.js";
|
|
4
|
+
import type { BeamMemoryState, Metadata } from "./types.js";
|
|
5
5
|
export type Vector = number[];
|
|
6
6
|
export type HybridWeights = readonly [vecWeight: number, ftsWeight: number, importanceWeight: number];
|
|
7
7
|
export interface VectorDistanceResult {
|
|
@@ -48,7 +48,7 @@ export declare function normalizeMetadata(input: unknown): Metadata;
|
|
|
48
48
|
export declare function metadataJson(input: unknown): string;
|
|
49
49
|
export declare function detectLanguage(text: string): string;
|
|
50
50
|
export declare function memoryRowMetadata(row: unknown): Metadata;
|
|
51
|
-
export { cosineSimilarity, hammingDistance, informationTheoreticScore, maximallyInformativeBinarization, quantizeInt8, } from "../binary-vectors";
|
|
51
|
+
export { cosineSimilarity, hammingDistance, informationTheoreticScore, maximallyInformativeBinarization, quantizeInt8, } from "../binary-vectors.js";
|
|
52
52
|
export { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, recallTokens, sha256Hex16 };
|
|
53
53
|
/** Identifies one freshly stored memory whose embedding still needs to be derived. */
|
|
54
54
|
export interface EmbedItem {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import type { BeamCaches, BeamConfig, BeamEvent, BeamMemoryOptions, BeamMemoryState, BeamStats, ImportStats, MemoriaRetrieveResult, Metadata, RecallEnhancedOptions, RecallOptions, RecallResult, RememberBatchItem, RememberBatchOptions, RememberOptions, SleepResult } from "./types";
|
|
3
|
-
export { initBeam } from "./schema";
|
|
4
|
-
export type * from "./types";
|
|
2
|
+
import type { BeamCaches, BeamConfig, BeamEvent, BeamMemoryOptions, BeamMemoryState, BeamStats, ImportStats, MemoriaRetrieveResult, Metadata, RecallEnhancedOptions, RecallOptions, RecallResult, RememberBatchItem, RememberBatchOptions, RememberOptions, SleepResult } from "./types.js";
|
|
3
|
+
export { initBeam } from "./schema.js";
|
|
4
|
+
export type * from "./types.js";
|
|
5
5
|
export declare class BeamMemory implements BeamMemoryState {
|
|
6
6
|
#private;
|
|
7
7
|
readonly db: Database;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BeamMemoryState, RecallEnhancedOptions, RecallOptions, RecallResult } from "./types";
|
|
1
|
+
import type { BeamMemoryState, RecallEnhancedOptions, RecallOptions, RecallResult } from "./types.js";
|
|
2
2
|
type RecallOptionsInternal = RecallOptions & {
|
|
3
3
|
source?: string | null;
|
|
4
4
|
topic?: string | null;
|
|
@@ -23,6 +23,26 @@ type FactRecallResult = RecallResult & {
|
|
|
23
23
|
subject?: string;
|
|
24
24
|
predicate?: string;
|
|
25
25
|
};
|
|
26
|
+
/**
|
|
27
|
+
* Default per-result content preview cap enforced by {@link recall}. Content
|
|
28
|
+
* longer than this is clipped and the last character replaced with `…` so
|
|
29
|
+
* callers see the truncation; the full row remains reachable via
|
|
30
|
+
* `Mnemopi.get()` (and, in the coding-agent, `memory://<id>`). Overridable per
|
|
31
|
+
* call via {@link RecallOptions.contentPreviewChars}.
|
|
32
|
+
*/
|
|
33
|
+
export declare const RECALL_CONTENT_PREVIEW_CHARS = 500;
|
|
34
|
+
/**
|
|
35
|
+
* Clip `content` to at most `limit` characters, replacing the tail with `…`
|
|
36
|
+
* when truncated so agents can distinguish a preview from a full row. Returns
|
|
37
|
+
* the original string (and `truncated: false`) when the limit is 0/negative or
|
|
38
|
+
* the content already fits. The single `…` occupies one character of the cap,
|
|
39
|
+
* so a 500-char cap yields at most 499 real characters plus the marker.
|
|
40
|
+
*/
|
|
41
|
+
export declare function clipRecallContent(content: string, limit?: number): {
|
|
42
|
+
content: string;
|
|
43
|
+
truncated: boolean;
|
|
44
|
+
fullLength: number;
|
|
45
|
+
};
|
|
26
46
|
export declare function parseQueryTime(value: RecallOptionsInternal["queryTime"]): Date;
|
|
27
47
|
export declare function temporalBoost(timestamp: unknown, queryTime: Date, halfLifeHours: number): number;
|
|
28
48
|
export declare function recall(beam: BeamMemoryState, query: string, topK?: number, options?: RecallOptionsInternal): Promise<RecallResult[]>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BeamMemoryState, BeamStats, ImportStats, RememberBatchItem, RememberBatchOptions, RememberOptions } from "./types";
|
|
1
|
+
import type { BeamMemoryState, BeamStats, ImportStats, RememberBatchItem, RememberBatchOptions, RememberOptions } from "./types.js";
|
|
2
2
|
type Row = Record<string, unknown>;
|
|
3
3
|
type StoreRememberOptions = RememberOptions & {
|
|
4
4
|
memoryId?: string;
|
|
@@ -148,6 +148,15 @@ export interface RecallOptions {
|
|
|
148
148
|
useIntent?: boolean;
|
|
149
149
|
useMmr?: boolean;
|
|
150
150
|
mmrLambda?: number;
|
|
151
|
+
/**
|
|
152
|
+
* Maximum characters of `content` returned per {@link RecallResult}. When the
|
|
153
|
+
* stored content exceeds this, the preview is clipped and the trailing
|
|
154
|
+
* character is replaced with `…` so callers can see it was truncated. The
|
|
155
|
+
* full row is always reachable via {@link BeamMemoryState.get}. `0` or a
|
|
156
|
+
* negative value disables clipping. Defaults to
|
|
157
|
+
* {@link RECALL_CONTENT_PREVIEW_CHARS} (500).
|
|
158
|
+
*/
|
|
159
|
+
contentPreviewChars?: number;
|
|
151
160
|
}
|
|
152
161
|
export interface RecallEnhancedOptions extends RecallOptions {
|
|
153
162
|
useCache?: boolean;
|
|
@@ -206,6 +215,15 @@ export type RecallResult = RecallRowFields & {
|
|
|
206
215
|
[key: string]: unknown;
|
|
207
216
|
id: string;
|
|
208
217
|
content: string;
|
|
218
|
+
/**
|
|
219
|
+
* True when {@link content} is a clipped preview of the stored row. The
|
|
220
|
+
* clip is capped at {@link RecallOptions.contentPreviewChars} (default 500)
|
|
221
|
+
* and the last character is replaced with `…`. Fetch the full row via
|
|
222
|
+
* `memory://<id>` (mnemopi backend) or the `Mnemopi.get(id)` API.
|
|
223
|
+
*/
|
|
224
|
+
truncated?: boolean;
|
|
225
|
+
/** Original character count of `content` before {@link truncated} clipping. */
|
|
226
|
+
full_length?: number;
|
|
209
227
|
score?: number;
|
|
210
228
|
distance?: number;
|
|
211
229
|
rank?: number;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { type VecType } from "../config";
|
|
3
|
-
import { type DatabasePath } from "../db";
|
|
4
|
-
export { cosineSimilarity } from "./vector-math";
|
|
2
|
+
import { type VecType } from "../config.js";
|
|
3
|
+
import { type DatabasePath } from "../db.js";
|
|
4
|
+
export { cosineSimilarity } from "./vector-math.js";
|
|
5
5
|
export declare const BITS_PER_BYTE = 8;
|
|
6
6
|
export declare const EMBEDDING_DIM: number;
|
|
7
7
|
export declare const BYTES_PER_VECTOR: number;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { EmbeddingModel } from "fastembed";
|
|
2
|
-
import { type EmbeddingOutput } from "./runtime-options";
|
|
3
|
-
export type { EmbeddingOutput } from "./runtime-options";
|
|
4
|
-
export { cosineSimilarity } from "./vector-math";
|
|
2
|
+
import { type EmbeddingOutput } from "./runtime-options.js";
|
|
3
|
+
export type { EmbeddingOutput } from "./runtime-options.js";
|
|
4
|
+
export { cosineSimilarity } from "./vector-math.js";
|
|
5
5
|
export type Vector = Float32Array;
|
|
6
6
|
export type EmbeddingMatrix = Vector[];
|
|
7
7
|
export interface EmbeddingProvider {
|
|
@@ -1,7 +1,34 @@
|
|
|
1
|
-
import { type RemoteLlmOptions } from "./local-llm";
|
|
1
|
+
import { type RemoteLlmOptions } from "./local-llm.js";
|
|
2
2
|
export declare const EXTRACTION_PROMPT_TEMPLATE: string;
|
|
3
3
|
export declare function buildExtractionPrompt(text: string, detectedLang?: string): string;
|
|
4
|
+
/** Parsed knowledge-graph edge emitted by the extractor LLM. */
|
|
5
|
+
export interface ExtractedKgTriple {
|
|
6
|
+
subject: string;
|
|
7
|
+
predicate: string;
|
|
8
|
+
object: string;
|
|
9
|
+
}
|
|
10
|
+
/** Category-preserving extraction result used by background memory routing. */
|
|
11
|
+
export interface ExtractedFactCategories {
|
|
12
|
+
facts: string[];
|
|
13
|
+
instructions: string[];
|
|
14
|
+
preferences: string[];
|
|
15
|
+
timelines: string[];
|
|
16
|
+
kg: ExtractedKgTriple[];
|
|
17
|
+
}
|
|
18
|
+
/** Flatten extracted string categories for legacy fact callers. */
|
|
19
|
+
export declare function flattenExtractedFactCategories(extracted: ExtractedFactCategories): string[];
|
|
20
|
+
/** Count string facts plus KG triples in a category-preserving extraction result. */
|
|
21
|
+
export declare function countExtractedFactCategories(extracted: ExtractedFactCategories): number;
|
|
22
|
+
/** Parse extractor output without discarding MEMORIA categories or KG triples. */
|
|
23
|
+
export declare function parseExtractedFactCategories(rawOutput: string | null | undefined): ExtractedFactCategories;
|
|
24
|
+
/** Parse extractor output into the legacy flat string fact list. */
|
|
4
25
|
export declare function parseFacts(rawOutput: string | null | undefined): string[];
|
|
5
26
|
export declare function heuristicExtractFacts(text: string): string[];
|
|
27
|
+
/** Extract fact categories from text using configured, host, local, or remote LLMs. */
|
|
28
|
+
export declare function extractFactCategories(text: string | null | undefined, options?: RemoteLlmOptions): Promise<ExtractedFactCategories>;
|
|
29
|
+
/** Extract legacy flat fact strings from text. */
|
|
6
30
|
export declare function extractFacts(text: string | null | undefined, options?: RemoteLlmOptions): Promise<string[]>;
|
|
31
|
+
/** Safely extract category-preserving facts, swallowing best-effort failures. */
|
|
32
|
+
export declare function extractFactCategoriesSafe(text: string | null | undefined): Promise<ExtractedFactCategories>;
|
|
33
|
+
/** Safely extract legacy flat fact strings, swallowing best-effort failures. */
|
|
7
34
|
export declare function extractFactsSafe(text: string | null | undefined): Promise<string[]>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { configureRecallFeatures, type RecallFeatureFlags } from "../config";
|
|
2
|
-
export * from "./banks";
|
|
3
|
-
export * from "./beam/index";
|
|
4
|
-
export { type LocalEmbeddingModel, type LocalModelInitializer, type LocalModelInitOptions, type StandardEmbeddingModel, setLocalModelInitializer, } from "./embeddings";
|
|
5
|
-
export * from "./memory";
|
|
6
|
-
export { addMemory, forget, get, getBank, getContext, getDefaultInstance, getStats, Mnemopi, query, recall, recallEnhanced, remember, resetDefaultInstanceForTests, resetMemoryForTests, resetModuleStateForTests, saveMemory, scratchpadClear, scratchpadRead, scratchpadWrite, search, setBank, sleep, sleepAllSessions, storeMemory, update, } from "./memory";
|
|
1
|
+
export { configureRecallFeatures, type RecallFeatureFlags } from "../config.js";
|
|
2
|
+
export * from "./banks.js";
|
|
3
|
+
export * from "./beam/index.js";
|
|
4
|
+
export { type LocalEmbeddingModel, type LocalModelInitializer, type LocalModelInitOptions, type StandardEmbeddingModel, setLocalModelInitializer, } from "./embeddings.js";
|
|
5
|
+
export * from "./memory.js";
|
|
6
|
+
export { addMemory, forget, get, getBank, getContext, getDefaultInstance, getStats, Mnemopi, query, recall, recallEnhanced, remember, resetDefaultInstanceForTests, resetMemoryForTests, resetModuleStateForTests, saveMemory, scratchpadClear, scratchpadRead, scratchpadWrite, search, setBank, sleep, sleepAllSessions, storeMemory, update, } from "./memory.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type FetchImpl } from "@oh-my-pi/pi-ai";
|
|
2
|
-
import { type CompleteOptions } from "./llm-backends";
|
|
3
|
-
import { type MnemopiLlmCompleteOptions } from "./runtime-options";
|
|
2
|
+
import { type CompleteOptions } from "./llm-backends.js";
|
|
3
|
+
import { type MnemopiLlmCompleteOptions } from "./runtime-options.js";
|
|
4
4
|
export interface RemoteLlmOptions {
|
|
5
5
|
fetch?: FetchImpl;
|
|
6
6
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
import type { Api, ApiKey, Model } from "@oh-my-pi/pi-ai";
|
|
3
|
-
import type { MemoryInput, Metadata } from "../types";
|
|
4
|
-
import { BeamMemory } from "./beam/index";
|
|
5
|
-
import type { RecallEnhancedOptions, RecallOptions, RecallResult, SleepResult } from "./beam/types";
|
|
6
|
-
import { type MnemopiEmbeddingRuntimeOptions, type MnemopiLlmCompletion, type MnemopiLlmRuntimeOptions, type ResolvedMnemopiRuntimeOptions } from "./runtime-options";
|
|
3
|
+
import type { MemoryInput, Metadata } from "../types.js";
|
|
4
|
+
import { BeamMemory } from "./beam/index.js";
|
|
5
|
+
import type { RecallEnhancedOptions, RecallOptions, RecallResult, SleepResult } from "./beam/types.js";
|
|
6
|
+
import { type MnemopiEmbeddingRuntimeOptions, type MnemopiLlmCompletion, type MnemopiLlmRuntimeOptions, type ResolvedMnemopiRuntimeOptions } from "./runtime-options.js";
|
|
7
7
|
export interface MnemopiOptions {
|
|
8
8
|
readonly db?: Database;
|
|
9
9
|
readonly dbPath?: string;
|
|
@@ -182,5 +182,5 @@ export declare function query(query: string, topK?: number, options?: ModuleReca
|
|
|
182
182
|
export declare function resetDefaultInstanceForTests(): void;
|
|
183
183
|
export declare function resetMemoryForTests(): void;
|
|
184
184
|
export declare function resetModuleStateForTests(): void;
|
|
185
|
-
export type { MemoryInput, MemoryStats } from "../types";
|
|
185
|
+
export type { MemoryInput, MemoryStats } from "../types.js";
|
|
186
186
|
export default Mnemopi;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { type DatabasePath } from "../../db";
|
|
2
|
+
import { type DatabasePath } from "../../db.js";
|
|
3
3
|
export declare const ANNOTATION_KINDS: readonly ["mentions", "fact", "occurred_on", "has_source"];
|
|
4
4
|
export type AnnotationKind = (typeof ANNOTATION_KINDS)[number];
|
|
5
5
|
export interface MigrationOptions {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from "./e6-triplestore-split";
|
|
1
|
+
export * from "./e6-triplestore-split.js";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { BeamMemoryState, RecallOptions, RecallResult } from "./beam/types";
|
|
2
|
-
import { type PolyphonicMemoryResult, type PolyphonicRecallOptions } from "./polyphonic-recall";
|
|
1
|
+
import type { BeamMemoryState, RecallOptions, RecallResult } from "./beam/types.js";
|
|
2
|
+
import { type PolyphonicMemoryResult, type PolyphonicRecallOptions } from "./polyphonic-recall.js";
|
|
3
3
|
export interface OrchestratorBeam extends BeamMemoryState {
|
|
4
4
|
recall?: (query: string, topK?: number, options?: RecallOptions) => Promise<RecallResult[]>;
|
|
5
5
|
recallEnhanced?: (query: string, topK?: number, options?: RecallOptions) => Promise<RecallResult[]>;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { type Env } from "../config";
|
|
3
|
-
import { type DatabasePath } from "../db";
|
|
4
|
-
import type { BeamMemoryState, JsonValue, Metadata, RecallResult } from "./beam/types";
|
|
5
|
-
import { EpisodicGraph } from "./episodic-graph";
|
|
6
|
-
import { VeracityConsolidator } from "./veracity-consolidation";
|
|
2
|
+
import { type Env } from "../config.js";
|
|
3
|
+
import { type DatabasePath } from "../db.js";
|
|
4
|
+
import type { BeamMemoryState, JsonValue, Metadata, RecallResult } from "./beam/types.js";
|
|
5
|
+
import { EpisodicGraph } from "./episodic-graph.js";
|
|
6
|
+
import { VeracityConsolidator } from "./veracity-consolidation.js";
|
|
7
7
|
export type PolyphonicVoice = "vector" | "graph" | "fact" | "temporal";
|
|
8
8
|
export interface VoiceRecallResult {
|
|
9
9
|
readonly memoryId: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { cosineSimilarity } from "./vector-math";
|
|
2
|
+
import { cosineSimilarity } from "./vector-math.js";
|
|
3
3
|
export { cosineSimilarity };
|
|
4
4
|
export declare const SHMR_BATCH_SIZE: number;
|
|
5
5
|
export declare const SHMR_MAX_ITERATIONS: number;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type QueryTime } from "../util/datetime";
|
|
1
|
+
import { type QueryTime } from "../util/datetime.js";
|
|
2
2
|
export type DatePrecision = "day" | "week" | "month" | "year" | "relative" | "unknown";
|
|
3
3
|
export type ParsedNaturalDate = [eventDate: Date, precision: Exclude<DatePrecision, "unknown">, temporalTags: string[]];
|
|
4
4
|
export interface TemporalInfo {
|
package/dist/types/dr/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from "./recovery";
|
|
1
|
+
export * from "./recovery.js";
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { configureRecallFeatures, type RecallFeatureFlags } from "./config";
|
|
2
|
-
export * from "./core/beam/index";
|
|
3
|
-
export * from "./core/embeddings";
|
|
4
|
-
export * from "./core/llm-backends";
|
|
5
|
-
export * from "./core/memory";
|
|
6
|
-
export { addMemory, flushExtractions, forget, get, getBank, getContext, getDefaultInstance, getStats, Mnemopi, query, recall, recallEnhanced, remember, resetDefaultInstanceForTests, resetMemoryForTests, resetModuleStateForTests, saveMemory, scratchpadClear, scratchpadRead, scratchpadWrite, search, setBank, sleep, sleepAllSessions, storeMemory, update, } from "./core/memory";
|
|
1
|
+
export { configureRecallFeatures, type RecallFeatureFlags } from "./config.js";
|
|
2
|
+
export * from "./core/beam/index.js";
|
|
3
|
+
export * from "./core/embeddings.js";
|
|
4
|
+
export * from "./core/llm-backends.js";
|
|
5
|
+
export * from "./core/memory.js";
|
|
6
|
+
export { addMemory, flushExtractions, forget, get, getBank, getContext, getDefaultInstance, getStats, Mnemopi, query, recall, recallEnhanced, remember, resetDefaultInstanceForTests, resetMemoryForTests, resetModuleStateForTests, saveMemory, scratchpadClear, scratchpadRead, scratchpadWrite, search, setBank, sleep, sleepAllSessions, storeMemory, update, } from "./core/memory.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from "../core/migrations/e6-triplestore-split";
|
|
1
|
+
export * from "../core/migrations/e6-triplestore-split.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from "./e6-triplestore-split";
|
|
1
|
+
export * from "./e6-triplestore-split.js";
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/pi-mnemopi",
|
|
4
|
-
"version": "16.3.
|
|
4
|
+
"version": "16.3.7",
|
|
5
5
|
"description": "Local SQLite memory engine for Oh My Pi agents",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -39,9 +39,9 @@
|
|
|
39
39
|
"fmt": "biome format --write ."
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@oh-my-pi/pi-ai": "16.3.
|
|
43
|
-
"@oh-my-pi/pi-catalog": "16.3.
|
|
44
|
-
"@oh-my-pi/pi-utils": "16.3.
|
|
42
|
+
"@oh-my-pi/pi-ai": "16.3.7",
|
|
43
|
+
"@oh-my-pi/pi-catalog": "16.3.7",
|
|
44
|
+
"@oh-my-pi/pi-utils": "16.3.7",
|
|
45
45
|
"lru-cache": "11.5.1"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
@@ -3,7 +3,7 @@ import { generateId, stableMemoryId } from "../../util/ids";
|
|
|
3
3
|
import { aaakEncode } from "../aaak";
|
|
4
4
|
import { REGEX_EXTRACTION_MAX_INPUT_CHARS } from "../entities";
|
|
5
5
|
import { EpisodicGraph } from "../episodic-graph";
|
|
6
|
-
import { heuristicExtractFacts } from "../extraction";
|
|
6
|
+
import { type ExtractedFactCategories, heuristicExtractFacts } from "../extraction";
|
|
7
7
|
import { clampVeracity } from "../veracity-consolidation";
|
|
8
8
|
import { scheduleEmbedding } from "./helpers";
|
|
9
9
|
import type { BeamMemoryState, BeamStats, JsonValue, MemoriaRetrieveResult, Metadata, SleepResult } from "./types";
|
|
@@ -292,7 +292,7 @@ function insertFactRows(
|
|
|
292
292
|
function insertTimeline(
|
|
293
293
|
beam: BeamMemoryState,
|
|
294
294
|
messageIdx: number,
|
|
295
|
-
date: string,
|
|
295
|
+
date: string | null,
|
|
296
296
|
description: string,
|
|
297
297
|
sourceMemoryId: string | null,
|
|
298
298
|
): void {
|
|
@@ -327,6 +327,38 @@ function insertKg(
|
|
|
327
327
|
});
|
|
328
328
|
}
|
|
329
329
|
|
|
330
|
+
function insertPreference(
|
|
331
|
+
beam: BeamMemoryState,
|
|
332
|
+
messageIdx: number,
|
|
333
|
+
preference: string,
|
|
334
|
+
topic: string | null,
|
|
335
|
+
sourceMemoryId: string | null,
|
|
336
|
+
): void {
|
|
337
|
+
beam.db.run(
|
|
338
|
+
`INSERT INTO memoria_preferences (session_id, message_idx, preference, topic, evolution, context_snippet, source_memory_id)
|
|
339
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
340
|
+
[sourceSession(beam), messageIdx, preference, topic, null, preference, sourceMemoryId],
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
function insertInstruction(
|
|
345
|
+
beam: BeamMemoryState,
|
|
346
|
+
messageIdx: number,
|
|
347
|
+
instruction: string,
|
|
348
|
+
context: string,
|
|
349
|
+
sourceMemoryId: string | null,
|
|
350
|
+
): void {
|
|
351
|
+
beam.db.run(
|
|
352
|
+
`INSERT INTO memoria_instructions (session_id, message_idx, instruction, active, topic, context_snippet, source_memory_id)
|
|
353
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
354
|
+
[sourceSession(beam), messageIdx, instruction, 1, null, context, sourceMemoryId],
|
|
355
|
+
);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function timelineDate(description: string): string | null {
|
|
359
|
+
return /\b\d{4}-\d{2}-\d{2}\b/.exec(description)?.[0] ?? null;
|
|
360
|
+
}
|
|
361
|
+
|
|
330
362
|
/**
|
|
331
363
|
* Populate the episodic graph (gists, facts, edges) for a freshly consolidated
|
|
332
364
|
* memory. The graph backs Polyphonic Recall's `graph` voice, which relies on
|
|
@@ -458,36 +490,68 @@ export function detectLanguage(_beam: BeamMemoryState, text: string): string {
|
|
|
458
490
|
}
|
|
459
491
|
return spanish >= 3 ? "es" : "en";
|
|
460
492
|
}
|
|
493
|
+
type StoreFactStringOptions = {
|
|
494
|
+
routeHeuristicCategories?: boolean;
|
|
495
|
+
};
|
|
496
|
+
|
|
461
497
|
export function storeFactStrings(
|
|
462
498
|
beam: BeamMemoryState,
|
|
463
499
|
facts: readonly string[],
|
|
464
500
|
messageIdx = 0,
|
|
465
501
|
sourceMemoryId: string | null = null,
|
|
466
502
|
importance = 0.7,
|
|
503
|
+
options: StoreFactStringOptions = {},
|
|
467
504
|
): number {
|
|
505
|
+
const routeHeuristicCategories = options.routeHeuristicCategories ?? true;
|
|
468
506
|
let stored = 0;
|
|
469
507
|
for (const fact of facts) {
|
|
470
508
|
insertFactRows(beam, messageIdx, "entity", "fact", fact, fact, importance, sourceMemoryId);
|
|
471
509
|
stored++;
|
|
510
|
+
if (!routeHeuristicCategories) continue;
|
|
472
511
|
const pref = /^The user (prefers|dislikes) (.+)$/i.exec(fact);
|
|
473
512
|
if (pref?.[2]) {
|
|
474
|
-
beam
|
|
475
|
-
`INSERT INTO memoria_preferences (session_id, message_idx, preference, topic, evolution, context_snippet, source_memory_id)
|
|
476
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
477
|
-
[sourceSession(beam), messageIdx, fact, pref[2], null, fact, sourceMemoryId],
|
|
478
|
-
);
|
|
513
|
+
insertPreference(beam, messageIdx, fact, pref[2], sourceMemoryId);
|
|
479
514
|
}
|
|
480
515
|
const instruction = /^Instruction: (.+)$/i.exec(fact);
|
|
481
516
|
if (instruction?.[1]) {
|
|
482
|
-
beam
|
|
483
|
-
`INSERT INTO memoria_instructions (session_id, message_idx, instruction, active, topic, context_snippet, source_memory_id)
|
|
484
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
485
|
-
[sourceSession(beam), messageIdx, instruction[1], 1, null, fact, sourceMemoryId],
|
|
486
|
-
);
|
|
517
|
+
insertInstruction(beam, messageIdx, instruction[1], fact, sourceMemoryId);
|
|
487
518
|
}
|
|
488
519
|
}
|
|
489
520
|
return stored;
|
|
490
521
|
}
|
|
522
|
+
|
|
523
|
+
/** Store category-preserving LLM extraction output in MEMORIA and KG tables. */
|
|
524
|
+
export function storeExtractedFactCategories(
|
|
525
|
+
beam: BeamMemoryState,
|
|
526
|
+
extracted: ExtractedFactCategories,
|
|
527
|
+
messageIdx = 0,
|
|
528
|
+
sourceMemoryId: string | null = null,
|
|
529
|
+
importance = 0.7,
|
|
530
|
+
): number {
|
|
531
|
+
let stored = storeFactStrings(beam, extracted.facts, messageIdx, sourceMemoryId, importance);
|
|
532
|
+
stored += storeFactStrings(beam, extracted.instructions, messageIdx, sourceMemoryId, importance, {
|
|
533
|
+
routeHeuristicCategories: false,
|
|
534
|
+
});
|
|
535
|
+
stored += storeFactStrings(beam, extracted.preferences, messageIdx, sourceMemoryId, importance, {
|
|
536
|
+
routeHeuristicCategories: false,
|
|
537
|
+
});
|
|
538
|
+
stored += storeFactStrings(beam, extracted.timelines, messageIdx, sourceMemoryId, importance, {
|
|
539
|
+
routeHeuristicCategories: false,
|
|
540
|
+
});
|
|
541
|
+
for (const instruction of extracted.instructions) {
|
|
542
|
+
insertInstruction(beam, messageIdx, instruction, instruction, sourceMemoryId);
|
|
543
|
+
}
|
|
544
|
+
for (const preference of extracted.preferences) {
|
|
545
|
+
insertPreference(beam, messageIdx, preference, null, sourceMemoryId);
|
|
546
|
+
}
|
|
547
|
+
for (const timeline of extracted.timelines) {
|
|
548
|
+
insertTimeline(beam, messageIdx, timelineDate(timeline), timeline, sourceMemoryId);
|
|
549
|
+
}
|
|
550
|
+
for (const triple of extracted.kg) {
|
|
551
|
+
insertKg(beam, messageIdx, triple.subject, triple.predicate, triple.object, sourceMemoryId);
|
|
552
|
+
}
|
|
553
|
+
return stored;
|
|
554
|
+
}
|
|
491
555
|
export function extractAndStoreFacts(
|
|
492
556
|
beam: BeamMemoryState,
|
|
493
557
|
content: string,
|
package/src/core/beam/recall.ts
CHANGED
|
@@ -69,6 +69,34 @@ const VERACITY_WEIGHTS: Record<string, number> = {
|
|
|
69
69
|
false: 0,
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Default per-result content preview cap enforced by {@link recall}. Content
|
|
74
|
+
* longer than this is clipped and the last character replaced with `…` so
|
|
75
|
+
* callers see the truncation; the full row remains reachable via
|
|
76
|
+
* `Mnemopi.get()` (and, in the coding-agent, `memory://<id>`). Overridable per
|
|
77
|
+
* call via {@link RecallOptions.contentPreviewChars}.
|
|
78
|
+
*/
|
|
79
|
+
export const RECALL_CONTENT_PREVIEW_CHARS = 500;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Clip `content` to at most `limit` characters, replacing the tail with `…`
|
|
83
|
+
* when truncated so agents can distinguish a preview from a full row. Returns
|
|
84
|
+
* the original string (and `truncated: false`) when the limit is 0/negative or
|
|
85
|
+
* the content already fits. The single `…` occupies one character of the cap,
|
|
86
|
+
* so a 500-char cap yields at most 499 real characters plus the marker.
|
|
87
|
+
*/
|
|
88
|
+
export function clipRecallContent(
|
|
89
|
+
content: string,
|
|
90
|
+
limit: number = RECALL_CONTENT_PREVIEW_CHARS,
|
|
91
|
+
): { content: string; truncated: boolean; fullLength: number } {
|
|
92
|
+
const fullLength = content.length;
|
|
93
|
+
if (limit <= 0 || fullLength <= limit) {
|
|
94
|
+
return { content, truncated: false, fullLength };
|
|
95
|
+
}
|
|
96
|
+
const head = content.slice(0, Math.max(0, limit - 1));
|
|
97
|
+
return { content: `${head}…`, truncated: true, fullLength };
|
|
98
|
+
}
|
|
99
|
+
|
|
72
100
|
const DEFAULT_LIMIT = 500;
|
|
73
101
|
const STOP_WORDS = new Set([
|
|
74
102
|
"a",
|
|
@@ -125,6 +153,7 @@ const FACT_QUERY_FILLER_WORDS = new Set([
|
|
|
125
153
|
]);
|
|
126
154
|
|
|
127
155
|
const FACT_CLITIC_FRAGMENTS = new Set(["d", "ll", "m", "re", "s", "t", "ve"]);
|
|
156
|
+
const FLAT_FACT_SEARCH_NOISE: Record<string, true> = { entity: true, fact: true };
|
|
128
157
|
|
|
129
158
|
function nowIso(): string {
|
|
130
159
|
return new Date().toISOString();
|
|
@@ -736,10 +765,11 @@ function scoreCandidate(
|
|
|
736
765
|
score *= tierWeight;
|
|
737
766
|
}
|
|
738
767
|
score *= veracityWeight * currentContentAdjustment(searchableContent, options.currentSensitive === true);
|
|
768
|
+
const preview = clipRecallContent(content, options.contentPreviewChars ?? RECALL_CONTENT_PREVIEW_CHARS);
|
|
739
769
|
const result: RecallResult = {
|
|
740
770
|
...candidate.row,
|
|
741
771
|
id: asString(candidate.row.id),
|
|
742
|
-
content: content
|
|
772
|
+
content: preview.content,
|
|
743
773
|
source: asNullableString(candidate.row.source),
|
|
744
774
|
timestamp: asNullableString(candidate.row.timestamp),
|
|
745
775
|
importance,
|
|
@@ -765,6 +795,8 @@ function scoreCandidate(
|
|
|
765
795
|
recency_decay: round4(decay),
|
|
766
796
|
temporal: round4(temporalScore),
|
|
767
797
|
},
|
|
798
|
+
truncated: preview.truncated,
|
|
799
|
+
full_length: preview.fullLength,
|
|
768
800
|
};
|
|
769
801
|
return result;
|
|
770
802
|
}
|
|
@@ -1019,7 +1051,7 @@ export async function recallEnhanced(
|
|
|
1019
1051
|
updateRecallCounts: false,
|
|
1020
1052
|
});
|
|
1021
1053
|
if (options.includeFacts === true) {
|
|
1022
|
-
const facts = factRecall(beam, query,
|
|
1054
|
+
const facts = factRecall(beam, query, factRecallLimit(topK));
|
|
1023
1055
|
results.push(...facts);
|
|
1024
1056
|
}
|
|
1025
1057
|
results.sort((left, right) => (right.score ?? 0) - (left.score ?? 0));
|
|
@@ -1028,20 +1060,36 @@ export async function recallEnhanced(
|
|
|
1028
1060
|
return finalResults;
|
|
1029
1061
|
}
|
|
1030
1062
|
|
|
1063
|
+
function factRecallLimit(topK: number): number {
|
|
1064
|
+
const requested = Math.max(0, Math.floor(topK));
|
|
1065
|
+
if (requested === 0) return 0;
|
|
1066
|
+
return Math.min(requested, Math.max(3, Math.ceil(requested / 2)));
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1031
1069
|
function sandwichOrder(results: readonly RecallResult[]): {
|
|
1032
1070
|
high: RecallResult[];
|
|
1033
1071
|
medium: RecallResult[];
|
|
1034
1072
|
closing: RecallResult[];
|
|
1035
1073
|
} {
|
|
1036
1074
|
const scored = [...results].sort((left, right) => (right.score ?? 0) - (left.score ?? 0));
|
|
1037
|
-
const
|
|
1038
|
-
const
|
|
1039
|
-
const
|
|
1040
|
-
|
|
1075
|
+
const highLimit = scored.length > 0 && scored.length < 4 ? 1 : 3;
|
|
1076
|
+
const high = scored.slice(0, highLimit);
|
|
1077
|
+
const medium = scored.slice(high.length, high.length + 5);
|
|
1078
|
+
const closing = scored.slice(high.length + medium.length, high.length + medium.length + 3);
|
|
1079
|
+
return { high, medium, closing };
|
|
1080
|
+
}
|
|
1081
|
+
function factSearchableText(subject: string, predicate: string, object: string): string {
|
|
1082
|
+
const objectText = object.trim();
|
|
1083
|
+
if (objectText.length === 0) return `${subject} ${predicate}`.trim();
|
|
1084
|
+
const structuralParts = [subject, predicate].filter(part => {
|
|
1085
|
+
const token = part.trim().toLowerCase();
|
|
1086
|
+
return token.length > 0 && FLAT_FACT_SEARCH_NOISE[token] !== true;
|
|
1087
|
+
});
|
|
1088
|
+
return [...structuralParts, objectText].join(" ").trim();
|
|
1041
1089
|
}
|
|
1042
1090
|
|
|
1043
1091
|
function factLine(result: RecallResult): string {
|
|
1044
|
-
const content = result.content.
|
|
1092
|
+
const content = clipRecallContent(result.content.trim(), 200).content;
|
|
1045
1093
|
const ts = typeof result.timestamp === "string" && result.timestamp.length > 0 ? result.timestamp.slice(0, 10) : "?";
|
|
1046
1094
|
const source = result.source ?? "unknown";
|
|
1047
1095
|
const score = result.score ?? result.importance ?? 0;
|
|
@@ -1140,7 +1188,7 @@ export function factRecall(beam: BeamMemoryState, query: string, topK = 30): Fac
|
|
|
1140
1188
|
const object = asString(row.object);
|
|
1141
1189
|
const confidence = asNumber(row.confidence, 0.5);
|
|
1142
1190
|
const content = object.length > 0 ? object : `${subject} ${predicate}`.trim();
|
|
1143
|
-
const searchable =
|
|
1191
|
+
const searchable = factSearchableText(subject, predicate, object);
|
|
1144
1192
|
const queryGroups = factExpandedTokenGroups(query, searchable);
|
|
1145
1193
|
const queryTokens = tokensFromGroups(queryGroups);
|
|
1146
1194
|
const lexical =
|
|
@@ -1171,6 +1219,7 @@ export function factRecall(beam: BeamMemoryState, query: string, topK = 30): Fac
|
|
|
1171
1219
|
};
|
|
1172
1220
|
return result;
|
|
1173
1221
|
})
|
|
1222
|
+
.filter(result => (result.score ?? 0) > 0)
|
|
1174
1223
|
.sort((left, right) => (right.score ?? 0) - (left.score ?? 0))
|
|
1175
1224
|
.slice(0, topK);
|
|
1176
1225
|
}
|
package/src/core/beam/store.ts
CHANGED
|
@@ -5,9 +5,9 @@ import { toUtcIso } from "../../util/datetime";
|
|
|
5
5
|
import { generateId } from "../../util/ids";
|
|
6
6
|
import { currentEmbeddingModel, embeddingsDisabled } from "../embeddings";
|
|
7
7
|
import { EpisodicGraph } from "../episodic-graph";
|
|
8
|
-
import {
|
|
8
|
+
import { countExtractedFactCategories, extractFactCategoriesSafe } from "../extraction";
|
|
9
9
|
import { getMnemopiRuntimeOptions, withMnemopiRuntimeOptions } from "../runtime-options";
|
|
10
|
-
import {
|
|
10
|
+
import { storeExtractedFactCategories } from "./consolidate";
|
|
11
11
|
import { type EmbedItem, scheduleEmbedding, vecAvailable, vecInsert } from "./helpers";
|
|
12
12
|
import type {
|
|
13
13
|
BeamEvent,
|
|
@@ -232,9 +232,9 @@ function proactiveLinkIfEnabled(
|
|
|
232
232
|
*/
|
|
233
233
|
async function runFactExtraction(beam: BeamMemoryState, memoryId: string, content: string): Promise<void> {
|
|
234
234
|
try {
|
|
235
|
-
const
|
|
236
|
-
if (
|
|
237
|
-
|
|
235
|
+
const extracted = await extractFactCategoriesSafe(content);
|
|
236
|
+
if (countExtractedFactCategories(extracted) === 0) return;
|
|
237
|
+
storeExtractedFactCategories(beam, extracted, 0, memoryId);
|
|
238
238
|
invalidateCaches(beam);
|
|
239
239
|
} catch {
|
|
240
240
|
// Background fact extraction is best-effort and never surfaces to the caller.
|
package/src/core/beam/types.ts
CHANGED
|
@@ -172,6 +172,15 @@ export interface RecallOptions {
|
|
|
172
172
|
useIntent?: boolean;
|
|
173
173
|
useMmr?: boolean;
|
|
174
174
|
mmrLambda?: number;
|
|
175
|
+
/**
|
|
176
|
+
* Maximum characters of `content` returned per {@link RecallResult}. When the
|
|
177
|
+
* stored content exceeds this, the preview is clipped and the trailing
|
|
178
|
+
* character is replaced with `…` so callers can see it was truncated. The
|
|
179
|
+
* full row is always reachable via {@link BeamMemoryState.get}. `0` or a
|
|
180
|
+
* negative value disables clipping. Defaults to
|
|
181
|
+
* {@link RECALL_CONTENT_PREVIEW_CHARS} (500).
|
|
182
|
+
*/
|
|
183
|
+
contentPreviewChars?: number;
|
|
175
184
|
}
|
|
176
185
|
|
|
177
186
|
export interface RecallEnhancedOptions extends RecallOptions {
|
|
@@ -238,6 +247,15 @@ export type RecallResult = RecallRowFields & {
|
|
|
238
247
|
[key: string]: unknown;
|
|
239
248
|
id: string;
|
|
240
249
|
content: string;
|
|
250
|
+
/**
|
|
251
|
+
* True when {@link content} is a clipped preview of the stored row. The
|
|
252
|
+
* clip is capped at {@link RecallOptions.contentPreviewChars} (default 500)
|
|
253
|
+
* and the last character is replaced with `…`. Fetch the full row via
|
|
254
|
+
* `memory://<id>` (mnemopi backend) or the `Mnemopi.get(id)` API.
|
|
255
|
+
*/
|
|
256
|
+
truncated?: boolean;
|
|
257
|
+
/** Original character count of `content` before {@link truncated} clipping. */
|
|
258
|
+
full_length?: number;
|
|
241
259
|
score?: number;
|
|
242
260
|
distance?: number;
|
|
243
261
|
rank?: number;
|
package/src/core/extraction.ts
CHANGED
|
@@ -82,48 +82,146 @@ function stripFence(raw: string): string {
|
|
|
82
82
|
return s.trim();
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
const FLAT_FACT_LIMIT = 5;
|
|
86
|
+
const STRUCTURED_CATEGORY_LIMIT = 5;
|
|
87
|
+
const STRING_CATEGORY_KEYS = ["facts", "instructions", "preferences", "timelines"] as const;
|
|
88
|
+
|
|
89
|
+
/** Parsed knowledge-graph edge emitted by the extractor LLM. */
|
|
90
|
+
export interface ExtractedKgTriple {
|
|
91
|
+
subject: string;
|
|
92
|
+
predicate: string;
|
|
93
|
+
object: string;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Category-preserving extraction result used by background memory routing. */
|
|
97
|
+
export interface ExtractedFactCategories {
|
|
98
|
+
facts: string[];
|
|
99
|
+
instructions: string[];
|
|
100
|
+
preferences: string[];
|
|
101
|
+
timelines: string[];
|
|
102
|
+
kg: ExtractedKgTriple[];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function emptyFactCategories(): ExtractedFactCategories {
|
|
106
|
+
return { facts: [], instructions: [], preferences: [], timelines: [], kg: [] };
|
|
107
|
+
}
|
|
108
|
+
|
|
85
109
|
function normalizeFact(fact: string): string {
|
|
86
110
|
const trimmed = fact.trim();
|
|
87
111
|
// Remove trailing sentence punctuation (. ! ?) if present
|
|
88
112
|
return trimmed.replace(/[.!?]+$/, "");
|
|
89
113
|
}
|
|
90
|
-
|
|
91
|
-
|
|
114
|
+
|
|
115
|
+
function normalizeFactArray(items: unknown): string[] {
|
|
116
|
+
if (!Array.isArray(items)) {
|
|
117
|
+
return [];
|
|
118
|
+
}
|
|
119
|
+
const out: string[] = [];
|
|
120
|
+
for (const item of items) {
|
|
121
|
+
if (item !== null && item !== undefined && String(item).trim() !== "") {
|
|
122
|
+
const normalized = normalizeFact(String(item));
|
|
123
|
+
if (normalized !== "") {
|
|
124
|
+
out.push(normalized);
|
|
125
|
+
if (out.length >= STRUCTURED_CATEGORY_LIMIT) break;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
133
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function triplePart(value: unknown): string {
|
|
137
|
+
return typeof value === "string" ? normalizeFact(value) : "";
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function normalizeKgTriple(item: unknown): ExtractedKgTriple | null {
|
|
141
|
+
let subject = "";
|
|
142
|
+
let predicate = "";
|
|
143
|
+
let object = "";
|
|
144
|
+
if (isRecord(item)) {
|
|
145
|
+
subject = triplePart(item.subject);
|
|
146
|
+
predicate = triplePart(item.predicate);
|
|
147
|
+
object = triplePart(item.object);
|
|
148
|
+
} else if (Array.isArray(item)) {
|
|
149
|
+
subject = triplePart(item[0]);
|
|
150
|
+
predicate = triplePart(item[1]);
|
|
151
|
+
object = triplePart(item[2]);
|
|
152
|
+
}
|
|
153
|
+
return subject !== "" && predicate !== "" && object !== "" ? { subject, predicate, object } : null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function normalizeKgArray(items: unknown): ExtractedKgTriple[] {
|
|
157
|
+
if (!Array.isArray(items)) {
|
|
92
158
|
return [];
|
|
93
159
|
}
|
|
160
|
+
const out: ExtractedKgTriple[] = [];
|
|
161
|
+
for (const item of items) {
|
|
162
|
+
const triple = normalizeKgTriple(item);
|
|
163
|
+
if (triple !== null) {
|
|
164
|
+
out.push(triple);
|
|
165
|
+
if (out.length >= STRUCTURED_CATEGORY_LIMIT) break;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return out;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Flatten extracted string categories for legacy fact callers. */
|
|
172
|
+
export function flattenExtractedFactCategories(extracted: ExtractedFactCategories): string[] {
|
|
173
|
+
const out: string[] = [];
|
|
174
|
+
for (const category of STRING_CATEGORY_KEYS) {
|
|
175
|
+
for (const item of extracted[category]) {
|
|
176
|
+
out.push(item);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return out;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Count string facts plus KG triples in a category-preserving extraction result. */
|
|
183
|
+
export function countExtractedFactCategories(extracted: ExtractedFactCategories): number {
|
|
184
|
+
return (
|
|
185
|
+
extracted.facts.length +
|
|
186
|
+
extracted.instructions.length +
|
|
187
|
+
extracted.preferences.length +
|
|
188
|
+
extracted.timelines.length +
|
|
189
|
+
extracted.kg.length
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Parse extractor output without discarding MEMORIA categories or KG triples. */
|
|
194
|
+
export function parseExtractedFactCategories(rawOutput: string | null | undefined): ExtractedFactCategories {
|
|
195
|
+
if (rawOutput === null || rawOutput === undefined) {
|
|
196
|
+
return emptyFactCategories();
|
|
197
|
+
}
|
|
94
198
|
const raw = rawOutput.trim();
|
|
95
199
|
if (raw === "" || raw.toUpperCase() === "NO_FACTS") {
|
|
96
|
-
return
|
|
200
|
+
return emptyFactCategories();
|
|
97
201
|
}
|
|
98
202
|
const rawClean = stripFence(raw);
|
|
99
203
|
if (rawClean.startsWith("{")) {
|
|
100
204
|
try {
|
|
101
|
-
const parsed = JSON.parse(rawClean)
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
const normalized = normalizeFact(String(item));
|
|
111
|
-
if (normalized !== "") {
|
|
112
|
-
out.push(normalized);
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
return out.slice(0, 5);
|
|
205
|
+
const parsed: unknown = JSON.parse(rawClean);
|
|
206
|
+
if (isRecord(parsed)) {
|
|
207
|
+
return {
|
|
208
|
+
facts: normalizeFactArray(parsed.facts),
|
|
209
|
+
instructions: normalizeFactArray(parsed.instructions),
|
|
210
|
+
preferences: normalizeFactArray(parsed.preferences),
|
|
211
|
+
timelines: normalizeFactArray(parsed.timelines),
|
|
212
|
+
kg: normalizeKgArray(parsed.kg),
|
|
213
|
+
};
|
|
119
214
|
}
|
|
120
215
|
} catch {
|
|
121
216
|
const matches = [...raw.matchAll(/"([^"]{10,})"/g)].map(m => m[1]).filter((v): v is string => v !== undefined);
|
|
122
217
|
if (matches.length > 0) {
|
|
123
|
-
return
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
218
|
+
return {
|
|
219
|
+
...emptyFactCategories(),
|
|
220
|
+
facts: matches
|
|
221
|
+
.map(normalizeFact)
|
|
222
|
+
.filter(f => f !== "")
|
|
223
|
+
.slice(0, FLAT_FACT_LIMIT),
|
|
224
|
+
};
|
|
127
225
|
}
|
|
128
226
|
}
|
|
129
227
|
}
|
|
@@ -136,8 +234,14 @@ export function parseFacts(rawOutput: string | null | undefined): string[] {
|
|
|
136
234
|
cleaned.push(normalized);
|
|
137
235
|
}
|
|
138
236
|
}
|
|
237
|
+
if (cleaned.length >= FLAT_FACT_LIMIT) break;
|
|
139
238
|
}
|
|
140
|
-
return
|
|
239
|
+
return { ...emptyFactCategories(), facts: cleaned };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Parse extractor output into the legacy flat string fact list. */
|
|
243
|
+
export function parseFacts(rawOutput: string | null | undefined): string[] {
|
|
244
|
+
return flattenExtractedFactCategories(parseExtractedFactCategories(rawOutput)).slice(0, FLAT_FACT_LIMIT);
|
|
141
245
|
}
|
|
142
246
|
function sentenceCase(value: string): string {
|
|
143
247
|
const trimmed = value.trim().replace(/[.!?]+$/, "");
|
|
@@ -205,39 +309,48 @@ async function tryHostExtraction(prompt: string): Promise<[boolean, string | nul
|
|
|
205
309
|
return [true, text === "" ? null : text];
|
|
206
310
|
}
|
|
207
311
|
|
|
208
|
-
async function localFallback(
|
|
312
|
+
async function localFallback(
|
|
313
|
+
prompt: string,
|
|
314
|
+
sourceText: string,
|
|
315
|
+
diag = getDiagnostics(),
|
|
316
|
+
): Promise<ExtractedFactCategories> {
|
|
209
317
|
diag.recordAttempt("local");
|
|
210
318
|
try {
|
|
211
319
|
const raw = await callLocalLlm(prompt);
|
|
212
320
|
if (raw !== null) {
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
321
|
+
const extracted = parseExtractedFactCategories(cleanOutput(raw));
|
|
322
|
+
const count = countExtractedFactCategories(extracted);
|
|
323
|
+
if (count > 0) {
|
|
324
|
+
diag.recordSuccess("local", count);
|
|
216
325
|
diag.recordCall({ succeeded: true });
|
|
217
|
-
return
|
|
326
|
+
return extracted;
|
|
218
327
|
}
|
|
219
328
|
diag.recordNoOutput("local");
|
|
220
329
|
}
|
|
221
330
|
} catch (exc) {
|
|
222
331
|
diag.recordFailure("local", exc, "local_llm_raised");
|
|
223
332
|
diag.recordCall({ succeeded: false });
|
|
224
|
-
return
|
|
333
|
+
return emptyFactCategories();
|
|
225
334
|
}
|
|
226
335
|
diag.recordFailure("local", undefined, "model_not_loaded");
|
|
227
336
|
const heuristic = heuristicExtractFacts(sourceText);
|
|
228
337
|
if (heuristic.length > 0) {
|
|
229
338
|
diag.recordSuccess("local", heuristic.length);
|
|
230
339
|
diag.recordCall({ succeeded: true });
|
|
231
|
-
return heuristic;
|
|
340
|
+
return { ...emptyFactCategories(), facts: heuristic };
|
|
232
341
|
}
|
|
233
342
|
diag.recordCall({ succeeded: false, allEmpty: true });
|
|
234
|
-
return
|
|
343
|
+
return emptyFactCategories();
|
|
235
344
|
}
|
|
236
345
|
|
|
237
|
-
|
|
346
|
+
/** Extract fact categories from text using configured, host, local, or remote LLMs. */
|
|
347
|
+
export async function extractFactCategories(
|
|
348
|
+
text: string | null | undefined,
|
|
349
|
+
options: RemoteLlmOptions = {},
|
|
350
|
+
): Promise<ExtractedFactCategories> {
|
|
238
351
|
const diag = getDiagnostics();
|
|
239
352
|
if (typeof text !== "string" || text.trim() === "") {
|
|
240
|
-
return
|
|
353
|
+
return emptyFactCategories();
|
|
241
354
|
}
|
|
242
355
|
const prompt = buildExtractionPrompt(text);
|
|
243
356
|
|
|
@@ -250,11 +363,12 @@ export async function extractFacts(text: string | null | undefined, options: Rem
|
|
|
250
363
|
try {
|
|
251
364
|
const raw = await callConfiguredCompletion(prompt, 0, { maxTokens: llmMaxTokens() });
|
|
252
365
|
if (typeof raw === "string" && raw.trim() !== "") {
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
366
|
+
const extracted = parseExtractedFactCategories(raw);
|
|
367
|
+
const count = countExtractedFactCategories(extracted);
|
|
368
|
+
if (count > 0) {
|
|
369
|
+
diag.recordSuccess("host", count);
|
|
256
370
|
diag.recordCall({ succeeded: true });
|
|
257
|
-
return
|
|
371
|
+
return extracted;
|
|
258
372
|
}
|
|
259
373
|
}
|
|
260
374
|
diag.recordNoOutput("host");
|
|
@@ -262,7 +376,7 @@ export async function extractFacts(text: string | null | undefined, options: Rem
|
|
|
262
376
|
diag.recordFailure("host", exc, "configured_completion_raised");
|
|
263
377
|
diag.recordCall({ succeeded: false });
|
|
264
378
|
console.warn(`extractFacts: configured completion raised: ${safeForLog(exc)}`);
|
|
265
|
-
return
|
|
379
|
+
return emptyFactCategories();
|
|
266
380
|
}
|
|
267
381
|
return localFallback(prompt, text, diag);
|
|
268
382
|
}
|
|
@@ -272,11 +386,12 @@ export async function extractFacts(text: string | null | undefined, options: Rem
|
|
|
272
386
|
if (attempted) {
|
|
273
387
|
diag.recordAttempt("host");
|
|
274
388
|
if (hostText !== null) {
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
389
|
+
const extracted = parseExtractedFactCategories(hostText);
|
|
390
|
+
const count = countExtractedFactCategories(extracted);
|
|
391
|
+
if (count > 0) {
|
|
392
|
+
diag.recordSuccess("host", count);
|
|
278
393
|
diag.recordCall({ succeeded: true });
|
|
279
|
-
return
|
|
394
|
+
return extracted;
|
|
280
395
|
}
|
|
281
396
|
}
|
|
282
397
|
diag.recordNoOutput("host");
|
|
@@ -287,7 +402,7 @@ export async function extractFacts(text: string | null | undefined, options: Rem
|
|
|
287
402
|
diag.recordFailure("host", exc, "host_adapter_raised");
|
|
288
403
|
diag.recordCall({ succeeded: false });
|
|
289
404
|
console.warn(`extractFacts: host LLM adapter raised: ${safeForLog(exc)}`);
|
|
290
|
-
return
|
|
405
|
+
return emptyFactCategories();
|
|
291
406
|
}
|
|
292
407
|
|
|
293
408
|
if (!llmAvailable()) {
|
|
@@ -296,22 +411,23 @@ export async function extractFacts(text: string | null | undefined, options: Rem
|
|
|
296
411
|
if (heuristic.length > 0) {
|
|
297
412
|
diag.recordSuccess("local", heuristic.length);
|
|
298
413
|
diag.recordCall({ succeeded: true });
|
|
299
|
-
return heuristic;
|
|
414
|
+
return { ...emptyFactCategories(), facts: heuristic };
|
|
300
415
|
}
|
|
301
416
|
diag.recordFailure("local", undefined, "llm_unavailable_at_call_site");
|
|
302
417
|
diag.recordCall({ succeeded: false });
|
|
303
|
-
return
|
|
418
|
+
return emptyFactCategories();
|
|
304
419
|
}
|
|
305
420
|
|
|
306
421
|
diag.recordAttempt("remote");
|
|
307
422
|
try {
|
|
308
423
|
const raw = await callRemoteLlm(prompt, 0, options);
|
|
309
424
|
if (raw !== null) {
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
|
|
425
|
+
const extracted = parseExtractedFactCategories(cleanOutput(raw));
|
|
426
|
+
const count = countExtractedFactCategories(extracted);
|
|
427
|
+
if (count > 0) {
|
|
428
|
+
diag.recordSuccess("remote", count);
|
|
313
429
|
diag.recordCall({ succeeded: true });
|
|
314
|
-
return
|
|
430
|
+
return extracted;
|
|
315
431
|
}
|
|
316
432
|
}
|
|
317
433
|
diag.recordNoOutput("remote");
|
|
@@ -323,14 +439,27 @@ export async function extractFacts(text: string | null | undefined, options: Rem
|
|
|
323
439
|
return localFallback(prompt, text, diag);
|
|
324
440
|
}
|
|
325
441
|
|
|
326
|
-
|
|
442
|
+
/** Extract legacy flat fact strings from text. */
|
|
443
|
+
export async function extractFacts(text: string | null | undefined, options: RemoteLlmOptions = {}): Promise<string[]> {
|
|
444
|
+
const extracted = await extractFactCategories(text, options);
|
|
445
|
+
return flattenExtractedFactCategories(extracted).slice(0, FLAT_FACT_LIMIT);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/** Safely extract category-preserving facts, swallowing best-effort failures. */
|
|
449
|
+
export async function extractFactCategoriesSafe(text: string | null | undefined): Promise<ExtractedFactCategories> {
|
|
327
450
|
try {
|
|
328
|
-
return await
|
|
451
|
+
return await extractFactCategories(text);
|
|
329
452
|
} catch (exc) {
|
|
330
453
|
const diag = getDiagnostics();
|
|
331
454
|
diag.recordFailure("wrapper", exc, "outer_wrapper_caught");
|
|
332
455
|
diag.recordCall({ succeeded: false });
|
|
333
456
|
console.warn(`extractFactsSafe: extractFacts() raised: ${safeForLog(exc)}`);
|
|
334
|
-
return
|
|
457
|
+
return emptyFactCategories();
|
|
335
458
|
}
|
|
336
459
|
}
|
|
460
|
+
|
|
461
|
+
/** Safely extract legacy flat fact strings, swallowing best-effort failures. */
|
|
462
|
+
export async function extractFactsSafe(text: string | null | undefined): Promise<string[]> {
|
|
463
|
+
const extracted = await extractFactCategoriesSafe(text);
|
|
464
|
+
return flattenExtractedFactCategories(extracted).slice(0, FLAT_FACT_LIMIT);
|
|
465
|
+
}
|
package/src/core/memory.ts
CHANGED
|
@@ -319,6 +319,7 @@ function toRecallOptions(options: RecallFacadeOptions): BeamRecallFacadeOptions
|
|
|
319
319
|
vecWeight: options.vecWeight ?? options.vec_weight ?? undefined,
|
|
320
320
|
ftsWeight: options.ftsWeight ?? options.fts_weight ?? undefined,
|
|
321
321
|
importanceWeight: options.importanceWeight ?? options.importance_weight ?? undefined,
|
|
322
|
+
contentPreviewChars: options.contentPreviewChars,
|
|
322
323
|
};
|
|
323
324
|
// Preserve the three-state semantics (`undefined` = auto-derive, `null` = explicitly
|
|
324
325
|
// FTS-only, `number[]` = caller-supplied) so callers can opt out of `recall()`'s
|