mem0ai 3.0.10 → 3.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +6 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -4
- package/dist/index.mjs.map +1 -1
- package/dist/oss/index.d.mts +20 -5
- package/dist/oss/index.d.ts +20 -5
- package/dist/oss/index.js +375 -130
- package/dist/oss/index.js.map +1 -1
- package/dist/oss/index.mjs +370 -126
- package/dist/oss/index.mjs.map +1 -1
- package/package.json +3 -2
package/dist/oss/index.js
CHANGED
|
@@ -44,6 +44,7 @@ __export(index_exports, {
|
|
|
44
44
|
LangchainEmbedder: () => LangchainEmbedder,
|
|
45
45
|
LangchainLLM: () => LangchainLLM,
|
|
46
46
|
LangchainVectorStore: () => LangchainVectorStore,
|
|
47
|
+
LiteLLM: () => LiteLLM,
|
|
47
48
|
Memory: () => Memory,
|
|
48
49
|
MemoryConfigSchema: () => MemoryConfigSchema,
|
|
49
50
|
MemoryVectorStore: () => MemoryVectorStore,
|
|
@@ -153,6 +154,11 @@ var OpenAIEmbedder = class {
|
|
|
153
154
|
...response.data.sort((a, b) => a.index - b.index).map((item) => item.embedding)
|
|
154
155
|
);
|
|
155
156
|
}
|
|
157
|
+
if (allEmbeddings.length !== texts.length) {
|
|
158
|
+
throw new Error(
|
|
159
|
+
`OpenAI embedBatch() returned ${allEmbeddings.length} embeddings for ${texts.length} texts using model '${this.model}'`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
156
162
|
return allEmbeddings;
|
|
157
163
|
}
|
|
158
164
|
};
|
|
@@ -1894,13 +1900,15 @@ var RedisDB = class {
|
|
|
1894
1900
|
}
|
|
1895
1901
|
async insert(vectors, ids, payloads) {
|
|
1896
1902
|
const data = vectors.map((vector, idx) => {
|
|
1903
|
+
var _a2, _b2;
|
|
1897
1904
|
const payload = toSnakeCase(payloads[idx]);
|
|
1898
1905
|
const id = ids[idx];
|
|
1906
|
+
const createdAt = payload.created_at ? new Date(payload.created_at).getTime() : 0;
|
|
1899
1907
|
const entry = {
|
|
1900
1908
|
memory_id: id,
|
|
1901
|
-
hash: payload.hash,
|
|
1902
|
-
memory: payload.data,
|
|
1903
|
-
created_at:
|
|
1909
|
+
hash: (_a2 = payload.hash) != null ? _a2 : "",
|
|
1910
|
+
memory: (_b2 = payload.data) != null ? _b2 : "",
|
|
1911
|
+
created_at: createdAt,
|
|
1904
1912
|
embedding: new Float32Array(vector).buffer
|
|
1905
1913
|
};
|
|
1906
1914
|
["agent_id", "run_id", "user_id"].forEach((field) => {
|
|
@@ -2079,13 +2087,16 @@ var RedisDB = class {
|
|
|
2079
2087
|
}
|
|
2080
2088
|
}
|
|
2081
2089
|
async update(vectorId, vector, payload) {
|
|
2090
|
+
var _a2, _b2;
|
|
2082
2091
|
const snakePayload = toSnakeCase(payload);
|
|
2092
|
+
const createdAt = snakePayload.created_at ? new Date(snakePayload.created_at).getTime() : 0;
|
|
2093
|
+
const updatedAt = snakePayload.updated_at ? new Date(snakePayload.updated_at).getTime() : 0;
|
|
2083
2094
|
const entry = {
|
|
2084
2095
|
memory_id: vectorId,
|
|
2085
|
-
hash: snakePayload.hash,
|
|
2086
|
-
memory: snakePayload.data,
|
|
2087
|
-
created_at:
|
|
2088
|
-
updated_at:
|
|
2096
|
+
hash: (_a2 = snakePayload.hash) != null ? _a2 : "",
|
|
2097
|
+
memory: (_b2 = snakePayload.data) != null ? _b2 : "",
|
|
2098
|
+
created_at: createdAt,
|
|
2099
|
+
updated_at: updatedAt,
|
|
2089
2100
|
embedding: Buffer.from(new Float32Array(vector).buffer)
|
|
2090
2101
|
};
|
|
2091
2102
|
["agent_id", "run_id", "user_id"].forEach((field) => {
|
|
@@ -2331,6 +2342,66 @@ var DeepSeekLLM = class extends OpenAILLM {
|
|
|
2331
2342
|
}
|
|
2332
2343
|
};
|
|
2333
2344
|
|
|
2345
|
+
// src/oss/src/llms/litellm.ts
|
|
2346
|
+
var LiteLLM = class extends OpenAILLM {
|
|
2347
|
+
constructor(config) {
|
|
2348
|
+
super({
|
|
2349
|
+
...config,
|
|
2350
|
+
apiKey: config.apiKey || process.env.LITELLM_API_KEY || "sk-anything",
|
|
2351
|
+
baseURL: config.baseURL || process.env.LITELLM_API_BASE || "http://localhost:4000",
|
|
2352
|
+
model: config.model || "gpt-5-mini"
|
|
2353
|
+
});
|
|
2354
|
+
}
|
|
2355
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2356
|
+
try {
|
|
2357
|
+
return await super.generateResponse(messages, responseFormat, tools);
|
|
2358
|
+
} catch (err) {
|
|
2359
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2360
|
+
throw new Error(`LiteLLM failed: ${message}`);
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
async generateChat(messages) {
|
|
2364
|
+
try {
|
|
2365
|
+
return await super.generateChat(messages);
|
|
2366
|
+
} catch (err) {
|
|
2367
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2368
|
+
throw new Error(`LiteLLM failed: ${message}`);
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
};
|
|
2372
|
+
|
|
2373
|
+
// src/oss/src/llms/minimax.ts
|
|
2374
|
+
var MiniMaxLLM = class extends OpenAILLM {
|
|
2375
|
+
constructor(config) {
|
|
2376
|
+
const apiKey = config.apiKey || process.env.MINIMAX_API_KEY;
|
|
2377
|
+
if (!apiKey) {
|
|
2378
|
+
throw new Error("MiniMax API key is required");
|
|
2379
|
+
}
|
|
2380
|
+
super({
|
|
2381
|
+
...config,
|
|
2382
|
+
apiKey,
|
|
2383
|
+
baseURL: config.baseURL || process.env.MINIMAX_API_BASE || "https://api.minimax.io/v1",
|
|
2384
|
+
model: config.model || "MiniMax-M2.7"
|
|
2385
|
+
});
|
|
2386
|
+
}
|
|
2387
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2388
|
+
try {
|
|
2389
|
+
return await super.generateResponse(messages, responseFormat, tools);
|
|
2390
|
+
} catch (err) {
|
|
2391
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2392
|
+
throw new Error(`MiniMax LLM failed: ${message}`);
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
async generateChat(messages) {
|
|
2396
|
+
try {
|
|
2397
|
+
return await super.generateChat(messages);
|
|
2398
|
+
} catch (err) {
|
|
2399
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2400
|
+
throw new Error(`MiniMax LLM failed: ${message}`);
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
};
|
|
2404
|
+
|
|
2334
2405
|
// src/oss/src/vector_stores/supabase.ts
|
|
2335
2406
|
var import_supabase_js = require("@supabase/supabase-js");
|
|
2336
2407
|
var SupabaseDB = class {
|
|
@@ -2844,9 +2915,8 @@ var GoogleLLM = class {
|
|
|
2844
2915
|
this.google = new import_genai2.GoogleGenAI({ apiKey: config.apiKey });
|
|
2845
2916
|
this.model = config.model || "gemini-2.0-flash";
|
|
2846
2917
|
}
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
const contents = messages.map((msg) => ({
|
|
2918
|
+
formatContents(messages) {
|
|
2919
|
+
return messages.map((msg) => ({
|
|
2850
2920
|
parts: [
|
|
2851
2921
|
{
|
|
2852
2922
|
text: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
|
|
@@ -2854,6 +2924,10 @@ var GoogleLLM = class {
|
|
|
2854
2924
|
],
|
|
2855
2925
|
role: msg.role === "system" ? "model" : "user"
|
|
2856
2926
|
}));
|
|
2927
|
+
}
|
|
2928
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2929
|
+
var _a2;
|
|
2930
|
+
const contents = this.formatContents(messages);
|
|
2857
2931
|
const config = {};
|
|
2858
2932
|
if (tools && tools.length > 0) {
|
|
2859
2933
|
config.tools = [
|
|
@@ -2885,20 +2959,22 @@ var GoogleLLM = class {
|
|
|
2885
2959
|
return text || "";
|
|
2886
2960
|
}
|
|
2887
2961
|
async generateChat(messages) {
|
|
2962
|
+
var _a2, _b2, _c;
|
|
2888
2963
|
const completion = await this.google.models.generateContent({
|
|
2889
|
-
contents: messages,
|
|
2964
|
+
contents: this.formatContents(messages),
|
|
2890
2965
|
model: this.model
|
|
2891
2966
|
});
|
|
2892
|
-
const response = completion.candidates[0].content;
|
|
2967
|
+
const response = (_b2 = (_a2 = completion.candidates) == null ? void 0 : _a2[0]) == null ? void 0 : _b2.content;
|
|
2968
|
+
const content = ((_c = response == null ? void 0 : response.parts) == null ? void 0 : _c.map((part) => part.text || "").join("")) || completion.text || "";
|
|
2893
2969
|
return {
|
|
2894
|
-
content
|
|
2895
|
-
role: response.role
|
|
2970
|
+
content,
|
|
2971
|
+
role: (response == null ? void 0 : response.role) || "assistant"
|
|
2896
2972
|
};
|
|
2897
2973
|
}
|
|
2898
2974
|
};
|
|
2899
2975
|
|
|
2900
2976
|
// src/oss/src/llms/azure.ts
|
|
2901
|
-
var
|
|
2977
|
+
var import_openai9 = require("openai");
|
|
2902
2978
|
var AzureOpenAILLM = class {
|
|
2903
2979
|
constructor(config) {
|
|
2904
2980
|
var _a2;
|
|
@@ -2906,7 +2982,7 @@ var AzureOpenAILLM = class {
|
|
|
2906
2982
|
throw new Error("Azure OpenAI requires both API key and endpoint");
|
|
2907
2983
|
}
|
|
2908
2984
|
const { endpoint, ...rest } = config.modelProperties;
|
|
2909
|
-
this.client = new
|
|
2985
|
+
this.client = new import_openai9.AzureOpenAI({
|
|
2910
2986
|
apiKey: config.apiKey,
|
|
2911
2987
|
endpoint,
|
|
2912
2988
|
...rest
|
|
@@ -2959,7 +3035,7 @@ var AzureOpenAILLM = class {
|
|
|
2959
3035
|
};
|
|
2960
3036
|
|
|
2961
3037
|
// src/oss/src/embeddings/azure.ts
|
|
2962
|
-
var
|
|
3038
|
+
var import_openai10 = require("openai");
|
|
2963
3039
|
var AzureOpenAIEmbedder = class {
|
|
2964
3040
|
constructor(config) {
|
|
2965
3041
|
var _a2;
|
|
@@ -2967,7 +3043,7 @@ var AzureOpenAIEmbedder = class {
|
|
|
2967
3043
|
throw new Error("Azure OpenAI requires both API key and endpoint");
|
|
2968
3044
|
}
|
|
2969
3045
|
const { endpoint, ...rest } = config.modelProperties;
|
|
2970
|
-
this.client = new
|
|
3046
|
+
this.client = new import_openai10.AzureOpenAI({
|
|
2971
3047
|
apiKey: config.apiKey,
|
|
2972
3048
|
endpoint,
|
|
2973
3049
|
...rest
|
|
@@ -3001,6 +3077,11 @@ var AzureOpenAIEmbedder = class {
|
|
|
3001
3077
|
...response.data.sort((a, b) => a.index - b.index).map((item) => item.embedding)
|
|
3002
3078
|
);
|
|
3003
3079
|
}
|
|
3080
|
+
if (allEmbeddings.length !== texts.length) {
|
|
3081
|
+
throw new Error(
|
|
3082
|
+
`Azure OpenAI embedBatch() returned ${allEmbeddings.length} embeddings for ${texts.length} texts using model '${this.model}'`
|
|
3083
|
+
);
|
|
3084
|
+
}
|
|
3004
3085
|
return allEmbeddings;
|
|
3005
3086
|
}
|
|
3006
3087
|
};
|
|
@@ -4604,24 +4685,59 @@ function buildFilterConditions(filters, startIndex) {
|
|
|
4604
4685
|
}
|
|
4605
4686
|
return { conditions, values, paramIndex };
|
|
4606
4687
|
}
|
|
4688
|
+
function getConnectionString(config) {
|
|
4689
|
+
var _a2;
|
|
4690
|
+
return ((_a2 = config.connectionString) == null ? void 0 : _a2.trim()) || void 0;
|
|
4691
|
+
}
|
|
4692
|
+
function validateConnectionConfig(config) {
|
|
4693
|
+
if (getConnectionString(config)) {
|
|
4694
|
+
return;
|
|
4695
|
+
}
|
|
4696
|
+
const missingFields = ["user", "password", "host", "port"].filter((field) => {
|
|
4697
|
+
const v = config[field];
|
|
4698
|
+
return v === void 0 || v === null || v === "";
|
|
4699
|
+
});
|
|
4700
|
+
if (missingFields.length > 0) {
|
|
4701
|
+
throw new Error(
|
|
4702
|
+
`PGVector requires either connectionString or ${missingFields.join(", ")}`
|
|
4703
|
+
);
|
|
4704
|
+
}
|
|
4705
|
+
}
|
|
4706
|
+
function buildClientConfig(config, database) {
|
|
4707
|
+
const connectionString = getConnectionString(config);
|
|
4708
|
+
if (connectionString) {
|
|
4709
|
+
return {
|
|
4710
|
+
connectionString,
|
|
4711
|
+
...config.ssl !== void 0 ? { ssl: config.ssl } : {}
|
|
4712
|
+
};
|
|
4713
|
+
}
|
|
4714
|
+
return {
|
|
4715
|
+
database,
|
|
4716
|
+
user: config.user,
|
|
4717
|
+
password: config.password,
|
|
4718
|
+
host: config.host,
|
|
4719
|
+
port: config.port,
|
|
4720
|
+
...config.ssl !== void 0 ? { ssl: config.ssl } : {}
|
|
4721
|
+
};
|
|
4722
|
+
}
|
|
4607
4723
|
var PGVector = class {
|
|
4608
4724
|
constructor(config) {
|
|
4725
|
+
validateConnectionConfig(config);
|
|
4609
4726
|
this.collectionName = validateIdentifier(
|
|
4610
4727
|
config.collectionName || "memories",
|
|
4611
4728
|
"collectionName"
|
|
4612
4729
|
);
|
|
4613
4730
|
this.useDiskann = config.diskann || false;
|
|
4614
4731
|
this.useHnsw = config.hnsw || false;
|
|
4615
|
-
this.
|
|
4732
|
+
this.useDirectConnection = !!getConnectionString(config);
|
|
4733
|
+
this.dbName = this.useDirectConnection ? "" : validateIdentifier(config.dbname || "vector_store", "dbname");
|
|
4616
4734
|
this.config = config;
|
|
4617
|
-
this.client = new Client(
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
port: config.port
|
|
4624
|
-
});
|
|
4735
|
+
this.client = new Client(
|
|
4736
|
+
buildClientConfig(
|
|
4737
|
+
config,
|
|
4738
|
+
this.useDirectConnection ? void 0 : "postgres"
|
|
4739
|
+
)
|
|
4740
|
+
);
|
|
4625
4741
|
this.initialize().catch(console.error);
|
|
4626
4742
|
}
|
|
4627
4743
|
col() {
|
|
@@ -4636,19 +4752,15 @@ var PGVector = class {
|
|
|
4636
4752
|
async _doInitialize() {
|
|
4637
4753
|
try {
|
|
4638
4754
|
await this.client.connect();
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
host: this.config.host,
|
|
4649
|
-
port: this.config.port
|
|
4650
|
-
});
|
|
4651
|
-
await this.client.connect();
|
|
4755
|
+
if (!this.useDirectConnection) {
|
|
4756
|
+
const dbExists = await this.checkDatabaseExists(this.dbName);
|
|
4757
|
+
if (!dbExists) {
|
|
4758
|
+
await this.createDatabase(this.dbName);
|
|
4759
|
+
}
|
|
4760
|
+
await this.client.end();
|
|
4761
|
+
this.client = new Client(buildClientConfig(this.config, this.dbName));
|
|
4762
|
+
await this.client.connect();
|
|
4763
|
+
}
|
|
4652
4764
|
await this.client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
4653
4765
|
await this.client.query(`
|
|
4654
4766
|
CREATE TABLE IF NOT EXISTS memory_migrations (
|
|
@@ -4919,6 +5031,10 @@ var LLMFactory = class {
|
|
|
4919
5031
|
return new LangchainLLM(config);
|
|
4920
5032
|
case "deepseek":
|
|
4921
5033
|
return new DeepSeekLLM(config);
|
|
5034
|
+
case "litellm":
|
|
5035
|
+
return new LiteLLM(config);
|
|
5036
|
+
case "minimax":
|
|
5037
|
+
return new MiniMaxLLM(config);
|
|
4922
5038
|
default:
|
|
4923
5039
|
throw new Error(`Unsupported LLM provider: ${provider}`);
|
|
4924
5040
|
}
|
|
@@ -5171,7 +5287,7 @@ var parse_vision_messages = async (messages) => {
|
|
|
5171
5287
|
};
|
|
5172
5288
|
|
|
5173
5289
|
// src/oss/src/utils/telemetry.ts
|
|
5174
|
-
var version = true ? "3.0.
|
|
5290
|
+
var version = true ? "3.0.13" : "dev";
|
|
5175
5291
|
var MEM0_TELEMETRY = true;
|
|
5176
5292
|
var _a, _b;
|
|
5177
5293
|
try {
|
|
@@ -6704,7 +6820,24 @@ var NON_SPECIFIC_ADJ = /* @__PURE__ */ new Set([
|
|
|
6704
6820
|
"collaborative",
|
|
6705
6821
|
"final",
|
|
6706
6822
|
"initial",
|
|
6707
|
-
"side"
|
|
6823
|
+
"side",
|
|
6824
|
+
"top"
|
|
6825
|
+
]);
|
|
6826
|
+
var TOPIC_PREFIX_WORDS = /* @__PURE__ */ new Set([
|
|
6827
|
+
"a",
|
|
6828
|
+
"an",
|
|
6829
|
+
"the",
|
|
6830
|
+
"my",
|
|
6831
|
+
"your",
|
|
6832
|
+
"our",
|
|
6833
|
+
"their",
|
|
6834
|
+
"his",
|
|
6835
|
+
"her",
|
|
6836
|
+
"its",
|
|
6837
|
+
"this",
|
|
6838
|
+
"that",
|
|
6839
|
+
"these",
|
|
6840
|
+
"those"
|
|
6708
6841
|
]);
|
|
6709
6842
|
var GENERIC_ENDINGS = /* @__PURE__ */ new Set([
|
|
6710
6843
|
"work",
|
|
@@ -6771,6 +6904,23 @@ var GENERIC_CAPS = /* @__PURE__ */ new Set([
|
|
|
6771
6904
|
"advantages",
|
|
6772
6905
|
"disadvantages"
|
|
6773
6906
|
]);
|
|
6907
|
+
var GENERIC_SINGLE_ENTITY_TERMS = /* @__PURE__ */ new Set([
|
|
6908
|
+
"user",
|
|
6909
|
+
"assistant",
|
|
6910
|
+
"agent",
|
|
6911
|
+
"customer",
|
|
6912
|
+
"client",
|
|
6913
|
+
"person",
|
|
6914
|
+
"people",
|
|
6915
|
+
"human",
|
|
6916
|
+
"memory",
|
|
6917
|
+
"message",
|
|
6918
|
+
"conversation",
|
|
6919
|
+
"chat",
|
|
6920
|
+
"session",
|
|
6921
|
+
"system",
|
|
6922
|
+
"top"
|
|
6923
|
+
]);
|
|
6774
6924
|
var FORMATTING_MARKERS = /* @__PURE__ */ new Set([
|
|
6775
6925
|
"*",
|
|
6776
6926
|
"-",
|
|
@@ -6817,22 +6967,64 @@ function stripGenericEnding(words) {
|
|
|
6817
6967
|
}
|
|
6818
6968
|
return words;
|
|
6819
6969
|
}
|
|
6820
|
-
function
|
|
6821
|
-
|
|
6822
|
-
|
|
6970
|
+
function stripTopicPrefix(words) {
|
|
6971
|
+
let start = 0;
|
|
6972
|
+
while (start < words.length && TOPIC_PREFIX_WORDS.has(words[start].toLowerCase())) {
|
|
6973
|
+
start++;
|
|
6823
6974
|
}
|
|
6824
|
-
|
|
6825
|
-
|
|
6826
|
-
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6975
|
+
return words.slice(start);
|
|
6976
|
+
}
|
|
6977
|
+
function cleanToken(token) {
|
|
6978
|
+
return token.replace(/^[^\w.]+|[^\w.]+$/g, "");
|
|
6979
|
+
}
|
|
6980
|
+
function tokenize(text) {
|
|
6981
|
+
var _a2;
|
|
6982
|
+
return (_a2 = text.match(
|
|
6983
|
+
/[A-Za-z_][\w-]*(?:\.[A-Za-z_][\w-]*)*|\d[\d,]*(?:\.\d+)?|[,:;.!?&]/g
|
|
6984
|
+
)) != null ? _a2 : [];
|
|
6985
|
+
}
|
|
6986
|
+
function isCapitalized(token) {
|
|
6987
|
+
return /^[A-Z]/.test(token) && /[A-Za-z]/.test(token);
|
|
6988
|
+
}
|
|
6989
|
+
function hasInternalCapOrDigit(token) {
|
|
6990
|
+
return /\d/.test(token) || /[A-Z]/.test(token.slice(1)) || /^[A-Z]{2,}$/.test(token);
|
|
6991
|
+
}
|
|
6992
|
+
function isBadSingleNameToken(token) {
|
|
6993
|
+
const lower = token.toLowerCase();
|
|
6994
|
+
return GENERIC_SINGLE_ENTITY_TERMS.has(lower) || GENERIC_CAPS.has(lower);
|
|
6995
|
+
}
|
|
6996
|
+
function looksLikeMetricCount(token) {
|
|
6997
|
+
return /^\d[\d,]*(?:\.\d+)?$/.test(token);
|
|
6998
|
+
}
|
|
6999
|
+
function isMetricListContext(tokens, idx) {
|
|
7000
|
+
const prev = idx > 0 ? tokens[idx - 1] : "";
|
|
7001
|
+
const next = idx + 1 < tokens.length ? tokens[idx + 1] : "";
|
|
7002
|
+
return [":", ",", ";"].includes(prev) || [",", ";"].includes(next);
|
|
7003
|
+
}
|
|
7004
|
+
function isSentenceStart(tokens, idx) {
|
|
7005
|
+
if (idx === 0) return true;
|
|
7006
|
+
return [".", "!", "?", ":"].includes(tokens[idx - 1]) || FORMATTING_MARKERS.has(tokens[idx - 1]);
|
|
7007
|
+
}
|
|
7008
|
+
function isListItemNameToken(tokens, idx) {
|
|
7009
|
+
const token = cleanToken(tokens[idx]);
|
|
7010
|
+
if (!isCapitalized(token) || isBadSingleNameToken(token)) return false;
|
|
7011
|
+
const next = idx + 1 < tokens.length ? cleanToken(tokens[idx + 1]) : "";
|
|
7012
|
+
if (!looksLikeMetricCount(next)) return false;
|
|
7013
|
+
return isMetricListContext(tokens, idx) || isMetricListContext(tokens, idx + 1);
|
|
7014
|
+
}
|
|
7015
|
+
function isNameToken(tokens, idx) {
|
|
7016
|
+
const token = cleanToken(tokens[idx]);
|
|
7017
|
+
if (!token || !isCapitalized(token) || isBadSingleNameToken(token))
|
|
7018
|
+
return false;
|
|
7019
|
+
if (hasInternalCapOrDigit(token) || isListItemNameToken(tokens, idx))
|
|
6833
7020
|
return true;
|
|
6834
|
-
|
|
6835
|
-
|
|
7021
|
+
return !isSentenceStart(tokens, idx);
|
|
7022
|
+
}
|
|
7023
|
+
function cleanEntityText(text) {
|
|
7024
|
+
return text.replace(/^\*+\s*|\s*\*+$/g, "").replace(/\s*:+$/g, "").replace(/^\d+\s*\.\s*/, "").replace(/\s+\d[\d,]*(?:\.\d+)?$/g, "").replace(/[.,;!?]+$/, "").trim().replace(/\s+/g, " ");
|
|
7025
|
+
}
|
|
7026
|
+
function isCoordinatedNameTopic(text) {
|
|
7027
|
+
return /\b[A-Z][\w-]+\s+and\s+[A-Z][\w-]+\b/.test(text);
|
|
6836
7028
|
}
|
|
6837
7029
|
function extractQuoted(text) {
|
|
6838
7030
|
const entities = [];
|
|
@@ -6853,62 +7045,57 @@ function extractQuoted(text) {
|
|
|
6853
7045
|
}
|
|
6854
7046
|
return entities;
|
|
6855
7047
|
}
|
|
7048
|
+
function extractIdentifiers(text) {
|
|
7049
|
+
const entities = [];
|
|
7050
|
+
const identifierRe = /\b[A-Za-z_][\w-]*(?:\.[A-Za-z_][\w-]*)+\b/g;
|
|
7051
|
+
let match;
|
|
7052
|
+
while ((match = identifierRe.exec(text)) !== null) {
|
|
7053
|
+
entities.push({ type: "IDENTIFIER", text: match[0] });
|
|
7054
|
+
}
|
|
7055
|
+
return entities;
|
|
7056
|
+
}
|
|
6856
7057
|
function extractProper(text) {
|
|
6857
7058
|
const entities = [];
|
|
6858
|
-
const tokens = text
|
|
6859
|
-
const
|
|
6860
|
-
"'s",
|
|
6861
|
-
"of",
|
|
6862
|
-
"the",
|
|
6863
|
-
"in",
|
|
6864
|
-
"and",
|
|
6865
|
-
"for",
|
|
6866
|
-
"at",
|
|
6867
|
-
"is"
|
|
6868
|
-
]);
|
|
7059
|
+
const tokens = tokenize(text);
|
|
7060
|
+
const innerConnectors = /* @__PURE__ */ new Set(["of", "the", "in", "for", "at"]);
|
|
6869
7061
|
let i = 0;
|
|
6870
7062
|
while (i < tokens.length) {
|
|
6871
|
-
const
|
|
6872
|
-
|
|
7063
|
+
const token = cleanToken(tokens[i]);
|
|
7064
|
+
const next = i + 1 < tokens.length ? tokens[i + 1] : "";
|
|
7065
|
+
const afterNext = i + 2 < tokens.length ? cleanToken(tokens[i + 2]) : "";
|
|
7066
|
+
if (token && next === "&" && afterNext && isCapitalized(token) && isCapitalized(afterNext) && !isBadSingleNameToken(token) && !isBadSingleNameToken(afterNext)) {
|
|
7067
|
+
entities.push({
|
|
7068
|
+
type: "PROPER",
|
|
7069
|
+
text: cleanEntityText(`${token} & ${afterNext}`)
|
|
7070
|
+
});
|
|
7071
|
+
i += 3;
|
|
7072
|
+
continue;
|
|
7073
|
+
}
|
|
7074
|
+
if (!isNameToken(tokens, i)) {
|
|
6873
7075
|
i++;
|
|
6874
7076
|
continue;
|
|
6875
7077
|
}
|
|
6876
|
-
const
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
const
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
|
|
6883
|
-
|
|
6884
|
-
const t = tokens[j];
|
|
6885
|
-
const tIsCap = t.length > 0 && t.charAt(0) === t.charAt(0).toUpperCase() && /[A-Z]/.test(t.charAt(0));
|
|
6886
|
-
if (tIsCap || functionWords.has(t.toLowerCase())) {
|
|
6887
|
-
seq.push({ token: t, idx: j });
|
|
6888
|
-
j++;
|
|
6889
|
-
} else {
|
|
6890
|
-
break;
|
|
6891
|
-
}
|
|
6892
|
-
}
|
|
6893
|
-
while (seq.length > 0 && functionWords.has(seq[seq.length - 1].token.toLowerCase())) {
|
|
6894
|
-
seq.pop();
|
|
7078
|
+
const span = [cleanToken(tokens[i])];
|
|
7079
|
+
let j = i + 1;
|
|
7080
|
+
while (j < tokens.length) {
|
|
7081
|
+
const current = cleanToken(tokens[j]);
|
|
7082
|
+
if (isNameToken(tokens, j)) {
|
|
7083
|
+
span.push(current);
|
|
7084
|
+
j++;
|
|
7085
|
+
continue;
|
|
6895
7086
|
}
|
|
6896
|
-
if (
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
});
|
|
6901
|
-
if (hasMidCap) {
|
|
6902
|
-
const phrase = seq.map((s) => s.token).join(" ");
|
|
6903
|
-
if (phrase.length > 2) {
|
|
6904
|
-
entities.push({ type: "PROPER", text: phrase });
|
|
6905
|
-
}
|
|
6906
|
-
}
|
|
7087
|
+
if (innerConnectors.has(current.toLowerCase()) && j + 1 < tokens.length && isNameToken(tokens, j + 1)) {
|
|
7088
|
+
span.push(current, cleanToken(tokens[j + 1]));
|
|
7089
|
+
j += 2;
|
|
7090
|
+
continue;
|
|
6907
7091
|
}
|
|
6908
|
-
|
|
6909
|
-
}
|
|
6910
|
-
|
|
7092
|
+
break;
|
|
7093
|
+
}
|
|
7094
|
+
const phrase = cleanEntityText(span.join(" "));
|
|
7095
|
+
if (phrase.length > 2) {
|
|
7096
|
+
entities.push({ type: "PROPER", text: phrase });
|
|
6911
7097
|
}
|
|
7098
|
+
i = Math.max(j, i + 1);
|
|
6912
7099
|
}
|
|
6913
7100
|
return entities;
|
|
6914
7101
|
}
|
|
@@ -6940,11 +7127,11 @@ function extractCompoundsWithNlp(text) {
|
|
|
6940
7127
|
const filtered = words.filter(
|
|
6941
7128
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6942
7129
|
);
|
|
6943
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7130
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6944
7131
|
if (cleaned.length >= 2) {
|
|
6945
|
-
const phrase = cleaned.join(" ");
|
|
7132
|
+
const phrase = cleanEntityText(cleaned.join(" "));
|
|
6946
7133
|
if (phrase.length > 3) {
|
|
6947
|
-
entities.push({ type: "
|
|
7134
|
+
entities.push({ type: "TOPIC", text: phrase });
|
|
6948
7135
|
}
|
|
6949
7136
|
}
|
|
6950
7137
|
}
|
|
@@ -6952,7 +7139,7 @@ function extractCompoundsWithNlp(text) {
|
|
|
6952
7139
|
}
|
|
6953
7140
|
function extractCompoundsRegex(text) {
|
|
6954
7141
|
const entities = [];
|
|
6955
|
-
const compoundRe = /\b([A-Z][a-z]+(?:\s+(?:of|
|
|
7142
|
+
const compoundRe = /\b([A-Z][a-z]+(?:\s+(?:of|the|for|in)\s+)?[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b/g;
|
|
6956
7143
|
let match;
|
|
6957
7144
|
while ((match = compoundRe.exec(text)) !== null) {
|
|
6958
7145
|
const phrase = match[1].trim();
|
|
@@ -6963,9 +7150,12 @@ function extractCompoundsRegex(text) {
|
|
|
6963
7150
|
const filtered = words.filter(
|
|
6964
7151
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6965
7152
|
);
|
|
6966
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7153
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6967
7154
|
if (cleaned.length >= 2) {
|
|
6968
|
-
entities.push({
|
|
7155
|
+
entities.push({
|
|
7156
|
+
type: "TOPIC",
|
|
7157
|
+
text: cleanEntityText(cleaned.join(" "))
|
|
7158
|
+
});
|
|
6969
7159
|
}
|
|
6970
7160
|
}
|
|
6971
7161
|
}
|
|
@@ -6987,9 +7177,12 @@ function extractCompoundsRegex(text) {
|
|
|
6987
7177
|
const filtered = words.filter(
|
|
6988
7178
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6989
7179
|
);
|
|
6990
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7180
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6991
7181
|
if (cleaned.length >= 2) {
|
|
6992
|
-
entities.push({
|
|
7182
|
+
entities.push({
|
|
7183
|
+
type: "TOPIC",
|
|
7184
|
+
text: cleanEntityText(cleaned.join(" "))
|
|
7185
|
+
});
|
|
6993
7186
|
}
|
|
6994
7187
|
}
|
|
6995
7188
|
}
|
|
@@ -7002,6 +7195,7 @@ function extractEntities(text) {
|
|
|
7002
7195
|
const raw = [];
|
|
7003
7196
|
raw.push(...extractQuoted(text));
|
|
7004
7197
|
raw.push(...extractProper(text));
|
|
7198
|
+
raw.push(...extractIdentifiers(text));
|
|
7005
7199
|
if (nlp) {
|
|
7006
7200
|
raw.push(...extractCompoundsWithNlp(text));
|
|
7007
7201
|
} else {
|
|
@@ -7019,13 +7213,13 @@ function extractEntities(text) {
|
|
|
7019
7213
|
const cleaned = [];
|
|
7020
7214
|
for (const entity of deduped) {
|
|
7021
7215
|
let txt = entity.text.trim();
|
|
7022
|
-
txt = txt
|
|
7023
|
-
txt = txt.replace(/\s*:+$/, "");
|
|
7024
|
-
txt = txt.replace(/^\d+\s*\.\s*/, "");
|
|
7025
|
-
txt = txt.replace(/[.,;!?]+$/, "").trim();
|
|
7216
|
+
txt = cleanEntityText(txt);
|
|
7026
7217
|
if (!txt || txt.length <= 2 || hasArtifacts(txt)) {
|
|
7027
7218
|
continue;
|
|
7028
7219
|
}
|
|
7220
|
+
if (entity.type === "TOPIC" && (/^\d/.test(txt) || isCoordinatedNameTopic(txt))) {
|
|
7221
|
+
continue;
|
|
7222
|
+
}
|
|
7029
7223
|
if (entity.type === "PROPER" && !txt.includes(" ") && GENERIC_CAPS.has(txt.toLowerCase())) {
|
|
7030
7224
|
continue;
|
|
7031
7225
|
}
|
|
@@ -7033,9 +7227,9 @@ function extractEntities(text) {
|
|
|
7033
7227
|
}
|
|
7034
7228
|
const typePriority = {
|
|
7035
7229
|
PROPER: 0,
|
|
7036
|
-
|
|
7230
|
+
IDENTIFIER: 1,
|
|
7037
7231
|
QUOTED: 2,
|
|
7038
|
-
|
|
7232
|
+
TOPIC: 3
|
|
7039
7233
|
};
|
|
7040
7234
|
const best = /* @__PURE__ */ new Map();
|
|
7041
7235
|
for (const entity of cleaned) {
|
|
@@ -7046,10 +7240,14 @@ function extractEntities(text) {
|
|
|
7046
7240
|
}
|
|
7047
7241
|
}
|
|
7048
7242
|
const bestEntities = Array.from(best.values());
|
|
7049
|
-
const allLower = bestEntities.map((e) => e.text.toLowerCase());
|
|
7050
7243
|
return bestEntities.filter(
|
|
7051
|
-
(entity) => !
|
|
7052
|
-
(other) =>
|
|
7244
|
+
(entity) => !bestEntities.some(
|
|
7245
|
+
(other) => {
|
|
7246
|
+
var _a3, _b3;
|
|
7247
|
+
return entity.text.toLowerCase() !== other.text.toLowerCase() && ((_a3 = typePriority[entity.type]) != null ? _a3 : 99) >= ((_b3 = typePriority[other.type]) != null ? _b3 : 99) && new RegExp(
|
|
7248
|
+
`(^|\\s)${entity.text.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(\\s|$)`
|
|
7249
|
+
).test(other.text.toLowerCase());
|
|
7250
|
+
}
|
|
7053
7251
|
)
|
|
7054
7252
|
);
|
|
7055
7253
|
}
|
|
@@ -7341,6 +7539,32 @@ var Memory = class _Memory {
|
|
|
7341
7539
|
if (payload.run_id) filters.run_id = payload.run_id;
|
|
7342
7540
|
return filters;
|
|
7343
7541
|
}
|
|
7542
|
+
_normalizeEntityText(value) {
|
|
7543
|
+
return value.trim().toLowerCase().replace(/\s+/g, " ");
|
|
7544
|
+
}
|
|
7545
|
+
async _existingEntitiesByText(entityStore, filters) {
|
|
7546
|
+
var _a2;
|
|
7547
|
+
const rowsByText = /* @__PURE__ */ new Map();
|
|
7548
|
+
let rows = [];
|
|
7549
|
+
try {
|
|
7550
|
+
const listed = await entityStore.list(filters, 1e4);
|
|
7551
|
+
rows = Array.isArray(listed) && Array.isArray(listed[0]) ? listed[0] : listed;
|
|
7552
|
+
} catch (e) {
|
|
7553
|
+
console.debug(
|
|
7554
|
+
`Exact entity lookup failed, falling back to semantic dedup: ${e}`
|
|
7555
|
+
);
|
|
7556
|
+
return rowsByText;
|
|
7557
|
+
}
|
|
7558
|
+
for (const row of rows) {
|
|
7559
|
+
const text = (_a2 = row.payload) == null ? void 0 : _a2.data;
|
|
7560
|
+
if (typeof text !== "string") continue;
|
|
7561
|
+
const key = this._normalizeEntityText(text);
|
|
7562
|
+
if (key && !rowsByText.has(key)) {
|
|
7563
|
+
rowsByText.set(key, row);
|
|
7564
|
+
}
|
|
7565
|
+
}
|
|
7566
|
+
return rowsByText;
|
|
7567
|
+
}
|
|
7344
7568
|
/**
|
|
7345
7569
|
* Remove `memoryId` from every entity record scoped to `filters`.
|
|
7346
7570
|
* If an entity's `linkedMemoryIds` becomes empty after removal, the
|
|
@@ -7418,6 +7642,10 @@ var Memory = class _Memory {
|
|
|
7418
7642
|
const entities = extractEntities(text);
|
|
7419
7643
|
if (entities.length === 0) return;
|
|
7420
7644
|
const entityStore = await this.getEntityStore();
|
|
7645
|
+
const exactMatches = await this._existingEntitiesByText(
|
|
7646
|
+
entityStore,
|
|
7647
|
+
filters
|
|
7648
|
+
);
|
|
7421
7649
|
for (const entity of entities) {
|
|
7422
7650
|
try {
|
|
7423
7651
|
let entityVec;
|
|
@@ -7428,12 +7656,18 @@ var Memory = class _Memory {
|
|
|
7428
7656
|
continue;
|
|
7429
7657
|
}
|
|
7430
7658
|
let matches = [];
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
|
|
7659
|
+
const exactMatch = exactMatches.get(
|
|
7660
|
+
this._normalizeEntityText(entity.text)
|
|
7661
|
+
);
|
|
7662
|
+
if (!exactMatch) {
|
|
7663
|
+
try {
|
|
7664
|
+
matches = await entityStore.search(entityVec, 1, filters);
|
|
7665
|
+
} catch (e) {
|
|
7666
|
+
}
|
|
7434
7667
|
}
|
|
7435
|
-
|
|
7436
|
-
|
|
7668
|
+
const semanticMatch = matches.length > 0 && ((_a2 = matches[0].score) != null ? _a2 : 0) >= 0.95 ? matches[0] : void 0;
|
|
7669
|
+
const match = exactMatch != null ? exactMatch : semanticMatch;
|
|
7670
|
+
if (match) {
|
|
7437
7671
|
const payload = match.payload || {};
|
|
7438
7672
|
const linked = new Set(
|
|
7439
7673
|
Array.isArray(payload.linkedMemoryIds) ? payload.linkedMemoryIds : []
|
|
@@ -7950,6 +8184,10 @@ var Memory = class _Memory {
|
|
|
7950
8184
|
}
|
|
7951
8185
|
if (valid.length > 0) {
|
|
7952
8186
|
const entityStore = await this.getEntityStore();
|
|
8187
|
+
const exactMatches = await this._existingEntitiesByText(
|
|
8188
|
+
entityStore,
|
|
8189
|
+
filters
|
|
8190
|
+
);
|
|
7953
8191
|
const toInsertVectors = [];
|
|
7954
8192
|
const toInsertIds = [];
|
|
7955
8193
|
const toInsertPayloads = [];
|
|
@@ -7957,12 +8195,16 @@ var Memory = class _Memory {
|
|
|
7957
8195
|
const { entityType, entityText, memoryIds } = globalEntities[key];
|
|
7958
8196
|
const entityVec = entityEmbeddings[j];
|
|
7959
8197
|
let matches = [];
|
|
7960
|
-
|
|
7961
|
-
|
|
7962
|
-
|
|
8198
|
+
const exactMatch = exactMatches.get(key);
|
|
8199
|
+
if (!exactMatch) {
|
|
8200
|
+
try {
|
|
8201
|
+
matches = await entityStore.search(entityVec, 1, filters);
|
|
8202
|
+
} catch (e) {
|
|
8203
|
+
}
|
|
7963
8204
|
}
|
|
7964
|
-
|
|
7965
|
-
|
|
8205
|
+
const semanticMatch = matches.length > 0 && ((_f = matches[0].score) != null ? _f : 0) >= 0.95 ? matches[0] : void 0;
|
|
8206
|
+
const match = exactMatch != null ? exactMatch : semanticMatch;
|
|
8207
|
+
if (match) {
|
|
7966
8208
|
const payload = match.payload || {};
|
|
7967
8209
|
const linked = new Set((_g = payload.linkedMemoryIds) != null ? _g : []);
|
|
7968
8210
|
for (const mid of memoryIds) linked.add(mid);
|
|
@@ -8334,7 +8576,9 @@ var Memory = class _Memory {
|
|
|
8334
8576
|
has_agent_id: !!config.agentId,
|
|
8335
8577
|
has_run_id: !!config.runId
|
|
8336
8578
|
});
|
|
8337
|
-
const
|
|
8579
|
+
const userId = validateAndTrimEntityId(config.userId, "userId");
|
|
8580
|
+
const agentId = validateAndTrimEntityId(config.agentId, "agentId");
|
|
8581
|
+
const runId = validateAndTrimEntityId(config.runId, "runId");
|
|
8338
8582
|
const filters = {};
|
|
8339
8583
|
if (userId) filters.user_id = userId;
|
|
8340
8584
|
if (agentId) filters.agent_id = agentId;
|
|
@@ -8692,6 +8936,7 @@ var Memory = class _Memory {
|
|
|
8692
8936
|
LangchainEmbedder,
|
|
8693
8937
|
LangchainLLM,
|
|
8694
8938
|
LangchainVectorStore,
|
|
8939
|
+
LiteLLM,
|
|
8695
8940
|
Memory,
|
|
8696
8941
|
MemoryConfigSchema,
|
|
8697
8942
|
MemoryVectorStore,
|