@xfabric/memory 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/index.d.ts +3 -0
- package/dist/chunking/index.d.ts.map +1 -0
- package/dist/chunking/index.js +3 -0
- package/dist/chunking/index.js.map +1 -0
- package/dist/chunking/markdown.d.ts +13 -0
- package/dist/chunking/markdown.d.ts.map +1 -0
- package/dist/chunking/markdown.js +106 -0
- package/dist/chunking/markdown.js.map +1 -0
- package/dist/chunking/session.d.ts +24 -0
- package/dist/chunking/session.d.ts.map +1 -0
- package/dist/chunking/session.js +173 -0
- package/dist/chunking/session.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/memory-manager.d.ts +189 -0
- package/dist/memory-manager.d.ts.map +1 -0
- package/dist/memory-manager.js +1055 -0
- package/dist/memory-manager.js.map +1 -0
- package/dist/providers/gemini.d.ts +6 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +73 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/index.d.ts +20 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +102 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/local.d.ts +14 -0
- package/dist/providers/local.d.ts.map +1 -0
- package/dist/providers/local.js +73 -0
- package/dist/providers/local.js.map +1 -0
- package/dist/providers/openai.d.ts +6 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +48 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/types.d.ts +62 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/search/fts.d.ts +11 -0
- package/dist/search/fts.d.ts.map +1 -0
- package/dist/search/fts.js +50 -0
- package/dist/search/fts.js.map +1 -0
- package/dist/search/hybrid.d.ts +16 -0
- package/dist/search/hybrid.d.ts.map +1 -0
- package/dist/search/hybrid.js +83 -0
- package/dist/search/hybrid.js.map +1 -0
- package/dist/search/index.d.ts +4 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +4 -0
- package/dist/search/index.js.map +1 -0
- package/dist/search/vector.d.ts +25 -0
- package/dist/search/vector.d.ts.map +1 -0
- package/dist/search/vector.js +152 -0
- package/dist/search/vector.js.map +1 -0
- package/dist/storage/index.d.ts +4 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +4 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/schema.d.ts +24 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +175 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/storage/sqlite-vec.d.ts +22 -0
- package/dist/storage/sqlite-vec.d.ts.map +1 -0
- package/dist/storage/sqlite-vec.js +85 -0
- package/dist/storage/sqlite-vec.js.map +1 -0
- package/dist/storage/sqlite.d.ts +206 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +352 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/sync/index.d.ts +4 -0
- package/dist/sync/index.d.ts.map +1 -0
- package/dist/sync/index.js +4 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/sync/minimatch.d.ts +6 -0
- package/dist/sync/minimatch.d.ts.map +1 -0
- package/dist/sync/minimatch.js +60 -0
- package/dist/sync/minimatch.js.map +1 -0
- package/dist/sync/session-monitor.d.ts +50 -0
- package/dist/sync/session-monitor.d.ts.map +1 -0
- package/dist/sync/session-monitor.js +126 -0
- package/dist/sync/session-monitor.js.map +1 -0
- package/dist/sync/watcher.d.ts +44 -0
- package/dist/sync/watcher.d.ts.map +1 -0
- package/dist/sync/watcher.js +110 -0
- package/dist/sync/watcher.js.map +1 -0
- package/dist/tools/index.d.ts +3 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +3 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/memory-get.d.ts +32 -0
- package/dist/tools/memory-get.d.ts.map +1 -0
- package/dist/tools/memory-get.js +53 -0
- package/dist/tools/memory-get.js.map +1 -0
- package/dist/tools/memory-search.d.ts +32 -0
- package/dist/tools/memory-search.d.ts.map +1 -0
- package/dist/tools/memory-search.js +56 -0
- package/dist/tools/memory-search.js.map +1 -0
- package/dist/types.d.ts +350 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +15 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/concurrency.d.ts +25 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +59 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/hash.d.ts +9 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +16 -0
- package/dist/utils/hash.js.map +1 -0
- package/dist/utils/index.d.ts +4 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +4 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/retry.d.ts +22 -0
- package/dist/utils/retry.d.ts.map +1 -0
- package/dist/utils/retry.js +48 -0
- package/dist/utils/retry.js.map +1 -0
- package/package.json +67 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,KAAK,sBAAsB,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAwB,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAA+B,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { MemoryChunk } from "../types.js";
|
|
2
|
+
export type ChunkingOptions = {
|
|
3
|
+
/** Approximate number of tokens per chunk (4 chars ≈ 1 token) */
|
|
4
|
+
tokens: number;
|
|
5
|
+
/** Number of overlap tokens between chunks */
|
|
6
|
+
overlap: number;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Chunk markdown content into overlapping text segments
|
|
10
|
+
* Uses character-based chunking (4 chars ≈ 1 token)
|
|
11
|
+
*/
|
|
12
|
+
export declare function chunkMarkdown(content: string, options: ChunkingOptions): MemoryChunk[];
|
|
13
|
+
//# sourceMappingURL=markdown.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/chunking/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,MAAM,MAAM,eAAe,GAAG;IAC5B,iEAAiE;IACjE,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,WAAW,EAAE,CAoHtF"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { hashText } from "../utils/hash.js";
|
|
2
|
+
/**
|
|
3
|
+
* Chunk markdown content into overlapping text segments
|
|
4
|
+
* Uses character-based chunking (4 chars ≈ 1 token)
|
|
5
|
+
*/
|
|
6
|
+
export function chunkMarkdown(content, options) {
|
|
7
|
+
if (!content || content.trim().length === 0) {
|
|
8
|
+
if (content.length > 0) {
|
|
9
|
+
// Content is only whitespace
|
|
10
|
+
return [
|
|
11
|
+
{
|
|
12
|
+
startLine: 1,
|
|
13
|
+
endLine: content.split("\n").length,
|
|
14
|
+
text: content,
|
|
15
|
+
hash: hashText(content),
|
|
16
|
+
},
|
|
17
|
+
];
|
|
18
|
+
}
|
|
19
|
+
return [];
|
|
20
|
+
}
|
|
21
|
+
const maxChars = Math.max(32, options.tokens * 4);
|
|
22
|
+
const overlapChars = Math.max(0, options.overlap * 4);
|
|
23
|
+
const lines = content.split("\n");
|
|
24
|
+
const chunks = [];
|
|
25
|
+
let currentChunk = [];
|
|
26
|
+
let currentChunkChars = 0;
|
|
27
|
+
let chunkStartLine = 1;
|
|
28
|
+
let currentLine = 1;
|
|
29
|
+
const flushChunk = (endLine) => {
|
|
30
|
+
if (currentChunk.length === 0)
|
|
31
|
+
return;
|
|
32
|
+
const text = currentChunk.join("\n");
|
|
33
|
+
chunks.push({
|
|
34
|
+
startLine: chunkStartLine,
|
|
35
|
+
endLine,
|
|
36
|
+
text,
|
|
37
|
+
hash: hashText(text),
|
|
38
|
+
});
|
|
39
|
+
};
|
|
40
|
+
const calculateOverlapLines = () => {
|
|
41
|
+
if (overlapChars <= 0 || currentChunk.length === 0) {
|
|
42
|
+
return { lines: [], startLine: currentLine + 1 };
|
|
43
|
+
}
|
|
44
|
+
// Work backwards to find lines to carry over
|
|
45
|
+
let overlapLength = 0;
|
|
46
|
+
const overlapLines = [];
|
|
47
|
+
let overlapStartLine = currentLine;
|
|
48
|
+
for (let i = currentChunk.length - 1; i >= 0 && overlapLength < overlapChars; i--) {
|
|
49
|
+
const line = currentChunk[i];
|
|
50
|
+
overlapLines.unshift(line);
|
|
51
|
+
overlapLength += line.length + 1; // +1 for newline
|
|
52
|
+
overlapStartLine = chunkStartLine + i;
|
|
53
|
+
}
|
|
54
|
+
return { lines: overlapLines, startLine: overlapStartLine };
|
|
55
|
+
};
|
|
56
|
+
for (const line of lines) {
|
|
57
|
+
const lineLength = line.length + 1; // +1 for newline
|
|
58
|
+
// Check if adding this line would exceed the chunk size
|
|
59
|
+
if (currentChunkChars + lineLength > maxChars && currentChunk.length > 0) {
|
|
60
|
+
// Flush current chunk
|
|
61
|
+
flushChunk(currentLine - 1);
|
|
62
|
+
// Calculate overlap
|
|
63
|
+
const overlap = calculateOverlapLines();
|
|
64
|
+
currentChunk = overlap.lines;
|
|
65
|
+
currentChunkChars = currentChunk.reduce((sum, l) => sum + l.length + 1, 0);
|
|
66
|
+
chunkStartLine = overlap.startLine;
|
|
67
|
+
}
|
|
68
|
+
// Handle very long lines by splitting them
|
|
69
|
+
if (lineLength > maxChars) {
|
|
70
|
+
// Flush any existing content first
|
|
71
|
+
if (currentChunk.length > 0) {
|
|
72
|
+
flushChunk(currentLine - 1);
|
|
73
|
+
currentChunk = [];
|
|
74
|
+
currentChunkChars = 0;
|
|
75
|
+
chunkStartLine = currentLine;
|
|
76
|
+
}
|
|
77
|
+
// Split the long line into segments
|
|
78
|
+
let offset = 0;
|
|
79
|
+
while (offset < line.length) {
|
|
80
|
+
const segment = line.slice(offset, offset + maxChars - 1);
|
|
81
|
+
const segmentText = segment;
|
|
82
|
+
chunks.push({
|
|
83
|
+
startLine: currentLine,
|
|
84
|
+
endLine: currentLine,
|
|
85
|
+
text: segmentText,
|
|
86
|
+
hash: hashText(segmentText),
|
|
87
|
+
});
|
|
88
|
+
offset += maxChars - 1 - overlapChars;
|
|
89
|
+
if (offset < 0)
|
|
90
|
+
offset = maxChars - 1;
|
|
91
|
+
}
|
|
92
|
+
chunkStartLine = currentLine + 1;
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
currentChunk.push(line);
|
|
96
|
+
currentChunkChars += lineLength;
|
|
97
|
+
}
|
|
98
|
+
currentLine++;
|
|
99
|
+
}
|
|
100
|
+
// Flush remaining content
|
|
101
|
+
if (currentChunk.length > 0) {
|
|
102
|
+
flushChunk(lines.length);
|
|
103
|
+
}
|
|
104
|
+
return chunks;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=markdown.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/chunking/markdown.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAS5C;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,OAAwB;IACrE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,6BAA6B;YAC7B,OAAO;gBACL;oBACE,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;oBACnC,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC;iBACxB;aACF,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,MAAM,UAAU,GAAG,CAAC,OAAe,EAAE,EAAE;QACrC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEtC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC;YACV,SAAS,EAAE,cAAc;YACzB,OAAO;YACP,IAAI;YACJ,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;SACrB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,GAA2C,EAAE;QACzE,IAAI,YAAY,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;QACnD,CAAC;QAED,6CAA6C;QAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,WAAW,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YAClF,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3B,aAAa,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;YACnD,gBAAgB,GAAG,cAAc,GAAG,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9D,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;QAErD,wDAAwD;QACxD,IAAI,iBAAiB,GAAG,UAAU,GAAG,QAAQ,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzE,sBAAsB;YACtB,UAAU,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;YAE5B,oBAAoB;YACpB,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAC;YACxC,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC;YAC7B,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC3E,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC;QACrC,CAAC;QAED,2CAA2C;QAC3C,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;YAC1B,mCAAmC;YACnC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,UAAU,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;gBAC5B,YAAY,GAAG,EAAE,CAAC;gBAClB,iBAAiB,GAAG,CAAC,CAAC;gBACtB,cAAc,GAAG,WAAW,CAAC;YAC/B,CAAC;YAED,oCAAoC;YACpC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAC1D,MAAM,WAAW,GAAG,OAAO,CAAC;gBAE5B,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,WAAW;oBACtB,OAAO,EAAE,WAAW;oBACpB,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC;iBAC5B,CAAC,CAAC;gBAEH,MAAM,IAAI,QAAQ,GAAG,CAAC,GAAG,YAAY,CAAC;gBACtC,IAAI,MAAM,GAAG,CAAC;oBAAE,MAAM,GAAG,QAAQ,GAAG,CAAC,CAAC;YACxC,CAAC;YAED,cAAc,GAAG,WAAW,GAAG,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,iBAAiB,IAAI,UAAU,CAAC;QAClC,CAAC;QAED,WAAW,EAAE,CAAC;IAChB,CAAC;IAED,0BAA0B;IAC1B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { SessionChunk } from "../types.js";
|
|
2
|
+
export type SessionChunkingOptions = {
|
|
3
|
+
/** Target tokens per chunk (4 chars ≈ 1 token) */
|
|
4
|
+
tokens?: number;
|
|
5
|
+
/** Overlap tokens between chunks */
|
|
6
|
+
overlap?: number;
|
|
7
|
+
/** Starting byte offset for delta tracking */
|
|
8
|
+
startByteOffset?: number;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Chunk session transcript into searchable chunks
|
|
12
|
+
*/
|
|
13
|
+
export declare function chunkSession(content: string, sessionId: string, options?: SessionChunkingOptions): SessionChunk[];
|
|
14
|
+
/**
|
|
15
|
+
* Merge consecutive messages from the same role for better context
|
|
16
|
+
*/
|
|
17
|
+
export declare function mergeConsecutiveMessages(messages: Array<{
|
|
18
|
+
role: "user" | "assistant";
|
|
19
|
+
text: string;
|
|
20
|
+
}>): Array<{
|
|
21
|
+
role: "user" | "assistant";
|
|
22
|
+
text: string;
|
|
23
|
+
}>;
|
|
24
|
+
//# sourceMappingURL=session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/chunking/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAGhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAgFF;;GAEG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,sBAA2B,GACnC,YAAY,EAAE,CAsDhB;AA2DD;;GAEG;AACH,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,GAC5D,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAkBrD"}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { hashText } from "../utils/hash.js";
|
|
2
|
+
/**
|
|
3
|
+
* Extract text content from a JSONL message
|
|
4
|
+
*/
|
|
5
|
+
function extractContent(message) {
|
|
6
|
+
if (!message?.content) {
|
|
7
|
+
return null;
|
|
8
|
+
}
|
|
9
|
+
if (typeof message.content === "string") {
|
|
10
|
+
return message.content;
|
|
11
|
+
}
|
|
12
|
+
// Handle array of content blocks
|
|
13
|
+
const textParts = message.content
|
|
14
|
+
.filter((block) => block.type === "text" && block.text)
|
|
15
|
+
.map((block) => block.text);
|
|
16
|
+
return textParts.length > 0 ? textParts.join("\n") : null;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Parse JSONL content and extract messages
|
|
20
|
+
*/
|
|
21
|
+
function parseMessages(content, startByteOffset) {
|
|
22
|
+
const messages = [];
|
|
23
|
+
const lines = content.split("\n");
|
|
24
|
+
let currentByteOffset = startByteOffset;
|
|
25
|
+
let lineNumber = 1;
|
|
26
|
+
for (const line of lines) {
|
|
27
|
+
const lineByteOffset = currentByteOffset;
|
|
28
|
+
if (line.trim()) {
|
|
29
|
+
try {
|
|
30
|
+
const msg = JSON.parse(line);
|
|
31
|
+
if (msg.type === "message" && msg.role && msg.message) {
|
|
32
|
+
const text = extractContent(msg.message);
|
|
33
|
+
if (text) {
|
|
34
|
+
messages.push({
|
|
35
|
+
role: msg.role,
|
|
36
|
+
text,
|
|
37
|
+
byteOffset: lineByteOffset,
|
|
38
|
+
lineNumber,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Skip invalid JSON lines
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// Add line length + newline for byte offset tracking
|
|
48
|
+
currentByteOffset += Buffer.byteLength(line, "utf-8") + 1;
|
|
49
|
+
lineNumber++;
|
|
50
|
+
}
|
|
51
|
+
return messages;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Chunk session transcript into searchable chunks
|
|
55
|
+
*/
|
|
56
|
+
export function chunkSession(content, sessionId, options = {}) {
|
|
57
|
+
const { tokens = 400, overlap = 80, startByteOffset = 0 } = options;
|
|
58
|
+
// Approximate chars per chunk (4 chars ≈ 1 token)
|
|
59
|
+
const targetChars = tokens * 4;
|
|
60
|
+
const overlapChars = overlap * 4;
|
|
61
|
+
// Parse messages from JSONL
|
|
62
|
+
const messages = parseMessages(content, startByteOffset);
|
|
63
|
+
if (messages.length === 0) {
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
const chunks = [];
|
|
67
|
+
let chunkIndex = 0;
|
|
68
|
+
// Process each message
|
|
69
|
+
for (const message of messages) {
|
|
70
|
+
const text = message.text;
|
|
71
|
+
// If message is small enough, keep it as one chunk
|
|
72
|
+
if (text.length <= targetChars) {
|
|
73
|
+
chunks.push({
|
|
74
|
+
sessionId,
|
|
75
|
+
role: message.role,
|
|
76
|
+
byteOffset: message.byteOffset,
|
|
77
|
+
chunkIndex: chunkIndex++,
|
|
78
|
+
startLine: message.lineNumber,
|
|
79
|
+
endLine: message.lineNumber,
|
|
80
|
+
text: formatChunkText(message.role, text),
|
|
81
|
+
hash: hashText(text),
|
|
82
|
+
});
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
// Split large messages into multiple chunks
|
|
86
|
+
const messageChunks = chunkLongText(text, targetChars, overlapChars);
|
|
87
|
+
for (const chunkText of messageChunks) {
|
|
88
|
+
chunks.push({
|
|
89
|
+
sessionId,
|
|
90
|
+
role: message.role,
|
|
91
|
+
byteOffset: message.byteOffset,
|
|
92
|
+
chunkIndex: chunkIndex++,
|
|
93
|
+
startLine: message.lineNumber,
|
|
94
|
+
endLine: message.lineNumber,
|
|
95
|
+
text: formatChunkText(message.role, chunkText),
|
|
96
|
+
hash: hashText(chunkText),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return chunks;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Format chunk text with role prefix for better context
|
|
104
|
+
*/
|
|
105
|
+
function formatChunkText(role, text) {
|
|
106
|
+
const roleLabel = role === "user" ? "[User]" : "[Assistant]";
|
|
107
|
+
return `${roleLabel} ${text}`;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Split long text into chunks with overlap
|
|
111
|
+
*/
|
|
112
|
+
function chunkLongText(text, targetChars, overlapChars) {
|
|
113
|
+
const chunks = [];
|
|
114
|
+
const lines = text.split("\n");
|
|
115
|
+
let currentChunk = "";
|
|
116
|
+
for (const line of lines) {
|
|
117
|
+
// Handle very long single lines by splitting them
|
|
118
|
+
if (line.length > targetChars) {
|
|
119
|
+
// Save current chunk if we have one
|
|
120
|
+
if (currentChunk.trim()) {
|
|
121
|
+
chunks.push(currentChunk.trim());
|
|
122
|
+
}
|
|
123
|
+
// Split the long line into multiple chunks
|
|
124
|
+
let pos = 0;
|
|
125
|
+
while (pos < line.length) {
|
|
126
|
+
const end = Math.min(pos + targetChars, line.length);
|
|
127
|
+
chunks.push(line.slice(pos, end).trim());
|
|
128
|
+
pos = end - overlapChars;
|
|
129
|
+
if (pos < 0)
|
|
130
|
+
pos = end; // Avoid infinite loop
|
|
131
|
+
}
|
|
132
|
+
currentChunk = "";
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
// If adding this line would exceed target, save current chunk
|
|
136
|
+
if (currentChunk.length + line.length + 1 > targetChars && currentChunk.length > 0) {
|
|
137
|
+
chunks.push(currentChunk.trim());
|
|
138
|
+
// Calculate overlap - keep last N characters
|
|
139
|
+
const overlapText = currentChunk.slice(-overlapChars);
|
|
140
|
+
currentChunk = overlapText + (overlapText ? "\n" : "") + line;
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
currentChunk += (currentChunk ? "\n" : "") + line;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
// Don't forget the last chunk
|
|
147
|
+
if (currentChunk.trim()) {
|
|
148
|
+
chunks.push(currentChunk.trim());
|
|
149
|
+
}
|
|
150
|
+
return chunks;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Merge consecutive messages from the same role for better context
|
|
154
|
+
*/
|
|
155
|
+
export function mergeConsecutiveMessages(messages) {
|
|
156
|
+
if (messages.length === 0)
|
|
157
|
+
return [];
|
|
158
|
+
const merged = [];
|
|
159
|
+
let current = { ...messages[0] };
|
|
160
|
+
for (let i = 1; i < messages.length; i++) {
|
|
161
|
+
const msg = messages[i];
|
|
162
|
+
if (msg.role === current.role) {
|
|
163
|
+
current.text += "\n\n" + msg.text;
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
merged.push(current);
|
|
167
|
+
current = { ...msg };
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
merged.push(current);
|
|
171
|
+
return merged;
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=session.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/chunking/session.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAoB5C;;GAEG;AACH,SAAS,cAAc,CAAC,OAAgC;IACtD,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,OAAO,CAAC;IACzB,CAAC;IAED,iCAAiC;IACjC,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO;SAC9B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC;SACtD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAK,CAAC,CAAC;IAE/B,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CACpB,OAAe,EACf,eAAuB;IAEvB,MAAM,QAAQ,GAKT,EAAE,CAAC;IAER,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,iBAAiB,GAAG,eAAe,CAAC;IACxC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,cAAc,GAAG,iBAAiB,CAAC;QAEzC,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC;gBAE7C,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;oBACtD,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACzC,IAAI,IAAI,EAAE,CAAC;wBACT,QAAQ,CAAC,IAAI,CAAC;4BACZ,IAAI,EAAE,GAAG,CAAC,IAAI;4BACd,IAAI;4BACJ,UAAU,EAAE,cAAc;4BAC1B,UAAU;yBACX,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,iBAAiB,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1D,UAAU,EAAE,CAAC;IACf,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAe,EACf,SAAiB,EACjB,UAAkC,EAAE;IAEpC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE,EAAE,eAAe,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAEpE,kDAAkD;IAClD,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,CAAC;IAC/B,MAAM,YAAY,GAAG,OAAO,GAAG,CAAC,CAAC;IAEjC,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAEzD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,uBAAuB;IACvB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QAE1B,mDAAmD;QACnD,IAAI,IAAI,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS;gBACT,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,UAAU,EAAE;gBACxB,SAAS,EAAE,OAAO,CAAC,UAAU;gBAC7B,OAAO,EAAE,OAAO,CAAC,UAAU;gBAC3B,IAAI,EAAE,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC;gBACzC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;aACrB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,4CAA4C;QAC5C,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;QAErE,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS;gBACT,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,UAAU,EAAE;gBACxB,SAAS,EAAE,OAAO,CAAC,UAAU;gBAC7B,OAAO,EAAE,OAAO,CAAC,UAAU;gBAC3B,IAAI,EAAE,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC;gBAC9C,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAA0B,EAAE,IAAY;IAC/D,MAAM,SAAS,GAAG,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC;IAC7D,OAAO,GAAG,SAAS,IAAI,IAAI,EAAE,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,WAAmB,EAAE,YAAoB;IAC5E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE/B,IAAI,YAAY,GAAG,EAAE,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,kDAAkD;QAClD,IAAI,IAAI,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YAC9B,oCAAoC;YACpC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;YAED,2CAA2C;YAC3C,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzC,GAAG,GAAG,GAAG,GAAG,YAAY,CAAC;gBACzB,IAAI,GAAG,GAAG,CAAC;oBAAE,GAAG,GAAG,GAAG,CAAC,CAAC,sBAAsB;YAChD,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QAED,8DAA8D;QAC9D,IAAI,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnF,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAEjC,6CAA6C;YAC7C,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;YACtD,YAAY,GAAG,WAAW,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,YAAY,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QACpD,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAA6D;IAE7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,MAAM,GAAwD,EAAE,CAAC;IACvE,IAAI,OAAO,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export type { MemoryConfig, MemorySearchResult, SearchOptions, MemoryFileEntry, MemoryChunk, StoredChunk, MemorySource, VectorSearchResult, KeywordSearchResult, HybridSearchOptions, MemorySearchParams, MemoryGetParams, MemorySearchResponse, MemoryGetResponse, MemoryIndexMeta, FileChangeEvent, SessionEvent, EmbeddingProviderType, SyncProgress, MemoryStatus, ProbeResult, SessionChunk, SessionFileState, BatchConfig, BatchJob, ExtendedMemoryConfig, } from "./types.js";
|
|
2
|
+
export { DEFAULT_CONFIG } from "./types.js";
|
|
3
|
+
export { MemoryManager } from "./memory-manager.js";
|
|
4
|
+
export { memorySearch, memoryGet, createMemorySearchToolDef, createMemoryGetToolDef, } from "./tools/index.js";
|
|
5
|
+
export type { EmbeddingProvider, EmbeddingProviderResult, ProviderConfig } from "./providers/index.js";
|
|
6
|
+
export { createEmbeddingProvider, createOpenAIProvider, createGeminiProvider, createLocalProvider, isLocalAvailable, computeProviderKey, } from "./providers/index.js";
|
|
7
|
+
export { MemoryStorage, createDatabase, type StorageOptions, type CreateDatabaseResult, } from "./storage/index.js";
|
|
8
|
+
export { ensureMemoryIndexSchema, migrateSchema, ensureColumn, SCHEMA_VERSION, type SchemaStatus, } from "./storage/index.js";
|
|
9
|
+
export { cosineSimilarity, loadSqliteVec, isSqliteVecAvailable } from "./storage/index.js";
|
|
10
|
+
export { chunkMarkdown, chunkSession, mergeConsecutiveMessages, type ChunkingOptions, type SessionChunkingOptions, } from "./chunking/index.js";
|
|
11
|
+
export { FileWatcher, type FileWatcherOptions } from "./sync/index.js";
|
|
12
|
+
export { SessionMonitor, type SessionMonitorOptions } from "./sync/index.js";
|
|
13
|
+
export { minimatch } from "./sync/index.js";
|
|
14
|
+
export { searchVector, searchVectorWithSqliteVec, syncVectorTable, searchFts, mergeHybridResults, buildFtsQuery, bm25RankToScore, } from "./search/index.js";
|
|
15
|
+
export { hashText, hashFile } from "./utils/index.js";
|
|
16
|
+
export { retry, RetryError, type RetryOptions } from "./utils/index.js";
|
|
17
|
+
export { Semaphore } from "./utils/index.js";
|
|
18
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EACV,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,WAAW,EACX,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,YAAY,EACZ,qBAAqB,EAErB,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACX,QAAQ,EACR,oBAAoB,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EACL,YAAY,EACZ,SAAS,EACT,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACvG,OAAO,EACL,uBAAuB,EACvB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,aAAa,EACb,cAAc,EACd,KAAK,cAAc,EACnB,KAAK,oBAAoB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,uBAAuB,EACvB,aAAa,EACb,YAAY,EACZ,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG3F,OAAO,EACL,aAAa,EACb,YAAY,EACZ,wBAAwB,EACxB,KAAK,eAAe,EACpB,KAAK,sBAAsB,GAC5B,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,WAAW,EAAE,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,OAAO,EACL,YAAY,EACZ,yBAAyB,EACzB,eAAe,EACf,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Main exports for @vibestudio/memory
|
|
2
|
+
export { DEFAULT_CONFIG } from "./types.js";
|
|
3
|
+
// Memory Manager
|
|
4
|
+
export { MemoryManager } from "./memory-manager.js";
|
|
5
|
+
// Tools
|
|
6
|
+
export { memorySearch, memoryGet, createMemorySearchToolDef, createMemoryGetToolDef, } from "./tools/index.js";
|
|
7
|
+
export { createEmbeddingProvider, createOpenAIProvider, createGeminiProvider, createLocalProvider, isLocalAvailable, computeProviderKey, } from "./providers/index.js";
|
|
8
|
+
// Storage
|
|
9
|
+
export { MemoryStorage, createDatabase, } from "./storage/index.js";
|
|
10
|
+
export { ensureMemoryIndexSchema, migrateSchema, ensureColumn, SCHEMA_VERSION, } from "./storage/index.js";
|
|
11
|
+
export { cosineSimilarity, loadSqliteVec, isSqliteVecAvailable } from "./storage/index.js";
|
|
12
|
+
// Chunking
|
|
13
|
+
export { chunkMarkdown, chunkSession, mergeConsecutiveMessages, } from "./chunking/index.js";
|
|
14
|
+
// Sync
|
|
15
|
+
export { FileWatcher } from "./sync/index.js";
|
|
16
|
+
export { SessionMonitor } from "./sync/index.js";
|
|
17
|
+
export { minimatch } from "./sync/index.js";
|
|
18
|
+
// Search
|
|
19
|
+
export { searchVector, searchVectorWithSqliteVec, syncVectorTable, searchFts, mergeHybridResults, buildFtsQuery, bm25RankToScore, } from "./search/index.js";
|
|
20
|
+
// Utilities
|
|
21
|
+
export { hashText, hashFile } from "./utils/index.js";
|
|
22
|
+
export { retry, RetryError } from "./utils/index.js";
|
|
23
|
+
export { Semaphore } from "./utils/index.js";
|
|
24
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,sCAAsC;AAiCtC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,iBAAiB;AACjB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,QAAQ;AACR,OAAO,EACL,YAAY,EACZ,SAAS,EACT,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EACL,uBAAuB,EACvB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAE9B,UAAU;AACV,OAAO,EACL,aAAa,EACb,cAAc,GAGf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,uBAAuB,EACvB,aAAa,EACb,YAAY,EACZ,cAAc,GAEf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE3F,WAAW;AACX,OAAO,EACL,aAAa,EACb,YAAY,EACZ,wBAAwB,GAGzB,MAAM,qBAAqB,CAAC;AAE7B,OAAO;AACP,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,cAAc,EAA8B,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,SAAS;AACT,OAAO,EACL,YAAY,EACZ,yBAAyB,EACzB,eAAe,EACf,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAE3B,YAAY;AACZ,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,UAAU,EAAqB,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import type { MemoryConfig, MemorySearchResult, SearchOptions, MemorySource, MemoryStatus, ProbeResult, ExtendedMemoryConfig } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Memory manager for indexing and searching workspace content
|
|
4
|
+
*/
|
|
5
|
+
export declare class MemoryManager {
|
|
6
|
+
private readonly config;
|
|
7
|
+
private storage;
|
|
8
|
+
private providerResult;
|
|
9
|
+
private watcher;
|
|
10
|
+
private syncPromise;
|
|
11
|
+
private embeddingSemaphore;
|
|
12
|
+
private closed;
|
|
13
|
+
private memoryDirty;
|
|
14
|
+
private sessionsDirty;
|
|
15
|
+
private memoryDirtyFiles;
|
|
16
|
+
private sessionsDirtyFiles;
|
|
17
|
+
private consecutiveEmbeddingFailures;
|
|
18
|
+
private fallbackActivated;
|
|
19
|
+
private fallbackProvider;
|
|
20
|
+
private batchDisabled;
|
|
21
|
+
private batchDisableReason?;
|
|
22
|
+
private sqliteVecLoaded;
|
|
23
|
+
private sqliteVecError?;
|
|
24
|
+
private syncIntervalTimer;
|
|
25
|
+
private lastSyncAt?;
|
|
26
|
+
private constructor();
|
|
27
|
+
/**
|
|
28
|
+
* Create or get a cached MemoryManager instance
|
|
29
|
+
*/
|
|
30
|
+
static create(config: MemoryConfig | ExtendedMemoryConfig): Promise<MemoryManager>;
|
|
31
|
+
/**
|
|
32
|
+
* Try to load sqlite-vec extension
|
|
33
|
+
*/
|
|
34
|
+
private tryLoadSqliteVec;
|
|
35
|
+
/**
|
|
36
|
+
* Initialize the embedding provider
|
|
37
|
+
*/
|
|
38
|
+
private initProvider;
|
|
39
|
+
/**
|
|
40
|
+
* Get the embedding provider (with fallback support)
|
|
41
|
+
*/
|
|
42
|
+
private get provider();
|
|
43
|
+
/**
|
|
44
|
+
* Get provider key for cache isolation
|
|
45
|
+
*/
|
|
46
|
+
private get providerKey();
|
|
47
|
+
/**
|
|
48
|
+
* Search memory for relevant content
|
|
49
|
+
*/
|
|
50
|
+
search(query: string, options?: SearchOptions & {
|
|
51
|
+
source?: MemorySource;
|
|
52
|
+
}): Promise<MemorySearchResult[]>;
|
|
53
|
+
/**
|
|
54
|
+
* Embed text with automatic fallback on provider failures
|
|
55
|
+
*/
|
|
56
|
+
private embedWithFallback;
|
|
57
|
+
/**
|
|
58
|
+
* Activate fallback provider
|
|
59
|
+
*/
|
|
60
|
+
private activateFallbackProvider;
|
|
61
|
+
/**
|
|
62
|
+
* Read file content with optional line range
|
|
63
|
+
*/
|
|
64
|
+
readFile(params: {
|
|
65
|
+
relPath: string;
|
|
66
|
+
from?: number;
|
|
67
|
+
lines?: number;
|
|
68
|
+
}): Promise<{
|
|
69
|
+
content: string;
|
|
70
|
+
startLine: number;
|
|
71
|
+
endLine: number;
|
|
72
|
+
}>;
|
|
73
|
+
/**
|
|
74
|
+
* Synchronize memory index with filesystem
|
|
75
|
+
*/
|
|
76
|
+
sync(): Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Perform atomic reindex into a temporary database
|
|
79
|
+
* On success, swap temp DB with main DB
|
|
80
|
+
* On failure, restore original
|
|
81
|
+
*/
|
|
82
|
+
reindex(): Promise<void>;
|
|
83
|
+
/**
|
|
84
|
+
* Index all files into a specific storage instance
|
|
85
|
+
*/
|
|
86
|
+
private indexAllFilesInto;
|
|
87
|
+
/**
|
|
88
|
+
* Perform full sync (for reindex operation)
|
|
89
|
+
*/
|
|
90
|
+
private performFullSync;
|
|
91
|
+
/**
|
|
92
|
+
* Perform the actual sync operation
|
|
93
|
+
*/
|
|
94
|
+
private performSync;
|
|
95
|
+
/**
|
|
96
|
+
* Sync memory markdown files
|
|
97
|
+
*/
|
|
98
|
+
private syncMemoryFiles;
|
|
99
|
+
/**
|
|
100
|
+
* Sync session files
|
|
101
|
+
*/
|
|
102
|
+
private syncSessionFiles;
|
|
103
|
+
/**
|
|
104
|
+
* Scan session directory for JSONL files
|
|
105
|
+
*/
|
|
106
|
+
private scanSessionFiles;
|
|
107
|
+
/**
|
|
108
|
+
* Index a session JSONL file with delta tracking
|
|
109
|
+
*/
|
|
110
|
+
private indexSessionFile;
|
|
111
|
+
/**
|
|
112
|
+
* Index sessions into a specific storage
|
|
113
|
+
*/
|
|
114
|
+
private indexSessionsInto;
|
|
115
|
+
/**
|
|
116
|
+
* Report sync progress
|
|
117
|
+
*/
|
|
118
|
+
private reportProgress;
|
|
119
|
+
/**
|
|
120
|
+
* Scan memory directory for markdown files
|
|
121
|
+
*/
|
|
122
|
+
private scanMemoryFiles;
|
|
123
|
+
/**
|
|
124
|
+
* Index a single file
|
|
125
|
+
*/
|
|
126
|
+
private indexFile;
|
|
127
|
+
/**
|
|
128
|
+
* Index a single file into a specific storage
|
|
129
|
+
*/
|
|
130
|
+
private indexFileInto;
|
|
131
|
+
/**
|
|
132
|
+
* Embed a batch of texts with caching and rate limiting
|
|
133
|
+
*/
|
|
134
|
+
private embedBatch;
|
|
135
|
+
/**
|
|
136
|
+
* Start watching for file changes
|
|
137
|
+
*/
|
|
138
|
+
private startWatching;
|
|
139
|
+
/**
|
|
140
|
+
* Start interval-based sync
|
|
141
|
+
*/
|
|
142
|
+
private startIntervalSync;
|
|
143
|
+
/**
|
|
144
|
+
* Stop interval-based sync
|
|
145
|
+
*/
|
|
146
|
+
private stopIntervalSync;
|
|
147
|
+
/**
|
|
148
|
+
* Stop watching for file changes
|
|
149
|
+
*/
|
|
150
|
+
stopWatching(): Promise<void>;
|
|
151
|
+
/**
|
|
152
|
+
* Close the manager and release resources
|
|
153
|
+
*/
|
|
154
|
+
close(): Promise<void>;
|
|
155
|
+
/**
|
|
156
|
+
* Get provider information
|
|
157
|
+
*/
|
|
158
|
+
getProviderInfo(): {
|
|
159
|
+
provider: string;
|
|
160
|
+
model: string;
|
|
161
|
+
fallbackFrom?: string;
|
|
162
|
+
fallbackReason?: string;
|
|
163
|
+
};
|
|
164
|
+
/**
|
|
165
|
+
* Get comprehensive status information
|
|
166
|
+
*/
|
|
167
|
+
getStatus(): MemoryStatus;
|
|
168
|
+
/**
|
|
169
|
+
* Probe vector search availability
|
|
170
|
+
*/
|
|
171
|
+
probeVectorAvailability(): ProbeResult;
|
|
172
|
+
/**
|
|
173
|
+
* Probe embedding API availability
|
|
174
|
+
*/
|
|
175
|
+
probeEmbeddingAvailability(): Promise<ProbeResult>;
|
|
176
|
+
/**
|
|
177
|
+
* Probe FTS5 availability
|
|
178
|
+
*/
|
|
179
|
+
probeFtsAvailability(): ProbeResult;
|
|
180
|
+
/**
|
|
181
|
+
* Mark memory files as dirty (for external triggers)
|
|
182
|
+
*/
|
|
183
|
+
markMemoryDirty(files?: string[]): void;
|
|
184
|
+
/**
|
|
185
|
+
* Mark session files as dirty (for external triggers)
|
|
186
|
+
*/
|
|
187
|
+
markSessionsDirty(files?: string[]): void;
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=memory-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-manager.d.ts","sourceRoot":"","sources":["../src/memory-manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,YAAY,EACZ,kBAAkB,EAClB,aAAa,EAEb,YAAY,EAEZ,YAAY,EACZ,WAAW,EACX,oBAAoB,EAErB,MAAM,YAAY,CAAC;AA2DpB;;GAEG;AACH,qBAAa,aAAa;IAkCtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAjCzB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,cAAc,CAAwC;IAC9D,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,kBAAkB,CAAY;IACtC,OAAO,CAAC,MAAM,CAAS;IAGvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,4BAA4B,CAAK;IACzC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,gBAAgB,CAAkC;IAG1D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,kBAAkB,CAAC,CAAS;IAGpC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,cAAc,CAAC,CAAS;IAGhC,OAAO,CAAC,iBAAiB,CAA+B;IAGxD,OAAO,CAAC,UAAU,CAAC,CAAS;IAE5B,OAAO;IAQP;;OAEG;WACU,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,oBAAoB,GAAG,OAAO,CAAC,aAAa,CAAC;IA2ExF;;OAEG;IACH,OAAO,CAAC,gBAAgB;IASxB;;OAEG;YACW,YAAY;IAiB1B;;OAEG;IACH,OAAO,KAAK,QAAQ,GAQnB;IAED;;OAEG;IACH,OAAO,KAAK,WAAW,GAMtB;IAED;;OAEG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAa,GAAG;QAAE,MAAM,CAAC,EAAE,YAAY,CAAA;KAAO,GACtD,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAiChC;;OAEG;YACW,iBAAiB;IA0B/B;;OAEG;YACW,wBAAwB;IAiCtC;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAwBpE;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAmF9B;;OAEG;YACW,iBAAiB;IA4B/B;;OAEG;YACW,eAAe;IAS7B;;OAEG;YACW,WAAW;IAgCzB;;OAEG;YACW,eAAe;IAiE7B;;OAEG;YACW,gBAAgB;IAiC9B;;OAEG;YACW,gBAAgB;IAiC9B;;OAEG;YACW,gBAAgB;IAmE9B;;OAEG;YACW,iBAAiB;IAoD/B;;OAEG;IACH,OAAO,CAAC,cAAc;IAMtB;;OAEG;YACW,eAAe;IA6B7B;;OAEG;YACW,SAAS;IAwCvB;;OAEG;YACW,aAAa;IAoC3B;;OAEG;YACW,UAAU;IA6ExB;;OAEG;YACW,aAAa;IAqB3B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAOnC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,eAAe,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB;IAiBD;;OAEG;IACH,SAAS,IAAI,YAAY;IAyDzB;;OAEG;IACH,uBAAuB,IAAI,WAAW;IA2BtC;;OAEG;IACG,0BAA0B,IAAI,OAAO,CAAC,WAAW,CAAC;IAkBxD;;OAEG;IACH,oBAAoB,IAAI,WAAW;IAkBnC;;OAEG;IACH,eAAe,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI;IAOvC;;OAEG;IACH,iBAAiB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI;CAM1C"}
|