memtrails 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/LICENSE +21 -0
- package/README.md +221 -2
- package/dist/core/consolidate.d.ts +144 -0
- package/dist/core/consolidate.js +660 -0
- package/dist/core/embedder.d.ts +30 -0
- package/dist/core/embedder.js +137 -0
- package/dist/core/episodes.d.ts +32 -0
- package/dist/core/episodes.js +53 -0
- package/dist/core/extract.d.ts +142 -0
- package/dist/core/extract.js +452 -0
- package/dist/core/gather.d.ts +17 -0
- package/dist/core/gather.js +98 -0
- package/dist/core/graph.d.ts +48 -0
- package/dist/core/graph.js +208 -0
- package/dist/core/local-embedder.d.ts +40 -0
- package/dist/core/local-embedder.js +98 -0
- package/dist/core/model.d.ts +16 -0
- package/dist/core/model.js +74 -0
- package/dist/core/paths.d.ts +1 -0
- package/dist/core/paths.js +7 -0
- package/dist/core/profiles.d.ts +30 -0
- package/dist/core/profiles.js +147 -0
- package/dist/core/qdrant-binary.d.ts +25 -0
- package/dist/core/qdrant-binary.js +108 -0
- package/dist/core/qdrant-client.d.ts +115 -0
- package/dist/core/qdrant-client.js +327 -0
- package/dist/core/qdrant-process.d.ts +30 -0
- package/dist/core/qdrant-process.js +160 -0
- package/dist/core/rebuild.d.ts +31 -0
- package/dist/core/rebuild.js +109 -0
- package/dist/core/recall-log.d.ts +29 -0
- package/dist/core/recall-log.js +82 -0
- package/dist/core/recall.d.ts +91 -0
- package/dist/core/recall.js +172 -0
- package/dist/core/redact.d.ts +2 -0
- package/dist/core/redact.js +48 -0
- package/dist/core/reranker.d.ts +26 -0
- package/dist/core/reranker.js +67 -0
- package/dist/core/room.d.ts +20 -0
- package/dist/core/room.js +63 -0
- package/dist/core/sleep.d.ts +44 -0
- package/dist/core/sleep.js +199 -0
- package/dist/core/store.d.ts +198 -0
- package/dist/core/store.js +190 -0
- package/dist/core/text.d.ts +31 -0
- package/dist/core/text.js +101 -0
- package/dist/core/words.d.ts +32 -0
- package/dist/core/words.js +127 -0
- package/dist/core/writer.d.ts +36 -0
- package/dist/core/writer.js +146 -0
- package/dist/importers/files.d.ts +31 -0
- package/dist/importers/files.js +157 -0
- package/dist/importers/git.d.ts +30 -0
- package/dist/importers/git.js +118 -0
- package/dist/importers/index.d.ts +14 -0
- package/dist/importers/index.js +26 -0
- package/dist/importers/jsonl.d.ts +20 -0
- package/dist/importers/jsonl.js +65 -0
- package/dist/importers/mbox.d.ts +20 -0
- package/dist/importers/mbox.js +347 -0
- package/dist/importers/slack.d.ts +21 -0
- package/dist/importers/slack.js +134 -0
- package/dist/importers/text.d.ts +31 -0
- package/dist/importers/text.js +191 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +22 -0
- package/dist/llm.d.ts +60 -0
- package/dist/llm.js +60 -0
- package/dist/memory.d.ts +106 -0
- package/dist/memory.js +262 -0
- package/dist/records.d.ts +62 -0
- package/dist/records.js +107 -0
- package/dist/tools/agent-tools.d.ts +52 -0
- package/dist/tools/agent-tools.js +142 -0
- package/package.json +78 -4
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
const NAMED = {
|
|
2
|
+
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', shy: '',
|
|
3
|
+
zwnj: '', zwj: '', lrm: '', rlm: '',
|
|
4
|
+
copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–', minus: '−',
|
|
5
|
+
lsquo: '‘', rsquo: '’', sbquo: '‚', ldquo: '“', rdquo: '”', bdquo: '„', laquo: '«', raquo: '»', lsaquo: '‹', rsaquo: '›',
|
|
6
|
+
euro: '€', pound: '£', yen: '¥', cent: '¢', curren: '¤', deg: '°', middot: '·', bull: '•', times: '×', divide: '÷',
|
|
7
|
+
sect: '§', para: '¶', iexcl: '¡', iquest: '¿', szlig: 'ß', aelig: 'æ', AElig: 'Æ', oelig: 'œ', OElig: 'Œ',
|
|
8
|
+
oslash: 'ø', Oslash: 'Ø', eth: 'ð', ETH: 'Ð', thorn: 'þ', THORN: 'Þ', ordf: 'ª', ordm: 'º', micro: 'µ', plusmn: '±',
|
|
9
|
+
frac12: '½', frac14: '¼', frac34: '¾', sup1: '¹', sup2: '²', sup3: '³', dagger: '†', Dagger: '‡', permil: '‰',
|
|
10
|
+
prime: '′', Prime: '″', le: '≤', ge: '≥', ne: '≠', infin: '∞', asymp: '≈', larr: '←', rarr: '→', uarr: '↑', darr: '↓',
|
|
11
|
+
harr: '↔', hearts: '♥', not: '¬', brvbar: '¦', uml: '¨', acute: '´', cedil: '¸', macr: '¯', circ: 'ˆ', tilde: '˜',
|
|
12
|
+
};
|
|
13
|
+
// Greek letters by name (Alpha … Omega, alpha … omega, sigmaf).
|
|
14
|
+
const GREEK = ['Alpha', 'Beta', 'Gamma', 'Delta', 'Epsilon', 'Zeta', 'Eta', 'Theta', 'Iota', 'Kappa', 'Lambda', 'Mu', 'Nu', 'Xi',
|
|
15
|
+
'Omicron', 'Pi', 'Rho', 'Sigma', 'Tau', 'Upsilon', 'Phi', 'Chi', 'Psi', 'Omega'];
|
|
16
|
+
GREEK.forEach((name, i) => {
|
|
17
|
+
const offset = i + (i >= 17 ? 1 : 0);
|
|
18
|
+
NAMED[name] = String.fromCodePoint(0x391 + offset);
|
|
19
|
+
NAMED[name.toLowerCase()] = String.fromCodePoint(0x3b1 + offset);
|
|
20
|
+
});
|
|
21
|
+
NAMED.sigmaf = 'ς';
|
|
22
|
+
// A Latin letter with a mark: é Ü ç š … composed from the letter and a combining mark.
|
|
23
|
+
const MARKS = {
|
|
24
|
+
acute: '́', grave: '̀', circ: '̂', tilde: '̃', uml: '̈', ring: '̊', cedil: '̧',
|
|
25
|
+
caron: '̌', macr: '̄', breve: '̆', ogon: '̨', dot: '̇', dblac: '̋',
|
|
26
|
+
};
|
|
27
|
+
const win1252 = new TextDecoder('windows-1252');
|
|
28
|
+
function namedEntity(name) {
|
|
29
|
+
if (name in NAMED)
|
|
30
|
+
return NAMED[name];
|
|
31
|
+
const m = /^([a-zA-Z])(acute|grave|circ|tilde|uml|ring|cedil|caron|macr|breve|ogon|dot|dblac)$/.exec(name);
|
|
32
|
+
if (!m)
|
|
33
|
+
return undefined;
|
|
34
|
+
const composed = (m[1] + MARKS[m[2]]).normalize('NFC');
|
|
35
|
+
return [...composed].length === 1 ? composed : undefined;
|
|
36
|
+
}
|
|
37
|
+
function codePoint(n) {
|
|
38
|
+
if (n >= 0x80 && n <= 0x9f)
|
|
39
|
+
return win1252.decode(new Uint8Array([n])); // what browsers do with – and the like
|
|
40
|
+
if (!Number.isFinite(n) || n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff))
|
|
41
|
+
return '�';
|
|
42
|
+
return String.fromCodePoint(n);
|
|
43
|
+
}
|
|
44
|
+
/** `&`, `é`, `😀`, `é` and the other common names, decoded; unknown names are left as written. */
|
|
45
|
+
export function decodeEntities(s) {
|
|
46
|
+
if (!s.includes('&'))
|
|
47
|
+
return s;
|
|
48
|
+
return s.replace(/&(#[xX][0-9a-fA-F]{1,6}|#\d{1,7}|[a-zA-Z][a-zA-Z0-9]{1,31});/g, (whole, ref) => {
|
|
49
|
+
if (ref[0] === '#')
|
|
50
|
+
return codePoint(ref[1] === 'x' || ref[1] === 'X' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10));
|
|
51
|
+
return namedEntity(ref) ?? whole;
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
const BLOCK = 'address|article|aside|blockquote|caption|center|dd|details|dialog|div|dl|dt|fieldset|figcaption|figure|footer|form|' +
|
|
55
|
+
'h[1-6]|header|hr|main|nav|ol|p|pre|section|summary|table|tbody|thead|tfoot|tr|ul';
|
|
56
|
+
/** Removes every `<tag …>…</tag>` (nested ones too), for `blockquote`: the quoted earlier message in an HTML reply. */
|
|
57
|
+
function dropElements(html, tag) {
|
|
58
|
+
const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, 'gi');
|
|
59
|
+
let out = '';
|
|
60
|
+
let depth = 0;
|
|
61
|
+
let from = 0;
|
|
62
|
+
for (let m = re.exec(html); m; m = re.exec(html)) {
|
|
63
|
+
if (!m[1]) {
|
|
64
|
+
if (depth === 0)
|
|
65
|
+
out += html.slice(from, m.index);
|
|
66
|
+
depth++;
|
|
67
|
+
}
|
|
68
|
+
else if (depth > 0) {
|
|
69
|
+
depth--;
|
|
70
|
+
if (depth === 0)
|
|
71
|
+
from = m.index + m[0].length;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return depth === 0 ? out + html.slice(from) : out;
|
|
75
|
+
}
|
|
76
|
+
/** The `<title>` of an HTML page, else its first `<h1>`, as text; undefined when it has neither. */
|
|
77
|
+
export function htmlTitle(html) {
|
|
78
|
+
for (const re of [/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i, /<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/i]) {
|
|
79
|
+
const m = re.exec(html);
|
|
80
|
+
const t = m ? decodeEntities(m[1].replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim() : '';
|
|
81
|
+
if (t)
|
|
82
|
+
return t;
|
|
83
|
+
}
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* HTML as plain text: scripts, styles, comments and the head dropped, blocks and line breaks kept as lines, list items
|
|
88
|
+
* as `- `, tags removed, entities decoded, `<pre>` kept as written. `dropQuotes` also drops `<blockquote>`s.
|
|
89
|
+
*/
|
|
90
|
+
export function htmlToText(html, opts = {}) {
|
|
91
|
+
let s = html
|
|
92
|
+
.replace(/<!--[\s\S]*?(-->|$)/g, ' ')
|
|
93
|
+
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
|
94
|
+
.replace(/<(script|style|head|noscript|template|svg|math|iframe|object)\b[\s\S]*?<\/\1\s*>/gi, ' ')
|
|
95
|
+
.replace(/<(script|style)\b[\s\S]*$/i, ' '); // one left open runs to the end
|
|
96
|
+
if (opts.dropQuotes)
|
|
97
|
+
s = dropElements(s, 'blockquote');
|
|
98
|
+
// Whitespace in the source is one space, except inside <pre>.
|
|
99
|
+
s = s.split(/(<pre\b[\s\S]*?<\/pre\s*>)/i)
|
|
100
|
+
.map((piece, i) => (i % 2 ? piece.replace(/\r\n?/g, '\n') : piece.replace(/\s+/g, ' ')))
|
|
101
|
+
.join('');
|
|
102
|
+
s = s
|
|
103
|
+
.replace(/<br\b[^>]*>/gi, '\n')
|
|
104
|
+
.replace(/<li\b[^>]*>/gi, '\n- ')
|
|
105
|
+
.replace(new RegExp(`</?(?:${BLOCK})\\b[^>]*>`, 'gi'), '\n')
|
|
106
|
+
.replace(/<\/t[dh]\s*>/gi, '\t')
|
|
107
|
+
.replace(/<\/?[a-zA-Z][^>]*>/g, '')
|
|
108
|
+
.replace(/<[!?][^>]*>/g, '');
|
|
109
|
+
s = decodeEntities(s);
|
|
110
|
+
return tidy(s);
|
|
111
|
+
}
|
|
112
|
+
/** Lines trimmed at the end, spaces and tabs inside a line kept to one, at most one empty line in a row. */
|
|
113
|
+
export function tidy(s) {
|
|
114
|
+
return s
|
|
115
|
+
.replace(/\r\n?/g, '\n')
|
|
116
|
+
.split('\n')
|
|
117
|
+
.map((l) => l.replace(/[ \t ]+/g, ' ').replace(/^ (?=\S)/, '').trimEnd())
|
|
118
|
+
.join('\n')
|
|
119
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
120
|
+
.trim();
|
|
121
|
+
}
|
|
122
|
+
/** Bytes as text in the charset named (any the platform knows); unnamed, ASCII or unknown: UTF-8, else Windows-1252. */
|
|
123
|
+
export function decodeBytes(bytes, charset) {
|
|
124
|
+
const label = (charset ?? '').trim().replace(/^["']|["']$/g, '').replace(/\*.*$/, '').toLowerCase();
|
|
125
|
+
if (label && !/^(us-?ascii|ascii|ansi_x3\.4-1968|utf-?8|unicode-1-1-utf-8|x-unknown|unknown(-8bit)?|default)$/.test(label)) {
|
|
126
|
+
try {
|
|
127
|
+
return new TextDecoder(label).decode(bytes);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
// A charset the platform does not know: read it as below.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return win1252.decode(bytes);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* A long text in parts of at most `maxChars`, each cut at a paragraph, else a line, else a sentence's end (in any
|
|
142
|
+
* script), else a space; never inside a character.
|
|
143
|
+
*/
|
|
144
|
+
export function splitText(text, maxChars) {
|
|
145
|
+
const max = Math.max(100, Math.floor(maxChars));
|
|
146
|
+
if (text.length <= max)
|
|
147
|
+
return [text];
|
|
148
|
+
const parts = [];
|
|
149
|
+
let rest = text;
|
|
150
|
+
while (rest.length > max) {
|
|
151
|
+
const window = rest.slice(0, max);
|
|
152
|
+
const half = Math.floor(max / 2);
|
|
153
|
+
let cut = window.lastIndexOf('\n\n');
|
|
154
|
+
if (cut < half)
|
|
155
|
+
cut = window.lastIndexOf('\n');
|
|
156
|
+
if (cut < half) {
|
|
157
|
+
const ends = [...window.slice(half).matchAll(/\p{Sentence_Terminal}+/gu)].at(-1);
|
|
158
|
+
cut = ends ? half + ends.index + ends[0].length : -1;
|
|
159
|
+
}
|
|
160
|
+
if (cut < half)
|
|
161
|
+
cut = window.search(/\s\S*$/);
|
|
162
|
+
if (cut < half)
|
|
163
|
+
cut = max;
|
|
164
|
+
const code = rest.charCodeAt(cut - 1);
|
|
165
|
+
if (code >= 0xd800 && code <= 0xdbff)
|
|
166
|
+
cut--; // keep a surrogate pair together
|
|
167
|
+
const part = rest.slice(0, cut).trim();
|
|
168
|
+
if (part)
|
|
169
|
+
parts.push(part);
|
|
170
|
+
rest = rest.slice(cut);
|
|
171
|
+
}
|
|
172
|
+
if (rest.trim())
|
|
173
|
+
parts.push(rest.trim());
|
|
174
|
+
return parts;
|
|
175
|
+
}
|
|
176
|
+
/** A record without its empty fields. */
|
|
177
|
+
export function record(r) {
|
|
178
|
+
const out = {};
|
|
179
|
+
for (const [k, v] of Object.entries(r)) {
|
|
180
|
+
if (v === undefined || (v === '' && k !== 'text'))
|
|
181
|
+
continue;
|
|
182
|
+
if (k === 'metadata' && v && typeof v === 'object') {
|
|
183
|
+
const m = Object.fromEntries(Object.entries(v).filter(([, x]) => x !== undefined && x !== ''));
|
|
184
|
+
if (Object.keys(m).length)
|
|
185
|
+
out.metadata = m;
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
out[k] = v;
|
|
189
|
+
}
|
|
190
|
+
return out;
|
|
191
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memtrails — retrieval and exploration for any text: e-mail, chat, notes, documents, news, commits, conversations.
|
|
3
|
+
*
|
|
4
|
+
* add records, conversations, or a stream from an importer — searchable at once, no model asked
|
|
5
|
+
* search dense + keyword, fused; reordered word by word on this machine; dates first when asked
|
|
6
|
+
* room the best sentences of what was found, for a small context
|
|
7
|
+
* open one item whole, with the records it was learned from
|
|
8
|
+
* explore the concept graph from a topic or items to what search does not find
|
|
9
|
+
* consolidate (with a language model) typed knowledge, dated observations, kept episodes
|
|
10
|
+
* sleep (with a language model) merge, supersede, link, profiles
|
|
11
|
+
* gather (with a language model) every instance a counting or listing question needs
|
|
12
|
+
* tools memory_search / memory_open / memory_explore for an agent
|
|
13
|
+
*/
|
|
14
|
+
export { openMemory, type Memory, type MemoryOptions, type SearchOptions } from './memory.js';
|
|
15
|
+
export { DEFAULT_COLLECTION, recordUid, timeOf, type Conversation, type MemoryRecord } from './records.js';
|
|
16
|
+
export type { RecalledItem, RecordRef, SearchQuery, Explored } from './core/recall.js';
|
|
17
|
+
export type { Gathered } from './core/gather.js';
|
|
18
|
+
export type { ConsolidationReport } from './core/consolidate.js';
|
|
19
|
+
export type { SleepReport } from './core/sleep.js';
|
|
20
|
+
export { openAiAsk, type Ask, type LlmRequest, type LlmAnswer, type LlmUsage, type OpenAiAskOptions } from './llm.js';
|
|
21
|
+
export { buildAgentTools, searchText, type AgentTool, type ToolProvider } from './tools/agent-tools.js';
|
|
22
|
+
export { ensureModel, defaultModelDir, MODEL_REPO, MODEL_REVISION } from './core/model.js';
|
|
23
|
+
export { ensureQdrant, QDRANT_VERSION } from './core/qdrant-binary.js';
|
|
24
|
+
export type { EmbedRoute } from './core/embedder.js';
|
|
25
|
+
export type { RerankRoute } from './core/reranker.js';
|
|
26
|
+
export { sentenceRoom, sentenceSpans } from './core/room.js';
|
|
27
|
+
export { maxSim, packWords, WordStore, WORD_LIMIT, type WordEncoder, type WordVectors, type PackedWords } from './core/words.js';
|
|
28
|
+
export { cacheDir } from './core/paths.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memtrails — retrieval and exploration for any text: e-mail, chat, notes, documents, news, commits, conversations.
|
|
3
|
+
*
|
|
4
|
+
* add records, conversations, or a stream from an importer — searchable at once, no model asked
|
|
5
|
+
* search dense + keyword, fused; reordered word by word on this machine; dates first when asked
|
|
6
|
+
* room the best sentences of what was found, for a small context
|
|
7
|
+
* open one item whole, with the records it was learned from
|
|
8
|
+
* explore the concept graph from a topic or items to what search does not find
|
|
9
|
+
* consolidate (with a language model) typed knowledge, dated observations, kept episodes
|
|
10
|
+
* sleep (with a language model) merge, supersede, link, profiles
|
|
11
|
+
* gather (with a language model) every instance a counting or listing question needs
|
|
12
|
+
* tools memory_search / memory_open / memory_explore for an agent
|
|
13
|
+
*/
|
|
14
|
+
export { openMemory } from './memory.js';
|
|
15
|
+
export { DEFAULT_COLLECTION, recordUid, timeOf } from './records.js';
|
|
16
|
+
export { openAiAsk } from './llm.js';
|
|
17
|
+
export { buildAgentTools, searchText } from './tools/agent-tools.js';
|
|
18
|
+
export { ensureModel, defaultModelDir, MODEL_REPO, MODEL_REVISION } from './core/model.js';
|
|
19
|
+
export { ensureQdrant, QDRANT_VERSION } from './core/qdrant-binary.js';
|
|
20
|
+
export { sentenceRoom, sentenceSpans } from './core/room.js';
|
|
21
|
+
export { maxSim, packWords, WordStore, WORD_LIMIT } from './core/words.js';
|
|
22
|
+
export { cacheDir } from './core/paths.js';
|
package/dist/llm.d.ts
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The language model memory asks, for consolidation, sleep and gathering — never for adding or searching. One structured
|
|
3
|
+
* ask: a system text, a prompt and a JSON Schema the answer must follow (strict structured output).
|
|
4
|
+
*
|
|
5
|
+
* `openAiAsk` speaks the OpenAI Responses API, so any provider or local server that serves it works (`baseUrl`). Bring
|
|
6
|
+
* any other model by passing your own `Ask`. Every call's tokens can be counted (`onUsage`).
|
|
7
|
+
*/
|
|
8
|
+
export interface LlmRequest {
|
|
9
|
+
/** What this is for, in a word (`memory`). */
|
|
10
|
+
purpose: string;
|
|
11
|
+
system: string;
|
|
12
|
+
prompt: string;
|
|
13
|
+
/** The answer's JSON Schema. */
|
|
14
|
+
schema: Record<string, unknown>;
|
|
15
|
+
/** What the structured answer is called. */
|
|
16
|
+
name?: string;
|
|
17
|
+
model?: string;
|
|
18
|
+
maxTokens?: number;
|
|
19
|
+
timeoutMs?: number;
|
|
20
|
+
/** What a failure cost, said after its reason ("so no summary was written"). */
|
|
21
|
+
consequence?: string;
|
|
22
|
+
}
|
|
23
|
+
export type LlmAnswer = {
|
|
24
|
+
answer: Record<string, unknown>;
|
|
25
|
+
model?: string;
|
|
26
|
+
}
|
|
27
|
+
/** Why it could not be asked, or did not answer, in words a person can read. */
|
|
28
|
+
| {
|
|
29
|
+
unavailable: string;
|
|
30
|
+
}
|
|
31
|
+
/** One for this purpose is already being asked. */
|
|
32
|
+
| {
|
|
33
|
+
busy: true;
|
|
34
|
+
};
|
|
35
|
+
export type Ask = (req: LlmRequest) => Promise<LlmAnswer>;
|
|
36
|
+
export interface LlmUsage {
|
|
37
|
+
model: string;
|
|
38
|
+
name?: string;
|
|
39
|
+
inputTokens: number;
|
|
40
|
+
cachedTokens: number;
|
|
41
|
+
outputTokens: number;
|
|
42
|
+
}
|
|
43
|
+
export interface OpenAiAskOptions {
|
|
44
|
+
/** Default: $MEMTRAILS_LLM_API_KEY, else $OPENAI_API_KEY. */
|
|
45
|
+
apiKey?: string;
|
|
46
|
+
/** Default: $MEMTRAILS_MODEL. */
|
|
47
|
+
model?: string;
|
|
48
|
+
/** Default: $MEMTRAILS_LLM_BASE_URL, else https://api.openai.com/v1. */
|
|
49
|
+
baseUrl?: string;
|
|
50
|
+
/** For reasoning models: how hard to think. Not sent when not given. */
|
|
51
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high';
|
|
52
|
+
/** Each answered call's tokens. */
|
|
53
|
+
onUsage?: (u: LlmUsage) => void;
|
|
54
|
+
/** Tries after a rate limit, a server error or a dropped connection (default 5), waiting longer each time. */
|
|
55
|
+
retries?: number;
|
|
56
|
+
fetchImpl?: typeof fetch;
|
|
57
|
+
}
|
|
58
|
+
export declare function openAiAsk(o?: OpenAiAskOptions): Ask;
|
|
59
|
+
/** The model the environment names (MEMTRAILS_MODEL and a key), asked when no `ask` is given. */
|
|
60
|
+
export declare const defaultAsk: Ask;
|
package/dist/llm.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The language model memory asks, for consolidation, sleep and gathering — never for adding or searching. One structured
|
|
3
|
+
* ask: a system text, a prompt and a JSON Schema the answer must follow (strict structured output).
|
|
4
|
+
*
|
|
5
|
+
* `openAiAsk` speaks the OpenAI Responses API, so any provider or local server that serves it works (`baseUrl`). Bring
|
|
6
|
+
* any other model by passing your own `Ask`. Every call's tokens can be counted (`onUsage`).
|
|
7
|
+
*/
|
|
8
|
+
const why = (reason, req) => `${reason}${req.consequence ? `, ${req.consequence}` : ''}`;
|
|
9
|
+
export function openAiAsk(o = {}) {
|
|
10
|
+
return async (req) => {
|
|
11
|
+
const model = req.model ?? o.model ?? process.env.MEMTRAILS_MODEL?.trim();
|
|
12
|
+
const key = o.apiKey ?? (process.env.MEMTRAILS_LLM_API_KEY?.trim() || process.env.OPENAI_API_KEY?.trim());
|
|
13
|
+
if (!model)
|
|
14
|
+
return { unavailable: why('no model is set (the model option, or MEMTRAILS_MODEL)', req) };
|
|
15
|
+
if (!key)
|
|
16
|
+
return { unavailable: why('no API key is set (the apiKey option, MEMTRAILS_LLM_API_KEY or OPENAI_API_KEY)', req) };
|
|
17
|
+
const base = (o.baseUrl ?? (process.env.MEMTRAILS_LLM_BASE_URL?.trim() || 'https://api.openai.com/v1')).replace(/\/+$/, '');
|
|
18
|
+
const body = JSON.stringify({
|
|
19
|
+
model, instructions: req.system, input: req.prompt, max_output_tokens: req.maxTokens ?? 12_000,
|
|
20
|
+
...(o.reasoningEffort ? { reasoning: { effort: o.reasoningEffort } } : {}),
|
|
21
|
+
text: { format: { type: 'json_schema', name: req.name ?? 'memory', strict: true, schema: req.schema } },
|
|
22
|
+
});
|
|
23
|
+
const tries = Math.max(1, (o.retries ?? 5) + 1);
|
|
24
|
+
let last = '';
|
|
25
|
+
for (let attempt = 0; attempt < tries; attempt++) {
|
|
26
|
+
if (attempt)
|
|
27
|
+
await new Promise((ok) => setTimeout(ok, Math.min(60_000, 1000 * 2 ** (attempt - 1)) * (0.5 + Math.random())));
|
|
28
|
+
let res;
|
|
29
|
+
try {
|
|
30
|
+
res = await (o.fetchImpl ?? fetch)(`${base}/responses`, {
|
|
31
|
+
method: 'POST', body, signal: AbortSignal.timeout(req.timeoutMs ?? 180_000),
|
|
32
|
+
headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
last = err.message || err.name;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (res.status === 429 || res.status >= 500) {
|
|
40
|
+
last = `HTTP ${res.status}`;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (!res.ok)
|
|
44
|
+
return { unavailable: why(`the model answered HTTP ${res.status}`, req) };
|
|
45
|
+
try {
|
|
46
|
+
const r = (await res.json());
|
|
47
|
+
o.onUsage?.({ model: r.model ?? model, name: req.name, inputTokens: r.usage?.input_tokens ?? 0,
|
|
48
|
+
cachedTokens: r.usage?.input_tokens_details?.cached_tokens ?? 0, outputTokens: r.usage?.output_tokens ?? 0 });
|
|
49
|
+
const text = (r.output ?? []).flatMap((x) => x.content ?? []).map((c) => c.text ?? '').join('');
|
|
50
|
+
return { model: r.model ?? model, answer: JSON.parse(text) };
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
return { unavailable: why(`the model's answer could not be read (${err.message})`, req) };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return { unavailable: why(`the model did not answer (${last})`, req) };
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/** The model the environment names (MEMTRAILS_MODEL and a key), asked when no `ask` is given. */
|
|
60
|
+
export const defaultAsk = (req) => openAiAsk()(req);
|
package/dist/memory.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { type ConsolidationReport } from './core/consolidate.js';
|
|
2
|
+
import { type EmbedRoute } from './core/embedder.js';
|
|
3
|
+
import { type Gathered } from './core/gather.js';
|
|
4
|
+
import { type Explored, type RecallDeps, type RecalledItem, type SearchQuery } from './core/recall.js';
|
|
5
|
+
import { type RerankRoute } from './core/reranker.js';
|
|
6
|
+
import { type SleepReport } from './core/sleep.js';
|
|
7
|
+
import { type Ask, type OpenAiAskOptions } from './llm.js';
|
|
8
|
+
import { type Conversation, type MemoryRecord } from './records.js';
|
|
9
|
+
import { type AgentTool } from './tools/agent-tools.js';
|
|
10
|
+
export interface MemoryOptions {
|
|
11
|
+
/** Where this memory lives: its logs, its index, its word vectors. */
|
|
12
|
+
home: string;
|
|
13
|
+
/** How texts are embedded (EmbeddingGemma-300m in every case: the index, prompts and measured thresholds are its):
|
|
14
|
+
* - default: on this machine, the model fetched on first use into the cache (218 MB);
|
|
15
|
+
* - `{ modelDir }`: on this machine, from a directory you provide;
|
|
16
|
+
* - `{ server }`: a server that serves it over the OpenAI embeddings API (text-embeddings-inference, Ollama, …).
|
|
17
|
+
* Word by word then needs `{ modelDir }` too, or it is off. */
|
|
18
|
+
embedding?: {
|
|
19
|
+
modelDir?: string;
|
|
20
|
+
server?: EmbedRoute;
|
|
21
|
+
threads?: number;
|
|
22
|
+
};
|
|
23
|
+
/** How the best 30 of a search are reordered: `'words'` (default with a local model: word by word, no server), a
|
|
24
|
+
* cross-encoder server (`RerankRoute`), your own function, or `false` for the search's own order. */
|
|
25
|
+
rerank?: 'words' | RerankRoute | RecallDeps['rerank'] | false;
|
|
26
|
+
/** The language model for consolidate, sleep and gather: your own `Ask`, or options for the OpenAI-compatible one.
|
|
27
|
+
* Default: the one the environment names (MEMTRAILS_MODEL and a key). */
|
|
28
|
+
llm?: Ask | OpenAiAskOptions;
|
|
29
|
+
/** Qdrant: default, a process of its own (the pinned binary, fetched on first use); or a server you run. */
|
|
30
|
+
qdrant?: {
|
|
31
|
+
url: string;
|
|
32
|
+
apiKey?: string;
|
|
33
|
+
collection?: string;
|
|
34
|
+
} | {
|
|
35
|
+
binary: string;
|
|
36
|
+
};
|
|
37
|
+
/** Windows the language model reads at once during consolidation (default 4). */
|
|
38
|
+
concurrency?: number;
|
|
39
|
+
log?: (line: string) => void;
|
|
40
|
+
}
|
|
41
|
+
export interface SearchOptions extends SearchQuery {
|
|
42
|
+
/** `relevance` (default) or `time` (oldest first) — for searchText. */
|
|
43
|
+
order?: 'relevance' | 'time';
|
|
44
|
+
}
|
|
45
|
+
export interface Memory {
|
|
46
|
+
/** Add or update records. Unchanged ones cost nothing. Searchable when this resolves. */
|
|
47
|
+
add(records: MemoryRecord | MemoryRecord[]): Promise<{
|
|
48
|
+
added: number;
|
|
49
|
+
unchanged: number;
|
|
50
|
+
}>;
|
|
51
|
+
/** Add or update a conversation: one record for each time the person speaks, with what was answered. */
|
|
52
|
+
addConversation(conversation: Conversation): Promise<{
|
|
53
|
+
added: number;
|
|
54
|
+
unchanged: number;
|
|
55
|
+
}>;
|
|
56
|
+
/** Add records from any source (an importer, a database cursor) in batches, streaming. */
|
|
57
|
+
import(records: AsyncIterable<MemoryRecord> | Iterable<MemoryRecord>, opts?: {
|
|
58
|
+
batch?: number;
|
|
59
|
+
onProgress?: (done: number) => void;
|
|
60
|
+
}): Promise<{
|
|
61
|
+
added: number;
|
|
62
|
+
unchanged: number;
|
|
63
|
+
}>;
|
|
64
|
+
/** Forget records: gone from search. What consolidation learned from them stays until sleep finds it unsupported. */
|
|
65
|
+
remove(ids: Array<string | {
|
|
66
|
+
id: string;
|
|
67
|
+
source?: string;
|
|
68
|
+
}>): Promise<number>;
|
|
69
|
+
/** Best first: records and what consolidation learned, each with its key text, dates, and the passage that matched. */
|
|
70
|
+
search(q: SearchOptions): Promise<RecalledItem[]>;
|
|
71
|
+
/** The same search as a reader is shown it: one line a memory, best first or oldest first. */
|
|
72
|
+
searchText(q: SearchOptions): Promise<string>;
|
|
73
|
+
/** What a small room should hold for a question: the best sentences of what search found, oldest first, in `chars`. */
|
|
74
|
+
room(q: SearchQuery & {
|
|
75
|
+
chars?: number;
|
|
76
|
+
}): Promise<string>;
|
|
77
|
+
/** One item whole, by the id search returned. */
|
|
78
|
+
open(id: string): Promise<Record<string, unknown> | null>;
|
|
79
|
+
/** Around a topic, or around items you found: what the concept graph reaches that search did not, and its concepts. */
|
|
80
|
+
explore(q: {
|
|
81
|
+
topic?: string;
|
|
82
|
+
from?: string[];
|
|
83
|
+
collections?: string[];
|
|
84
|
+
limit?: number;
|
|
85
|
+
}): Promise<Explored>;
|
|
86
|
+
/** A set for a question that counts, lists or totals: a wide search, then the language model keeps every instance. */
|
|
87
|
+
gather(q: SearchQuery): Promise<Gathered>;
|
|
88
|
+
/** Read what was added since last time and keep typed knowledge, dated observations and kept episodes. Needs a
|
|
89
|
+
* language model. Returns how many windows were read. */
|
|
90
|
+
consolidate(): Promise<ConsolidationReport>;
|
|
91
|
+
/** The offline pass: merge near-duplicates, retire what was superseded, link, write profiles. Needs a language model. */
|
|
92
|
+
sleep(): Promise<SleepReport>;
|
|
93
|
+
/** Encode what word by word and the room do not hold yet (done after every add, consolidate and sleep). */
|
|
94
|
+
indexWords(): Promise<number>;
|
|
95
|
+
/** Write the index again from the logs. */
|
|
96
|
+
rebuild(): Promise<void>;
|
|
97
|
+
/** Tool definitions an agent can be handed: memory_search, memory_open, memory_explore. */
|
|
98
|
+
tools(): AgentTool[];
|
|
99
|
+
stats(): {
|
|
100
|
+
records: number;
|
|
101
|
+
items: number;
|
|
102
|
+
words: number;
|
|
103
|
+
};
|
|
104
|
+
close(): Promise<void>;
|
|
105
|
+
}
|
|
106
|
+
export declare function openMemory(o: MemoryOptions): Promise<Memory>;
|