@pinkpixel/memi 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +298 -2
- package/dist/cli/commands/config.js +38 -0
- package/dist/cli/commands/doctor.js +53 -0
- package/dist/cli/commands/memories.js +146 -0
- package/dist/cli/commands/reindex.js +34 -0
- package/dist/cli/commands/serve.js +9 -0
- package/dist/cli/commands/ui.js +26 -0
- package/dist/cli/index.js +20 -0
- package/dist/cli/util.js +46 -0
- package/dist/core/config.js +89 -0
- package/dist/core/db.js +21 -0
- package/dist/core/format.js +16 -0
- package/dist/core/memories.js +192 -0
- package/dist/core/project.js +53 -0
- package/dist/core/rows.js +38 -0
- package/dist/core/runtime.js +10 -0
- package/dist/core/schema.js +60 -0
- package/dist/core/search.js +86 -0
- package/dist/core/types.js +12 -0
- package/dist/core/vectors.js +102 -0
- package/dist/embed/http.js +20 -0
- package/dist/embed/index.js +9 -0
- package/dist/embed/ollama.js +24 -0
- package/dist/embed/openai.js +22 -0
- package/dist/embed/types.js +2 -0
- package/dist/lib/app-info.js +5 -0
- package/dist/mcp/server.js +24 -0
- package/dist/mcp/tools.js +152 -0
- package/dist/ui/assets.js +38 -0
- package/dist/ui/routes.js +91 -0
- package/dist/ui/security.js +47 -0
- package/dist/ui/server.js +33 -0
- package/package.json +63 -4
- package/skills/memi-curate/SKILL.md +89 -0
- package/skills/memi-memory/SKILL.md +141 -0
- package/skills/memi-memory/references/examples.md +50 -0
- package/skills/memi-memory/references/rubrics.md +44 -0
- package/skills/memi-setup/SKILL.md +159 -0
- package/ui/css/controls.css +274 -0
- package/ui/css/dialogs.css +145 -0
- package/ui/css/layout.css +237 -0
- package/ui/css/memories.css +168 -0
- package/ui/css/tokens.css +125 -0
- package/ui/favicon.svg +1 -0
- package/ui/fonts/OFL.txt +192 -0
- package/ui/fonts/geist-latin-wght-normal.woff2 +0 -0
- package/ui/fonts/geist-mono-latin-wght-normal.woff2 +0 -0
- package/ui/index.html +75 -0
- package/ui/js/api.js +30 -0
- package/ui/js/confirm.js +29 -0
- package/ui/js/dom.js +49 -0
- package/ui/js/editor.js +133 -0
- package/ui/js/icons.js +29 -0
- package/ui/js/list.js +66 -0
- package/ui/js/main.js +244 -0
- package/ui/js/sidebar.js +85 -0
- package/ui/js/state.js +61 -0
- package/ui/js/toast.js +10 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { formatHits, formatMemories, formatMemory } from "../../core/format.js";
|
|
2
|
+
import { pickScope, resolveProject } from "../../core/project.js";
|
|
3
|
+
import { openRuntime } from "../../core/runtime.js";
|
|
4
|
+
import { MemiError } from "../../core/types.js";
|
|
5
|
+
import { action, confirm, printJson, splitTags, toInt } from "../util.js";
|
|
6
|
+
const SCOPES = ["both", "project", "global", "all"];
|
|
7
|
+
const checkScope = (v) => {
|
|
8
|
+
if (!SCOPES.includes(v))
|
|
9
|
+
throw new MemiError(`Scope must be one of: ${SCOPES.join(", ")}`);
|
|
10
|
+
return v;
|
|
11
|
+
};
|
|
12
|
+
const currentProject = (rt, project) => resolveProject({ project, agent: rt.agent })?.project ?? null;
|
|
13
|
+
export const registerMemories = (program) => {
|
|
14
|
+
program
|
|
15
|
+
.command("add <content...>")
|
|
16
|
+
.description("Save a memory")
|
|
17
|
+
.option("-c, --category <name>", "category (default: context)")
|
|
18
|
+
.option("-i, --importance <1-5>", "importance", toInt("Importance"))
|
|
19
|
+
.option("-t, --tags <a,b>", "comma-separated tags")
|
|
20
|
+
.option("--pin", "load this at the start of every session")
|
|
21
|
+
.option("--global", "save globally instead of for this project")
|
|
22
|
+
.option("-p, --project <name>", "project name (default: git repo or agent name)")
|
|
23
|
+
.option("--agent <name>", "agent name to record")
|
|
24
|
+
.action(action(async (content, o) => {
|
|
25
|
+
const rt = openRuntime();
|
|
26
|
+
const agent = o.agent ?? rt.agent;
|
|
27
|
+
const target = pickScope({ scope: o.global ? "global" : undefined, project: o.project, agent });
|
|
28
|
+
const r = await rt.store.add({
|
|
29
|
+
content: content.join(" "),
|
|
30
|
+
...target,
|
|
31
|
+
agent,
|
|
32
|
+
category: o.category,
|
|
33
|
+
importance: o.importance,
|
|
34
|
+
tags: splitTags(o.tags),
|
|
35
|
+
pinned: o.pin,
|
|
36
|
+
});
|
|
37
|
+
console.log(`Saved #${r.memory.id} (${target.scope === "global" ? "global" : `project ${target.project}`}, ${r.memory.category}, importance ${r.memory.importance}).`);
|
|
38
|
+
if (r.similar.length)
|
|
39
|
+
console.log(`\nSimilar memories already exist:\n\n${formatMemories(r.similar.map((s) => s.memory))}`);
|
|
40
|
+
if (r.warning)
|
|
41
|
+
console.error(`\nNote: ${r.warning}`);
|
|
42
|
+
}));
|
|
43
|
+
program
|
|
44
|
+
.command("search <query...>")
|
|
45
|
+
.description("Search memories by meaning and keyword")
|
|
46
|
+
.option("--mode <mode>", "hybrid, semantic or text", "hybrid")
|
|
47
|
+
.option("--scope <scope>", "both, project, global or all", checkScope, "both")
|
|
48
|
+
.option("-p, --project <name>", "project name")
|
|
49
|
+
.option("-c, --category <name>")
|
|
50
|
+
.option("--min-importance <1-5>", "", toInt("Importance"))
|
|
51
|
+
.option("-n, --limit <n>", "", toInt("Limit"), 10)
|
|
52
|
+
.option("--json", "print JSON")
|
|
53
|
+
.action(action(async (query, o) => {
|
|
54
|
+
const rt = openRuntime();
|
|
55
|
+
const r = await rt.store.search({
|
|
56
|
+
query: query.join(" "),
|
|
57
|
+
mode: o.mode,
|
|
58
|
+
scope: o.scope,
|
|
59
|
+
project: currentProject(rt, o.project),
|
|
60
|
+
category: o.category,
|
|
61
|
+
minImportance: o.minImportance,
|
|
62
|
+
limit: o.limit,
|
|
63
|
+
});
|
|
64
|
+
if (o.json)
|
|
65
|
+
return printJson(r);
|
|
66
|
+
console.log(r.hits.length ? formatHits(r.hits) : "No matching memories.");
|
|
67
|
+
if (r.note)
|
|
68
|
+
console.error(`\nNote: ${r.note}`);
|
|
69
|
+
}));
|
|
70
|
+
program
|
|
71
|
+
.command("list")
|
|
72
|
+
.description("List memories")
|
|
73
|
+
.option("--scope <scope>", "both, project, global or all", checkScope, "both")
|
|
74
|
+
.option("-p, --project <name>", "project name")
|
|
75
|
+
.option("-c, --category <name>")
|
|
76
|
+
.option("--min-importance <1-5>", "", toInt("Importance"))
|
|
77
|
+
.option("--pinned", "only pinned memories")
|
|
78
|
+
.option("--order <order>", "recent, oldest or importance", "recent")
|
|
79
|
+
.option("-n, --limit <n>", "", toInt("Limit"), 50)
|
|
80
|
+
.option("--offset <n>", "", toInt("Offset"), 0)
|
|
81
|
+
.option("--json", "print JSON")
|
|
82
|
+
.action(action(async (o) => {
|
|
83
|
+
const rt = openRuntime();
|
|
84
|
+
const r = rt.store.list({
|
|
85
|
+
scope: o.scope,
|
|
86
|
+
project: currentProject(rt, o.project),
|
|
87
|
+
category: o.category,
|
|
88
|
+
minImportance: o.minImportance,
|
|
89
|
+
pinned: o.pinned ? true : undefined,
|
|
90
|
+
order: o.order,
|
|
91
|
+
limit: o.limit,
|
|
92
|
+
offset: o.offset,
|
|
93
|
+
});
|
|
94
|
+
if (o.json)
|
|
95
|
+
return printJson(r);
|
|
96
|
+
console.log(r.items.length ? `${r.items.length} of ${r.total}\n\n${formatMemories(r.items)}` : "No memories found.");
|
|
97
|
+
}));
|
|
98
|
+
program
|
|
99
|
+
.command("update <id>")
|
|
100
|
+
.description("Change a memory")
|
|
101
|
+
.option("--content <text>")
|
|
102
|
+
.option("-c, --category <name>")
|
|
103
|
+
.option("-i, --importance <1-5>", "", toInt("Importance"))
|
|
104
|
+
.option("-t, --tags <a,b>", "replaces existing tags")
|
|
105
|
+
.option("--pin")
|
|
106
|
+
.option("--unpin")
|
|
107
|
+
.option("--global", "move to global")
|
|
108
|
+
.option("-p, --project <name>", "move to this project")
|
|
109
|
+
.action(action(async (id, o) => {
|
|
110
|
+
const rt = openRuntime();
|
|
111
|
+
const scope = o.global ? "global" : o.project ? "project" : undefined;
|
|
112
|
+
const r = await rt.store.update(toInt("Id")(id), {
|
|
113
|
+
content: o.content,
|
|
114
|
+
category: o.category,
|
|
115
|
+
importance: o.importance,
|
|
116
|
+
tags: splitTags(o.tags),
|
|
117
|
+
pinned: o.pin ? true : o.unpin ? false : undefined,
|
|
118
|
+
scope,
|
|
119
|
+
project: o.project,
|
|
120
|
+
});
|
|
121
|
+
if (!r)
|
|
122
|
+
throw new MemiError(`No memory with id ${id}.`);
|
|
123
|
+
console.log(`Updated.\n\n${formatMemory(r.memory)}`);
|
|
124
|
+
if (r.warning)
|
|
125
|
+
console.error(`\nNote: ${r.warning}`);
|
|
126
|
+
}));
|
|
127
|
+
program
|
|
128
|
+
.command("forget <id>")
|
|
129
|
+
.description("Delete a memory")
|
|
130
|
+
.option("-y, --yes", "skip the confirmation")
|
|
131
|
+
.action(action(async (id, o) => {
|
|
132
|
+
const rt = openRuntime();
|
|
133
|
+
const memory = rt.store.get(toInt("Id")(id));
|
|
134
|
+
if (!memory)
|
|
135
|
+
throw new MemiError(`No memory with id ${id}.`);
|
|
136
|
+
console.error(formatMemory(memory) + "\n");
|
|
137
|
+
if (!(await confirm("Delete this memory permanently?", o.yes)))
|
|
138
|
+
return console.error("Kept.");
|
|
139
|
+
rt.store.remove(memory.id);
|
|
140
|
+
console.log(`Deleted #${memory.id}.`);
|
|
141
|
+
}));
|
|
142
|
+
program
|
|
143
|
+
.command("categories")
|
|
144
|
+
.description("List categories and how many memories each holds")
|
|
145
|
+
.action(action(() => console.log(openRuntime().store.categories().map((c) => `${c.name} (${c.count})${c.description ? `: ${c.description}` : ""}`).join("\n"))));
|
|
146
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { openRuntime } from "../../core/runtime.js";
|
|
2
|
+
import { MemiError } from "../../core/types.js";
|
|
3
|
+
import { action, confirm } from "../util.js";
|
|
4
|
+
export const registerReindex = (program) => {
|
|
5
|
+
program
|
|
6
|
+
.command("reindex")
|
|
7
|
+
.description("Rebuild the vector index with the current embedder")
|
|
8
|
+
.option("--missing", "only embed memories that have no vector yet, nothing is dropped")
|
|
9
|
+
.option("-y, --yes", "skip the confirmation")
|
|
10
|
+
.action(action(async (o) => {
|
|
11
|
+
const { store } = openRuntime();
|
|
12
|
+
const status = store.status();
|
|
13
|
+
if (!status.embedder)
|
|
14
|
+
throw new MemiError("No embedder configured.");
|
|
15
|
+
const name = `${status.embedder.provider}/${status.embedder.model}`;
|
|
16
|
+
if (o.missing) {
|
|
17
|
+
if (status.state.status === "mismatch")
|
|
18
|
+
throw new MemiError(status.warning ?? "The embedder changed. Run a full reindex.");
|
|
19
|
+
console.log(`Embedded ${await store.backfill()} memories with ${name}.`);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
if (status.state.status === "mismatch") {
|
|
23
|
+
const { provider, model, dimensions } = status.state.stored;
|
|
24
|
+
console.error(`The embedder changed from ${provider}/${model} (${dimensions} dims) to ${name}.\n`);
|
|
25
|
+
}
|
|
26
|
+
console.error(`This drops the vector table and rebuilds it for ${name}, re-embedding all ${status.total} memories.\n` +
|
|
27
|
+
"The memories themselves are kept. If the embedder is unreachable, nothing is changed.\n");
|
|
28
|
+
if (!(await confirm("Rebuild the vector index?", o.yes)))
|
|
29
|
+
return console.error("Nothing changed.");
|
|
30
|
+
const result = await store.reindex((done, total) => process.stderr.write(`\rEmbedding ${done}/${total}`));
|
|
31
|
+
process.stderr.write("\n");
|
|
32
|
+
console.log(`Rebuilt ${result.count} vectors with ${name} (${result.dimensions} dimensions).`);
|
|
33
|
+
}));
|
|
34
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { openRuntime } from "../../core/runtime.js";
|
|
2
|
+
import { serveStdio } from "../../mcp/server.js";
|
|
3
|
+
import { action } from "../util.js";
|
|
4
|
+
export const registerServe = (program) => {
|
|
5
|
+
program
|
|
6
|
+
.command("serve")
|
|
7
|
+
.description("Run the MCP server over stdio")
|
|
8
|
+
.action(action(async () => serveStdio(openRuntime())));
|
|
9
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { openRuntime } from "../../core/runtime.js";
|
|
3
|
+
import { startUi } from "../../ui/server.js";
|
|
4
|
+
import { action, toInt } from "../util.js";
|
|
5
|
+
function openBrowser(url) {
|
|
6
|
+
const [cmd, args] = process.platform === "darwin" ? ["open", [url]] : process.platform === "win32" ? ["cmd", ["/c", "start", "", url]] : ["xdg-open", [url]];
|
|
7
|
+
spawn(cmd, args, { detached: true, stdio: "ignore" })
|
|
8
|
+
.on("error", () => console.error(`Could not open a browser. Visit ${url}`))
|
|
9
|
+
.unref();
|
|
10
|
+
}
|
|
11
|
+
export const registerUi = (program) => {
|
|
12
|
+
program
|
|
13
|
+
.command("ui")
|
|
14
|
+
.description("Run the local memory manager")
|
|
15
|
+
.option("-p, --port <n>", "port (default: ui.port from config, 4747)", toInt("Port"))
|
|
16
|
+
.option("--open", "open it in your browser")
|
|
17
|
+
.action(action(async (o) => {
|
|
18
|
+
const rt = openRuntime();
|
|
19
|
+
const ui = await startUi(rt, { port: o.port ?? rt.config.ui.port });
|
|
20
|
+
console.log(`memi UI running at ${ui.url}\nPress Ctrl+C to stop.`);
|
|
21
|
+
if (o.open)
|
|
22
|
+
openBrowser(ui.url);
|
|
23
|
+
process.once("SIGINT", () => ui.close().then(() => process.exit(0)));
|
|
24
|
+
process.once("SIGTERM", () => ui.close().then(() => process.exit(0)));
|
|
25
|
+
}));
|
|
26
|
+
};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from "commander";
|
|
3
|
+
import { APP_VERSION } from "../lib/app-info.js";
|
|
4
|
+
import { registerConfig } from "./commands/config.js";
|
|
5
|
+
import { registerDoctor } from "./commands/doctor.js";
|
|
6
|
+
import { registerMemories } from "./commands/memories.js";
|
|
7
|
+
import { registerReindex } from "./commands/reindex.js";
|
|
8
|
+
import { registerServe } from "./commands/serve.js";
|
|
9
|
+
import { registerUi } from "./commands/ui.js";
|
|
10
|
+
import { exitOnKnownError } from "./util.js";
|
|
11
|
+
const program = new Command().name("memi").description("Local-first memory for AI agents").version(APP_VERSION);
|
|
12
|
+
for (const register of [registerServe, registerMemories, registerReindex, registerConfig, registerDoctor, registerUi])
|
|
13
|
+
register(program);
|
|
14
|
+
try {
|
|
15
|
+
// Option parsers such as --scope and --importance throw while parsing, before any command action runs.
|
|
16
|
+
await program.parseAsync();
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
exitOnKnownError(err);
|
|
20
|
+
}
|
package/dist/cli/util.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { createInterface } from "node:readline/promises";
|
|
2
|
+
import { MemiError } from "../core/types.js";
|
|
3
|
+
import { EmbedError } from "../embed/types.js";
|
|
4
|
+
/** Prints expected failures as one clean line and exits 1. Anything else is a real bug and is rethrown. */
|
|
5
|
+
export function exitOnKnownError(err) {
|
|
6
|
+
const e = err;
|
|
7
|
+
if (e instanceof MemiError || e instanceof EmbedError || e.name === "ZodError") {
|
|
8
|
+
console.error(`memi: ${e.message}`);
|
|
9
|
+
process.exit(1);
|
|
10
|
+
}
|
|
11
|
+
throw err;
|
|
12
|
+
}
|
|
13
|
+
/** Wraps a command action so expected failures print one clean line instead of a stack trace. */
|
|
14
|
+
export function action(fn) {
|
|
15
|
+
return async (...args) => {
|
|
16
|
+
try {
|
|
17
|
+
await fn(...args);
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
exitOnKnownError(err);
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export async function confirm(question, yes) {
|
|
25
|
+
if (yes)
|
|
26
|
+
return true;
|
|
27
|
+
if (!process.stdin.isTTY)
|
|
28
|
+
throw new MemiError("This needs confirmation and there is no terminal to ask in. Pass --yes to continue.");
|
|
29
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
30
|
+
try {
|
|
31
|
+
return /^y(es)?$/i.test((await rl.question(`${question} [y/N] `)).trim());
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
rl.close();
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
export const splitTags = (v) => (v ? v.split(",").map((t) => t.trim()).filter(Boolean) : undefined);
|
|
38
|
+
export function toInt(name) {
|
|
39
|
+
return (v) => {
|
|
40
|
+
const n = Number(v);
|
|
41
|
+
if (!Number.isInteger(n))
|
|
42
|
+
throw new MemiError(`${name} must be a whole number.`);
|
|
43
|
+
return n;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export const printJson = (v) => console.log(JSON.stringify(v, null, 2));
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
export const ProviderSchema = z.enum(["ollama", "openai"]);
|
|
6
|
+
const ConfigSchema = z.object({
|
|
7
|
+
embedder: z
|
|
8
|
+
.object({
|
|
9
|
+
provider: ProviderSchema.default("ollama"),
|
|
10
|
+
model: z.string().optional(),
|
|
11
|
+
baseUrl: z.string().optional(),
|
|
12
|
+
})
|
|
13
|
+
.default({ provider: "ollama" }),
|
|
14
|
+
agent: z.string().optional(),
|
|
15
|
+
ui: z.object({ port: z.number().int().default(4747) }).default({ port: 4747 }),
|
|
16
|
+
});
|
|
17
|
+
export function memiHome() {
|
|
18
|
+
return process.env.MEMI_HOME || join(homedir(), ".memi");
|
|
19
|
+
}
|
|
20
|
+
export const dbPath = () => join(memiHome(), "memi.db");
|
|
21
|
+
export const configPath = () => join(memiHome(), "config.json");
|
|
22
|
+
export function loadConfig() {
|
|
23
|
+
let raw = {};
|
|
24
|
+
if (existsSync(configPath())) {
|
|
25
|
+
try {
|
|
26
|
+
raw = JSON.parse(readFileSync(configPath(), "utf8"));
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
throw new Error(`Could not read ${configPath()}: ${err.message}`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return ConfigSchema.parse(raw);
|
|
33
|
+
}
|
|
34
|
+
export function saveConfig(config) {
|
|
35
|
+
mkdirSync(memiHome(), { recursive: true });
|
|
36
|
+
writeFileSync(configPath(), JSON.stringify(config, null, 2) + "\n");
|
|
37
|
+
}
|
|
38
|
+
const DEFAULTS = {
|
|
39
|
+
ollama: { model: "nomic-embed-text", baseUrl: "http://localhost:11434" },
|
|
40
|
+
openai: { model: "text-embedding-3-small", baseUrl: "https://api.openai.com/v1" },
|
|
41
|
+
};
|
|
42
|
+
/** Config file first, then env overrides. The OpenAI key only ever comes from the environment. */
|
|
43
|
+
export function resolveEmbedder(config = loadConfig()) {
|
|
44
|
+
const provider = ProviderSchema.parse(process.env.MEMI_EMBED_PROVIDER ?? config.embedder.provider);
|
|
45
|
+
const defaults = DEFAULTS[provider];
|
|
46
|
+
const configured = config.embedder.provider === provider ? config.embedder : undefined;
|
|
47
|
+
const envBase = provider === "ollama" ? process.env.MEMI_OLLAMA_URL : process.env.OPENAI_BASE_URL;
|
|
48
|
+
return {
|
|
49
|
+
provider,
|
|
50
|
+
model: process.env.MEMI_EMBED_MODEL ?? configured?.model ?? defaults.model,
|
|
51
|
+
baseUrl: (envBase ?? configured?.baseUrl ?? defaults.baseUrl).replace(/\/+$/, ""),
|
|
52
|
+
apiKey: provider === "openai" ? process.env.OPENAI_API_KEY : undefined,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export function resolveAgent(config = loadConfig()) {
|
|
56
|
+
return process.env.MEMI_AGENT?.trim() || config.agent?.trim() || null;
|
|
57
|
+
}
|
|
58
|
+
export const CONFIG_KEYS = ["embedder.provider", "embedder.model", "embedder.baseUrl", "agent", "ui.port"];
|
|
59
|
+
/** Returns a copy of the config with one key changed. Switching provider clears the old provider's model and URL. */
|
|
60
|
+
export function setConfigValue(config, key, value) {
|
|
61
|
+
const next = structuredClone(config);
|
|
62
|
+
switch (key) {
|
|
63
|
+
case "embedder.provider": {
|
|
64
|
+
const provider = ProviderSchema.safeParse(value);
|
|
65
|
+
if (!provider.success)
|
|
66
|
+
throw new Error(`embedder.provider must be one of: ${ProviderSchema.options.join(", ")}`);
|
|
67
|
+
next.embedder = { provider: provider.data };
|
|
68
|
+
return next;
|
|
69
|
+
}
|
|
70
|
+
case "embedder.model":
|
|
71
|
+
next.embedder.model = value;
|
|
72
|
+
return next;
|
|
73
|
+
case "embedder.baseUrl":
|
|
74
|
+
next.embedder.baseUrl = value;
|
|
75
|
+
return next;
|
|
76
|
+
case "agent":
|
|
77
|
+
next.agent = value;
|
|
78
|
+
return next;
|
|
79
|
+
case "ui.port": {
|
|
80
|
+
const port = Number(value);
|
|
81
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535)
|
|
82
|
+
throw new Error("ui.port must be a number from 1 to 65535");
|
|
83
|
+
next.ui.port = port;
|
|
84
|
+
return next;
|
|
85
|
+
}
|
|
86
|
+
default:
|
|
87
|
+
throw new Error(`Unknown key "${key}". Keys: ${CONFIG_KEYS.join(", ")}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
package/dist/core/db.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import { mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import * as sqliteVec from "sqlite-vec";
|
|
5
|
+
import { dbPath } from "./config.js";
|
|
6
|
+
import { migrate } from "./schema.js";
|
|
7
|
+
import { BUILTIN_CATEGORIES } from "./types.js";
|
|
8
|
+
export function openDb(path = dbPath()) {
|
|
9
|
+
if (path !== ":memory:")
|
|
10
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
11
|
+
const db = new Database(path);
|
|
12
|
+
db.pragma("journal_mode = WAL");
|
|
13
|
+
db.pragma("foreign_keys = ON");
|
|
14
|
+
db.pragma("busy_timeout = 5000");
|
|
15
|
+
sqliteVec.load(db);
|
|
16
|
+
migrate(db);
|
|
17
|
+
const seed = db.prepare("INSERT OR IGNORE INTO categories(name, description, builtin) VALUES (?, ?, 1)");
|
|
18
|
+
for (const [name, description] of Object.entries(BUILTIN_CATEGORIES))
|
|
19
|
+
seed.run(name, description);
|
|
20
|
+
return db;
|
|
21
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export function formatMemory(m, score) {
|
|
2
|
+
const where = m.scope === "global" ? "global" : `project ${m.project}`;
|
|
3
|
+
const meta = [
|
|
4
|
+
m.category,
|
|
5
|
+
`importance ${m.importance}`,
|
|
6
|
+
m.pinned ? "pinned" : null,
|
|
7
|
+
where,
|
|
8
|
+
m.tags.length ? `tags ${m.tags.join(", ")}` : null,
|
|
9
|
+
m.updatedAt.slice(0, 10),
|
|
10
|
+
m.agent ? `by ${m.agent}` : null,
|
|
11
|
+
score !== undefined ? `score ${score.toFixed(4)}` : null,
|
|
12
|
+
].filter(Boolean);
|
|
13
|
+
return `#${m.id} | ${meta.join(" | ")}\n${m.content}`;
|
|
14
|
+
}
|
|
15
|
+
export const formatMemories = (list) => list.map((m) => formatMemory(m)).join("\n\n");
|
|
16
|
+
export const formatHits = (hits) => hits.map((h) => formatMemory(h.memory, h.score)).join("\n\n");
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { rowToMemory, whereClause } from "./rows.js";
|
|
2
|
+
import { DEFAULT_CATEGORY, MAX_CONTENT_LENGTH, MemiError, } from "./types.js";
|
|
3
|
+
import { search } from "./search.js";
|
|
4
|
+
import { deleteVector, embedMissing, embedderState, ensureVectors, knn, mismatchMessage, putVector, rebuildVectors, scopeKey } from "./vectors.js";
|
|
5
|
+
/** Cosine distance under which a new memory is reported as similar to an existing one. */
|
|
6
|
+
export const SIMILAR_DISTANCE = 0.15;
|
|
7
|
+
const ORDER = {
|
|
8
|
+
recent: "m.updated_at DESC, m.id DESC",
|
|
9
|
+
oldest: "m.created_at ASC, m.id ASC",
|
|
10
|
+
importance: "m.importance DESC, m.updated_at DESC",
|
|
11
|
+
};
|
|
12
|
+
export function normalizeCategory(raw) {
|
|
13
|
+
const name = raw.trim().toLowerCase().replace(/\s+/g, "-");
|
|
14
|
+
if (!/^[a-z0-9][a-z0-9_-]{0,31}$/.test(name)) {
|
|
15
|
+
throw new MemiError(`Invalid category "${raw}". Use up to 32 letters, numbers, dashes or underscores.`);
|
|
16
|
+
}
|
|
17
|
+
return name;
|
|
18
|
+
}
|
|
19
|
+
const normalizeTags = (tags = []) => [...new Set(tags.map((t) => t.trim().toLowerCase()).filter(Boolean))].slice(0, 10);
|
|
20
|
+
function checkImportance(n) {
|
|
21
|
+
if (!Number.isInteger(n) || n < 1 || n > 5)
|
|
22
|
+
throw new MemiError("Importance must be a whole number from 1 to 5.");
|
|
23
|
+
return n;
|
|
24
|
+
}
|
|
25
|
+
function checkContent(content) {
|
|
26
|
+
const text = content.trim();
|
|
27
|
+
if (!text)
|
|
28
|
+
throw new MemiError("Memory content cannot be empty.");
|
|
29
|
+
if (text.length > MAX_CONTENT_LENGTH)
|
|
30
|
+
throw new MemiError(`Memory content is too long (max ${MAX_CONTENT_LENGTH} characters).`);
|
|
31
|
+
return text;
|
|
32
|
+
}
|
|
33
|
+
export class MemoryStore {
|
|
34
|
+
db;
|
|
35
|
+
embedder;
|
|
36
|
+
constructor(db, embedder = null) {
|
|
37
|
+
this.db = db;
|
|
38
|
+
this.embedder = embedder;
|
|
39
|
+
}
|
|
40
|
+
ensureCategory(name) {
|
|
41
|
+
this.db.prepare("INSERT OR IGNORE INTO categories(name, description, builtin) VALUES (?, '', 0)").run(name);
|
|
42
|
+
}
|
|
43
|
+
resolveScope(scope, project) {
|
|
44
|
+
const s = scope ?? (project ? "project" : "global");
|
|
45
|
+
if (s === "project" && !project)
|
|
46
|
+
throw new MemiError("A project memory needs a project name.");
|
|
47
|
+
return { scope: s, project: s === "project" ? project : null };
|
|
48
|
+
}
|
|
49
|
+
get(id) {
|
|
50
|
+
const row = this.db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
|
|
51
|
+
return row ? rowToMemory(row) : null;
|
|
52
|
+
}
|
|
53
|
+
getMany(ids) {
|
|
54
|
+
if (!ids.length)
|
|
55
|
+
return [];
|
|
56
|
+
const rows = this.db.prepare(`SELECT * FROM memories WHERE id IN (${ids.map(() => "?").join(",")})`).all(...ids);
|
|
57
|
+
return rows.map(rowToMemory);
|
|
58
|
+
}
|
|
59
|
+
/** Embeds text if possible. Never throws: a memory is saved even when the embedder is down. */
|
|
60
|
+
async tryEmbed(text) {
|
|
61
|
+
if (!this.embedder)
|
|
62
|
+
return { vec: null, warning: "No embedder configured, saved for text search only." };
|
|
63
|
+
try {
|
|
64
|
+
const state = await ensureVectors(this.db, this.embedder);
|
|
65
|
+
if (state.status === "mismatch")
|
|
66
|
+
return { vec: null, warning: mismatchMessage(state, this.embedder) };
|
|
67
|
+
const [vec] = await this.embedder.embed([text], "document");
|
|
68
|
+
return { vec: vec ?? null };
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
return { vec: null, warning: `Saved without a vector (${err.message}). Run \`memi reindex --missing\` later to backfill.` };
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async add(input) {
|
|
75
|
+
const content = checkContent(input.content);
|
|
76
|
+
const { scope, project } = this.resolveScope(input.scope, input.project);
|
|
77
|
+
const category = normalizeCategory(input.category ?? DEFAULT_CATEGORY);
|
|
78
|
+
const importance = checkImportance(input.importance ?? 3);
|
|
79
|
+
const now = new Date().toISOString();
|
|
80
|
+
const { vec, warning } = await this.tryEmbed(content);
|
|
81
|
+
const key = scopeKey(scope, project);
|
|
82
|
+
const similarHits = vec ? knn(this.db, vec, 5, key).filter((n) => n.distance <= SIMILAR_DISTANCE) : [];
|
|
83
|
+
const id = this.db.transaction(() => {
|
|
84
|
+
this.ensureCategory(category);
|
|
85
|
+
const res = this.db
|
|
86
|
+
.prepare(`INSERT INTO memories(scope, project, agent, category, content, tags, importance, pinned, created_at, updated_at)
|
|
87
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
88
|
+
.run(scope, project, input.agent ?? null, category, content, JSON.stringify(normalizeTags(input.tags)), importance, input.pinned ? 1 : 0, now, now);
|
|
89
|
+
const newId = Number(res.lastInsertRowid);
|
|
90
|
+
if (vec)
|
|
91
|
+
putVector(this.db, newId, key, vec);
|
|
92
|
+
return newId;
|
|
93
|
+
})();
|
|
94
|
+
const byId = new Map(this.getMany(similarHits.map((h) => h.id)).map((m) => [m.id, m]));
|
|
95
|
+
return {
|
|
96
|
+
memory: this.get(id),
|
|
97
|
+
similar: similarHits.flatMap((h) => (byId.has(h.id) ? [{ memory: byId.get(h.id), distance: h.distance }] : [])),
|
|
98
|
+
embedded: vec !== null,
|
|
99
|
+
warning,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
async update(id, patch) {
|
|
103
|
+
const current = this.get(id);
|
|
104
|
+
if (!current)
|
|
105
|
+
return null;
|
|
106
|
+
const content = patch.content !== undefined ? checkContent(patch.content) : current.content;
|
|
107
|
+
const target = this.resolveScope(patch.scope ?? current.scope, patch.project !== undefined ? patch.project : current.project);
|
|
108
|
+
const category = patch.category !== undefined ? normalizeCategory(patch.category) : current.category;
|
|
109
|
+
const importance = patch.importance !== undefined ? checkImportance(patch.importance) : current.importance;
|
|
110
|
+
const tags = patch.tags !== undefined ? normalizeTags(patch.tags) : current.tags;
|
|
111
|
+
const pinned = patch.pinned ?? current.pinned;
|
|
112
|
+
const needsVector = content !== current.content || target.scope !== current.scope || target.project !== current.project;
|
|
113
|
+
const embedded = needsVector ? await this.tryEmbed(content) : { vec: null, warning: undefined };
|
|
114
|
+
this.db.transaction(() => {
|
|
115
|
+
this.ensureCategory(category);
|
|
116
|
+
this.db
|
|
117
|
+
.prepare("UPDATE memories SET scope=?, project=?, category=?, content=?, tags=?, importance=?, pinned=?, updated_at=? WHERE id=?")
|
|
118
|
+
.run(target.scope, target.project, category, content, JSON.stringify(tags), importance, pinned ? 1 : 0, new Date().toISOString(), id);
|
|
119
|
+
if (needsVector) {
|
|
120
|
+
deleteVector(this.db, id);
|
|
121
|
+
if (embedded.vec)
|
|
122
|
+
putVector(this.db, id, scopeKey(target.scope, target.project), embedded.vec);
|
|
123
|
+
}
|
|
124
|
+
})();
|
|
125
|
+
return { memory: this.get(id), warning: embedded.warning };
|
|
126
|
+
}
|
|
127
|
+
remove(id) {
|
|
128
|
+
return this.db.transaction(() => {
|
|
129
|
+
deleteVector(this.db, id);
|
|
130
|
+
return this.db.prepare("DELETE FROM memories WHERE id = ?").run(id).changes > 0;
|
|
131
|
+
})();
|
|
132
|
+
}
|
|
133
|
+
list(opts = {}) {
|
|
134
|
+
const { sql, params } = whereClause(opts);
|
|
135
|
+
const total = this.db.prepare(`SELECT COUNT(*) AS n FROM memories m WHERE ${sql}`).get(params).n;
|
|
136
|
+
const rows = this.db
|
|
137
|
+
.prepare(`SELECT m.* FROM memories m WHERE ${sql} ORDER BY ${ORDER[opts.order ?? "recent"]} LIMIT @limit OFFSET @offset`)
|
|
138
|
+
.all({ ...params, limit: Math.min(opts.limit ?? 50, 200), offset: opts.offset ?? 0 });
|
|
139
|
+
return { items: rows.map(rowToMemory), total };
|
|
140
|
+
}
|
|
141
|
+
search(opts) {
|
|
142
|
+
return search(this.db, this.embedder, opts);
|
|
143
|
+
}
|
|
144
|
+
/** Pinned and high-importance memories: what an agent should load at the start of a session. */
|
|
145
|
+
context(opts = {}) {
|
|
146
|
+
const { sql, params } = whereClause({ scope: "both", project: opts.project });
|
|
147
|
+
const rows = this.db
|
|
148
|
+
.prepare(`SELECT m.* FROM memories m WHERE ${sql} AND (m.pinned = 1 OR m.importance >= 4)
|
|
149
|
+
ORDER BY m.pinned DESC, m.importance DESC, m.updated_at DESC LIMIT @limit`)
|
|
150
|
+
.all({ ...params, limit: opts.limit ?? 20 });
|
|
151
|
+
return rows.map(rowToMemory);
|
|
152
|
+
}
|
|
153
|
+
categories() {
|
|
154
|
+
const rows = this.db
|
|
155
|
+
.prepare(`SELECT c.name, c.description, c.builtin, COUNT(m.id) AS count
|
|
156
|
+
FROM categories c LEFT JOIN memories m ON m.category = c.name
|
|
157
|
+
GROUP BY c.name ORDER BY c.builtin DESC, c.name`)
|
|
158
|
+
.all();
|
|
159
|
+
return rows.map((r) => ({ ...r, builtin: r.builtin === 1 }));
|
|
160
|
+
}
|
|
161
|
+
projects() {
|
|
162
|
+
return this.db
|
|
163
|
+
.prepare("SELECT project, COUNT(*) AS count FROM memories WHERE scope = 'project' GROUP BY project ORDER BY project")
|
|
164
|
+
.all();
|
|
165
|
+
}
|
|
166
|
+
everything() {
|
|
167
|
+
return this.db.prepare("SELECT * FROM memories ORDER BY id").all().map(rowToMemory);
|
|
168
|
+
}
|
|
169
|
+
/** Drops and rebuilds the vector table with the current embedder. Memories are kept. */
|
|
170
|
+
async reindex(onProgress) {
|
|
171
|
+
if (!this.embedder)
|
|
172
|
+
throw new MemiError("No embedder configured.");
|
|
173
|
+
return rebuildVectors(this.db, this.embedder, this.everything(), onProgress);
|
|
174
|
+
}
|
|
175
|
+
async backfill() {
|
|
176
|
+
if (!this.embedder)
|
|
177
|
+
throw new MemiError("No embedder configured.");
|
|
178
|
+
return embedMissing(this.db, this.embedder, this.everything());
|
|
179
|
+
}
|
|
180
|
+
status() {
|
|
181
|
+
const total = this.db.prepare("SELECT COUNT(*) AS n FROM memories").get().n;
|
|
182
|
+
const state = this.embedder ? embedderState(this.db, this.embedder) : { status: "uninitialized" };
|
|
183
|
+
const vectors = state.status === "uninitialized" ? 0 : this.db.prepare("SELECT COUNT(*) AS n FROM memory_vec").get().n;
|
|
184
|
+
return {
|
|
185
|
+
total,
|
|
186
|
+
vectors,
|
|
187
|
+
embedder: this.embedder ? { provider: this.embedder.provider, model: this.embedder.model } : null,
|
|
188
|
+
state,
|
|
189
|
+
warning: this.embedder && state.status === "mismatch" ? mismatchMessage(state, this.embedder) : undefined,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
3
|
+
import { MemiError } from "./types.js";
|
|
4
|
+
export function findGitRoot(start) {
|
|
5
|
+
let dir = resolve(start);
|
|
6
|
+
for (;;) {
|
|
7
|
+
if (existsSync(join(dir, ".git")))
|
|
8
|
+
return dir;
|
|
9
|
+
const parent = dirname(dir);
|
|
10
|
+
if (parent === dir)
|
|
11
|
+
return null;
|
|
12
|
+
dir = parent;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Project id order: explicit argument, MEMI_PROJECT, git root folder name, then the agent name.
|
|
17
|
+
* Returns null when nothing identifies a project, so callers can fall back to global or ask.
|
|
18
|
+
*/
|
|
19
|
+
export function resolveProject(opts) {
|
|
20
|
+
const explicit = opts.project?.trim();
|
|
21
|
+
if (explicit)
|
|
22
|
+
return { project: explicit, source: "explicit" };
|
|
23
|
+
const env = process.env.MEMI_PROJECT?.trim();
|
|
24
|
+
if (env)
|
|
25
|
+
return { project: env, source: "env" };
|
|
26
|
+
const root = findGitRoot(opts.cwd ?? process.cwd());
|
|
27
|
+
if (root)
|
|
28
|
+
return { project: basename(root), source: "git" };
|
|
29
|
+
const agent = opts.agent?.trim();
|
|
30
|
+
if (agent)
|
|
31
|
+
return { project: `agent:${agent}`, source: "agent" };
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
export function requireProject(opts) {
|
|
35
|
+
const info = resolveProject(opts);
|
|
36
|
+
if (!info) {
|
|
37
|
+
throw new MemiError("Could not work out which project this is. There is no git repo here and no agent name set. " +
|
|
38
|
+
"Pass a project name, set MEMI_PROJECT, or set an agent name with MEMI_AGENT.");
|
|
39
|
+
}
|
|
40
|
+
return info;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Where a new memory goes. With no explicit scope it lands in the current project when one can be
|
|
44
|
+
* worked out, and in global otherwise.
|
|
45
|
+
*/
|
|
46
|
+
export function pickScope(opts) {
|
|
47
|
+
if (opts.scope === "global")
|
|
48
|
+
return { scope: "global", project: null };
|
|
49
|
+
if (opts.scope === "project")
|
|
50
|
+
return { scope: "project", project: requireProject(opts).project };
|
|
51
|
+
const info = resolveProject(opts);
|
|
52
|
+
return info ? { scope: "project", project: info.project } : { scope: "global", project: null };
|
|
53
|
+
}
|