@bubstack/moe-memory 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/version.d.ts
CHANGED
|
@@ -9,5 +9,5 @@
|
|
|
9
9
|
* constant instead. `test/version-consistency.test.ts` asserts it equals
|
|
10
10
|
* package.json, which is the invariant the generator existed to guarantee.
|
|
11
11
|
*/
|
|
12
|
-
export declare const VERSION = "0.1.
|
|
12
|
+
export declare const VERSION = "0.1.2";
|
|
13
13
|
//# sourceMappingURL=version.d.ts.map
|
package/dist/version.js
CHANGED
package/package.json
CHANGED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* One-way importer for private-journal-mcp's `.embedding` sidecars.
|
|
3
|
-
*
|
|
4
|
-
* Upstream wrote, beside every `<entry>.md`, an `<entry>.embedding` JSON file:
|
|
5
|
-
*
|
|
6
|
-
* { embedding: number[], text: string, sections: string[], timestamp: number, path: string }
|
|
7
|
-
*
|
|
8
|
-
* Two reasons that data cannot simply be loaded into the new index:
|
|
9
|
-
*
|
|
10
|
-
* 1. **The vectors are from a different encoder.** Sidecars were produced by
|
|
11
|
-
* `Xenova/all-MiniLM-L6-v2`; this package embeds with `Xenova/bge-small-en-v1.5`
|
|
12
|
-
* at dtype q8 and an asymmetric query prefix. Both are 384-dimensional, so a
|
|
13
|
-
* vec0 column accepts either without complaint and a mixed corpus ranks
|
|
14
|
-
* wrongly with no error. So the sidecar's `embedding` array is DISCARDED and
|
|
15
|
-
* the entry is re-embedded from its markdown.
|
|
16
|
-
* 2. **`path` was baked in absolute.** Upstream's search returned the path stored
|
|
17
|
-
* inside the JSON rather than the path it had just walked, so renaming the
|
|
18
|
-
* journal directory produced results that `read_journal_entry` then refused.
|
|
19
|
-
* So the sidecar's `path` is DISCARDED too, and the walk decides.
|
|
20
|
-
*
|
|
21
|
-
* Which leaves nothing worth importing from the sidecar itself. What this
|
|
22
|
-
* function actually does is find the orphans — `.md` files with a sidecar whose
|
|
23
|
-
* day-directory is not where the new layout expects it — and, mainly, delete the
|
|
24
|
-
* sidecars once the entry is indexed, so the old and new indexes cannot drift.
|
|
25
|
-
*
|
|
26
|
-
* It is opt-in (`moe-memory journal import-legacy`) and never runs on its own:
|
|
27
|
-
* deleting a file the user might still want read by an upstream install is not
|
|
28
|
-
* something to do silently.
|
|
29
|
-
*
|
|
30
|
-
* FIXED: it searched only the CURRENT roots. The command exists because the
|
|
31
|
-
* paths moved on import — `<project>/.private-journal` → `<project>/.moe-journal`
|
|
32
|
-
* and `~/.private-journal` → `<data dir>/journal` — so on any install that had
|
|
33
|
-
* not already hand-copied its journal across, it walked exactly the two
|
|
34
|
-
* directories the sidecars provably are not in, found nothing, and printed
|
|
35
|
-
* "Legacy .embedding sidecars found: 0". Indistinguishable from success. It now
|
|
36
|
-
* also surveys the upstream directories (`findLegacyJournalRoots`) and reports
|
|
37
|
-
* them in `legacy`, so the caller can tell the user their data is over there.
|
|
38
|
-
*
|
|
39
|
-
* It reports rather than migrates, deliberately, matching `findLegacyDataDir`.
|
|
40
|
-
* Journal entries are private reflections; quietly relocating them is worse
|
|
41
|
-
* than quietly relocating a conversation archive, and that was already judged
|
|
42
|
-
* too much to do behind someone's back.
|
|
43
|
-
*/
|
|
44
|
-
import type Database from "better-sqlite3";
|
|
45
|
-
import type { JournalStore } from "./store.js";
|
|
46
|
-
export interface LegacyImportResult {
|
|
47
|
-
/** Sidecars found on disk. */
|
|
48
|
-
found: number;
|
|
49
|
-
/** Sidecars whose entry is now present in the SQLite index. */
|
|
50
|
-
indexed: number;
|
|
51
|
-
/** Sidecars deleted (only when `remove` is set and the entry is indexed). */
|
|
52
|
-
removed: number;
|
|
53
|
-
/** Sidecars with no surviving `.md` beside them. */
|
|
54
|
-
orphaned: string[];
|
|
55
|
-
/**
|
|
56
|
-
* Upstream journal directories that exist but are NOT being walked, with what
|
|
57
|
-
* is sitting in them.
|
|
58
|
-
*
|
|
59
|
-
* Empty is the normal case. Non-empty means this command cannot do its job
|
|
60
|
-
* yet: the paths moved on import (`.private-journal` → `.moe-journal`,
|
|
61
|
-
* `~/.private-journal` → `<data dir>/journal`), so the sidecars are still
|
|
62
|
-
* over there and nothing here can see them. Reported rather than migrated —
|
|
63
|
-
* see `findLegacyJournalRoots`.
|
|
64
|
-
*/
|
|
65
|
-
legacy: Array<{
|
|
66
|
-
root: string;
|
|
67
|
-
entries: number;
|
|
68
|
-
sidecars: number;
|
|
69
|
-
}>;
|
|
70
|
-
}
|
|
71
|
-
export declare function importLegacyJournalSidecars(db: Database.Database, store: JournalStore, options?: {
|
|
72
|
-
remove?: boolean;
|
|
73
|
-
}): Promise<LegacyImportResult>;
|
|
74
|
-
//# sourceMappingURL=legacy-sidecars.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"legacy-sidecars.d.ts","sourceRoot":"","sources":["../../src/journal/legacy-sidecars.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAIH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAI3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;;;;OASG;IACH,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpE;AA4BD,wBAAsB,2BAA2B,CAC/C,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GACjC,OAAO,CAAC,kBAAkB,CAAC,CA2E7B"}
|
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* One-way importer for private-journal-mcp's `.embedding` sidecars.
|
|
3
|
-
*
|
|
4
|
-
* Upstream wrote, beside every `<entry>.md`, an `<entry>.embedding` JSON file:
|
|
5
|
-
*
|
|
6
|
-
* { embedding: number[], text: string, sections: string[], timestamp: number, path: string }
|
|
7
|
-
*
|
|
8
|
-
* Two reasons that data cannot simply be loaded into the new index:
|
|
9
|
-
*
|
|
10
|
-
* 1. **The vectors are from a different encoder.** Sidecars were produced by
|
|
11
|
-
* `Xenova/all-MiniLM-L6-v2`; this package embeds with `Xenova/bge-small-en-v1.5`
|
|
12
|
-
* at dtype q8 and an asymmetric query prefix. Both are 384-dimensional, so a
|
|
13
|
-
* vec0 column accepts either without complaint and a mixed corpus ranks
|
|
14
|
-
* wrongly with no error. So the sidecar's `embedding` array is DISCARDED and
|
|
15
|
-
* the entry is re-embedded from its markdown.
|
|
16
|
-
* 2. **`path` was baked in absolute.** Upstream's search returned the path stored
|
|
17
|
-
* inside the JSON rather than the path it had just walked, so renaming the
|
|
18
|
-
* journal directory produced results that `read_journal_entry` then refused.
|
|
19
|
-
* So the sidecar's `path` is DISCARDED too, and the walk decides.
|
|
20
|
-
*
|
|
21
|
-
* Which leaves nothing worth importing from the sidecar itself. What this
|
|
22
|
-
* function actually does is find the orphans — `.md` files with a sidecar whose
|
|
23
|
-
* day-directory is not where the new layout expects it — and, mainly, delete the
|
|
24
|
-
* sidecars once the entry is indexed, so the old and new indexes cannot drift.
|
|
25
|
-
*
|
|
26
|
-
* It is opt-in (`moe-memory journal import-legacy`) and never runs on its own:
|
|
27
|
-
* deleting a file the user might still want read by an upstream install is not
|
|
28
|
-
* something to do silently.
|
|
29
|
-
*
|
|
30
|
-
* FIXED: it searched only the CURRENT roots. The command exists because the
|
|
31
|
-
* paths moved on import — `<project>/.private-journal` → `<project>/.moe-journal`
|
|
32
|
-
* and `~/.private-journal` → `<data dir>/journal` — so on any install that had
|
|
33
|
-
* not already hand-copied its journal across, it walked exactly the two
|
|
34
|
-
* directories the sidecars provably are not in, found nothing, and printed
|
|
35
|
-
* "Legacy .embedding sidecars found: 0". Indistinguishable from success. It now
|
|
36
|
-
* also surveys the upstream directories (`findLegacyJournalRoots`) and reports
|
|
37
|
-
* them in `legacy`, so the caller can tell the user their data is over there.
|
|
38
|
-
*
|
|
39
|
-
* It reports rather than migrates, deliberately, matching `findLegacyDataDir`.
|
|
40
|
-
* Journal entries are private reflections; quietly relocating them is worse
|
|
41
|
-
* than quietly relocating a conversation archive, and that was already judged
|
|
42
|
-
* too much to do behind someone's back.
|
|
43
|
-
*/
|
|
44
|
-
import fs from "node:fs/promises";
|
|
45
|
-
import path from "node:path";
|
|
46
|
-
import { getJournalIndexState } from "../db.js";
|
|
47
|
-
import { findLegacyJournalRoots } from "../paths.js";
|
|
48
|
-
import { DAY_DIR_PATTERN, journalEntryId } from "./markdown.js";
|
|
49
|
-
/** Count `.md` entries and `.embedding` sidecars under a journal root. */
|
|
50
|
-
async function surveyRoot(root) {
|
|
51
|
-
let entries = 0;
|
|
52
|
-
let sidecars = 0;
|
|
53
|
-
let dayDirs;
|
|
54
|
-
try {
|
|
55
|
-
dayDirs = await fs.readdir(root);
|
|
56
|
-
}
|
|
57
|
-
catch {
|
|
58
|
-
return { entries, sidecars };
|
|
59
|
-
}
|
|
60
|
-
for (const dayDir of dayDirs) {
|
|
61
|
-
if (!DAY_DIR_PATTERN.test(dayDir))
|
|
62
|
-
continue;
|
|
63
|
-
let files;
|
|
64
|
-
try {
|
|
65
|
-
files = await fs.readdir(path.join(root, dayDir));
|
|
66
|
-
}
|
|
67
|
-
catch {
|
|
68
|
-
continue;
|
|
69
|
-
}
|
|
70
|
-
for (const file of files) {
|
|
71
|
-
if (file.endsWith(".md"))
|
|
72
|
-
entries++;
|
|
73
|
-
else if (file.endsWith(".embedding"))
|
|
74
|
-
sidecars++;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return { entries, sidecars };
|
|
78
|
-
}
|
|
79
|
-
export async function importLegacyJournalSidecars(db, store, options = {}) {
|
|
80
|
-
const result = {
|
|
81
|
-
found: 0,
|
|
82
|
-
indexed: 0,
|
|
83
|
-
removed: 0,
|
|
84
|
-
orphaned: [],
|
|
85
|
-
legacy: [],
|
|
86
|
-
};
|
|
87
|
-
// Look where the data actually is before reporting zero.
|
|
88
|
-
for (const legacyRoot of findLegacyJournalRoots()) {
|
|
89
|
-
const survey = await surveyRoot(legacyRoot);
|
|
90
|
-
if (survey.entries === 0 && survey.sidecars === 0)
|
|
91
|
-
continue;
|
|
92
|
-
result.legacy.push({ root: legacyRoot, ...survey });
|
|
93
|
-
}
|
|
94
|
-
// Re-index first: the markdown files are the source of truth, and every
|
|
95
|
-
// sidecar's entry has to exist in the new index before we consider deleting it.
|
|
96
|
-
await store.indexJournal(db);
|
|
97
|
-
const state = getJournalIndexState(db);
|
|
98
|
-
for (const root of store.roots()) {
|
|
99
|
-
let dayDirs;
|
|
100
|
-
try {
|
|
101
|
-
dayDirs = await fs.readdir(root.path);
|
|
102
|
-
}
|
|
103
|
-
catch {
|
|
104
|
-
continue;
|
|
105
|
-
}
|
|
106
|
-
for (const dayDir of dayDirs) {
|
|
107
|
-
if (!DAY_DIR_PATTERN.test(dayDir))
|
|
108
|
-
continue;
|
|
109
|
-
const dayPath = path.join(root.path, dayDir);
|
|
110
|
-
let files;
|
|
111
|
-
try {
|
|
112
|
-
files = await fs.readdir(dayPath);
|
|
113
|
-
}
|
|
114
|
-
catch {
|
|
115
|
-
continue;
|
|
116
|
-
}
|
|
117
|
-
for (const file of files) {
|
|
118
|
-
if (!file.endsWith(".embedding"))
|
|
119
|
-
continue;
|
|
120
|
-
result.found++;
|
|
121
|
-
const sidecarPath = path.join(dayPath, file);
|
|
122
|
-
const entryPath = sidecarPath.replace(/\.embedding$/, ".md");
|
|
123
|
-
try {
|
|
124
|
-
await fs.access(entryPath);
|
|
125
|
-
}
|
|
126
|
-
catch {
|
|
127
|
-
result.orphaned.push(sidecarPath);
|
|
128
|
-
continue;
|
|
129
|
-
}
|
|
130
|
-
// The scope the walk would assign is not knowable here without reading
|
|
131
|
-
// the entry, so check both possibilities.
|
|
132
|
-
const indexed = state.has(journalEntryId("project", root.path, entryPath)) ||
|
|
133
|
-
state.has(journalEntryId("user", root.path, entryPath));
|
|
134
|
-
if (!indexed)
|
|
135
|
-
continue;
|
|
136
|
-
result.indexed++;
|
|
137
|
-
if (options.remove) {
|
|
138
|
-
try {
|
|
139
|
-
await fs.unlink(sidecarPath);
|
|
140
|
-
result.removed++;
|
|
141
|
-
}
|
|
142
|
-
catch (error) {
|
|
143
|
-
console.error(`moe-memory: could not remove legacy sidecar ${sidecarPath}: ${String(error)}`);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
return result;
|
|
150
|
-
}
|
|
151
|
-
//# sourceMappingURL=legacy-sidecars.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"legacy-sidecars.js","sourceRoot":"","sources":["../../src/journal/legacy-sidecars.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAyBhE,0EAA0E;AAC1E,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS;QAC5C,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;iBAC/B,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAAE,QAAQ,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,EAAqB,EACrB,KAAmB,EACnB,UAAgC,EAAE;IAElC,MAAM,MAAM,GAAuB;QACjC,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,EAAE;QACZ,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,yDAAyD;IACzD,KAAK,MAAM,UAAU,IAAI,sBAAsB,EAAE,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,SAAS;QAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,wEAAwE;IACxE,gFAAgF;IAChF,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,EAAE,CAAC,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjC,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,SAAS;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,IAAI,KAAe,CAAC;YACpB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;oBAAE,SAAS;gBAC3C,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC7C,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAE7D,IAAI,CAAC;oBACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC7B,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAClC,SAAS;gBACX,CAAC;gBAED,uEAAuE;gBACvE,0CAA0C;gBAC1C,MAAM,OAAO,GACX,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC1D,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;gBAC1D,IAAI,CAAC,OAAO;oBAAE,SAAS;gBACvB,MAAM,CAAC,OAAO,EAAE,CAAC;gBAEjB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;wBAC7B,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CACX,+CAA+C,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/E,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|