@context-vault/core 2.17.1 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capture.d.ts +21 -0
- package/dist/capture.d.ts.map +1 -0
- package/dist/capture.js +269 -0
- package/dist/capture.js.map +1 -0
- package/dist/categories.d.ts +6 -0
- package/dist/categories.d.ts.map +1 -0
- package/dist/categories.js +50 -0
- package/dist/categories.js.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +190 -0
- package/dist/config.js.map +1 -0
- package/dist/constants.d.ts +33 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +23 -0
- package/dist/constants.js.map +1 -0
- package/dist/db.d.ts +13 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +191 -0
- package/dist/db.js.map +1 -0
- package/dist/embed.d.ts +5 -0
- package/dist/embed.d.ts.map +1 -0
- package/dist/embed.js +78 -0
- package/dist/embed.js.map +1 -0
- package/dist/files.d.ts +13 -0
- package/dist/files.d.ts.map +1 -0
- package/dist/files.js +66 -0
- package/dist/files.js.map +1 -0
- package/dist/formatters.d.ts +8 -0
- package/dist/formatters.d.ts.map +1 -0
- package/dist/formatters.js +18 -0
- package/dist/formatters.js.map +1 -0
- package/dist/frontmatter.d.ts +12 -0
- package/dist/frontmatter.d.ts.map +1 -0
- package/dist/frontmatter.js +101 -0
- package/dist/frontmatter.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +297 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest-url.d.ts +20 -0
- package/dist/ingest-url.d.ts.map +1 -0
- package/dist/ingest-url.js +113 -0
- package/dist/ingest-url.js.map +1 -0
- package/dist/main.d.ts +14 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +25 -0
- package/dist/main.js.map +1 -0
- package/dist/search.d.ts +18 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +238 -0
- package/dist/search.js.map +1 -0
- package/dist/types.d.ts +176 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +66 -16
- package/src/capture.ts +308 -0
- package/src/categories.ts +54 -0
- package/src/{core/config.js → config.ts} +34 -33
- package/src/{constants.js → constants.ts} +6 -3
- package/src/db.ts +229 -0
- package/src/{index/embed.js → embed.ts} +10 -35
- package/src/{core/files.js → files.ts} +15 -20
- package/src/{capture/formatters.js → formatters.ts} +13 -11
- package/src/{core/frontmatter.js → frontmatter.ts} +26 -33
- package/src/index.ts +353 -0
- package/src/ingest-url.ts +99 -0
- package/src/main.ts +111 -0
- package/src/{retrieve/index.js → search.ts} +62 -150
- package/src/types.ts +166 -0
- package/src/capture/file-ops.js +0 -99
- package/src/capture/import-pipeline.js +0 -46
- package/src/capture/importers.js +0 -387
- package/src/capture/index.js +0 -250
- package/src/capture/ingest-url.js +0 -252
- package/src/consolidation/index.js +0 -112
- package/src/core/categories.js +0 -73
- package/src/core/error-log.js +0 -54
- package/src/core/linking.js +0 -161
- package/src/core/migrate-dirs.js +0 -196
- package/src/core/status.js +0 -350
- package/src/core/telemetry.js +0 -90
- package/src/core/temporal.js +0 -146
- package/src/index/db.js +0 -586
- package/src/index/index.js +0 -583
- package/src/index.js +0 -71
- package/src/server/helpers.js +0 -44
- package/src/server/tools/clear-context.js +0 -47
- package/src/server/tools/context-status.js +0 -182
- package/src/server/tools/create-snapshot.js +0 -200
- package/src/server/tools/delete-context.js +0 -60
- package/src/server/tools/get-context.js +0 -765
- package/src/server/tools/ingest-project.js +0 -244
- package/src/server/tools/ingest-url.js +0 -88
- package/src/server/tools/list-buckets.js +0 -116
- package/src/server/tools/list-context.js +0 -163
- package/src/server/tools/save-context.js +0 -632
- package/src/server/tools/session-start.js +0 -285
- package/src/server/tools.js +0 -172
- package/src/sync/sync.js +0 -235
|
@@ -1,252 +0,0 @@
|
|
|
1
|
-
export function htmlToMarkdown(html) {
|
|
2
|
-
let md = html;
|
|
3
|
-
|
|
4
|
-
// Remove scripts, styles, nav, header, footer, aside
|
|
5
|
-
md = md.replace(/<script[\s\S]*?<\/script>/gi, "");
|
|
6
|
-
md = md.replace(/<style[\s\S]*?<\/style>/gi, "");
|
|
7
|
-
md = md.replace(/<nav[\s\S]*?<\/nav>/gi, "");
|
|
8
|
-
md = md.replace(/<header[\s\S]*?<\/header>/gi, "");
|
|
9
|
-
md = md.replace(/<footer[\s\S]*?<\/footer>/gi, "");
|
|
10
|
-
md = md.replace(/<aside[\s\S]*?<\/aside>/gi, "");
|
|
11
|
-
|
|
12
|
-
// Convert headings
|
|
13
|
-
md = md.replace(
|
|
14
|
-
/<h1[^>]*>([\s\S]*?)<\/h1>/gi,
|
|
15
|
-
(_, c) => `\n# ${stripTags(c).trim()}\n`,
|
|
16
|
-
);
|
|
17
|
-
md = md.replace(
|
|
18
|
-
/<h2[^>]*>([\s\S]*?)<\/h2>/gi,
|
|
19
|
-
(_, c) => `\n## ${stripTags(c).trim()}\n`,
|
|
20
|
-
);
|
|
21
|
-
md = md.replace(
|
|
22
|
-
/<h3[^>]*>([\s\S]*?)<\/h3>/gi,
|
|
23
|
-
(_, c) => `\n### ${stripTags(c).trim()}\n`,
|
|
24
|
-
);
|
|
25
|
-
md = md.replace(
|
|
26
|
-
/<h4[^>]*>([\s\S]*?)<\/h4>/gi,
|
|
27
|
-
(_, c) => `\n#### ${stripTags(c).trim()}\n`,
|
|
28
|
-
);
|
|
29
|
-
md = md.replace(
|
|
30
|
-
/<h5[^>]*>([\s\S]*?)<\/h5>/gi,
|
|
31
|
-
(_, c) => `\n##### ${stripTags(c).trim()}\n`,
|
|
32
|
-
);
|
|
33
|
-
md = md.replace(
|
|
34
|
-
/<h6[^>]*>([\s\S]*?)<\/h6>/gi,
|
|
35
|
-
(_, c) => `\n###### ${stripTags(c).trim()}\n`,
|
|
36
|
-
);
|
|
37
|
-
|
|
38
|
-
// Convert links
|
|
39
|
-
md = md.replace(
|
|
40
|
-
/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi,
|
|
41
|
-
(_, href, text) => {
|
|
42
|
-
const cleanText = stripTags(text).trim();
|
|
43
|
-
return cleanText ? `[${cleanText}](${href})` : "";
|
|
44
|
-
},
|
|
45
|
-
);
|
|
46
|
-
|
|
47
|
-
// Convert code blocks
|
|
48
|
-
md = md.replace(
|
|
49
|
-
/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi,
|
|
50
|
-
(_, c) => `\n\`\`\`\n${decodeEntities(c).trim()}\n\`\`\`\n`,
|
|
51
|
-
);
|
|
52
|
-
md = md.replace(
|
|
53
|
-
/<pre[^>]*>([\s\S]*?)<\/pre>/gi,
|
|
54
|
-
(_, c) => `\n\`\`\`\n${decodeEntities(stripTags(c)).trim()}\n\`\`\`\n`,
|
|
55
|
-
);
|
|
56
|
-
|
|
57
|
-
// Convert inline code
|
|
58
|
-
md = md.replace(
|
|
59
|
-
/<code[^>]*>([\s\S]*?)<\/code>/gi,
|
|
60
|
-
(_, c) => `\`${decodeEntities(c).trim()}\``,
|
|
61
|
-
);
|
|
62
|
-
|
|
63
|
-
// Convert strong/em
|
|
64
|
-
md = md.replace(
|
|
65
|
-
/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi,
|
|
66
|
-
(_, __, c) => `**${stripTags(c).trim()}**`,
|
|
67
|
-
);
|
|
68
|
-
md = md.replace(
|
|
69
|
-
/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi,
|
|
70
|
-
(_, __, c) => `*${stripTags(c).trim()}*`,
|
|
71
|
-
);
|
|
72
|
-
|
|
73
|
-
// Convert list items
|
|
74
|
-
md = md.replace(
|
|
75
|
-
/<li[^>]*>([\s\S]*?)<\/li>/gi,
|
|
76
|
-
(_, c) => `- ${stripTags(c).trim()}\n`,
|
|
77
|
-
);
|
|
78
|
-
|
|
79
|
-
// Convert paragraphs and line breaks
|
|
80
|
-
md = md.replace(/<br\s*\/?>/gi, "\n");
|
|
81
|
-
md = md.replace(
|
|
82
|
-
/<p[^>]*>([\s\S]*?)<\/p>/gi,
|
|
83
|
-
(_, c) => `\n${stripTags(c).trim()}\n`,
|
|
84
|
-
);
|
|
85
|
-
md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, c) => {
|
|
86
|
-
return (
|
|
87
|
-
"\n" +
|
|
88
|
-
stripTags(c)
|
|
89
|
-
.trim()
|
|
90
|
-
.split("\n")
|
|
91
|
-
.map((l) => `> ${l}`)
|
|
92
|
-
.join("\n") +
|
|
93
|
-
"\n"
|
|
94
|
-
);
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
// Remove remaining HTML tags
|
|
98
|
-
md = stripTags(md);
|
|
99
|
-
|
|
100
|
-
// Decode HTML entities
|
|
101
|
-
md = decodeEntities(md);
|
|
102
|
-
|
|
103
|
-
// Clean up whitespace
|
|
104
|
-
md = md.replace(/\n{3,}/g, "\n\n").trim();
|
|
105
|
-
|
|
106
|
-
return md;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
function stripTags(html) {
|
|
110
|
-
return html.replace(/<[^>]+>/g, "");
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function decodeEntities(text) {
|
|
114
|
-
return text
|
|
115
|
-
.replace(/&/g, "&")
|
|
116
|
-
.replace(/</g, "<")
|
|
117
|
-
.replace(/>/g, ">")
|
|
118
|
-
.replace(/"/g, '"')
|
|
119
|
-
.replace(/'/g, "'")
|
|
120
|
-
.replace(/ /g, " ")
|
|
121
|
-
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
|
|
122
|
-
.replace(/&#x([0-9a-f]+);/gi, (_, n) =>
|
|
123
|
-
String.fromCharCode(parseInt(n, 16)),
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/**
|
|
128
|
-
* Extract the main readable content from an HTML page.
|
|
129
|
-
* Prefers <article> or <main>, falls back to <body>.
|
|
130
|
-
*
|
|
131
|
-
* @param {string} html
|
|
132
|
-
* @param {string} url
|
|
133
|
-
* @returns {{ title: string, body: string }}
|
|
134
|
-
*/
|
|
135
|
-
export function extractHtmlContent(html, url) {
|
|
136
|
-
// Extract <title>
|
|
137
|
-
const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
138
|
-
const title = titleMatch
|
|
139
|
-
? stripTags(decodeEntities(titleMatch[1])).trim()
|
|
140
|
-
: "";
|
|
141
|
-
|
|
142
|
-
// Try to extract main content area
|
|
143
|
-
let contentHtml = "";
|
|
144
|
-
|
|
145
|
-
const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
|
|
146
|
-
const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
|
|
147
|
-
|
|
148
|
-
if (articleMatch) {
|
|
149
|
-
contentHtml = articleMatch[1];
|
|
150
|
-
} else if (mainMatch) {
|
|
151
|
-
contentHtml = mainMatch[1];
|
|
152
|
-
} else {
|
|
153
|
-
// Fall back to <body>
|
|
154
|
-
const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
|
|
155
|
-
contentHtml = bodyMatch ? bodyMatch[1] : html;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
const body = htmlToMarkdown(contentHtml);
|
|
159
|
-
|
|
160
|
-
return { title, body };
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Fetch a URL, extract readable content, and return an EntryData object.
|
|
165
|
-
*
|
|
166
|
-
* @param {string} url
|
|
167
|
-
* @param {{ kind?: string, tags?: string[], source?: string, maxBodyLength?: number, timeoutMs?: number }} [opts]
|
|
168
|
-
* @returns {Promise<{ kind: string, title: string, body: string, tags: string[], meta: object, source: string }>}
|
|
169
|
-
*/
|
|
170
|
-
export async function ingestUrl(url, opts = {}) {
|
|
171
|
-
const {
|
|
172
|
-
kind = "reference",
|
|
173
|
-
tags = [],
|
|
174
|
-
source,
|
|
175
|
-
maxBodyLength = 50000,
|
|
176
|
-
timeoutMs = 15000,
|
|
177
|
-
} = opts;
|
|
178
|
-
|
|
179
|
-
let domain;
|
|
180
|
-
try {
|
|
181
|
-
domain = new URL(url).hostname;
|
|
182
|
-
} catch {
|
|
183
|
-
throw new Error(`Invalid URL: ${url}`);
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
const controller = new AbortController();
|
|
187
|
-
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
188
|
-
|
|
189
|
-
let response;
|
|
190
|
-
try {
|
|
191
|
-
response = await fetch(url, {
|
|
192
|
-
signal: controller.signal,
|
|
193
|
-
headers: {
|
|
194
|
-
"User-Agent":
|
|
195
|
-
"ContextVault/1.0 (+https://github.com/fellanH/context-vault)",
|
|
196
|
-
Accept: "text/html,application/xhtml+xml,text/plain,*/*",
|
|
197
|
-
},
|
|
198
|
-
});
|
|
199
|
-
} catch (err) {
|
|
200
|
-
if (err.name === "AbortError") {
|
|
201
|
-
throw new Error(`Request timed out after ${timeoutMs}ms`);
|
|
202
|
-
}
|
|
203
|
-
throw new Error(`Fetch failed: ${err.message}`);
|
|
204
|
-
} finally {
|
|
205
|
-
clearTimeout(timeout);
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
if (!response.ok) {
|
|
209
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
const contentType = response.headers.get("content-type") || "";
|
|
213
|
-
const html = await response.text();
|
|
214
|
-
|
|
215
|
-
let title, body;
|
|
216
|
-
|
|
217
|
-
if (
|
|
218
|
-
contentType.includes("text/html") ||
|
|
219
|
-
contentType.includes("application/xhtml")
|
|
220
|
-
) {
|
|
221
|
-
const extracted = extractHtmlContent(html, url);
|
|
222
|
-
title = extracted.title;
|
|
223
|
-
body = extracted.body;
|
|
224
|
-
} else {
|
|
225
|
-
// Plain text or other — use as-is
|
|
226
|
-
title = domain;
|
|
227
|
-
body = html;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Truncate if too long
|
|
231
|
-
if (body.length > maxBodyLength) {
|
|
232
|
-
body = body.slice(0, maxBodyLength) + "\n\n[Content truncated]";
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
if (!body.trim()) {
|
|
236
|
-
throw new Error("No readable content extracted from URL");
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
return {
|
|
240
|
-
kind,
|
|
241
|
-
title: title || domain,
|
|
242
|
-
body,
|
|
243
|
-
tags: [...tags, "web-import"],
|
|
244
|
-
meta: {
|
|
245
|
-
url,
|
|
246
|
-
domain,
|
|
247
|
-
fetched_at: new Date().toISOString(),
|
|
248
|
-
content_type: contentType.split(";")[0].trim() || "text/html",
|
|
249
|
-
},
|
|
250
|
-
source: source || domain,
|
|
251
|
-
};
|
|
252
|
-
}
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Consolidation utilities — identifies tags and entries that warrant maintenance.
|
|
3
|
-
*
|
|
4
|
-
* These are pure DB queries with no LLM calls. The caller decides what to do
|
|
5
|
-
* with the results (e.g. run create_snapshot, archive entries, report to user).
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Identifies tags that have accumulated enough entries to warrant consolidation.
|
|
10
|
-
*
|
|
11
|
-
* A tag is "hot" when it has >= tagThreshold non-superseded entries AND no
|
|
12
|
-
* brief/snapshot was saved for it within the last maxSnapshotAgeDays days.
|
|
13
|
-
*
|
|
14
|
-
* @param {import('node:sqlite').DatabaseSync} db
|
|
15
|
-
* @param {{ tagThreshold?: number, maxSnapshotAgeDays?: number }} [opts]
|
|
16
|
-
* @returns {{ tag: string, entryCount: number, lastSnapshotAge: number | null }[]}
|
|
17
|
-
*/
|
|
18
|
-
export function findHotTags(
|
|
19
|
-
db,
|
|
20
|
-
{ tagThreshold = 10, maxSnapshotAgeDays = 7 } = {},
|
|
21
|
-
) {
|
|
22
|
-
const rows = db
|
|
23
|
-
.prepare(
|
|
24
|
-
`SELECT id, tags, kind FROM vault
|
|
25
|
-
WHERE superseded_by IS NULL
|
|
26
|
-
AND tags IS NOT NULL
|
|
27
|
-
AND tags != '[]'`,
|
|
28
|
-
)
|
|
29
|
-
.all();
|
|
30
|
-
|
|
31
|
-
const tagCounts = new Map();
|
|
32
|
-
|
|
33
|
-
for (const row of rows) {
|
|
34
|
-
let tags;
|
|
35
|
-
try {
|
|
36
|
-
tags = JSON.parse(row.tags);
|
|
37
|
-
} catch {
|
|
38
|
-
continue;
|
|
39
|
-
}
|
|
40
|
-
if (!Array.isArray(tags)) continue;
|
|
41
|
-
|
|
42
|
-
for (const tag of tags) {
|
|
43
|
-
if (typeof tag !== "string" || !tag) continue;
|
|
44
|
-
tagCounts.set(tag, (tagCounts.get(tag) ?? 0) + 1);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
const hotTags = [];
|
|
49
|
-
|
|
50
|
-
for (const [tag, count] of tagCounts) {
|
|
51
|
-
if (count < tagThreshold) continue;
|
|
52
|
-
|
|
53
|
-
const snapshotRow = db
|
|
54
|
-
.prepare(
|
|
55
|
-
`SELECT created_at FROM vault
|
|
56
|
-
WHERE kind = 'brief'
|
|
57
|
-
AND tags LIKE ?
|
|
58
|
-
AND created_at > datetime('now', '-' || ? || ' days')
|
|
59
|
-
ORDER BY created_at DESC
|
|
60
|
-
LIMIT 1`,
|
|
61
|
-
)
|
|
62
|
-
.get(`%"${tag}"%`, String(maxSnapshotAgeDays));
|
|
63
|
-
|
|
64
|
-
if (snapshotRow) continue;
|
|
65
|
-
|
|
66
|
-
const lastSnapshotAny = db
|
|
67
|
-
.prepare(
|
|
68
|
-
`SELECT created_at FROM vault
|
|
69
|
-
WHERE kind = 'brief'
|
|
70
|
-
AND tags LIKE ?
|
|
71
|
-
ORDER BY created_at DESC
|
|
72
|
-
LIMIT 1`,
|
|
73
|
-
)
|
|
74
|
-
.get(`%"${tag}"%`);
|
|
75
|
-
|
|
76
|
-
let lastSnapshotAge = null;
|
|
77
|
-
if (lastSnapshotAny) {
|
|
78
|
-
const ms = Date.now() - new Date(lastSnapshotAny.created_at).getTime();
|
|
79
|
-
lastSnapshotAge = Math.floor(ms / (1000 * 60 * 60 * 24));
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
hotTags.push({ tag, entryCount: count, lastSnapshotAge });
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
hotTags.sort((a, b) => b.entryCount - a.entryCount);
|
|
86
|
-
|
|
87
|
-
return hotTags;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Identifies cold entries (old, never or rarely accessed) that can be archived.
|
|
92
|
-
*
|
|
93
|
-
* Returns IDs of entries that are old enough, have low hit counts, are not
|
|
94
|
-
* superseded, and are not in permanent kinds (decision, architecture, brief).
|
|
95
|
-
*
|
|
96
|
-
* @param {import('node:sqlite').DatabaseSync} db
|
|
97
|
-
* @param {{ maxAgeDays?: number, maxHitCount?: number }} [opts]
|
|
98
|
-
* @returns {string[]} Entry IDs eligible for archiving
|
|
99
|
-
*/
|
|
100
|
-
export function findColdEntries(db, { maxAgeDays = 90, maxHitCount = 0 } = {}) {
|
|
101
|
-
const rows = db
|
|
102
|
-
.prepare(
|
|
103
|
-
`SELECT id FROM vault
|
|
104
|
-
WHERE hit_count <= ?
|
|
105
|
-
AND created_at < datetime('now', '-' || ? || ' days')
|
|
106
|
-
AND superseded_by IS NULL
|
|
107
|
-
AND kind NOT IN ('decision', 'architecture', 'brief')`,
|
|
108
|
-
)
|
|
109
|
-
.all(maxHitCount, String(maxAgeDays));
|
|
110
|
-
|
|
111
|
-
return rows.map((r) => r.id);
|
|
112
|
-
}
|
package/src/core/categories.js
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* categories.js — Static kind→category mapping
|
|
3
|
-
*
|
|
4
|
-
* Three categories with distinct write semantics:
|
|
5
|
-
* knowledge — append-only, enduring (default)
|
|
6
|
-
* entity — upsert by identity_key, enduring
|
|
7
|
-
* event — append-only, decaying relevance
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
const KIND_CATEGORY = {
|
|
11
|
-
// Knowledge — append-only, enduring
|
|
12
|
-
insight: "knowledge",
|
|
13
|
-
decision: "knowledge",
|
|
14
|
-
pattern: "knowledge",
|
|
15
|
-
prompt: "knowledge",
|
|
16
|
-
note: "knowledge",
|
|
17
|
-
document: "knowledge",
|
|
18
|
-
reference: "knowledge",
|
|
19
|
-
// Entity — upsert, enduring
|
|
20
|
-
contact: "entity",
|
|
21
|
-
project: "entity",
|
|
22
|
-
tool: "entity",
|
|
23
|
-
source: "entity",
|
|
24
|
-
bucket: "entity",
|
|
25
|
-
// Event — append-only, decaying
|
|
26
|
-
event: "event",
|
|
27
|
-
conversation: "event",
|
|
28
|
-
message: "event",
|
|
29
|
-
session: "event",
|
|
30
|
-
task: "event",
|
|
31
|
-
log: "event",
|
|
32
|
-
feedback: "event",
|
|
33
|
-
};
|
|
34
|
-
|
|
35
|
-
/** Map category name → directory name on disk */
|
|
36
|
-
const CATEGORY_DIR_NAMES = {
|
|
37
|
-
knowledge: "knowledge",
|
|
38
|
-
entity: "entities",
|
|
39
|
-
event: "events",
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
/** Set of valid category directory names (for reindex discovery) */
|
|
43
|
-
export const CATEGORY_DIRS = new Set(Object.values(CATEGORY_DIR_NAMES));
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Staleness thresholds (in days) per knowledge kind.
|
|
47
|
-
* Kinds not listed here are considered enduring (no staleness threshold).
|
|
48
|
-
* Based on updated_at; falls back to created_at if updated_at is null.
|
|
49
|
-
*/
|
|
50
|
-
export const KIND_STALENESS_DAYS = {
|
|
51
|
-
pattern: 180,
|
|
52
|
-
decision: 365,
|
|
53
|
-
reference: 90,
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
const DURABLE_KINDS = new Set(["decision", "architecture", "pattern"]);
|
|
57
|
-
const EPHEMERAL_KINDS = new Set(["session", "observation"]);
|
|
58
|
-
|
|
59
|
-
export function categoryFor(kind) {
|
|
60
|
-
return KIND_CATEGORY[kind] || "knowledge";
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export function defaultTierFor(kind) {
|
|
64
|
-
if (DURABLE_KINDS.has(kind)) return "durable";
|
|
65
|
-
if (EPHEMERAL_KINDS.has(kind)) return "ephemeral";
|
|
66
|
-
return "working";
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/** Returns the category directory name for a given kind (e.g. "insight" → "knowledge") */
|
|
70
|
-
export function categoryDirFor(kind) {
|
|
71
|
-
const cat = categoryFor(kind);
|
|
72
|
-
return CATEGORY_DIR_NAMES[cat] || "knowledge";
|
|
73
|
-
}
|
package/src/core/error-log.js
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
appendFileSync,
|
|
3
|
-
existsSync,
|
|
4
|
-
mkdirSync,
|
|
5
|
-
readFileSync,
|
|
6
|
-
statSync,
|
|
7
|
-
writeFileSync,
|
|
8
|
-
} from "node:fs";
|
|
9
|
-
import { join } from "node:path";
|
|
10
|
-
|
|
11
|
-
const MAX_LOG_SIZE = 1024 * 1024; // 1MB
|
|
12
|
-
|
|
13
|
-
export function errorLogPath(dataDir) {
|
|
14
|
-
return join(dataDir, "error.log");
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Append a structured JSON entry to the startup error log.
|
|
19
|
-
* Rotates the file if it exceeds MAX_LOG_SIZE.
|
|
20
|
-
* Never throws — logging failures must not mask the original error.
|
|
21
|
-
*
|
|
22
|
-
* @param {string} dataDir
|
|
23
|
-
* @param {object} entry
|
|
24
|
-
*/
|
|
25
|
-
export function appendErrorLog(dataDir, entry) {
|
|
26
|
-
try {
|
|
27
|
-
mkdirSync(dataDir, { recursive: true });
|
|
28
|
-
const logPath = errorLogPath(dataDir);
|
|
29
|
-
if (existsSync(logPath) && statSync(logPath).size >= MAX_LOG_SIZE) {
|
|
30
|
-
writeFileSync(logPath, "");
|
|
31
|
-
}
|
|
32
|
-
appendFileSync(logPath, JSON.stringify(entry) + "\n");
|
|
33
|
-
} catch {
|
|
34
|
-
// intentionally swallowed
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Return number of log lines in the error log, or 0 if absent.
|
|
40
|
-
*
|
|
41
|
-
* @param {string} dataDir
|
|
42
|
-
* @returns {number}
|
|
43
|
-
*/
|
|
44
|
-
export function errorLogCount(dataDir) {
|
|
45
|
-
try {
|
|
46
|
-
const logPath = errorLogPath(dataDir);
|
|
47
|
-
if (!existsSync(logPath)) return 0;
|
|
48
|
-
return readFileSync(logPath, "utf-8")
|
|
49
|
-
.split("\n")
|
|
50
|
-
.filter((l) => l.trim()).length;
|
|
51
|
-
} catch {
|
|
52
|
-
return 0;
|
|
53
|
-
}
|
|
54
|
-
}
|
package/src/core/linking.js
DELETED
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* linking.js — Pure graph traversal for related_to links.
|
|
3
|
-
*
|
|
4
|
-
* All functions accept a db handle and return data — no side effects.
|
|
5
|
-
* The calling layer (get-context handler) is responsible for I/O wiring.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Parse a `related_to` JSON string from the DB into an array of ID strings.
|
|
10
|
-
* Returns an empty array on any parse failure or null input.
|
|
11
|
-
*
|
|
12
|
-
* @param {string|null|undefined} raw
|
|
13
|
-
* @returns {string[]}
|
|
14
|
-
*/
|
|
15
|
-
export function parseRelatedTo(raw) {
|
|
16
|
-
if (!raw) return [];
|
|
17
|
-
try {
|
|
18
|
-
const parsed = JSON.parse(raw);
|
|
19
|
-
if (!Array.isArray(parsed)) return [];
|
|
20
|
-
return parsed.filter((id) => typeof id === "string" && id.trim());
|
|
21
|
-
} catch {
|
|
22
|
-
return [];
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Fetch vault entries by their IDs, scoped to a user.
|
|
28
|
-
* Returns only entries that exist and are not expired or superseded.
|
|
29
|
-
*
|
|
30
|
-
* @param {import('node:sqlite').DatabaseSync} db
|
|
31
|
-
* @param {string[]} ids
|
|
32
|
-
* @param {string|null|undefined} userId
|
|
33
|
-
* @returns {object[]} Matching DB rows
|
|
34
|
-
*/
|
|
35
|
-
export function resolveLinks(db, ids, userId) {
|
|
36
|
-
if (!ids.length) return [];
|
|
37
|
-
const unique = [...new Set(ids)];
|
|
38
|
-
const placeholders = unique.map(() => "?").join(",");
|
|
39
|
-
// When userId is defined (hosted mode), scope to that user.
|
|
40
|
-
// When userId is undefined (local mode), no user scoping — all entries accessible.
|
|
41
|
-
const userClause =
|
|
42
|
-
userId !== undefined && userId !== null ? "AND user_id = ?" : "";
|
|
43
|
-
const params =
|
|
44
|
-
userId !== undefined && userId !== null ? [...unique, userId] : unique;
|
|
45
|
-
try {
|
|
46
|
-
return db
|
|
47
|
-
.prepare(
|
|
48
|
-
`SELECT * FROM vault
|
|
49
|
-
WHERE id IN (${placeholders})
|
|
50
|
-
${userClause}
|
|
51
|
-
AND (expires_at IS NULL OR expires_at > datetime('now'))
|
|
52
|
-
AND superseded_by IS NULL`,
|
|
53
|
-
)
|
|
54
|
-
.all(...params);
|
|
55
|
-
} catch {
|
|
56
|
-
return [];
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* Find all entries that declare `entryId` in their `related_to` field
|
|
62
|
-
* (i.e. entries that point *to* this entry — backlinks).
|
|
63
|
-
* Scoped to the same user. Excludes expired and superseded entries.
|
|
64
|
-
*
|
|
65
|
-
* @param {import('node:sqlite').DatabaseSync} db
|
|
66
|
-
* @param {string} entryId
|
|
67
|
-
* @param {string|null|undefined} userId
|
|
68
|
-
* @returns {object[]} Entries with a backlink to entryId
|
|
69
|
-
*/
|
|
70
|
-
export function resolveBacklinks(db, entryId, userId) {
|
|
71
|
-
if (!entryId) return [];
|
|
72
|
-
// When userId is defined (hosted mode), scope to that user.
|
|
73
|
-
// When userId is undefined (local mode), no user scoping — all entries accessible.
|
|
74
|
-
const userClause =
|
|
75
|
-
userId !== undefined && userId !== null ? "AND user_id = ?" : "";
|
|
76
|
-
const likePattern = `%"${entryId}"%`;
|
|
77
|
-
const params =
|
|
78
|
-
userId !== undefined && userId !== null
|
|
79
|
-
? [likePattern, userId]
|
|
80
|
-
: [likePattern];
|
|
81
|
-
try {
|
|
82
|
-
return db
|
|
83
|
-
.prepare(
|
|
84
|
-
`SELECT * FROM vault
|
|
85
|
-
WHERE related_to LIKE ?
|
|
86
|
-
${userClause}
|
|
87
|
-
AND (expires_at IS NULL OR expires_at > datetime('now'))
|
|
88
|
-
AND superseded_by IS NULL`,
|
|
89
|
-
)
|
|
90
|
-
.all(...params);
|
|
91
|
-
} catch {
|
|
92
|
-
return [];
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* For a set of primary entry IDs, collect all forward links (entries pointed
|
|
98
|
-
* to by `related_to`) and backlinks (entries that point back to any primary).
|
|
99
|
-
*
|
|
100
|
-
* Returns a Map of id → entry row for all linked entries, excluding entries
|
|
101
|
-
* already present in `primaryIds`.
|
|
102
|
-
*
|
|
103
|
-
* @param {import('node:sqlite').DatabaseSync} db
|
|
104
|
-
* @param {object[]} primaryEntries - Full entry rows (must have id + related_to fields)
|
|
105
|
-
* @param {string|null|undefined} userId
|
|
106
|
-
* @returns {{ forward: object[], backward: object[] }}
|
|
107
|
-
*/
|
|
108
|
-
export function collectLinkedEntries(db, primaryEntries, userId) {
|
|
109
|
-
const primaryIds = new Set(primaryEntries.map((e) => e.id));
|
|
110
|
-
|
|
111
|
-
// Forward: resolve all IDs from related_to fields
|
|
112
|
-
const forwardIds = [];
|
|
113
|
-
for (const entry of primaryEntries) {
|
|
114
|
-
const ids = parseRelatedTo(entry.related_to);
|
|
115
|
-
for (const id of ids) {
|
|
116
|
-
if (!primaryIds.has(id)) forwardIds.push(id);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
const forwardEntries = resolveLinks(db, forwardIds, userId).filter(
|
|
120
|
-
(e) => !primaryIds.has(e.id),
|
|
121
|
-
);
|
|
122
|
-
|
|
123
|
-
// Backward: find all entries that link to any primary entry
|
|
124
|
-
const backwardSeen = new Set();
|
|
125
|
-
const backwardEntries = [];
|
|
126
|
-
const forwardIds2 = new Set(forwardEntries.map((e) => e.id));
|
|
127
|
-
for (const entry of primaryEntries) {
|
|
128
|
-
const backlinks = resolveBacklinks(db, entry.id, userId);
|
|
129
|
-
for (const bl of backlinks) {
|
|
130
|
-
if (!primaryIds.has(bl.id) && !backwardSeen.has(bl.id)) {
|
|
131
|
-
backwardSeen.add(bl.id);
|
|
132
|
-
backwardEntries.push(bl);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return { forward: forwardEntries, backward: backwardEntries };
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Validate a `related_to` value from user input.
|
|
142
|
-
* Must be an array of non-empty strings (ULID-like IDs).
|
|
143
|
-
* Returns an error message string if invalid, or null if valid.
|
|
144
|
-
*
|
|
145
|
-
* @param {unknown} relatedTo
|
|
146
|
-
* @returns {string|null}
|
|
147
|
-
*/
|
|
148
|
-
export function validateRelatedTo(relatedTo) {
|
|
149
|
-
if (relatedTo === undefined || relatedTo === null) return null;
|
|
150
|
-
if (!Array.isArray(relatedTo))
|
|
151
|
-
return "related_to must be an array of entry IDs";
|
|
152
|
-
for (const id of relatedTo) {
|
|
153
|
-
if (typeof id !== "string" || !id.trim()) {
|
|
154
|
-
return "each related_to entry must be a non-empty string ID";
|
|
155
|
-
}
|
|
156
|
-
if (id.length > 32) {
|
|
157
|
-
return `related_to ID too long (max 32 chars): "${id.slice(0, 32)}..."`;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
return null;
|
|
161
|
-
}
|