djournal 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,93 @@
1
+ "use strict";
2
+
3
+ const MiniSearch = require("minisearch");
4
+
5
+ const INDEX_VERSION = "minisearch-v1";
6
+ const INDEX_OPTIONS = Object.freeze({
7
+ idField: "id",
8
+ fields: ["path", "workTitle", "workDescription", "title", "summary", "headings", "body"],
9
+ storeFields: [
10
+ "rawId", "kind", "path", "workSlug", "workItemId", "workTitle", "title", "summary",
11
+ "entryType", "status", "visibility", "createdAt", "updatedAt", "links", "incomingLinks",
12
+ ],
13
+ });
14
+
15
+ function createSearchIndex(documents = []) {
16
+ const index = new MiniSearch(INDEX_OPTIONS);
17
+ if (documents.length) index.addAll(documents);
18
+ return index;
19
+ }
20
+
21
+ function loadSearchIndex(value) {
22
+ return MiniSearch.loadJSON(JSON.stringify(value), INDEX_OPTIONS);
23
+ }
24
+
25
+ function serializeSearchIndex(index) {
26
+ return JSON.parse(JSON.stringify(index));
27
+ }
28
+
29
+ function list(value) {
30
+ if (!value) return [];
31
+ return Array.isArray(value) ? value : String(value).split(",").map((item) => item.trim()).filter(Boolean);
32
+ }
33
+
34
+ function resultShape(result, incomingLinks) {
35
+ return {
36
+ id: result.rawId,
37
+ documentId: result.id,
38
+ path: result.path,
39
+ workSlug: result.workSlug,
40
+ workItemId: result.workItemId,
41
+ title: result.title,
42
+ summary: result.summary,
43
+ entryType: result.entryType,
44
+ status: result.status,
45
+ visibility: result.visibility,
46
+ createdAt: result.createdAt,
47
+ updatedAt: result.updatedAt,
48
+ score: result.score,
49
+ terms: result.terms,
50
+ match: result.match,
51
+ links: result.links || [],
52
+ incomingLinks: incomingLinks?.[result.rawId] || [],
53
+ };
54
+ }
55
+
56
+ function searchIndex(index, query, options = {}) {
57
+ if (typeof query !== "string" || !query.trim()) {
58
+ const error = new Error("recall search requires a non-empty query");
59
+ error.code = "USAGE";
60
+ throw error;
61
+ }
62
+ const types = new Set(list(options.types || options.type));
63
+ const kinds = new Set(list(options.kinds || options.kind));
64
+ const statuses = new Set(list(options.status));
65
+ const visibilities = new Set(list(options.visibility));
66
+ const limit = Number.isInteger(options.limit) && options.limit > 0 ? Math.min(options.limit, 100) : 10;
67
+ const results = index.search(query.trim(), {
68
+ boost: { title: 7, workTitle: 5, summary: 4, path: 3, workDescription: 2, headings: 2, body: 1 },
69
+ prefix: (term) => term.length >= 3,
70
+ fuzzy: (term) => term.length >= 5 ? 0.2 : false,
71
+ filter: (result) => {
72
+ if (options.work && result.workSlug !== options.work) return false;
73
+ if (types.size && !types.has(result.entryType)) return false;
74
+ if (kinds.size && !kinds.has(result.kind)) return false;
75
+ if (statuses.size && !statuses.has(result.status)) return false;
76
+ if (visibilities.size && !visibilities.has(result.visibility)) return false;
77
+ return true;
78
+ },
79
+ });
80
+ return {
81
+ workItems: results.filter((result) => result.kind === "work").slice(0, limit).map((result) => resultShape(result, options.incomingLinks)),
82
+ entries: results.filter((result) => result.kind === "entry").slice(0, limit).map((result) => resultShape(result, options.incomingLinks)),
83
+ };
84
+ }
85
+
86
+ module.exports = {
87
+ INDEX_OPTIONS,
88
+ INDEX_VERSION,
89
+ createSearchIndex,
90
+ loadSearchIndex,
91
+ searchIndex,
92
+ serializeSearchIndex,
93
+ };
@@ -0,0 +1,304 @@
1
+ "use strict";
2
+
3
+ const crypto = require("node:crypto");
4
+ const fs = require("node:fs");
5
+ const path = require("node:path");
6
+ const {
7
+ buildCorpus,
8
+ discoverSourceFiles,
9
+ documentFromDescriptor,
10
+ readWorkContexts,
11
+ sameFingerprint,
12
+ sha256,
13
+ } = require("./corpus.js");
14
+ const {
15
+ INDEX_VERSION,
16
+ createSearchIndex,
17
+ loadSearchIndex,
18
+ serializeSearchIndex,
19
+ } = require("./search.js");
20
+
21
+ const SNAPSHOT_SCHEMA_VERSION = 1;
22
+ const MAX_SNAPSHOT_BYTES = 256 * 1024 * 1024;
23
+ const LOCK_STALE_MS = 30_000;
24
+
25
+ class RecallSnapshotError extends Error {
26
+ constructor(message, code = "RECALL_SNAPSHOT_ERROR") {
27
+ super(message);
28
+ this.name = "RecallSnapshotError";
29
+ this.code = code;
30
+ }
31
+ }
32
+
33
+ function projectIdentity(context) {
34
+ return context.projectKey || `local-${sha256(path.resolve(context.root || context.target)).slice(0, 16)}`;
35
+ }
36
+
37
+ function cachePaths(context, options = {}) {
38
+ const directory = path.resolve(options.cacheDir || path.join(context.root || context.target, "cache", "recall"));
39
+ return { directory, snapshot: path.join(directory, "snapshot.json"), lock: path.join(directory, "refresh.lock") };
40
+ }
41
+
42
+ function warning(code, detail, file) {
43
+ return { code, ...(file ? { path: file } : {}), ...(detail ? { detail } : {}) };
44
+ }
45
+
46
+ function validManifest(value) {
47
+ return value && typeof value === "object" && !Array.isArray(value) && Object.values(value).every((item) => (
48
+ item && typeof item === "object" && Number.isFinite(item.size) && Number.isFinite(item.mtimeMs) &&
49
+ typeof item.sha256 === "string" && typeof item.documentId === "string"
50
+ ));
51
+ }
52
+
53
+ function validateSnapshot(value, context) {
54
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
55
+ if (value.schemaVersion !== SNAPSHOT_SCHEMA_VERSION || value.indexVersion !== INDEX_VERSION) return false;
56
+ if (value.projectKey !== projectIdentity(context) || typeof value.builtAt !== "string") return false;
57
+ if (!validManifest(value.files) || !value.documents || typeof value.documents !== "object" || !value.index) return false;
58
+ return true;
59
+ }
60
+
61
+ function loadSnapshot(context, options = {}) {
62
+ const file = cachePaths(context, options).snapshot;
63
+ try {
64
+ const stat = fs.statSync(file);
65
+ if (!stat.isFile() || stat.size > MAX_SNAPSHOT_BYTES) {
66
+ return { snapshot: null, warnings: [warning("RECALL_CACHE_INVALID", "snapshot is not a bounded regular file")] };
67
+ }
68
+ const value = JSON.parse(fs.readFileSync(file, "utf8"));
69
+ if (!validateSnapshot(value, context)) {
70
+ return { snapshot: null, warnings: [warning("RECALL_CACHE_INCOMPATIBLE", "snapshot schema, index version, or project identity differs")] };
71
+ }
72
+ return { snapshot: value, warnings: [] };
73
+ } catch (error) {
74
+ if (error.code === "ENOENT") return { snapshot: null, warnings: [] };
75
+ return { snapshot: null, warnings: [warning("RECALL_CACHE_INVALID", error.message)] };
76
+ }
77
+ }
78
+
79
+ function acquireLock(paths, options = {}) {
80
+ fs.mkdirSync(paths.directory, { recursive: true, mode: 0o700 });
81
+ try { fs.chmodSync(paths.directory, 0o700); } catch {}
82
+ try {
83
+ const descriptor = fs.openSync(paths.lock, "wx", 0o600);
84
+ fs.writeFileSync(descriptor, `${JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() })}\n`);
85
+ fs.closeSync(descriptor);
86
+ return () => { try { fs.unlinkSync(paths.lock); } catch {} };
87
+ } catch (error) {
88
+ if (error.code !== "EEXIST") throw error;
89
+ try {
90
+ const stat = fs.statSync(paths.lock);
91
+ const now = Number(options.now ?? Date.now());
92
+ if (Number.isFinite(now) && now - stat.mtimeMs > (options.lockStaleMs || LOCK_STALE_MS)) {
93
+ fs.unlinkSync(paths.lock);
94
+ return acquireLock(paths, options);
95
+ }
96
+ } catch (retryError) {
97
+ if (retryError.code === "ENOENT") return acquireLock(paths, options);
98
+ }
99
+ return null;
100
+ }
101
+ }
102
+
103
+ function inspectSnapshot(context, options = {}) {
104
+ const paths = cachePaths(context, options);
105
+ const loaded = loadSnapshot(context, options);
106
+ const exists = fs.existsSync(paths.snapshot);
107
+ let persistenceAvailable = false;
108
+ try {
109
+ let probe = paths.directory;
110
+ while (!fs.existsSync(probe) && path.dirname(probe) !== probe) probe = path.dirname(probe);
111
+ fs.accessSync(probe, fs.constants.W_OK);
112
+ persistenceAvailable = true;
113
+ } catch {}
114
+ return {
115
+ path: paths.snapshot,
116
+ exists,
117
+ valid: Boolean(loaded.snapshot),
118
+ persistenceAvailable,
119
+ schemaVersion: loaded.snapshot?.schemaVersion,
120
+ indexVersion: loaded.snapshot?.indexVersion,
121
+ documentCount: loaded.snapshot ? Object.keys(loaded.snapshot.documents).length : 0,
122
+ builtAt: loaded.snapshot?.builtAt,
123
+ warnings: loaded.warnings,
124
+ };
125
+ }
126
+
127
+ function writeSnapshot(context, snapshot, options = {}) {
128
+ const paths = cachePaths(context, options);
129
+ let release;
130
+ try {
131
+ release = acquireLock(paths, options);
132
+ if (!release) return { persisted: false, warning: warning("RECALL_CACHE_LOCKED", "another process is refreshing the cache") };
133
+ const temporary = path.join(paths.directory, `.snapshot.${process.pid}.${crypto.randomBytes(4).toString("hex")}.tmp`);
134
+ try {
135
+ fs.writeFileSync(temporary, `${JSON.stringify(snapshot)}\n`, { mode: 0o600 });
136
+ fs.renameSync(temporary, paths.snapshot);
137
+ } catch (error) {
138
+ try { fs.unlinkSync(temporary); } catch {}
139
+ throw error;
140
+ }
141
+ return { persisted: true };
142
+ } catch (error) {
143
+ return { persisted: false, warning: warning("RECALL_CACHE_UNWRITABLE", error.message) };
144
+ } finally {
145
+ if (release) release();
146
+ }
147
+ }
148
+
149
+ function storedDocument(document) {
150
+ const { body, headings, workDescription, sourceRelative, ...stored } = document;
151
+ return stored;
152
+ }
153
+
154
+ function incomingLinks(documents) {
155
+ const incoming = {};
156
+ for (const document of Object.values(documents)) {
157
+ for (const link of document.links || []) {
158
+ if (!link.toEntryId) continue;
159
+ if (!incoming[link.toEntryId]) incoming[link.toEntryId] = [];
160
+ incoming[link.toEntryId].push({
161
+ fromEntryId: document.rawId,
162
+ relation: link.relation,
163
+ sourcePath: document.path,
164
+ });
165
+ }
166
+ }
167
+ return incoming;
168
+ }
169
+
170
+ function snapshotFromRecords(context, records, index) {
171
+ const files = {};
172
+ const documents = {};
173
+ for (const record of records) {
174
+ files[record.document.sourceRelative] = record.manifest;
175
+ documents[record.document.id] = storedDocument(record.document);
176
+ }
177
+ return {
178
+ schemaVersion: SNAPSHOT_SCHEMA_VERSION,
179
+ indexVersion: INDEX_VERSION,
180
+ projectKey: projectIdentity(context),
181
+ builtAt: new Date().toISOString(),
182
+ files,
183
+ documents,
184
+ incomingLinks: incomingLinks(documents),
185
+ index: serializeSearchIndex(index),
186
+ };
187
+ }
188
+
189
+ function fullBuild(context, discovery) {
190
+ const corpus = buildCorpus(context, discovery);
191
+ const index = createSearchIndex(corpus.records.map((record) => record.document));
192
+ return { index, snapshot: snapshotFromRecords(context, corpus.records, index), warnings: corpus.warnings };
193
+ }
194
+
195
+ function changedPaths(snapshot, discovery, force) {
196
+ const current = new Map(discovery.files.map((file) => [file.relative, file]));
197
+ const changed = new Set();
198
+ const removed = [];
199
+ if (force) for (const relative of current.keys()) changed.add(relative);
200
+ else {
201
+ for (const descriptor of discovery.files) {
202
+ if (!sameFingerprint(snapshot.files[descriptor.relative], descriptor)) changed.add(descriptor.relative);
203
+ }
204
+ for (const relative of Object.keys(snapshot.files)) if (!current.has(relative)) removed.push(relative);
205
+ }
206
+ const affectedWork = new Set();
207
+ for (const relative of changed) if (current.get(relative)?.kind === "work") affectedWork.add(current.get(relative).workSlug);
208
+ for (const relative of removed) if (snapshot.files[relative]?.kind === "work") affectedWork.add(snapshot.files[relative].workSlug);
209
+ for (const descriptor of discovery.files) if (affectedWork.has(descriptor.workSlug)) changed.add(descriptor.relative);
210
+ return { changed, removed, current };
211
+ }
212
+
213
+ function incrementalBuild(context, prior, discovery) {
214
+ const warnings = [...discovery.warnings];
215
+ const index = loadSearchIndex(prior.index);
216
+ const files = { ...prior.files };
217
+ const documents = { ...prior.documents };
218
+ const changes = changedPaths(prior, discovery, false);
219
+ if (!changes.changed.size && !changes.removed.length) return { index, snapshot: prior, warnings, changed: false };
220
+
221
+ for (const relative of changes.removed) {
222
+ const old = files[relative];
223
+ if (old?.documentId && index.has(old.documentId)) index.discard(old.documentId);
224
+ if (old?.documentId) delete documents[old.documentId];
225
+ delete files[relative];
226
+ }
227
+
228
+ const workContexts = readWorkContexts(discovery.files, warnings);
229
+ for (const relative of [...changes.changed].sort()) {
230
+ const descriptor = changes.current.get(relative);
231
+ if (!descriptor) continue;
232
+ const old = files[relative];
233
+ if (old?.documentId && index.has(old.documentId)) index.discard(old.documentId);
234
+ if (old?.documentId) delete documents[old.documentId];
235
+ const record = documentFromDescriptor(descriptor, workContexts, warnings);
236
+ if (index.has(record.document.id)) {
237
+ warnings.push(warning("DUPLICATE_DOCUMENT_ID", record.document.id, record.document.path));
238
+ record.document.id = `${record.document.kind}:path_${sha256(record.document.sourceRelative).slice(0, 24)}`;
239
+ record.manifest.documentId = record.document.id;
240
+ }
241
+ index.add(record.document);
242
+ files[relative] = record.manifest;
243
+ documents[record.document.id] = storedDocument(record.document);
244
+ }
245
+
246
+ const snapshot = {
247
+ schemaVersion: SNAPSHOT_SCHEMA_VERSION,
248
+ indexVersion: INDEX_VERSION,
249
+ projectKey: projectIdentity(context),
250
+ builtAt: new Date().toISOString(),
251
+ files,
252
+ documents,
253
+ incomingLinks: incomingLinks(documents),
254
+ index: serializeSearchIndex(index),
255
+ };
256
+ return { index, snapshot, warnings, changed: true };
257
+ }
258
+
259
+ function ensureSnapshot(context, options = {}) {
260
+ const discovery = discoverSourceFiles(context);
261
+ const loaded = options.noCache || options.rebuild ? { snapshot: null, warnings: [] } : loadSnapshot(context, options);
262
+ let built;
263
+ let status;
264
+ if (!loaded.snapshot || options.rebuild || options.noCache) {
265
+ built = fullBuild(context, discovery);
266
+ status = "rebuilt";
267
+ } else {
268
+ try {
269
+ built = incrementalBuild(context, loaded.snapshot, discovery);
270
+ status = built.changed ? "refreshed" : "hit";
271
+ } catch (error) {
272
+ loaded.warnings.push(warning("RECALL_CACHE_REBUILD", error.message));
273
+ built = fullBuild(context, discovery);
274
+ status = "rebuilt";
275
+ }
276
+ }
277
+
278
+ const warnings = [...loaded.warnings, ...built.warnings];
279
+ let mode = "persistent";
280
+ if (options.noCache) mode = "memory";
281
+ else if (status !== "hit") {
282
+ const written = writeSnapshot(context, built.snapshot, options);
283
+ if (!written.persisted) {
284
+ mode = "memory";
285
+ if (written.warning) warnings.push(written.warning);
286
+ }
287
+ }
288
+ return { index: built.index, snapshot: built.snapshot, mode, status, warnings };
289
+ }
290
+
291
+ module.exports = {
292
+ LOCK_STALE_MS,
293
+ MAX_SNAPSHOT_BYTES,
294
+ RecallSnapshotError,
295
+ SNAPSHOT_SCHEMA_VERSION,
296
+ cachePaths,
297
+ ensureSnapshot,
298
+ incomingLinks,
299
+ inspectSnapshot,
300
+ loadSnapshot,
301
+ projectIdentity,
302
+ validateSnapshot,
303
+ writeSnapshot,
304
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "djournal",
3
- "version": "0.7.0",
3
+ "version": "0.9.0",
4
4
  "description": "Durable project memory for coding agents, stored as linked Markdown",
5
5
  "type": "commonjs",
6
6
  "bin": {
@@ -24,9 +24,10 @@
24
24
  "scripts": {
25
25
  "check:package": "node scripts/check-package.cjs",
26
26
  "check:pr-title": "node scripts/check-pr-title.js",
27
+ "benchmark:recall": "node scripts/benchmark-recall.cjs",
27
28
  "link:global": "npm link",
28
29
  "release": "semantic-release",
29
- "test": "node tests/update-checker.test.js && node tests/journal-hook.test.js && node tests/pi-adapter.test.js && node tests/installer/installer.test.js && node tests/release-policy.test.js && node tests/skill-instructions.test.js && npm run check:package",
30
+ "test": "node tests/update-checker.test.js && node tests/journal-hook.test.js && node tests/pi-adapter.test.js && node tests/recall.test.js && node tests/installer/installer.test.js && node tests/release-policy.test.js && node tests/skill-instructions.test.js && npm run check:package",
30
31
  "prepublishOnly": "npm test"
31
32
  },
32
33
  "repository": {
@@ -56,6 +57,8 @@
56
57
  "semantic-release": "25.0.3"
57
58
  },
58
59
  "dependencies": {
59
- "semver": "7.8.5"
60
+ "minisearch": "7.2.0",
61
+ "semver": "7.8.5",
62
+ "yaml": "2.9.0"
60
63
  }
61
64
  }