@hyperspell/openclaw-hyperspell 0.13.1 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/client.js CHANGED
@@ -214,6 +214,30 @@ export class HyperspellClient {
214
214
  log.debugResponse("memories.get", { resourceId, hasData: "data" in raw });
215
215
  return raw;
216
216
  }
217
+ /**
218
+ * Delete a memory by resource id. Memory-sync uploads land in the "vault"
219
+ * source (user-added documents), so that is the default. Best-effort: a
220
+ * 404 (already gone) is treated as success so callers can prune their
221
+ * local manifest unconditionally.
222
+ */
223
+ async deleteMemory(resourceId, options) {
224
+ const source = options?.source ?? "vault";
225
+ log.debugRequest("memories.delete", { resourceId, source });
226
+ try {
227
+ await this.client.memories.delete(resourceId, { source }, this.requestOptions(options?.userId));
228
+ log.debugResponse("memories.delete", { resourceId, deleted: true });
229
+ return { deleted: true };
230
+ }
231
+ catch (err) {
232
+ const status = err?.status;
233
+ if (status === 404) {
234
+ log.debug(`Memory ${resourceId} already absent (404) — treating as deleted`);
235
+ return { deleted: true };
236
+ }
237
+ log.error(`Failed to delete memory ${resourceId}`, err);
238
+ return { deleted: false };
239
+ }
240
+ }
217
241
  async sendTrace(history, options) {
218
242
  log.debugRequest("sessions.add", {
219
243
  historyLength: history.length,
@@ -4,7 +4,7 @@ import * as path from "node:path";
4
4
  import { userInfo } from "node:os";
5
5
  import * as p from "@clack/prompts";
6
6
  import Hyperspell from "hyperspell";
7
- import { syncAllMemoryFiles, getMemoryFiles } from "../sync/markdown.js";
7
+ import { syncAllFilesSectionized, syncAllMemoryFiles, getMemoryFiles, } from "../sync/markdown.js";
8
8
  import { openInBrowser } from "../lib/browser.js";
9
9
  import { HyperspellClient } from "../client.js";
10
10
  import { getWorkspaceDir, parseConfig, resolveConfigPath } from "../config.js";
@@ -271,6 +271,12 @@ async function runSetup() {
271
271
  autoTrace: { enabled: false, extract: ["procedure"] },
272
272
  emotionalContext: false,
273
273
  syncMemories: true,
274
+ syncMemoriesConfig: {
275
+ enabled: true,
276
+ sectionize: true,
277
+ watchPaths: [],
278
+ debounceMs: 2000,
279
+ },
274
280
  sources: [],
275
281
  maxResults: 10,
276
282
  relevanceThreshold: 0.6,
@@ -284,15 +290,18 @@ async function runSetup() {
284
290
  loopsQuery: "open tasks pending questions unfinished promised need to follow up",
285
291
  },
286
292
  });
287
- const result = await syncAllMemoryFiles(hyperspellClient, workspaceDir);
293
+ // Use the same sectionized path the runtime uses, so a fresh setup
294
+ // doesn't whole-file-upload everything and then get re-synced as
295
+ // sections on first gateway start (= duplicate memories).
296
+ const result = await syncAllFilesSectionized(hyperspellClient, workspaceDir);
288
297
  if (result.failed > 0) {
289
- s3.stop(`Synced ${result.synced} files, ${result.failed} failed`);
298
+ s3.stop(`Synced ${result.synced} section(s), ${result.skipped} unchanged, ${result.failed} failed`);
290
299
  for (const error of result.errors) {
291
300
  p.log.error(` ${error}`);
292
301
  }
293
302
  }
294
303
  else {
295
- s3.stop(`Synced ${result.synced} memory files`);
304
+ s3.stop(`Synced ${result.synced} section(s) (${result.skipped} unchanged)`);
296
305
  }
297
306
  }
298
307
  else {
@@ -552,7 +561,12 @@ export function registerCliCommands(program, pluginConfig) {
552
561
  const cfg = parseConfig(pluginConfig);
553
562
  const client = new HyperspellClient(cfg);
554
563
  const workspaceDir = getWorkspaceDir();
555
- const result = await syncAllMemoryFiles(client, workspaceDir);
564
+ // Mirror the runtime's sync mode so this manual command can't
565
+ // produce whole-file blobs that the gateway then re-syncs as
566
+ // sections (= duplicate memories).
567
+ const result = cfg.syncMemoriesConfig.sectionize
568
+ ? await syncAllFilesSectionized(client, workspaceDir)
569
+ : await syncAllMemoryFiles(client, workspaceDir);
556
570
  process.stdout.write(`Synced ${result.synced} files, ${result.failed} failed\n`);
557
571
  if (result.errors.length > 0) {
558
572
  for (const error of result.errors) {
package/dist/config.js CHANGED
@@ -235,6 +235,22 @@ export function parseConfig(raw) {
235
235
  const kgRaw = (cfg.knowledgeGraph ?? {});
236
236
  const atRaw = (cfg.autoTrace ?? {});
237
237
  const soRaw = (cfg.startupOrientation ?? {});
238
+ // syncMemories can be a boolean (legacy) or an object (new)
239
+ const smRaw = cfg.syncMemories;
240
+ const syncMemoriesEnabled = typeof smRaw === "boolean"
241
+ ? smRaw
242
+ : typeof smRaw === "object" && smRaw !== null
243
+ ? smRaw.enabled ?? true
244
+ : false;
245
+ const smObj = typeof smRaw === "object" && smRaw !== null
246
+ ? smRaw
247
+ : {};
248
+ // The rest of parseConfig is strict about unknown keys; the syncMemories
249
+ // object form must be too, or a typo (sectionise/debounceMS) is silently
250
+ // ignored and the user gets default behavior they didn't ask for.
251
+ if (typeof smRaw === "object" && smRaw !== null && !Array.isArray(smRaw)) {
252
+ assertAllowedKeys(smObj, ["enabled", "sectionize", "watchPaths", "debounceMs"], "hyperspell.syncMemories");
253
+ }
238
254
  return {
239
255
  apiKey,
240
256
  userId: cfg.userId,
@@ -256,7 +272,15 @@ export function parseConfig(raw) {
256
272
  loopsQuery: soRaw.loopsQuery ??
257
273
  "open tasks pending questions unfinished promised need to follow up",
258
274
  },
259
- syncMemories: cfg.syncMemories ?? false,
275
+ syncMemories: syncMemoriesEnabled,
276
+ syncMemoriesConfig: {
277
+ enabled: syncMemoriesEnabled,
278
+ sectionize: smObj.sectionize ?? true,
279
+ watchPaths: Array.isArray(smObj.watchPaths)
280
+ ? smObj.watchPaths
281
+ : [],
282
+ debounceMs: smObj.debounceMs ?? 2000,
283
+ },
260
284
  sources: parseSources(cfg.sources),
261
285
  maxResults: cfg.maxResults ?? 10,
262
286
  relevanceThreshold: cfg.relevanceThreshold ?? 0.6,
@@ -1,54 +1,134 @@
1
1
  import * as path from "node:path";
2
2
  import { getWorkspaceDir } from "../config.js";
3
3
  import { log } from "../logger.js";
4
- import { syncMarkdownFile, syncAllMemoryFiles } from "../sync/markdown.js";
4
+ import { syncMarkdownFile, syncMarkdownFileSectionized, syncAllMemoryFiles, syncAllFilesSectionized, } from "../sync/markdown.js";
5
5
  /**
6
- * Build a handler for file change events that syncs markdown files to Hyperspell
6
+ * Build a handler for file change events that syncs markdown files to Hyperspell.
7
+ *
8
+ * When `sectionize` is enabled (default for new config), files are split by
9
+ * ## headings and each section is synced as a separate memory. Unchanged
10
+ * sections (tracked by content hash) are skipped.
11
+ *
12
+ * When `sectionize` is false (legacy mode), entire files are uploaded as
13
+ * single memories.
7
14
  */
8
15
  export function buildFileSyncHandler(client, cfg) {
9
16
  const workspaceDir = getWorkspaceDir();
10
17
  const memoryDir = path.join(workspaceDir, "memory");
11
18
  const syncUserId = cfg.multiUser?.sharedUserId;
12
- return async (event) => {
13
- const filePath = event.file_path;
14
- if (!filePath)
15
- return;
16
- // Only process markdown files in the workspace's memory directory
17
- if (!filePath.startsWith(memoryDir) || !filePath.endsWith(".md")) {
18
- return;
19
+ const sectionize = cfg.syncMemoriesConfig.sectionize;
20
+ const watchPaths = cfg.syncMemoriesConfig.watchPaths;
21
+ const debounceMs = cfg.syncMemoriesConfig.debounceMs;
22
+ // Resolve additional watch paths to absolute paths for matching
23
+ const resolvedWatchPaths = (watchPaths ?? []).map((wp) => wp.startsWith("/") ? wp : path.join(workspaceDir, wp));
24
+ // Debounce map: filePath -> timeout handle
25
+ const debounceTimers = new Map();
26
+ /**
27
+ * Check whether a file path is syncable (in memory/ or in a watchPath).
28
+ */
29
+ function isSyncable(filePath) {
30
+ if (!filePath.endsWith(".md"))
31
+ return false;
32
+ // Always sync memory/ files
33
+ if (filePath.startsWith(memoryDir + path.sep))
34
+ return true;
35
+ // Check additional watch paths
36
+ for (const wp of resolvedWatchPaths) {
37
+ if (filePath === wp || filePath.startsWith(wp + "/")) {
38
+ return true;
39
+ }
19
40
  }
41
+ return false;
42
+ }
43
+ async function doSync(filePath) {
20
44
  const fileName = path.basename(filePath);
21
45
  log.info(`Memory file changed: ${fileName}`);
22
46
  try {
23
- const result = await syncMarkdownFile(client, filePath, { userId: syncUserId });
24
- if (result.success) {
25
- log.info(`Synced ${fileName} -> ${result.resourceId}`);
47
+ if (sectionize) {
48
+ const result = await syncMarkdownFileSectionized(client, filePath, workspaceDir, { userId: syncUserId });
49
+ if (result.synced > 0 || result.removed > 0) {
50
+ log.info(`Section-synced ${fileName}: ${result.synced} synced, ${result.skipped} unchanged, ${result.removed} removed`);
51
+ }
52
+ else if (result.skipped > 0) {
53
+ log.debug(`${fileName}: all ${result.skipped} sections unchanged`);
54
+ }
55
+ if (result.failed > 0) {
56
+ log.error(`${fileName}: ${result.failed} sections failed:`);
57
+ for (const err of result.errors) {
58
+ log.error(` - ${err}`);
59
+ }
60
+ }
26
61
  }
27
62
  else {
28
- log.error(`Failed to sync ${fileName}: ${result.error}`);
63
+ // Legacy whole-file sync
64
+ const result = await syncMarkdownFile(client, filePath, { userId: syncUserId });
65
+ if (result.success) {
66
+ log.info(`Synced ${fileName} -> ${result.resourceId}`);
67
+ }
68
+ else {
69
+ log.error(`Failed to sync ${fileName}: ${result.error}`);
70
+ }
29
71
  }
30
72
  }
31
73
  catch (err) {
32
74
  log.error(`Error syncing ${fileName}`, err);
33
75
  }
76
+ }
77
+ return async (event) => {
78
+ const filePath = event.file_path;
79
+ if (!filePath)
80
+ return;
81
+ if (!isSyncable(filePath))
82
+ return;
83
+ // Debounce: if the file changes rapidly (agent writing in chunks),
84
+ // wait until writes settle before syncing.
85
+ if (debounceMs > 0) {
86
+ const existing = debounceTimers.get(filePath);
87
+ if (existing)
88
+ clearTimeout(existing);
89
+ debounceTimers.set(filePath, setTimeout(() => {
90
+ debounceTimers.delete(filePath);
91
+ doSync(filePath).catch((err) => log.error("Debounced sync failed", err));
92
+ }, debounceMs));
93
+ }
94
+ else {
95
+ await doSync(filePath);
96
+ }
34
97
  };
35
98
  }
36
99
  /**
37
- * Sync all existing memory files on startup
100
+ * Sync all existing memory files on startup.
101
+ * Uses section-level sync when sectionize is enabled.
38
102
  */
39
103
  export async function syncMemoriesOnStartup(client, workspaceDir, options) {
40
104
  log.info("Syncing existing memory files...");
41
- const result = await syncAllMemoryFiles(client, workspaceDir, { userId: options?.userId });
42
- if (result.synced > 0) {
43
- log.info(`Synced ${result.synced} memory files`);
44
- }
45
- if (result.failed > 0) {
46
- log.error(`Failed to sync ${result.failed} files:`);
47
- for (const error of result.errors) {
48
- log.error(` - ${error}`);
105
+ if (options?.sectionize) {
106
+ const result = await syncAllFilesSectionized(client, workspaceDir, {
107
+ userId: options.userId,
108
+ watchPaths: options.watchPaths,
109
+ });
110
+ log.info(`Section sync complete: ${result.synced} synced, ${result.skipped} unchanged, ${result.removed} removed, ${result.failed} failed`);
111
+ if (result.failed > 0) {
112
+ for (const error of result.errors) {
113
+ log.error(` - ${error}`);
114
+ }
49
115
  }
50
116
  }
51
- if (result.synced === 0 && result.failed === 0) {
52
- log.info("No memory files found in memory/ directory");
117
+ else {
118
+ const result = await syncAllMemoryFiles(client, workspaceDir, {
119
+ userId: options?.userId,
120
+ });
121
+ if (result.synced > 0) {
122
+ log.info(`Synced ${result.synced} memory files`);
123
+ }
124
+ if (result.failed > 0) {
125
+ log.error(`Failed to sync ${result.failed} files:`);
126
+ for (const error of result.errors) {
127
+ log.error(` - ${error}`);
128
+ }
129
+ }
130
+ if (result.synced === 0 && result.failed === 0) {
131
+ log.info("No memory files found in memory/ directory");
132
+ }
53
133
  }
54
134
  }
package/dist/index.js CHANGED
@@ -104,6 +104,7 @@ export default {
104
104
  if (cfg.syncMemories) {
105
105
  const fileSyncHandler = buildFileSyncHandler(client, cfg);
106
106
  api.on("file_changed", fileSyncHandler);
107
+ api.logger.info(`hyperspell: memory sync enabled (sectionize=${cfg.syncMemoriesConfig.sectionize}, watchPaths=${cfg.syncMemoriesConfig.watchPaths.length})`);
107
108
  }
108
109
  // Register memory network tools
109
110
  if (cfg.knowledgeGraph.enabled) {
@@ -121,6 +122,8 @@ export default {
121
122
  const workspaceDir = getWorkspaceDir();
122
123
  await syncMemoriesOnStartup(client, workspaceDir, {
123
124
  userId: cfg.multiUser?.sharedUserId,
125
+ sectionize: cfg.syncMemoriesConfig.sectionize,
126
+ watchPaths: cfg.syncMemoriesConfig.watchPaths,
124
127
  });
125
128
  }
126
129
  },
@@ -1,10 +1,13 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
+ import * as crypto from "node:crypto";
3
4
  import { log } from "../logger.js";
4
5
  const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/;
5
- /**
6
- * Parse frontmatter from markdown content
7
- */
6
+ const MANIFEST_VERSION = 1;
7
+ const HASH_FILE_NAME = ".hyperspell-sync-hashes.json";
8
+ // ---------------------------------------------------------------------------
9
+ // Frontmatter helpers (unchanged from original)
10
+ // ---------------------------------------------------------------------------
8
11
  function parseFrontmatter(content) {
9
12
  const match = content.match(FRONTMATTER_REGEX);
10
13
  if (!match) {
@@ -23,16 +26,10 @@ function parseFrontmatter(content) {
23
26
  }
24
27
  return { frontmatter, body };
25
28
  }
26
- /**
27
- * Serialize frontmatter back to string
28
- */
29
29
  function serializeFrontmatter(frontmatter) {
30
30
  const lines = Object.entries(frontmatter).map(([key, value]) => `${key}: ${value}`);
31
31
  return `---\n${lines.join("\n")}\n---\n`;
32
32
  }
33
- /**
34
- * Read a markdown file and parse its content
35
- */
36
33
  function readMarkdownFile(filePath) {
37
34
  try {
38
35
  const content = fs.readFileSync(filePath, "utf-8");
@@ -50,9 +47,6 @@ function readMarkdownFile(filePath) {
50
47
  return null;
51
48
  }
52
49
  }
53
- /**
54
- * Update the hyperspell_id in the frontmatter of a markdown file
55
- */
56
50
  function updateFrontmatterId(filePath, hyperspellId) {
57
51
  try {
58
52
  const content = fs.readFileSync(filePath, "utf-8");
@@ -66,9 +60,152 @@ function updateFrontmatterId(filePath, hyperspellId) {
66
60
  log.error(`Failed to update frontmatter in ${filePath}`, err);
67
61
  }
68
62
  }
63
+ // ---------------------------------------------------------------------------
64
+ // Section-level parsing
65
+ // ---------------------------------------------------------------------------
66
+ function contentHash(text) {
67
+ return crypto.createHash("sha256").update(text.trim()).digest("hex").slice(0, 16);
68
+ }
69
69
  /**
70
- * Get all markdown files from the memory directory, including subdirectories
70
+ * Split markdown content into sections delimited by ## headings.
71
+ * Content before the first ## heading becomes a section titled after the
72
+ * file-level # heading (or the filename).
73
+ *
74
+ * Sections shorter than `minLength` characters are merged into the previous
75
+ * section to avoid noisy micro-memories.
71
76
  */
77
+ export function parseMarkdownSections(content, fallbackTitle, minLength = 80) {
78
+ const lines = content.split("\n");
79
+ const raw = [];
80
+ let fileTitle = fallbackTitle;
81
+ let currentSection = null;
82
+ for (let i = 0; i < lines.length; i++) {
83
+ const line = lines[i];
84
+ // Capture file-level title from # heading
85
+ if (/^# /.test(line) && currentSection === null) {
86
+ fileTitle = line.replace(/^# /, "").trim();
87
+ continue;
88
+ }
89
+ // New section on ## heading
90
+ if (/^## /.test(line)) {
91
+ if (currentSection) {
92
+ raw.push(currentSection);
93
+ }
94
+ currentSection = {
95
+ title: line.replace(/^## /, "").trim(),
96
+ lines: [],
97
+ startLine: i + 1,
98
+ };
99
+ continue;
100
+ }
101
+ if (currentSection) {
102
+ currentSection.lines.push(line);
103
+ }
104
+ else {
105
+ // Content before any ## heading — becomes preamble section
106
+ if (!raw.length) {
107
+ currentSection = { title: fileTitle, lines: [line], startLine: i + 1 };
108
+ }
109
+ }
110
+ }
111
+ if (currentSection) {
112
+ raw.push(currentSection);
113
+ }
114
+ // Merge short sections into the previous one
115
+ const merged = [];
116
+ for (const section of raw) {
117
+ const text = section.lines.join("\n").trim();
118
+ if (!text)
119
+ continue;
120
+ if (text.length < minLength && merged.length > 0) {
121
+ // Append to previous section
122
+ const prev = merged[merged.length - 1];
123
+ prev.content += `\n\n### ${section.title}\n${text}`;
124
+ prev.contentHash = contentHash(prev.content);
125
+ prev.endLine = section.startLine + section.lines.length;
126
+ }
127
+ else {
128
+ merged.push({
129
+ title: section.title,
130
+ content: text,
131
+ contentHash: contentHash(text),
132
+ startLine: section.startLine,
133
+ endLine: section.startLine + section.lines.length,
134
+ });
135
+ }
136
+ }
137
+ return merged;
138
+ }
139
+ /**
140
+ * Section titles key the manifest per file, so two sections sharing the same
141
+ * ## heading in one file would collide — one resourceId tracked, the other
142
+ * silently orphaned and re-uploaded on every sync. Disambiguate repeats so
143
+ * each section maps to a stable, unique key.
144
+ */
145
+ export function dedupeTitles(sections) {
146
+ const seen = new Map();
147
+ return sections.map((s) => {
148
+ const n = (seen.get(s.title) ?? 0) + 1;
149
+ seen.set(s.title, n);
150
+ return n === 1 ? s : { ...s, title: `${s.title} (${n})` };
151
+ });
152
+ }
153
+ // ---------------------------------------------------------------------------
154
+ // Sync manifest (persisted to workspace, workspace-relative keys)
155
+ // ---------------------------------------------------------------------------
156
+ function manifestPath(workspaceDir) {
157
+ return path.join(workspaceDir, HASH_FILE_NAME);
158
+ }
159
+ /** Workspace-relative, posix-normalized key so the manifest is portable. */
160
+ export function fileKey(workspaceDir, filePath) {
161
+ return path.relative(workspaceDir, filePath).split(path.sep).join("/");
162
+ }
163
+ function emptyManifest() {
164
+ return { version: MANIFEST_VERSION, files: {} };
165
+ }
166
+ export function loadManifest(workspaceDir) {
167
+ const p = manifestPath(workspaceDir);
168
+ try {
169
+ if (fs.existsSync(p)) {
170
+ const parsed = JSON.parse(fs.readFileSync(p, "utf-8"));
171
+ // Self-healing: an unrecognized/legacy shape is discarded. This feature
172
+ // has never shipped a released on-disk format, so nothing to migrate —
173
+ // a mismatch just triggers one re-sync, which is idempotent.
174
+ if (parsed && parsed.version === MANIFEST_VERSION && parsed.files) {
175
+ return { version: MANIFEST_VERSION, files: parsed.files };
176
+ }
177
+ }
178
+ }
179
+ catch (err) {
180
+ log.error("Failed to read sync manifest", err);
181
+ }
182
+ return emptyManifest();
183
+ }
184
+ export function saveManifest(workspaceDir, manifest) {
185
+ const p = manifestPath(workspaceDir);
186
+ try {
187
+ fs.writeFileSync(p, JSON.stringify(manifest, null, 2));
188
+ }
189
+ catch (err) {
190
+ log.error("Failed to write sync manifest", err);
191
+ }
192
+ }
193
+ /**
194
+ * Serializes manifest read-modify-write across the whole process. Startup bulk
195
+ * sync and the live file_changed handler (plus independent per-file debounce
196
+ * timers) would otherwise interleave load → await addMemory → save and lose
197
+ * resourceId entries, causing the next sync to re-upload sections as brand-new
198
+ * memories. Every load→sync→save runs inside this critical section.
199
+ */
200
+ let syncChain = Promise.resolve();
201
+ export function withSyncLock(fn) {
202
+ const run = syncChain.then(fn, fn);
203
+ syncChain = run.then(() => undefined, () => undefined);
204
+ return run;
205
+ }
206
+ // ---------------------------------------------------------------------------
207
+ // Public API — file-level sync (original, kept for backward compat)
208
+ // ---------------------------------------------------------------------------
72
209
  export function getMemoryFiles(workspaceDir) {
73
210
  const memoryDir = path.join(workspaceDir, "memory");
74
211
  if (!fs.existsSync(memoryDir)) {
@@ -95,7 +232,47 @@ export function getMemoryFiles(workspaceDir) {
95
232
  return results;
96
233
  }
97
234
  /**
98
- * Sync a single markdown file to Hyperspell
235
+ * Collect all syncable files based on configuration.
236
+ * Includes memory/*.md by default, plus any additional watchPaths.
237
+ */
238
+ export function getSyncableFiles(workspaceDir, watchPaths) {
239
+ const files = getMemoryFiles(workspaceDir);
240
+ if (watchPaths) {
241
+ for (const wp of watchPaths) {
242
+ const resolved = wp.startsWith("/") ? wp : path.join(workspaceDir, wp);
243
+ if (!fs.existsSync(resolved))
244
+ continue;
245
+ const stat = fs.statSync(resolved);
246
+ if (stat.isFile() && resolved.endsWith(".md")) {
247
+ if (!files.includes(resolved)) {
248
+ files.push(resolved);
249
+ }
250
+ }
251
+ else if (stat.isDirectory()) {
252
+ // Walk directory for .md files
253
+ const walk = (dir) => {
254
+ try {
255
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
256
+ const fullPath = path.join(dir, entry.name);
257
+ if (entry.isDirectory()) {
258
+ walk(fullPath);
259
+ }
260
+ else if (entry.name.endsWith(".md") && !files.includes(fullPath)) {
261
+ files.push(fullPath);
262
+ }
263
+ }
264
+ }
265
+ catch (_err) { /* skip unreadable dirs */ }
266
+ };
267
+ walk(resolved);
268
+ }
269
+ }
270
+ }
271
+ return files;
272
+ }
273
+ /**
274
+ * Original whole-file sync (kept for backward compatibility when
275
+ * sectionize is false).
99
276
  */
100
277
  export async function syncMarkdownFile(client, filePath, options) {
101
278
  const file = readMarkdownFile(filePath);
@@ -116,7 +293,6 @@ export async function syncMarkdownFile(client, filePath, options) {
116
293
  },
117
294
  userId: options?.userId,
118
295
  });
119
- // Update frontmatter with new resource ID if it changed or was newly created
120
296
  if (result.resourceId !== file.hyperspellId) {
121
297
  updateFrontmatterId(filePath, result.resourceId);
122
298
  }
@@ -128,9 +304,156 @@ export async function syncMarkdownFile(client, filePath, options) {
128
304
  return { success: false, error: errorMsg };
129
305
  }
130
306
  }
307
+ // ---------------------------------------------------------------------------
308
+ // Section-level sync (new)
309
+ // ---------------------------------------------------------------------------
131
310
  /**
132
- * Sync all markdown files in the memory directory
311
+ * Sync a single markdown file to Hyperspell at section granularity.
312
+ * Each ## section becomes a separate memory. Content hashes are tracked
313
+ * so unchanged sections are skipped on subsequent syncs.
314
+ *
315
+ * The section title is prepended with the file-level context (e.g.
316
+ * "2026-05-09 — The day David's mum got sick") for better retrieval.
133
317
  */
318
+ export async function syncMarkdownFileSectionized(client, filePath, workspaceDir, options) {
319
+ return withSyncLock(async () => {
320
+ const empty = { synced: 0, skipped: 0, failed: 0, removed: 0, errors: [] };
321
+ try {
322
+ const file = readMarkdownFile(filePath);
323
+ if (!file || !file.content) {
324
+ return file ? empty : { ...empty, errors: ["Failed to read file"] };
325
+ }
326
+ const sections = dedupeTitles(parseMarkdownSections(file.content, file.title));
327
+ if (sections.length === 0) {
328
+ return empty;
329
+ }
330
+ const manifest = loadManifest(workspaceDir);
331
+ const key = fileKey(workspaceDir, filePath);
332
+ const prevSections = manifest.files[key]?.sections ?? {};
333
+ const fileName = path.basename(filePath, ".md");
334
+ let synced = 0;
335
+ let skipped = 0;
336
+ let failed = 0;
337
+ let removed = 0;
338
+ let dirty = false;
339
+ const errors = [];
340
+ const currentTitles = new Set(sections.map((s) => s.title));
341
+ // Sections in the manifest but absent from the current parse are either
342
+ // renames (same content hash resurfacing under a new title) or true
343
+ // deletions. Index leftovers by content hash so a renamed section can
344
+ // reclaim its existing Hyperspell resource instead of creating a
345
+ // duplicate + orphan. Hash collisions among removed sections are rare
346
+ // and resolved best-effort (last writer wins).
347
+ const orphanByHash = new Map();
348
+ for (const [title, rec] of Object.entries(prevSections)) {
349
+ if (!currentTitles.has(title)) {
350
+ orphanByHash.set(rec.hash, { title, resourceId: rec.resourceId });
351
+ }
352
+ }
353
+ const newSections = {};
354
+ for (const section of sections) {
355
+ const prev = prevSections[section.title];
356
+ // Unchanged: keep the record, skip the upload.
357
+ if (prev && prev.hash === section.contentHash) {
358
+ newSections[section.title] = prev;
359
+ skipped++;
360
+ continue;
361
+ }
362
+ // New title: if identical content was just removed under a different
363
+ // heading, this is a rename — reuse the resource and claim it so it is
364
+ // not also deleted as an orphan below.
365
+ let reuseResourceId = prev?.resourceId;
366
+ if (!prev) {
367
+ const renamedFrom = orphanByHash.get(section.contentHash);
368
+ if (renamedFrom) {
369
+ reuseResourceId = renamedFrom.resourceId;
370
+ orphanByHash.delete(section.contentHash);
371
+ log.info(`Section renamed: "${renamedFrom.title}" -> "${section.title}" (${fileName}) — updating in place`);
372
+ }
373
+ }
374
+ const memoryTitle = file.title !== section.title ? `${file.title} — ${section.title}` : section.title;
375
+ try {
376
+ const result = await client.addMemory(section.content, {
377
+ title: memoryTitle,
378
+ resourceId: reuseResourceId,
379
+ collection: "openclaw",
380
+ metadata: {
381
+ openclaw_source: "memory_sync_section",
382
+ file_path: filePath,
383
+ file_name: fileName,
384
+ section_title: section.title,
385
+ content_hash: section.contentHash,
386
+ },
387
+ userId: options?.userId,
388
+ });
389
+ newSections[section.title] = {
390
+ hash: section.contentHash,
391
+ resourceId: result.resourceId,
392
+ };
393
+ synced++;
394
+ dirty = true;
395
+ log.info(`Synced section: ${memoryTitle} -> ${result.resourceId}`);
396
+ }
397
+ catch (err) {
398
+ const errorMsg = err instanceof Error ? err.message : String(err);
399
+ errors.push(`${memoryTitle}: ${errorMsg}`);
400
+ failed++;
401
+ // Preserve the prior record so a transient failure does not drop the
402
+ // resourceId and force a duplicate upload next run.
403
+ if (prev)
404
+ newSections[section.title] = prev;
405
+ }
406
+ }
407
+ // Whatever remains is a genuine deletion: gone from source, no rename
408
+ // reclaimed it. Delete the remote memory so retrieval does not keep
409
+ // serving content the user removed.
410
+ for (const { title, resourceId } of orphanByHash.values()) {
411
+ if (!resourceId) {
412
+ // Never got a resourceId — nothing to delete, just drop the record.
413
+ dirty = true;
414
+ continue;
415
+ }
416
+ const { deleted } = await client.deleteMemory(resourceId, {
417
+ userId: options?.userId,
418
+ });
419
+ if (deleted) {
420
+ removed++;
421
+ dirty = true;
422
+ log.info(`Removed deleted section "${title}" (${fileName}) -> ${resourceId}`);
423
+ }
424
+ else {
425
+ // Keep the record so the delete is retried next run rather than
426
+ // silently leaking the orphan into the retrieval layer. Count it
427
+ // as a failure so the stats are consistent and callers that gate
428
+ // logging on `failed > 0` actually surface it.
429
+ newSections[title] = prevSections[title];
430
+ failed++;
431
+ errors.push(`delete "${title}": failed`);
432
+ }
433
+ }
434
+ if (Object.keys(newSections).length > 0) {
435
+ manifest.files[key] = { sections: newSections };
436
+ }
437
+ else {
438
+ delete manifest.files[key];
439
+ }
440
+ if (dirty) {
441
+ saveManifest(workspaceDir, manifest);
442
+ }
443
+ return { synced, skipped, failed, removed, errors };
444
+ }
445
+ catch (err) {
446
+ // The body is defensive everywhere above; this guards the lock chain
447
+ // against an unexpected throw so one bad file cannot wedge the queue.
448
+ const msg = err instanceof Error ? err.message : String(err);
449
+ log.error(`Sectionized sync crashed for ${filePath}`, err);
450
+ return { ...empty, errors: [msg] };
451
+ }
452
+ });
453
+ }
454
+ // ---------------------------------------------------------------------------
455
+ // Bulk sync (original whole-file approach)
456
+ // ---------------------------------------------------------------------------
134
457
  export async function syncAllMemoryFiles(client, workspaceDir, options) {
135
458
  const files = getMemoryFiles(workspaceDir);
136
459
  let synced = 0;
@@ -149,3 +472,35 @@ export async function syncAllMemoryFiles(client, workspaceDir, options) {
149
472
  }
150
473
  return { synced, failed, errors };
151
474
  }
475
+ // ---------------------------------------------------------------------------
476
+ // Bulk sync (section-level approach)
477
+ // ---------------------------------------------------------------------------
478
+ /**
479
+ * Sync all syncable files at section granularity.
480
+ * Unchanged sections (by content hash) are skipped.
481
+ */
482
+ export async function syncAllFilesSectionized(client, workspaceDir, options) {
483
+ const files = getSyncableFiles(workspaceDir, options?.watchPaths);
484
+ let totalSynced = 0;
485
+ let totalSkipped = 0;
486
+ let totalFailed = 0;
487
+ let totalRemoved = 0;
488
+ const allErrors = [];
489
+ for (const filePath of files) {
490
+ const result = await syncMarkdownFileSectionized(client, filePath, workspaceDir, {
491
+ userId: options?.userId,
492
+ });
493
+ totalSynced += result.synced;
494
+ totalSkipped += result.skipped;
495
+ totalFailed += result.failed;
496
+ totalRemoved += result.removed;
497
+ allErrors.push(...result.errors);
498
+ }
499
+ return {
500
+ synced: totalSynced,
501
+ skipped: totalSkipped,
502
+ failed: totalFailed,
503
+ removed: totalRemoved,
504
+ errors: allErrors,
505
+ };
506
+ }
@@ -71,7 +71,7 @@
71
71
  },
72
72
  "syncMemories": {
73
73
  "label": "Sync Memory Files",
74
- "help": "Automatically sync markdown files in workspace/memory/ to Hyperspell",
74
+ "help": "Automatically sync markdown files to Hyperspell. Set to true for defaults, or use an object for fine-grained control: { sectionize: true, watchPaths: ['MEMORY.md', 'notes/'], debounceMs: 2000 }",
75
75
  "advanced": true
76
76
  },
77
77
  "knowledgeGraph": {
@@ -144,7 +144,30 @@
144
144
  "type": "boolean"
145
145
  },
146
146
  "syncMemories": {
147
- "type": "boolean"
147
+ "oneOf": [
148
+ { "type": "boolean" },
149
+ {
150
+ "type": "object",
151
+ "properties": {
152
+ "enabled": { "type": "boolean" },
153
+ "sectionize": {
154
+ "type": "boolean",
155
+ "description": "Split files by ## headings into separate memories for better retrieval granularity (default: true)"
156
+ },
157
+ "watchPaths": {
158
+ "type": "array",
159
+ "items": { "type": "string" },
160
+ "description": "Additional paths to watch beyond memory/ (relative to workspace or absolute)"
161
+ },
162
+ "debounceMs": {
163
+ "type": "number",
164
+ "minimum": 0,
165
+ "maximum": 30000,
166
+ "description": "Debounce file changes in ms to avoid syncing mid-write (default: 2000)"
167
+ }
168
+ }
169
+ }
170
+ ]
148
171
  },
149
172
  "knowledgeGraph": {
150
173
  "type": "object",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hyperspell/openclaw-hyperspell",
3
- "version": "0.13.1",
3
+ "version": "0.14.0",
4
4
  "type": "module",
5
5
  "description": "OpenClaw Hyperspell memory plugin",
6
6
  "license": "MIT",
@@ -38,7 +38,7 @@
38
38
  "check-types": "tsc --noEmit",
39
39
  "lint": "bunx @biomejs/biome ci .",
40
40
  "lint:fix": "bunx @biomejs/biome check --write .",
41
- "test": "node --test --experimental-strip-types lib/sender.test.ts config.test.ts hooks/auto-trace.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts"
41
+ "test": "node --test --experimental-strip-types lib/sender.test.ts config.test.ts sync/markdown.test.ts hooks/auto-trace.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts"
42
42
  },
43
43
  "openclaw": {
44
44
  "extensions": [