dsh-memory-recall 0.1.0

package/src/snapshot.js

@@ -0,0 +1,63 @@
+// What a new chat's system prompt says about memory.
+//
+// Rendered ONCE, when the chat's agent starts, and never again for that chat.
+// dsh keeps the prompt prefix cacheable only while its text is unchanged —
+// "any change may invalidate reuse from the first changed token" — so a
+// memory saved mid-chat is written to disk at once but reaches the prompt in
+// the next new chat. The guidance says so, so the model does not expect to
+// see its own save reflected, and does not save twice.
+
+import { TARGETS, joinEntries, sizeOf } from './store.js';
+
+/** After prompt-notes (950) and every first-party instruction, before the environment suffix (10000). */
+export const SECTION_ORDER = 9500;
+export const SECTION_NAME = 'dsh-memory-recall:memory';
+
+const RULE = '═'.repeat(46);
+
+export const GUIDANCE = [
+  '# Memory',
+  '',
+  'You have a persistent memory that carries across chats: the blocks below, when they hold anything. It is fixed for this chat —',
+  'what you save now is on disk at once but appears here from the next new chat, so do not re-save something because you do not see it yet.',
+  '',
+  "- Save with the `memory` tool. Target 'user' is who the user is: name, role, preferences, how they like to work.",
+  "  Target 'memory' is your own notes: environment facts, conventions, tool quirks, lessons that apply to every task.",
+  '- Save durable facts, one short fact per entry, when you learn them — not task progress, one-off details, or anything easy to look up again.',
+  '- The user sees each save as a "Saved to memory" note in the chat and can edit or delete entries in the Memory panel.',
+  '  Never save secrets or credentials, or anything the user would not want written down.',
+  '- When the user refers to an earlier conversation ("what did we decide about…", "like last time"), use `session_search`.',
+  '  It asks the user for permission to read past chats first. If they decline, say so and carry on without it; do not ask again in the same reply.',
+  '- A user message carrying [workspace "Name" — folder] means the user picked that workspace with @: pass it as `workspace` to',
+  '  session_search. A chat the user @-mentions arrives as a "Referenced sessions" snapshot — use that directly, no search needed.',
+].join('\n');
+
+/**
+ * The note when the tools failed to load: the memory below can be read, but
+ * nothing can be saved or searched — and the model must say so rather than
+ * improvise a substitute (it once wrote a notes file instead).
+ */
+export const GUIDANCE_NO_TOOLS = [
+  '# Memory',
+  '',
+  'You have a persistent memory that carries across chats: the blocks below, when they hold anything.',
+  'Its tools (`memory` and `session_search`) failed to load in this dsh, so right now you cannot save to memory or search',
+  'earlier chats. If the user asks you to remember something, tell them memory saving is unavailable in this session and',
+  'that they can add it themselves in the Memory panel in the sidebar. Do not substitute a file or anything else for it.',
+].join('\n');
+
+/** One store as a prompt block, or '' when it is empty. */
+export function renderBlock(target, entries, limit) {
+  if (!entries.length) return '';
+  const chars = sizeOf(entries);
+  const percent = limit > 0 ? Math.min(100, Math.floor((chars / limit) * 100)) : 0;
+  const usage = `${percent}% — ${chars.toLocaleString('en-US')}/${limit.toLocaleString('en-US')} chars`;
+  return `${RULE}\n${TARGETS[target].title} [${usage}]\n${RULE}\n${joinEntries(entries)}`;
+}
+
+/** The whole section: guidance, then the user profile, then the notes. */
+export function renderSection({ user = [], memory = [] }, limits, { tools = true } = {}) {
+  const guidance = tools ? GUIDANCE : GUIDANCE_NO_TOOLS;
+  const blocks = [renderBlock('user', user, limits.user), renderBlock('memory', memory, limits.memory)].filter(Boolean);
+  return blocks.length ? `${guidance}\n\n${blocks.join('\n\n')}` : `${guidance}\n\n(Nothing is saved yet.)`;
+}

package/src/store.js

@@ -0,0 +1,298 @@
+// The two memory files, and every change made to them.
+//
+//   USER.md    who the user is — name, role, preferences, how they like to work
+//   MEMORY.md  the model's own notes — environment facts, conventions, lessons
+//
+// Each file is a list of short entries separated by a line holding only `§`,
+// the format Hermes Agent uses, so the files stay readable and hand-editable.
+// Each has a character budget: everything in them is pasted into the system
+// prompt of every new chat, so they must stay small, and a full store makes
+// the model consolidate (merge or drop stale entries) rather than grow.
+//
+// Every change re-reads the file first, so an edit made by hand or in the
+// Memory panel is never overwritten by a stale copy; writes go to a temporary
+// file and are renamed into place, so a crash never leaves half a file; and
+// changes run one at a time.
+
+import { readFileSync } from 'node:fs';
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { firstThreatMessage } from './threats.js';
+
+export const ENTRY_DELIMITER = '\n§\n';
+const SPLIT = /\r?\n§\r?\n/;
+
+export const TARGETS = {
+  user: { file: 'USER.md', label: 'user profile', title: 'USER PROFILE (who the user is)' },
+  memory: { file: 'MEMORY.md', label: 'memory', title: 'MEMORY (your personal notes)' },
+};
+
+/** Hermes Agent's defaults: 2,200 characters of notes, 1,375 of user profile. */
+export const DEFAULT_LIMITS = { memory: 2200, user: 1375 };
+
+/** Entries in a file's text: trimmed, blanks dropped, exact duplicates dropped. */
+export function parseEntries(raw) {
+  if (typeof raw !== 'string') return [];
+  const text = raw.replace(/^\uFEFF/, ''); // a BOM from Notepad would stick to the first entry
+  const out = [];
+  for (const part of text.split(SPLIT)) {
+    const entry = part.trim();
+    if (entry !== '' && !out.includes(entry)) out.push(entry);
+  }
+  return out;
+}
+
+export const joinEntries = (entries) => entries.join(ENTRY_DELIMITER);
+
+/** Characters an entry list costs, as the prompt will carry it. */
+export const sizeOf = (entries) => joinEntries(entries).length;
+
+function isTarget(target) {
+  return Object.hasOwn(TARGETS, target);
+}
+
+/** What makes an entry unusable as an entry, or null. */
+function entryProblem(content) {
+  if (typeof content !== 'string' || content.trim() === '') return 'the entry is empty';
+  if (/(^|\r?\n)§(\r?\n|$)/.test(content.trim())) return 'an entry must not contain a line holding only §, which separates entries';
+  return null;
+}
+
+export class MemoryStore {
+  /**
+   * @param {object} options
+   * @param {string} options.dir     where USER.md and MEMORY.md live
+   * @param {object} [options.limits] {memory, user} character budgets
+   */
+  constructor({ dir, limits = {} }) {
+    this.dir = dir;
+    this.limits = { ...DEFAULT_LIMITS, ...limits };
+    this.entries = { user: [], memory: [] };
+    this.queue = Promise.resolve();
+  }
+
+  pathFor(target) {
+    return join(this.dir, TARGETS[target].file);
+  }
+
+  limit(target) {
+    return this.limits[target];
+  }
+
+  /** `{chars, limit, percent}` for a target, or for a hypothetical entry list. */
+  usage(target, entries = this.entries[target]) {
+    const chars = sizeOf(entries);
+    const limit = this.limit(target);
+    return { chars, limit, percent: limit > 0 ? Math.min(100, Math.floor((chars / limit) * 100)) : 0 };
+  }
+
+  /** A copy of both lists. */
+  snapshot() {
+    return { user: [...this.entries.user], memory: [...this.entries.memory] };
+  }
+
+  async readTarget(target) {
+    try {
+      return parseEntries(await readFile(this.pathFor(target), 'utf8'));
+    } catch (error) {
+      if (error?.code === 'ENOENT') return [];
+      throw error;
+    }
+  }
+
+  /**
+   * Read both files synchronously — two small files, read once when a chat
+   * starts, so the snapshot is in place before that chat's first prompt is
+   * assembled rather than racing it. A file that cannot be read keeps the last
+   * good copy instead of briefing the chat with an empty memory.
+   */
+  loadSync() {
+    for (const target of Object.keys(TARGETS)) {
+      try {
+        this.entries[target] = parseEntries(readFileSync(this.pathFor(target), 'utf8'));
+      } catch (error) {
+        if (error?.code === 'ENOENT') this.entries[target] = [];
+      }
+    }
+    return this.snapshot();
+  }
+
+  /** Read both files from disk. */
+  async load() {
+    this.entries.user = await this.readTarget('user');
+    this.entries.memory = await this.readTarget('memory');
+    return this.snapshot();
+  }
+
+  async writeTarget(target, entries) {
+    await mkdir(this.dir, { recursive: true });
+    const path = this.pathFor(target);
+    const temp = `${path}.${process.pid}.${Date.now()}.tmp`;
+    await writeFile(temp, entries.length ? `${joinEntries(entries)}\n` : '', 'utf8');
+    await rename(temp, path);
+  }
+
+  /** Run `work` after every earlier change has finished, whatever its outcome. */
+  serial(work) {
+    const run = this.queue.then(work, work);
+    this.queue = run.catch(() => {});
+    return run;
+  }
+
+  /**
+   * Re-read `target`, let `mutate(entries)` return `{entries, message, changed}`
+   * or a failure, then save. The read is fresh every time so a hand edit made
+   * since the last change is kept.
+   */
+  change(target, action, mutate) {
+    return this.serial(async () => {
+      if (!isTarget(target)) return this.fail(target, `unknown target '${target}' — use 'memory' or 'user'`);
+      let current;
+      try {
+        current = await this.readTarget(target);
+      } catch (error) {
+        return this.fail(target, `could not read ${TARGETS[target].file} (${error?.code || error?.message || error}); nothing was changed`);
+      }
+      this.entries[target] = current;
+      const result = mutate([...current]);
+      if (result.success === false) return result;
+      try {
+        await this.writeTarget(target, result.entries);
+      } catch (error) {
+        return this.fail(target, `could not save ${TARGETS[target].file} (${error?.code || error?.message || error}); nothing was changed`);
+      }
+      this.entries[target] = result.entries;
+      return {
+        success: true,
+        target,
+        action,
+        message: result.message,
+        changed: result.changed,
+        usage: this.usage(target),
+        entryCount: result.entries.length,
+      };
+    });
+  }
+
+  fail(target, error, extra = {}) {
+    const known = isTarget(target);
+    return {
+      success: false,
+      target,
+      error,
+      ...(known ? { usage: this.usage(target) } : {}),
+      ...extra,
+    };
+  }
+
+  /** A failure that carries the live entries, so the model can consolidate. */
+  failWithEntries(target, entries, error) {
+    return this.fail(target, error, { currentEntries: entries, usage: this.usage(target, entries) });
+  }
+
+  /** The index of the one entry containing `oldText`, or a failure. */
+  findOne(target, entries, oldText) {
+    if (typeof oldText !== 'string' || oldText.trim() === '') {
+      return this.failWithEntries(target, entries, 'old_text is required: a short piece of the entry to change, unique to it');
+    }
+    const needle = oldText.trim();
+    const hits = entries.map((e, i) => (e.includes(needle) ? i : -1)).filter((i) => i >= 0);
+    if (hits.length === 0) return this.failWithEntries(target, entries, `no ${TARGETS[target].label} entry contains "${needle}"`);
+    if (hits.length > 1) {
+      return this.failWithEntries(target, entries, `"${needle}" is in ${hits.length} entries — use a longer piece that is unique to one`);
+    }
+    return hits[0];
+  }
+
+  /** Apply one operation to an entry list in memory; no budget check, no save. */
+  step(target, entries, op, { scan }) {
+    const action = op?.action;
+    const content = typeof op?.content === 'string' ? op.content : typeof op?.new_text === 'string' ? op.new_text : undefined;
+    if (action === 'add' || action === 'replace') {
+      const problem = entryProblem(content);
+      if (problem) return this.failWithEntries(target, entries, `${action}: ${problem}`);
+      if (scan) {
+        const threat = firstThreatMessage(content);
+        if (threat) return this.fail(target, threat, { blocked: true });
+      }
+    }
+    if (action === 'add') {
+      const entry = content.trim();
+      if (entries.includes(entry)) return { entries, changed: { added: [] }, note: 'already saved' };
+      return { entries: [...entries, entry], changed: { added: [entry] } };
+    }
+    if (action === 'replace' || action === 'remove') {
+      const at = this.findOne(target, entries, op.old_text);
+      if (typeof at !== 'number') return at;
+      const before = entries[at];
+      if (action === 'remove') {
+        return { entries: entries.filter((_, i) => i !== at), changed: { removed: [before] } };
+      }
+      const after = content.trim();
+      const next = [...entries];
+      if (next.includes(after) && after !== before) next.splice(at, 1); // merging into an existing entry
+      else next[at] = after;
+      return { entries: next, changed: { replaced: [{ from: before, to: after }] } };
+    }
+    return this.failWithEntries(target, entries, `unknown action '${action}' — use add, replace or remove`);
+  }
+
+  overBudget(target, entries, what) {
+    const { chars, limit } = this.usage(target, entries);
+    return this.failWithEntries(target, this.entries[target],
+      `${what} would bring the ${TARGETS[target].label} to ${chars.toLocaleString('en-US')}/${limit.toLocaleString('en-US')} characters — over its budget. `
+      + 'Consolidate in ONE memory call: an operations batch that replaces overlapping entries with shorter merged ones or removes stale ones, '
+      + 'and adds the new entry, all together. The budget is checked on the final result.');
+  }
+
+  /** @param {{scan?: boolean}} [options] scan=false for edits the user makes themselves */
+  add(target, content, { scan = true } = {}) {
+    return this.change(target, 'add', (entries) => {
+      const r = this.step(target, entries, { action: 'add', content }, { scan });
+      if (r.success === false) return r;
+      if (r.note === 'already saved') return { entries, message: 'That entry is already saved — nothing changed.', changed: { added: [] } };
+      if (sizeOf(r.entries) > this.limit(target)) return this.overBudget(target, r.entries, 'Adding this entry');
+      return { entries: r.entries, message: 'Entry added.', changed: r.changed };
+    });
+  }
+
+  replace(target, oldText, content, { scan = true } = {}) {
+    return this.change(target, 'replace', (entries) => {
+      const r = this.step(target, entries, { action: 'replace', old_text: oldText, content }, { scan });
+      if (r.success === false) return r;
+      if (sizeOf(r.entries) > this.limit(target)) return this.overBudget(target, r.entries, 'This replacement');
+      return { entries: r.entries, message: 'Entry replaced.', changed: r.changed };
+    });
+  }
+
+  remove(target, oldText) {
+    return this.change(target, 'remove', (entries) => {
+      const r = this.step(target, entries, { action: 'remove', old_text: oldText }, { scan: false });
+      if (r.success === false) return r;
+      return { entries: r.entries, message: 'Entry removed.', changed: r.changed };
+    });
+  }
+
+  /**
+   * Several operations as one change: applied in order to a copy, the budget
+   * checked once on the final list, and nothing written unless every step
+   * works. This is what lets a full store make room and add in one call.
+   */
+  applyBatch(target, operations, { scan = true } = {}) {
+    return this.change(target, 'batch', (entries) => {
+      if (!Array.isArray(operations) || operations.length === 0) {
+        return this.failWithEntries(target, entries, 'operations must be a non-empty list of {action, content?, old_text?}');
+      }
+      let list = entries;
+      const changed = { added: [], removed: [], replaced: [] };
+      for (const [index, op] of operations.entries()) {
+        const r = this.step(target, list, op, { scan });
+        if (r.success === false) return { ...r, error: `operation ${index + 1} (${op?.action ?? '?'}): ${r.error} — nothing was changed` };
+        list = r.entries;
+        for (const key of Object.keys(changed)) if (r.changed?.[key]) changed[key].push(...r.changed[key]);
+      }
+      if (sizeOf(list) > this.limit(target)) return this.overBudget(target, list, 'This batch');
+      return { entries: list, message: `${operations.length} change(s) applied.`, changed };
+    });
+  }
+}

package/src/threats.js

@@ -0,0 +1,113 @@
+// Refuse memory writes that read like a prompt injection or an exfiltration.
+//
+// Memory is pasted into the system prompt of every chat that starts after it
+// is saved, so a poisoned entry is not a one-off: it keeps instructing the
+// model in every future session until someone notices and deletes it. Every
+// write the MODEL makes is scanned here first; a hit refuses the write and
+// says why. (Edits you make yourself in the Memory panel are yours and are not
+// scanned.)
+//
+// Ported from Hermes Agent's tools/threat_patterns.py — the scope it applies to
+// memory writes, which is every pattern it has — with Hermes-specific paths
+// swapped for dsh's. Hermes Agent is MIT licensed, (c) 2025 Nous Research.
+// The rule it states for new patterns applies here too: anchor on attack
+// vocabulary or unambiguous attack behaviour, not on bossy English ("you must"
+// is common in legitimate notes).
+
+/** Scanners are advisory: bound the worst case. */
+export const MAX_SCAN_CHARS = 65_536;
+
+// Bounded filler between key words; an unbounded `(?:\w+\s+)*` backtracks badly.
+const F = String.raw`(?:\w+\s+){0,8}`;
+// An env var whose name ENDS in a secret-ish word ($OPENAI_API_KEY, ${TOKEN}),
+// so a benign $SOME_KEYBOARD_URL passes.
+const SECRET_VAR = String.raw`\$\{?\w*(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)S?\b`;
+const MODIFY = String.raw`(update|modify|edit|write|change|append|add\s+to)\s+[^\n]{0,2048}`;
+
+/** [pattern source, id] — all matched case-insensitively. */
+const PATTERNS = [
+  // Classic prompt injection
+  [String.raw`ignore\s+${F}(previous|all|above|prior)\s+${F}instructions`, 'prompt_injection'],
+  [String.raw`system\s+prompt\s+override`, 'sys_prompt_override'],
+  [String.raw`disregard\s+${F}(your|all|any)\s+${F}(instructions|rules|guidelines)`, 'disregard_rules'],
+  [String.raw`act\s+as\s+(if|though)\s+${F}you\s+${F}(have\s+no|don't\s+have)\s+${F}(restrictions|limits|rules)`, 'bypass_restrictions'],
+  [String.raw`<!--[^>]{0,512}(?:ignore|override|system|secret|hidden)[^>]{0,512}-->`, 'html_comment_injection'],
+  [String.raw`<\s*div\s+style\s*=\s*["'][^>]{0,2048}display\s*:\s*none`, 'hidden_div'],
+  [String.raw`translate\s+[^\n]{0,512}\s+into\s+\w+(?:[\s-]+\w+){0,2}\s+and\s+(execute|run|eval)\b`, 'translate_execute'],
+  [String.raw`do\s+not\s+${F}tell\s+${F}the\s+user`, 'deception_hide'],
+
+  // Role-play / identity hijack
+  [String.raw`you\s+are\s+${F}now\s+(?:a|an|the)\s+`, 'role_hijack'],
+  [String.raw`pretend\s+${F}(you\s+are|to\s+be)\s+`, 'role_pretend'],
+  [String.raw`output\s+${F}(system|initial)\s+prompt`, 'leak_system_prompt'],
+  [String.raw`(respond|answer|reply)\s+without\s+${F}(restrictions|limitations|filters|safety)`, 'remove_filters'],
+  [String.raw`you\s+have\s+been\s+${F}(updated|upgraded|patched)\s+to`, 'fake_update'],
+  [String.raw`\bname\s+yourself\s+\w+`, 'identity_override'],
+
+  // Command-and-control style promptware
+  [String.raw`register\s+(as\s+)?a?\s*node`, 'c2_node_registration'],
+  [String.raw`(heartbeat|beacon|check[\s\-]?in)\s+(to|with)\s+`, 'c2_heartbeat'],
+  [String.raw`pull\s+(down\s+)?(?:new\s+)?task(?:ing|s)?\b`, 'c2_task_pull'],
+  [String.raw`connect\s+to\s+the\s+network\b`, 'c2_network_connect'],
+  [String.raw`you\s+must\s+(?:\w+\s+){0,3}(register|connect|report|beacon)\b`, 'forced_action'],
+  [String.raw`only\s+use\s+one[\s\-]?liners?\b`, 'anti_forensic_oneliner'],
+  [String.raw`never\s+${F}(?:create|write)\s+${F}(?:script|file)\s+${F}disk`, 'anti_forensic_disk'],
+  [String.raw`unset\s+\w*(?:CLAUDE|CODEX|HERMES|AGENT|OPENAI|ANTHROPIC|DEEPSEEK|DSH)\w*`, 'env_var_unset_agent'],
+  [String.raw`\b(?:cobalt\s*strike|sliver|havoc|mythic|metasploit|brainworm)\b`, 'known_c2_framework'],
+  [String.raw`\bc2\s+(?:server|channel|infrastructure|beacon)\b`, 'c2_explicit'],
+  [String.raw`\bcommand\s+and\s+control\b`, 'c2_explicit_long'],
+
+  // Exfiltration
+  [String.raw`curl\s+[^\n]{0,2048}${SECRET_VAR}`, 'exfil_curl'],
+  [String.raw`wget\s+[^\n]{0,2048}${SECRET_VAR}`, 'exfil_wget'],
+  [String.raw`cat\s+[^\n]{0,2048}(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)`, 'read_secrets'],
+  [String.raw`(send|post|upload|transmit)\s+[^\n]{0,2048}\s+(to|at)\s+https?://`, 'send_to_url'],
+  [String.raw`(include|output|print|share)\s+${F}(conversation|chat\s+history|previous\s+messages|full\s+context|entire\s+context)`, 'context_exfil'],
+
+  // Persistence / backdoors, and tampering with the agent's own configuration
+  [String.raw`authorized_keys`, 'ssh_backdoor'],
+  [String.raw`\$HOME/\.ssh|~/\.ssh`, 'ssh_access'],
+  [String.raw`\$HOME/\.dsh/\.env|~/\.dsh/\.env`, 'dsh_env'],
+  [String.raw`${MODIFY}(?:AGENTS\.md|CLAUDE\.md|\.cursorrules|\.clinerules)`, 'agent_config_mod'],
+  [String.raw`${MODIFY}\.dsh/(settings\.yaml|cordis\.patch\.yml|cordis\.yml)`, 'dsh_config_mod'],
+
+  // Hardcoded secrets
+  [String.raw`(?:api[_-]?key|token|secret|password)\s*[=:]\s*["'][A-Za-z0-9+/=_-]{20,}`, 'hardcoded_secret'],
+].map(([source, id]) => [new RegExp(source, 'i'), id]);
+
+/**
+ * Zero-width and bidirectional-control characters: invisible in any editor,
+ * used to hide instructions inside innocent-looking text.
+ */
+export const INVISIBLE_CHARS = new Set([
+  '\u200B', '\u200C', '\u200D', '\u2060', '\u2062', '\u2063', '\u2064', '\uFEFF',
+  '\u202A', '\u202B', '\u202C', '\u202D', '\u202E', '\u2066', '\u2067', '\u2068', '\u2069',
+]);
+
+/** Every pattern id found in `content`; invisible characters as `invisible_unicode_U+XXXX`. */
+export function scanForThreats(content) {
+  if (typeof content !== 'string' || content === '') return [];
+  const text = content.slice(0, MAX_SCAN_CHARS);
+  const found = [];
+  // On the raw text: NFKC below can remove some of these code points.
+  for (const ch of new Set(text)) {
+    if (INVISIBLE_CHARS.has(ch)) found.push(`invisible_unicode_U+${ch.codePointAt(0).toString(16).toUpperCase().padStart(4, '0')}`);
+  }
+  // NFKC folds full-width and compatibility forms (ｃａｔ → cat), so a
+  // homograph spelling does not slip past. Cross-script look-alikes
+  // (Cyrillic а) are not folded — that needs a confusables table.
+  const folded = text.normalize('NFKC');
+  for (const [pattern, id] of PATTERNS) if (pattern.test(folded)) found.push(id);
+  return found;
+}
+
+/** A user-facing refusal for the first threat found, or null. */
+export function firstThreatMessage(content) {
+  const [first] = scanForThreats(content);
+  if (first === undefined) return null;
+  if (first.startsWith('invisible_unicode_')) {
+    return `Blocked: the entry contains an invisible unicode character ${first.slice('invisible_unicode_'.length)} (possible hidden instruction).`;
+  }
+  return `Blocked: the entry matches the threat pattern '${first}'. Memory is placed in the system prompt of every future chat, `
+    + 'so it must not contain injection or exfiltration instructions.';
+}