@ijfw/memory-server 1.4.4 → 1.5.0

package/src/memory/obsidian-parser.js

@@ -0,0 +1,91 @@
+// IJFW v1.5.0 -- Obsidian-grade markdown extractor.
+//
+// Pulls three structured signals out of memory body text:
+//   1. [[wikilinks]]            -> links: [{ target, line }]
+//   2. #nested/tags             -> tags:  [{ path, depth }]
+//   3. [key:: value] (Dataview) -> meta:  [{ key, value }]
+//
+// Code fences and inline-code spans are masked before extraction so that
+// example syntax never produces fake edges. DB writes happen in
+// indexObsidianRelations (idempotent re-index: clears prior rows for the
+// memory id before re-inserting).
+
+const FENCE_RE = /```[\s\S]*?```/g;
+const INLINE_CODE_RE = /`[^`\n]+`/g;
+// eslint-disable-next-line security/detect-unsafe-regex -- parses developer-authored markdown notes on local disk; negated [^\]\n] classes bound match to one line per token
+const WIKILINK_RE = /\[\[([^\]\n|]+)(?:\|[^\]\n]+)?\]\]/g;
+const TAG_RE = /(?:^|[^\w&])#([\w/-]+)/g;
+const META_RE = /\[([A-Za-z_][\w-]*)::\s*([^\]\n]+?)\]/g;
+
+function maskCode(text) {
+  return text
+    .replace(FENCE_RE, (m) => ' '.repeat(m.length))
+    .replace(INLINE_CODE_RE, (m) => ' '.repeat(m.length));
+}
+
+function lineOf(masked, idx) {
+  let line = 1;
+  for (let i = 0; i < idx; i++) if (masked.charCodeAt(i) === 10) line++;
+  return line;
+}
+
+function normaliseLinkTarget(raw) {
+  return raw
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, '-')
+    .replace(/[^a-z0-9/_-]/g, '');
+}
+
+export function parseObsidian(text) {
+  if (typeof text !== 'string' || text.length === 0) {
+    return { links: [], tags: [], meta: [] };
+  }
+  const masked = maskCode(text);
+
+  const links = [];
+  for (const m of masked.matchAll(WIKILINK_RE)) {
+    const target = normaliseLinkTarget(m[1]);
+    if (target) links.push({ target, line: lineOf(masked, m.index) });
+  }
+
+  const tagSet = new Map();
+  for (const m of masked.matchAll(TAG_RE)) {
+    const path = m[1].toLowerCase().replace(/^\/+|\/+$/g, '');
+    if (!path) continue;
+    const depth = path.split('/').length;
+    if (!tagSet.has(path)) tagSet.set(path, { path, depth });
+  }
+
+  const meta = [];
+  for (const m of masked.matchAll(META_RE)) {
+    meta.push({ key: m[1].toLowerCase(), value: m[2].trim() });
+  }
+
+  return { links, tags: [...tagSet.values()], meta };
+}
+
+export function indexObsidianRelations(db, memoryId, text) {
+  const parsed = parseObsidian(text);
+  const insLink = db.prepare(
+    'INSERT OR IGNORE INTO memory_links (from_id, to_target, line) VALUES (?, ?, ?)',
+  );
+  const insTag = db.prepare(
+    'INSERT OR IGNORE INTO memory_tags (memory_id, tag_path, depth) VALUES (?, ?, ?)',
+  );
+  const insMeta = db.prepare(
+    'INSERT OR IGNORE INTO memory_meta (memory_id, key, value) VALUES (?, ?, ?)',
+  );
+  const tx = db.transaction(() => {
+    db.prepare('DELETE FROM memory_links WHERE from_id=?').run(memoryId);
+    db.prepare('DELETE FROM memory_tags WHERE memory_id=?').run(memoryId);
+    db.prepare('DELETE FROM memory_meta WHERE memory_id=?').run(memoryId);
+    for (const l of parsed.links) insLink.run(memoryId, l.target, l.line);
+    for (const t of parsed.tags) insTag.run(memoryId, t.path, t.depth);
+    for (const m of parsed.meta) insMeta.run(memoryId, m.key, m.value);
+  });
+  tx();
+  return parsed;
+}
+
+export default { parseObsidian, indexObsidianRelations };

package/src/memory/query-dataview.js

@@ -0,0 +1,86 @@
+// IJFW v1.5.0 -- Dataview-grade declarative query for memory_entries.
+//
+// Supported grammar (v1 — intentionally minimal; expand in v1.6+):
+//   tag = #path[/sub]*           prefix-matches memory_tags.tag_path
+//   linked_to = "target"         matches memory_links.to_target
+//   created_after = <unix-secs>  memory_entries.created_at > N
+//   created_before = <unix-secs> memory_entries.created_at < N
+//
+// Multiple filters AND together via case-insensitive " and ".
+// Whitespace-tolerant. Unrecognised clauses are silently skipped
+// (caller can inspect parsed.filters for __unrecognised entries).
+
+const CLAUSE_AND = /\s+and\s+/i;
+const TAG_RE = /^\s*tag\s*=\s*#?([\w/_-]+)\s*$/i;
+const LINKED_RE = /^\s*linked_to\s*=\s*"([^"\n]+)"\s*$/i;
+const CREATED_AFTER_RE = /^\s*created_after\s*=\s*(\d+)\s*$/i;
+const CREATED_BEFORE_RE = /^\s*created_before\s*=\s*(\d+)\s*$/i;
+
+export function parseDataviewQuery(input) {
+  const clauses = String(input || '').split(CLAUSE_AND);
+  const out = { filters: [] };
+  for (const raw of clauses) {
+    const c = raw.trim();
+    if (!c) continue;
+    let m;
+    if ((m = c.match(TAG_RE))) {
+      out.tag = m[1].toLowerCase().replace(/^\/+|\/+$/g, '');
+      continue;
+    }
+    if ((m = c.match(LINKED_RE))) {
+      out.filters.push({ field: 'linked_to', op: '=', value: m[1] });
+      continue;
+    }
+    if ((m = c.match(CREATED_AFTER_RE))) {
+      out.filters.push({ field: 'created_after', op: '=', value: Number(m[1]) });
+      continue;
+    }
+    if ((m = c.match(CREATED_BEFORE_RE))) {
+      out.filters.push({ field: 'created_before', op: '=', value: Number(m[1]) });
+      continue;
+    }
+    out.filters.push({ field: '__unrecognised', op: '=', value: c });
+  }
+  return out;
+}
+
+export function runDataviewQuery(db, parsed) {
+  const where = [];
+  const params = [];
+  let join = '';
+  if (parsed.tag) {
+    join += ' JOIN memory_tags t ON t.memory_id = e.id ';
+    where.push('(t.tag_path = ? OR t.tag_path LIKE ?)');
+    params.push(parsed.tag, `${parsed.tag}/%`);
+  }
+  for (const f of parsed.filters) {
+    switch (f.field) {
+      case 'linked_to':
+        join += ' JOIN memory_links l ON l.from_id = e.id ';
+        where.push('l.to_target = ?');
+        params.push(f.value);
+        break;
+      case 'created_after':
+        where.push('e.created_at > ?');
+        params.push(f.value);
+        break;
+      case 'created_before':
+        where.push('e.created_at < ?');
+        params.push(f.value);
+        break;
+      default: /* __unrecognised silently skipped */
+    }
+  }
+  // Note: memory_entries (migration 001 schema) has columns:
+  // id (INTEGER PK), body, source, session_id, created_at. No title column.
+  // memory_links/_tags/_meta store TEXT memory_id — SQLite coerces across
+  // the type boundary at JOIN time (SQLite is permissive without strict FKs).
+  const sql = `SELECT DISTINCT e.id, e.body, e.source, e.created_at
+               FROM memory_entries e ${join}
+               ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
+               ORDER BY e.created_at DESC`;
+  const rows = db.prepare(sql).all(...params);
+  return { rows, rowCount: rows.length, parsed, sql };
+}
+
+export default { parseDataviewQuery, runDataviewQuery };

package/src/memory/search.js

@@ -58,10 +58,20 @@ async function loadMemoryMigrationsSync() {
   const v1 = await import('./migrations/001-fts5-init.js');
   const v2 = await import('./migrations/002-tier-semantic.js');
   const v3 = await import('./migrations/003-stale-candidate.js');
+  const v4 = await import('./migrations/004-bitemporal.js');
+  const v5 = await import('./migrations/005-vector-cache.js');
+  const v6 = await import('./migrations/006-obsidian-graph.js');
+  const v7 = await import('./migrations/007-skill-telemetry.js');
+  const v8 = await import('./migrations/008-write-provenance.js');
   return [
     { version: v1.VERSION, description: v1.DESCRIPTION, up: v1.up },
     { version: v2.VERSION, description: v2.DESCRIPTION, up: v2.up },
     { version: v3.VERSION, description: v3.DESCRIPTION, up: v3.up },
+    { version: v4.VERSION, description: v4.DESCRIPTION, up: v4.up },
+    { version: v5.VERSION, description: v5.DESCRIPTION, up: v5.up },
+    { version: v6.VERSION, description: v6.DESCRIPTION, up: v6.up },
+    { version: v7.VERSION, description: v7.DESCRIPTION, up: v7.up },
+    { version: v8.VERSION, description: v8.DESCRIPTION, up: v8.up },
   ].sort((a, b) => a.version - b.version);
 }