@try-works/dsh-memory 0.1.31 → 0.2.0

package/src/entry.js

@@ -1,10 +1,10 @@
 // Entry format (Format 2), parsing, and id allocation. Pure (no fs) — unit-testable.
-export const LAYER_CODES = { semantic: 'sem', episodic: 'epi', procedural: 'pro', learning: 'learning', codebase: 'code', history: 'hist', decision: 'dec' }
-export const LAYER_FILES = { semantic: 'semantic.md', episodic: 'episodic.md', procedural: 'procedural.md', learning: 'learnings.md', codebase: 'codebase.md', history: 'history.md', decision: 'decisions.md' }
+export const LAYER_CODES = { semantic: 'sem', episodic: 'epi', procedural: 'pro', learning: 'learning', codebase: 'code', history: 'hist', decision: 'dec', asset: 'ast' }
+export const LAYER_FILES = { semantic: 'semantic.md', episodic: 'episodic.md', procedural: 'procedural.md', learning: 'learnings.md', codebase: 'codebase.md', history: 'history.md', decision: 'decisions.md', asset: 'assets.md' }
 export const LAYERS = Object.keys(LAYER_CODES)
 
 /** Sub-directory holding the per-topic files for each layered kind. */
-export const LAYER_DIRS = { semantic: 'semantic', episodic: 'episodic', procedural: 'procedural', learning: 'learnings', codebase: 'codebase', history: 'history', decision: 'decisions' }
+export const LAYER_DIRS = { semantic: 'semantic', episodic: 'episodic', procedural: 'procedural', learning: 'learnings', codebase: 'codebase', history: 'history', decision: 'decisions', asset: 'assets' }
 
 /** Layered kinds, in a stable order. */
 export const LAYERED = Object.keys(LAYER_DIRS)
@@ -84,6 +84,39 @@ export function formatEntry({ id, source, date, importance = 2, status = 'active
   return lines.join('\n') + '\n\n' + content
 }
 
+/**
+ * Section titles used inside an asset entry's body. They are RENDERED as H3 (`### What it is`,
+ * `### When to read it`, `### Notes`) because `##` is Format 2's entry delimiter; H2 titles are
+ * still accepted defensively for entries captured before that decision, so an asset entry is
+ * recognised before one of these is allowed to start a new entry.
+ */
+export const ASSET_SECTION_TITLES = ['What it is', 'When to read it', 'Notes']
+
+function isAssetSection(title) {
+  return ASSET_SECTION_TITLES.some((section) => section.toLowerCase() === String(title ?? '').trim().toLowerCase())
+}
+
+/** True when `line` starts a NEW entry for the entry whose header lines begin at `from`. */
+function isAssetSectionHeading(current, line) {
+  if (current === null || current.meta['Asset-Sha256'] === undefined) return false
+  const heading = line.match(/^##\s+(.+)$/)
+  return heading !== null && isAssetSection(heading[1])
+}
+
+/**
+ * Find the line index where the entry containing `idIdx` ends: the next `## ` line that is not an
+ * asset body section. Shared by the in-place editors so a refresh cannot cut an asset body apart.
+ */
+function entryEndLine(lines, idIdx) {
+  for (let i = idIdx + 1; i < lines.length; i++) {
+    const heading = lines[i].match(/^##\s+(.+)$/)
+    if (!heading) continue
+    const isAsset = lines.slice(idIdx, i).some((line) => /^\*\*Asset-Sha256:\*\*/.test(line))
+    if (!(isAsset && isAssetSection(heading[1]))) return i
+  }
+  return lines.length
+}
+
 /**
  * Parse a layer file into entries: each '## heading' starts an entry, following
  * '**Key:** value' lines populate its metadata and the rest is the body. Returns
@@ -98,10 +131,14 @@ export function parseEntries(text) {
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i]
     const h = line.match(/^##\s+(.+)$/)
-    if (h) { push(); current = { heading: h[1].trim(), startLine: i + 1, endLine: undefined, lastLine: i + 1, meta: {}, body: [] }; continue }
+    const assetBodySection = isAssetSectionHeading(current, line)
+    if (h && !assetBodySection) { push(); current = { heading: h[1].trim(), startLine: i + 1, endLine: undefined, lastLine: i + 1, meta: {}, body: [] }; continue }
     if (!current) continue
     current.lastLine = i + 1
-    const m = line.match(/^\*\*([A-Za-z-]+):\*\*\s*(.*)$/)
+    // Header keys may contain digits (`Asset-Sha256`, `Asset-Bytes`): the original
+    // [A-Za-z-]+ class silently pushed those lines into the body, so the asset headers could be
+    // rendered but never parsed back.
+    const m = line.match(/^\*\*([A-Za-z0-9_-]+):\*\*\s*(.*)$/)
     if (m) current.meta[m[1]] = m[2].trim()
     else current.body.push(line)
   }
@@ -111,6 +148,10 @@ export function parseEntries(text) {
     .map((e) => ({
       id: e.meta['Id'],
       heading: e.heading,
+      // The raw header map, exposed because feature-specific headers (Asset-*, and any future
+      // kind) cannot all be projected here without turning this module into a schema of every
+      // feature. The known fields below stay the stable projection.
+      meta: e.meta,
       importance: e.meta['Importance'] ? Number(e.meta['Importance']) : 2,
       status: e.meta['Status'] ?? 'active',
       date: e.meta['Date'],
@@ -133,8 +174,7 @@ export function setEntryStatus(text, id, status, supersededBy) {
   const lines = String(text).split('\n')
   const idIdx = lines.findIndex((l) => l.includes('**Id:** ' + id))
   if (idIdx < 0) return String(text)
-  let end = lines.length
-  for (let i = idIdx + 1; i < lines.length; i++) { if (/^##\s+/.test(lines[i])) { end = i; break } }
+  const end = entryEndLine(lines, idIdx)
   for (let i = idIdx; i < end; i++) {
     if (/^\*\*Status:\*\*/.test(lines[i])) {
       lines[i] = '**Status:** ' + status
@@ -153,8 +193,7 @@ export function setEntryBody(text, id, newBody) {
   const lines = String(text).split('\n')
   const idIdx = lines.findIndex((l) => l.includes('**Id:** ' + id))
   if (idIdx < 0) return String(text)
-  let end = lines.length
-  for (let i = idIdx + 1; i < lines.length; i++) { if (/^##\s+/.test(lines[i])) { end = i; break } }
+  const end = entryEndLine(lines, idIdx)
   // A refresh rewrites the body because the fact was RE-CONFIRMED, so it must reset the
   // freshness clock too. Verified is the age basis, so bumping only Updated would leave a
   // rewritten entry permanently stale - the refresh pass could never cure anything.
@@ -169,11 +208,17 @@ export function setEntryBody(text, id, newBody) {
     const updatedAt = lines.findIndex((l, i) => i > idIdx && i < end && /^\*\*Updated:\*\*/.test(l))
     if (updatedAt >= 0) lines.splice(updatedAt + 1, 0, '**Verified:** ' + stamp)
   }
-  let bodyStart = end
-  for (let i = idIdx; i < end; i++) {
+  // Recompute the boundary AFTER the insertion above: it shifted every later line by one, and a
+  // stale end left the last old body line behind on every rewrite of an entry without Verified.
+  const bodyEnd = entryEndLine(lines, idIdx)
+  let bodyStart = bodyEnd
+  for (let i = idIdx; i < bodyEnd; i++) {
     if (/^\*\*Format:\*\*/.test(lines[i])) { bodyStart = i + 1; break }
   }
-  while (bodyStart < end && lines[bodyStart].trim() === '') bodyStart++
-  lines.splice(bodyStart, end - bodyStart, ...String(newBody).split('\n'))
+  // Asset headers are written after Format, so a plain "skip blank lines" body rewrite would
+  // replace the metadata (sha256, mime, bytes, origin) together with the documentation. The body
+  // starts after the last Asset-* header; a non-asset entry is unaffected.
+  while (bodyStart < bodyEnd && (lines[bodyStart].trim() === '' || /^\*\*Asset-[A-Za-z0-9-]+:\*\*/.test(lines[bodyStart]))) bodyStart++
+  lines.splice(bodyStart, bodyEnd - bodyStart, ...String(newBody).split('\n'))
   return lines.join('\n')
 }

package/src/host.js

@@ -24,6 +24,7 @@ import { buildMap } from './map.js'
 import { backupTasks, defaultSchedulePath, planRestore, readScheduleStore } from './tasks.js'
 import { autoAssociateSession } from './session.js'
 import { readAssociations } from './workspace.js'
+import { describeAsset, usedForFrom } from './assets/describe.js'
 
 const PLUGIN = 'dsh-memory'
 
@@ -95,7 +96,7 @@ function resolveTarget(agent) {
   return undefined
 }
 
-export function wireHost(ctx, store, cfg) {
+export function wireHost(ctx, store, cfg, options = {}) {
   const keyBySession = new Map()
   const scheduler = new SyncScheduler({ commitDebounceMs: cfg.commitDebounceMs, syncIntervalMs: cfg.autoSyncIntervalMs })
   const captureThrottle = new Map()
@@ -110,6 +111,15 @@ export function wireHost(ctx, store, cfg) {
   let lastHistoryAttempt = 0
   let syncInFlight = false
 
+  // The asset store module is loaded lazily, at the first asset operation. `options.assets` is the
+  // test/embedding seam; production passes nothing and the real module is imported.
+  let assetsModulePromise
+  async function assetsApi() {
+    if (options.assets) return options.assets
+    assetsModulePromise = assetsModulePromise ?? import('./assets/index.js')
+    return assetsModulePromise
+  }
+
   const git = makeGitRunner(ctx)
   const llmService = ctx.get('llm')
 
@@ -309,6 +319,183 @@ export function wireHost(ctx, store, cfg) {
       .catch((error) => ctx.logger.warn('dsh-memory: capture failed: ' + (error && error.message)))
   }
 
+  // --- automatic attachment capture ---------------------------------------------------------
+  // An image or file the user puts in a message is stored automatically when its class toggle is
+  // on. The design is explicit that this does NOT prompt: the session event has no caller waiting
+  // for an answer, so a quota wall is recorded as a notice memory_status can surface rather than a
+  // question that would block a turn. The bytes are read from the attachment service AT USE, and
+  // never at wire time.
+  const capturedAttachments = new Set()
+  const IMAGE_EXTENSIONS = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/gif': 'gif' }
+
+  function userMessageText(content) {
+    const parts = Array.isArray(content) ? content : []
+    return parts
+      .filter((part) => part && part.type === 'text' && typeof part.text === 'string')
+      .map((part) => part.text)
+      .join(' ')
+      .replace(/\s+/g, ' ')
+      .trim()
+      .slice(0, 500)
+  }
+
+  /** The durable attachment references one user message carries, image and file parts alike. */
+  function attachmentRefs(content) {
+    const parts = Array.isArray(content) ? content : []
+    const refs = []
+    for (const part of parts) {
+      if (!part || typeof part !== 'object') continue
+      if (part.type !== 'image' && part.type !== 'file') continue
+      if (!part.attachment || typeof part.attachment !== 'object') continue
+      refs.push({ kind: part.type, ref: part.attachment })
+    }
+    return refs
+  }
+
+  /** The reference resolved to a host path (cheap) or verified bytes (fallback). */
+  async function readAttachment(item) {
+    const attachments = ctx.get('attachments')
+    if (!attachments) return undefined
+    try {
+      if (item.kind === 'image') {
+        const hostPath = typeof attachments.imageHostPath === 'function' ? attachments.imageHostPath(item.ref) : undefined
+        if (hostPath && existsSync(hostPath)) return { path: hostPath }
+        if (typeof attachments.readImage !== 'function') return undefined
+        const stored = await attachments.readImage(item.ref)
+        return { bytes: stored?.data }
+      }
+      const hostPath = typeof attachments.fileHostPath === 'function' ? attachments.fileHostPath(item.ref) : undefined
+      if (hostPath && existsSync(hostPath)) return { path: hostPath }
+      if (typeof attachments.readFileStream !== 'function') return undefined
+      const chunks = []
+      for await (const chunk of attachments.readFileStream(item.ref)) chunks.push(Buffer.from(chunk))
+      return { bytes: Buffer.concat(chunks) }
+    } catch (error) {
+      ctx.logger.debug('dsh-memory: could not read a user attachment: ' + (error && error.message))
+      return undefined
+    }
+  }
+
+  function maybeCaptureAttachments(session, event) {
+    if (isSubagentSession(session)) return
+    const data = event?.data
+    // Only a HUMAN message: plugin-injected context and question replies carry their own sources.
+    // A missing source is treated as human because a harness-written user/message is the human
+    // turn; every non-human producer in this deployment names its kind.
+    const sourceKind = data?.source?.kind
+    if (!data || (sourceKind !== undefined && sourceKind !== 'user')) return
+    const refs = attachmentRefs(data.content)
+    if (refs.length === 0) return
+    // DEFERRED, like the map publisher: this handler runs inside an append, and capture reads
+    // bytes and may run a model call. A macrotask lands after the append has settled.
+    setTimeout(() => {
+      void captureAttachments(session, data, refs).catch((error) => ctx.logger.warn('dsh-memory: asset capture failed: ' + (error && error.message)))
+    }, 0)
+  }
+
+  async function captureAttachments(session, data, refs) {
+    const assets = await assetsApi()
+    const key = await resolveKey(session)
+    const scope = key === 'global' ? 'global' : 'workspace'
+    const keyArg = scope === 'global' ? undefined : key
+    const toggles = assets.resolveToggles(cfg, store.root, { scope, ...(keyArg ? { key: keyArg } : {}) })
+    const userMessage = userMessageText(data.content)
+    // assetsDescribe: false means no model call is spent documenting a capture; the store's
+    // deterministic floor writes the two sections instead.
+    const describe = cfg.assetsDescribe === false
+      ? undefined
+      : (context) => describeAsset(context, { llm: llmService ? (prompt) => runLlm(undefined, prompt, 'asset-describe', 700) : undefined })
+    for (const item of refs) {
+      const attachmentId = String(item.ref?.attachmentId ?? '')
+      const dedupeKey = session.id + ':' + attachmentId
+      if (attachmentId === '' || capturedAttachments.has(dedupeKey)) continue
+
+      const mime = item.kind === 'image' && typeof item.ref.mediaType === 'string' ? item.ref.mediaType : undefined
+      const name = typeof item.ref.name === 'string' && item.ref.name !== ''
+        ? item.ref.name
+        : item.kind === 'image' ? ('image.' + (IMAGE_EXTENSIONS[mime] ?? 'png')) : 'attachment'
+      const assetClass = assets.classify({ name, ...(mime ? { mime } : {}) })
+      const reason = !toggles.enabled
+        ? 'feature-off'
+        : assetClass === undefined ? 'unsupported-class'
+          : !toggles[assetClass] ? 'toggle-off' : undefined
+      if (reason) {
+        // A counted skip, never a silent drop: "why wasn't this stored?" has an answer.
+        record(store.root, 'asset-skipped', {
+          count: 1,
+          detail: { class: assetClass ?? 'unknown', reason, scope, ...(keyArg ? { key: keyArg } : {}), name },
+        })
+        continue
+      }
+
+      const source = await readAttachment(item)
+      if (source === undefined) {
+        record(store.root, 'asset-skipped', {
+          count: 1,
+          detail: { class: assetClass, reason: 'attachments-unavailable', scope, ...(keyArg ? { key: keyArg } : {}), name },
+        })
+        continue
+      }
+
+      const captureContext = {
+        ...(userMessage ? { userMessage } : {}),
+        toolCall: 'session/event: user attachment',
+        name,
+        ...(mime ? { mime } : {}),
+        class: assetClass,
+      }
+      const input = {
+        ...(source.path ? { path: source.path } : { bytes: source.bytes }),
+        name,
+        ...(mime ? { mime } : {}),
+        class: assetClass,
+        scope,
+        ...(keyArg ? { key: keyArg } : {}),
+        context: captureContext,
+        usedFor: usedForFrom(captureContext),
+        ...(describe ? { describe } : {}),
+        runner: (argv, args) => git.run(argv, args),
+      }
+      let result
+      try { result = await assets.putAsset(store, cfg, input) } catch (error) {
+        result = { ok: false, code: 'write-failed', message: String(error?.message ?? error) }
+      }
+      if (result && result.ok) {
+        // Mark the attachment only AFTER it was stored: a failed capture must be retryable if the
+        // same file is sent again, and `putAsset` already recorded `asset-added` itself.
+        capturedAttachments.add(dedupeKey)
+        // The scheduler still needs to know the store changed, or the bytes would wait for the
+        // next cadence to be committed and synced.
+        scheduler.markDirty()
+        continue
+      }
+      const code = String(result?.code ?? 'write-failed')
+      const detail = result?.detail && typeof result.detail === 'object' ? result.detail : {}
+      if (code === 'quota-exceeded') {
+        try {
+          assets.addQuotaNotice(store, {
+            scope,
+            ...(keyArg ? { key: keyArg } : {}),
+            usedBytes: detail.usedBytes,
+            quotaBytes: detail.quotaBytes,
+          })
+        } catch { /* a diagnostic notice must never fail the turn */ }
+      }
+      record(store.root, 'asset-skipped', {
+        count: 1,
+        detail: {
+          class: assetClass,
+          reason: code,
+          scope,
+          ...(keyArg ? { key: keyArg } : {}),
+          ...(code === 'quota-exceeded'
+            ? { usedBytes: Number(detail.usedBytes) || 0, quotaBytes: Number(detail.quotaBytes) || 0 }
+            : {}),
+        },
+      })
+    }
+  }
+
   /**
    * What the consolidation cadence should act on now.
    *
@@ -465,7 +652,7 @@ export function wireHost(ctx, store, cfg) {
    * states mean different things and only some of them are work. 'abandoned' and 'orphaned' in
    * particular are the answers to 'did anything quietly give up', which nothing could answer before.
    */
-  function statusNow(agent) {
+  async function statusNow(agent) {
     const base = store.status()
     const states = summarizeQueue(store.root)
     const counts = {}
@@ -484,6 +671,15 @@ export function wireHost(ctx, store, cfg) {
     // 1), then a sample of ids to act on. The full list is one query away for anyone who wants it.
     const attention = groupProblems(problems(store.root))
     const work = due(store.root)
+    // The asset numbers and any quota wall an automatic capture met. assetBytesReport reads entries
+    // and the inventory only; readQuotaNotices is the pending board for "why wasn't that stored?".
+    let assets
+    try {
+      const api = await assetsApi()
+      const report = api.assetBytesReport(store, cfg)
+      const quotaNotices = typeof api.readQuotaNotices === 'function' ? api.readQuotaNotices(store) : []
+      assets = { ...report, ...(quotaNotices.length > 0 ? { quotaNotices } : {}) }
+    } catch { /* the asset layer is optional; its absence must not hide the rest of the status */ }
     return {
       root: base.root,
       pendingDigests: pendingDigests(store).length,
@@ -500,7 +696,7 @@ export function wireHost(ctx, store, cfg) {
       conflicted: Boolean(base.sync && base.sync.conflicted),
       lastSync: base.sync && base.sync.last ? base.sync.last : null,
       metrics: readMetrics(store.root),
-      ...(agent ? {} : {}),
+      ...(assets ? { assets } : {}),
     }
   }
 
@@ -878,6 +1074,9 @@ export function wireHost(ctx, store, cfg) {
     } else if (event.type === 'turn/end') {
       maybeCapture(session)
       setTimeout(() => publishMemoryMap(session), 0)
+    } else if (event.type === 'user/message') {
+      // Automatic attachment capture. Deferred inside the helper; never asks a question.
+      maybeCaptureAttachments(session, event)
     }
   })
 

package/src/refresh.js

@@ -41,9 +41,14 @@ export function parseRefreshDecisions(text) {
 
 export async function runRefresh(store, { llm, limit = 3, scope, key, nowMs = Date.now() } = {}) {
   const ledger = buildLedger(store.root)
+  // Assets are IMMUTABLE in phase 1: memory_asset_forget is the only thing that removes one. A
+  // refresh supersede would hide the entry from listAssets while its inventory refcount never
+  // dropped, leaving the blob in the store and in LFS with nothing pointing at it. The layer is
+  // excluded before the freshness scan, so the model is never even asked about it.
+  const refreshable = ledger.filter((e) => e.layer !== 'asset')
   const scoped = scope === 'workspace' && key
-    ? ledger.filter((e) => e.file.startsWith('workspaces/' + key + '/'))
-    : ledger
+    ? refreshable.filter((e) => e.file.startsWith('workspaces/' + key + '/'))
+    : refreshable
   const flagged = scanFreshness(scoped, nowMs).slice(0, limit)
   if (flagged.length === 0) return { refreshed: 0, updated: 0, superseded: 0 }
   const text = await llm(buildRefreshPrompt(flagged))

package/src/search.js

@@ -4,15 +4,47 @@
 // trigram index serves SUBSTRING queries - scripts written without spaces, which the word
 // tokenizer cannot express at all. A word query that finds nothing also retries as a substring,
 // so a partial word still matches without paying for the trigram scan on every search.
-import { existsSync } from 'node:fs'
+import { existsSync, readFileSync } from 'node:fs'
+import { join, sep } from 'node:path'
 import { DatabaseSync } from 'node:sqlite'
 import { buildFtsQuery, buildIndex, buildLikeQuery, buildTrigramQuery, indexPath, needsSubstring, SUBSTRING_TABLE } from './fts.js'
+import { parseEntries } from './entry.js'
+import { layerOfFile } from './files.js'
 import { buildLedgerFor } from './ledger.js'
 import { rankBlend } from './rank.js'
 
 /** Rows pulled from the word index before ranking. */
 const CANDIDATES = 200
 
+/**
+ * The `asset` block for a hit that is an asset entry: { sha256, mime, name, bytes, class }.
+ *
+ * It is read from the hit's ENTRY (headers + headings), never from the blob, and never by walking
+ * the object tree: search stays a documentation lookup. The block is what tells an agent that the
+ * bytes exist and can be fetched with memory_asset_get.
+ */
+function assetBlockFor(root, hit, cache) {
+  const file = join(root, String(hit.file).split('/').join(sep))
+  if (layerOfFile(root, file) !== 'asset') return undefined
+  let entries = cache.get(file)
+  if (entries === undefined) {
+    try { entries = parseEntries(readFileSync(file, 'utf8')) } catch { entries = [] }
+    cache.set(file, entries)
+  }
+  const entry = entries.find((candidate) => candidate.id === hit.id)
+    ?? entries.find((candidate) => hit.line >= candidate.startLine && hit.line <= candidate.endLine)
+  const meta = entry && entry.meta
+  const sha256 = meta && meta['Asset-Sha256']
+  if (typeof sha256 !== 'string' || sha256 === '') return undefined
+  return {
+    sha256,
+    mime: String(meta['Asset-Mime'] ?? ''),
+    name: String(meta['Asset-Name'] ?? entry.heading ?? ''),
+    bytes: Number(meta['Asset-Bytes']) || 0,
+    class: String(meta['Asset-Class'] ?? ''),
+  }
+}
+
 /**
  * A LIKE scan over the indexed lines. Used only when a MATCH cannot express the query: a
  * no-space script shorter than the trigram minimum. Bounded by the same candidate cap.
@@ -69,8 +101,20 @@ export function search(store, query, { limit = 10, scope, key } = {}) {
     const hits = filtered.map((r) => ({ file: r.file, line: r.line, section: r.section, snippet: String(r.content).trim() }))
     // Ranking needs importance and recency only for the files that matched. Building the whole
     // ledger here read ~8,000 files for a handful of hits and made one query cost ~15 s.
-    return rankBlend(hits, buildLedgerFor(store.root, hits.map((h) => h.file))).slice(0, limit)
-      .map((h) => ({ file: h.file, line: h.line, section: h.section ?? '', snippet: h.snippet, id: h.id ?? '', score: Math.round(h.score * 1000) / 1000 }))
+    const ranked = rankBlend(hits, buildLedgerFor(store.root, hits.map((h) => h.file))).slice(0, limit)
+    const assetCache = new Map()
+    return ranked.map((h) => {
+      const asset = assetBlockFor(store.root, h, assetCache)
+      return {
+        file: h.file,
+        line: h.line,
+        section: h.section ?? '',
+        snippet: h.snippet,
+        id: h.id ?? '',
+        score: Math.round(h.score * 1000) / 1000,
+        ...(asset ? { asset } : {}),
+      }
+    })
   } finally {
     db.close()
   }