@try-works/dsh-memory 0.1.32 → 0.2.0

package/STATUS.md

@@ -3,10 +3,35 @@
 A tracker and planner: what exists, what is verified working, what is written but not
 loaded, and what is left. Kept as a record of MEASURED state, not intention.
 
-**Last measured: 2026-11-01** (409 tests passing; plugin at 0.1.30, published and installed
+**Last measured: 2026-11-01** (522 tests passing; plugin at 0.2.0, published and installed
 byte-identical on harness 0.2.1-rc.2; queue terminal-clean; the Memory tab reads the published
 map under the new event name; the FTS index is 148.6 MB after a rebuild that used to be 436 MB,
-search measures 46-274 ms and an injection 3.3 s).
+search measures 46-274 ms and an injection 3.3 s; the asset store ships phase 1).
+
+## Asset store, phase 1 (0.2.0)
+
+Built from `docs/asset-store-proposal.md` and `docs/asset-api-contract.md` by three agents working
+in parallel against one contract, with TDD throughout (failing test first; the acceptance suite is
+independent of the implementation).
+
+What exists: a new `assets` layer in both scopes; content-addressed blobs under
+`assets/objects/<ab>/<sha256>.<ext>` carried by Git LFS (`.gitattributes`, `git lfs install
+--local`, pointer detection); five tools (`memory_asset_put/get/list/forget/policy`); automatic
+attachment capture on a human message when the class toggle is on, with no prompt; three class
+toggles resolved as config (hard off) → workspace policy → global policy; 50 MB per file and 1 GB
+per scope with refusals that surface the figures and ask the user through `ctx.userQuestions`
+(forgive / raise / forget / cancel) instead of pruning; one model description from capture context
+with a deterministic fallback; `memory_status` reporting asset bytes and quota notices; search
+hits on asset entries carrying `{ sha256, mime, name, bytes, class }`.
+
+What does NOT exist, by design: content indexing. The store documents a file ("What it is", "When
+to read it"); it never extracts or indexes the file's contents, and a token that appears only
+inside a PDF is deliberately not searchable (pinned by the acceptance suite).
+
+Two integration decisions worth keeping: `refresh` skips the asset layer entirely (a supersede
+would hide the entry while its inventory refcount stayed, orphaning the blob), and an automatic
+capture over quota records a notice instead of asking, because `session/event` is reentrancy-
+sensitive and has no answerer; only `memory_asset_put` asks.
 
 ## Store performance, 2026-11-01
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@try-works/dsh-memory",
-  "version": "0.1.32",
+  "version": "0.2.0",
   "description": "Git-backed, workspace-scoped memory for the DeepSeek Harness (global layer + per-workspace layers)",
   "type": "module",
   "main": "src/index.js",

package/src/assets/describe.js

@@ -0,0 +1,181 @@
+// The one model call that documents a captured asset.
+//
+// An asset is stored as bytes plus an entry, and the entry is what search finds later. At capture
+// time the agent still has the context that made the file relevant - the user's request, the tool
+// call, the task - and a later reader does not. This module turns that context (and ONLY that
+// context: the file itself is never opened) into the two sections of the entry body:
+//
+//   ## What it is
+//   ## When to read it
+//
+// The model is advisory. Capture must never fail for want of a description, so every failure
+// (no model routed, a throw, an empty reply, an unparseable one) falls back to a deterministic
+// pair of lines derived from the same context. "Deterministic" is a promise: the same context
+// yields byte-identical text, with no timestamps and no randomness.
+
+/** Bound on each generated section, so a verbose model cannot bloat the entry and the index. */
+export const MAX_SECTION_CHARS = 1200
+/** Bound on the excerpt of the user message quoted in the deterministic fallback. */
+export const MAX_EXCERPT_CHARS = 160
+
+/** Normalize any context value to bounded, single-line text without throwing on weird input. */
+function text(value, maxChars = MAX_SECTION_CHARS) {
+  if (value === undefined || value === null) return ''
+  let rendered
+  if (typeof value === 'string') rendered = value
+  else {
+    try { rendered = JSON.stringify(value) } catch { rendered = String(value) }
+  }
+  return String(rendered ?? '').replace(/\s+/g, ' ').trim().slice(0, maxChars)
+}
+
+/**
+ * The fields this module documents from. Anything else in the input - a path, bytes, a contents
+ * field, a count - is deliberately ignored: this module never reads the file, so it cannot
+ * describe it.
+ *
+ * putAsset calls the describe hook with a small META object ({ class, mime, name, bytes,
+ * context: {userMessage, toolCall, task} }), while a direct caller passes the bare context. Both
+ * shapes are accepted here so the hook and the unit-test seam are the same function.
+ */
+function captureFacts(context = {}) {
+  const source = context && typeof context === 'object' ? context : {}
+  const nested = source.context && typeof source.context === 'object' ? source.context : {}
+  return {
+    name: text(source.name, 240) || 'untitled file',
+    mime: text(source.mime, 120) || undefined,
+    class: text(source.class, 40) || undefined,
+    userMessage: text(source.userMessage ?? nested.userMessage, MAX_EXCERPT_CHARS) || undefined,
+    toolCall: text(source.toolCall ?? nested.toolCall, 120) || undefined,
+    task: text(source.task ?? nested.task, MAX_EXCERPT_CHARS) || undefined,
+  }
+}
+
+/**
+ * The deterministic `Asset-Used-For:` line.
+ *
+ * This is ALWAYS written to the entry, model or no model: it is the one sentence that survives a
+ * deployment with no routed model, and it records the situation in which the file was captured.
+ * Pure and deterministic for a given context.
+ */
+export function usedForFrom(context = {}) {
+  const facts = captureFacts(context)
+  const parts = []
+  if (facts.userMessage) parts.push('while answering "' + facts.userMessage + '"')
+  if (facts.task) parts.push('for the task "' + facts.task + '"')
+  if (facts.toolCall) parts.push('(via ' + facts.toolCall + ')')
+  if (parts.length === 0) {
+    return 'Captured from the session context; the description was written without reading the file.'
+  }
+  return 'Captured ' + parts.join(' ') + '; the description was written without reading the file.'
+}
+
+/**
+ * The deterministic fallback description. Public because tests (and any caller that wants to
+ * know what the fallback will say before spending a model call) must be able to pin it.
+ * @returns {{whatItIs: string, whenToRead: string, usedFor: string}}
+ */
+export function fallbackDescription(context = {}) {
+  const facts = captureFacts(context)
+  const kind = facts.class ? facts.class.replace(/s$/, '') + ' file' : 'file'
+  const type = facts.mime ? ' (' + facts.mime + ')' : ''
+  const whatItIs = 'A ' + kind + ' named "' + facts.name + '"' + type
+    + ', stored from the context of this session. Its contents were not read, so this line is a pointer rather than a summary.'
+  const trigger = facts.userMessage ? 'the request "' + facts.userMessage + '"'
+    : facts.task ? 'the task "' + facts.task + '"'
+      : 'a later request in this area'
+  const whenToRead = 'Read this when a task returns to ' + trigger
+    + ' or explicitly needs "' + facts.name + '". Open it with the appropriate reader; nothing from inside the file is indexed.'
+  return { whatItIs, whenToRead, usedFor: usedForFrom(context) }
+}
+
+/**
+ * The single prompt sent to the model. It spoils nothing beyond the capture context, and says so
+ * explicitly: a model asked to "describe this PDF" without being told it cannot see it will
+ * invent a plausible table of contents, which is exactly the failure this store can least afford.
+ */
+export function buildDescribePrompt(context = {}) {
+  const facts = captureFacts(context)
+  const lines = [
+    'Write the catalogue entry for a file being kept in a memory store.',
+    'You have NOT read the file and cannot see its contents. Describe the file only from the capture context below, and say nothing that would require opening it. Do not invent sections, tables, numbers, or page counts. Write plain sentences: no markdown headings and no bullet lists.',
+    '',
+    'Reply in exactly this form, with no preamble and no extra sections:',
+    '',
+    'WHAT IT IS:',
+    '<one or two sentences on what this file is, from the context>',
+    '',
+    'WHEN TO READ IT:',
+    '<one or two sentences on which future requests should open it, and which should not>',
+    '',
+    'Capture context:',
+    '- name: ' + facts.name,
+  ]
+  if (facts.mime) lines.push('- mime type: ' + facts.mime)
+  if (facts.class) lines.push('- class: ' + facts.class)
+  if (facts.userMessage) lines.push('- what the user was doing: ' + facts.userMessage)
+  if (facts.task) lines.push('- task: ' + facts.task)
+  if (facts.toolCall) lines.push('- tool call: ' + facts.toolCall)
+  return lines.join('\n')
+}
+
+/**
+ * Parse the two sections out of a model reply.
+ *
+ * Tolerant of the forms a model actually produces: `WHAT IT IS:` / `When to read it:`, with or
+ * without markdown heading markers. Returns undefined unless BOTH sections yield non-empty text,
+ * because half an entry is worse than the deterministic fallback: a missing "When to read it"
+ * turns the asset invisible to the only query that matters. An optional NOTES section is carried
+ * through when the model volunteered one; the text itself never keeps its section heading.
+ * @returns {{whatItIs: string, whenToRead: string, notes?: string}|undefined}
+ */
+export function parseDescription(reply) {
+  const body = String(reply ?? '').trim()
+  if (body === '') return undefined
+  const section = (heading) => {
+    const pattern = new RegExp(
+      '(?:^|\\n)\\s*(?:#{1,6}\\s*)?' + heading + '\\s*:?\\s*\\n([\\s\\S]*?)(?=(?:\\n\\s*(?:#{1,6}\\s*)?(?:WHAT IT IS|WHEN TO READ IT|NOTES)\\s*:?\\s*(?:\\n|$))|$)',
+      'i',
+    )
+    const match = pattern.exec(body)
+    const value = match ? match[1].trim() : ''
+    return value === '' ? undefined : value.slice(0, MAX_SECTION_CHARS)
+  }
+  const whatItIs = section('WHAT IT IS')
+  const whenToRead = section('WHEN TO READ IT')
+  if (!whatItIs || !whenToRead) return undefined
+  const notes = section('NOTES')
+  return notes === undefined ? { whatItIs, whenToRead } : { whatItIs, whenToRead, notes }
+}
+
+/**
+ * Describe one captured asset: one model call, or the deterministic fallback.
+ *
+ * The returned text has NO heading markers of its own - the entry renderer owns the `### What it
+ * is` / `### When to read it` headings and demotes anything heading-like inside the text.
+ *
+ * @param context - capture context or putAsset's describe-hook meta. Only the documentation facts
+ *   are used; a path or bytes in the input is ignored, because this function never reads the file.
+ * @param options.llm - optional async (prompt) => text. Called at most once.
+ * @returns {{whatItIs: string, whenToRead: string, usedFor: string, source: 'model'|'fallback', reason?: string, notes?: string}}
+ */
+export async function describeAsset(context = {}, { llm } = {}) {
+  const fallback = fallbackDescription(context)
+  if (typeof llm !== 'function') return { ...fallback, source: 'fallback', reason: 'no model available' }
+  let reply
+  try {
+    reply = await llm(buildDescribePrompt(context))
+  } catch (error) {
+    return { ...fallback, source: 'fallback', reason: 'the model call failed: ' + text(error && error.message ? error.message : error, 160) }
+  }
+  if (typeof reply !== 'string' || reply.trim() === '') {
+    return { ...fallback, source: 'fallback', reason: 'the model returned nothing' }
+  }
+  const parsed = parseDescription(reply)
+  if (!parsed) {
+    return { ...fallback, source: 'fallback', reason: 'the model reply did not contain both sections' }
+  }
+  return { ...parsed, usedFor: fallback.usedFor, source: 'model' }
+}
+
+export default describeAsset

package/src/assets/entry.js

@@ -0,0 +1,169 @@
+// Asset entries: an ordinary Format 2 entry in the `asset` layer, plus Asset-* headers and the
+// documentation sections search relies on.
+//
+// Rendering and parsing are pure, so the round trip is testable without a store. The bytes never
+// enter the entry: it holds metadata and the documentation a later reader searches for.
+import { formatEntry } from '../entry.js'
+
+/** Body section titles. Search relies on exactly these headings. */
+export const SECTION_WHAT = 'What it is'
+export const SECTION_WHEN = 'When to read it'
+export const SECTION_NOTES = 'Notes'
+
+/**
+ * Body sections are H3, not H2: `## ` is Format 2's entry delimiter, so an H2 section would split
+ * an asset into an entry plus ignored pseudo-entries. The parser still accepts H2 section titles for
+ * entries written before that decision (and for hand-written ones), which is why parseEntries also
+ * carries the defensive H2-as-body rule for entries that already hold `Asset-Sha256`.
+ */
+const SECTION_PREFIX = '### '
+
+/** Collapse a value onto one line: headers are one field per line, and a newline would end it. */
+function oneLine(value) {
+  return String(value ?? '').replace(/\r?\n+/g, ' ').trim()
+}
+
+function optionalNumber(value) {
+  const number = Number(value)
+  return Number.isFinite(number) && number >= 0 ? number : 0
+}
+
+function assetHeaderLines(asset = {}) {
+  const lines = [
+    '**Asset-Class:** ' + oneLine(asset.class),
+    '**Asset-Mime:** ' + oneLine(asset.mime),
+    '**Asset-Bytes:** ' + String(optionalNumber(asset.bytes)),
+    '**Asset-Sha256:** ' + oneLine(asset.sha256),
+    '**Asset-Name:** ' + oneLine(asset.name),
+    '**Asset-Origin:** ' + oneLine(asset.origin ?? 'inline'),
+    '**Asset-Used-For:** ' + oneLine(asset.usedFor),
+  ]
+  const description = oneLine(asset.description)
+  if (description !== '') lines.push('**Asset-Description:** ' + description)
+  return lines
+}
+
+/**
+ * Normalize a documentation body: demote any markdown heading inside it. A model-supplied
+ * description may contain `## something`; at column zero that would terminate the entry (and, for
+ * the three reserved titles, the section), so nested headings are lifted to `###`.
+ */
+function sectionText(value) {
+  return String(value ?? '')
+    .replace(/\r\n/g, '\n')
+    .split('\n')
+    .map((line) => line.replace(/^(\s*)#{1,6}\s+/, '$1### '))
+    .join('\n')
+    .trim()
+}
+
+function section(title, text) {
+  const body = sectionText(text)
+  return body === '' ? '' : SECTION_PREFIX + title + '\n\n' + body
+}
+
+/**
+ * Render one asset entry block, heading included.
+ * @param input.id - Format 2 id (`global-ast-0001`).
+ * @param input.asset - { class, mime, bytes, sha256, name, origin, usedFor, description? }.
+ * @param input.documentation - { whatItIs, whenToRead, notes? }; the two named sections are the
+ *   search surface and are always written by putAsset (which supplies a deterministic fallback).
+ * @returns the markdown block, ready for appendToTopic.
+ */
+export function renderAssetEntry({
+  id,
+  source,
+  date,
+  importance = 2,
+  status = 'active',
+  volatility,
+  verified,
+  until,
+  heading,
+  asset,
+  documentation = {},
+} = {}) {
+  const title = oneLine(heading ?? asset?.name) || 'asset'
+  const sections = [
+    section(SECTION_WHAT, documentation.whatItIs),
+    section(SECTION_WHEN, documentation.whenToRead),
+  ]
+  const notes = section(SECTION_NOTES, documentation.notes)
+  if (notes !== '') sections.push(notes)
+  const content = [assetHeaderLines(asset).join('\n'), ...sections.filter((part) => part !== '')]
+    .filter((part) => part !== '')
+    .join('\n\n')
+  const metadata = formatEntry({
+    id,
+    source,
+    date,
+    importance,
+    status,
+    volatility,
+    verified,
+    until,
+    content,
+  })
+  return '## ' + title + '\n\n' + metadata
+}
+
+/** Split an entry body into the three documented sections. H3 is the format; H2 is tolerated. */
+function parseSections(content) {
+  const sections = { whatItIs: undefined, whenToRead: undefined, notes: undefined }
+  const buffers = { whatItIs: [], whenToRead: [], notes: [] }
+  let current
+  const flush = () => {
+    if (current === undefined) return
+    const text = buffers[current].join('\n').trim()
+    if (text !== '') sections[current] = text
+  }
+  for (const line of String(content ?? '').split('\n')) {
+    const heading = line.match(/^#{2,3}\s+(.+)$/)
+    if (heading) {
+      flush()
+      const title = heading[1].trim().toLowerCase()
+      current = title === SECTION_WHAT.toLowerCase()
+        ? 'whatItIs'
+        : title === SECTION_WHEN.toLowerCase()
+          ? 'whenToRead'
+          : title === SECTION_NOTES.toLowerCase()
+            ? 'notes'
+            : undefined
+      if (current !== undefined) buffers[current] = []
+      continue
+    }
+    if (current !== undefined) buffers[current].push(line)
+  }
+  flush()
+  return sections
+}
+
+/**
+ * Parse an asset entry from the projection parseEntries returns (raw `meta` + body).
+ * @returns the asset, or undefined when the entry is not an asset entry.
+ */
+export function parseAssetEntry(entry) {
+  if (entry === null || entry === undefined || entry.meta === null || entry.meta === undefined) return undefined
+  const meta = entry.meta
+  const sha256 = meta['Asset-Sha256']
+  if (!sha256) return undefined
+  const sections = parseSections(entry.content)
+  return {
+    id: entry.id,
+    heading: entry.heading,
+    date: entry.date,
+    updated: entry.updated,
+    status: entry.status,
+    name: meta['Asset-Name'],
+    class: meta['Asset-Class'],
+    mime: meta['Asset-Mime'],
+    bytes: optionalNumber(meta['Asset-Bytes']),
+    sha256,
+    origin: meta['Asset-Origin'],
+    usedFor: meta['Asset-Used-For'],
+    description: meta['Asset-Description'],
+    whatItIs: sections.whatItIs,
+    whenToRead: sections.whenToRead,
+    notes: sections.notes,
+  }
+}