dsh-instruction-memory 1.4.0 → 1.4.2

package/lib/index.js

@@ -30,7 +30,7 @@ import { readFileSync, watch } from 'node:fs'
 import { randomUUID } from 'node:crypto'
 import { isIP } from 'node:net'
 import { homedir, hostname } from 'node:os'
-import { basename, dirname, isAbsolute, join } from 'node:path'
+import { basename, dirname, isAbsolute, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import {
   FEEDBACK_ENV,
@@ -93,6 +93,15 @@ const MAX_SETS = 20
 const MAX_SET_NAME = 30
 /** Rolling history snapshots (F14): how many are kept PER writing device. */
 const MAX_HISTORY = 20
+/**
+ * Byte ceiling for one device's snapshots (F14). The count alone bounded
+ * nothing in practice: at the 200 × 6,000-character cap a single snapshot is
+ * ~3.6 MB, so twenty of them were ~72 MB per device — and a custom backup
+ * directory inside a cloud-synced folder meant that much upload traffic for
+ * one checkbox toggle. Newest snapshots win; the newest is always kept even
+ * when it alone exceeds the budget.
+ */
+const MAX_HISTORY_BYTES = 32 * 1024 * 1024
 /**
  * F7: the longest single pause. Doubles as the setTimeout guard — 7 days is
  * far below the 2^31-1 ms (≈24.8 days) timer ceiling, so the delay never
@@ -165,7 +174,14 @@ const PLUGIN_VERSION = (() => {
  */
 export function resolveStorePath(env = process.env, home = homedir()) {
   const configured = typeof env.DSH_HOME === 'string' ? env.DSH_HOME.trim() : ''
-  const root = configured !== '' ? configured : join(home, '.dsh')
+  // A relative override (`DSH_HOME=./data`) is resolved ONCE against the
+  // process cwd. Without this the store path stayed relative for the whole
+  // process: the settings page promised an absolute path (README), the
+  // watcher's dirname() became '.', and the "same file" identity depended on
+  // how the path was spelled. Resolving does not make a relative override
+  // cwd-independent — it cannot be — but it pins one absolute location for
+  // this process and keeps every displayed path absolute.
+  const root = configured !== '' ? resolve(configured) : join(home, '.dsh')
   return join(root, STORE_DIR_NAME, STORE_FILE_NAME)
 }
 
@@ -512,6 +528,41 @@ export function exportFileName(epoch) {
   return 'dsh-instruction-memory-' + stamp.slice(0, 8) + '-' + stamp.slice(9, 15) + 'Z.json'
 }
 
+/**
+ * Which snapshots survive retention, for ONE writing device.
+ *
+ * Two ceilings apply together: the count (MAX_HISTORY) and a byte budget
+ * (MAX_HISTORY_BYTES). Counting alone bounded nothing at the top end — see the
+ * constant's note — while a byte budget alone would let a flood of tiny saves
+ * grow the directory without limit. The newest snapshot is always kept, even
+ * when it alone exceeds the byte budget: a history that prunes its own latest
+ * version is worse than a large one.
+ *
+ * Pure and exported so the policy is unit-testable without writing 32 MB.
+ *
+ * @param {{ name: string, size: number }[]} items ascending by name (chronological)
+ * @returns {{ keep: string[], remove: string[] }} `remove` is oldest-first
+ */
+export function planHistoryPrune(items, maxCount = MAX_HISTORY, maxBytes = MAX_HISTORY_BYTES) {
+  const list = (Array.isArray(items) ? items : [])
+    .filter((item) => item !== null && typeof item === 'object' && typeof item.name === 'string' && item.name !== '')
+  const keep = []
+  let bytes = 0
+  for (let index = list.length - 1; index >= 0; index -= 1) {
+    const size = Number.isFinite(list[index].size) ? Math.max(0, list[index].size) : 0
+    // keep.length === 0 keeps the newest unconditionally (see above).
+    if (keep.length === 0 || (keep.length < maxCount && bytes + size <= maxBytes)) {
+      keep.push(list[index].name)
+      bytes += size
+    }
+  }
+  const kept = new Set(keep)
+  return {
+    keep,
+    remove: list.filter((item) => !kept.has(item.name)).map((item) => item.name),
+  }
+}
+
 /**
  * Which fields of a raw entry had to be cut to fit the stored limits.
  *
@@ -748,6 +799,28 @@ function normalizeData(raw) {
   return loaded
 }
 
+/**
+ * The scalar half of the store shape, in ONE place.
+ *
+ * The boot state, normalizeData, persistableData, cloneData and snapshot() all
+ * spell these fields out. A new field added to four of them and forgotten in
+ * the fifth is dropped on save, lost on rollback, or invisible to the panel —
+ * the boot state's own comment already warns about exactly that drift. The
+ * array halves stay explicit because each caller transforms them differently
+ * (copy, filter to the active set, or map to {id,name}).
+ */
+const storeScalars = (data) => ({
+  version: FORMAT_VERSION,
+  rev: typeof data.rev === 'number' && Number.isFinite(data.rev) && data.rev > 0 ? data.rev : 0,
+  enabled: data.enabled === true,
+  disabledUntil: typeof data.disabledUntil === 'number' && Number.isFinite(data.disabledUntil) && data.disabledUntil > 0
+    ? data.disabledUntil
+    : 0,
+  budgetChars: clampBudget(data.budgetChars),
+  backupDir: typeof data.backupDir === 'string' ? data.backupDir : '',
+  activeSetId: data.activeSetId,
+})
+
 /**
  * A structural copy of the store, shallow per entry.
  *
@@ -759,15 +832,7 @@ function normalizeData(raw) {
  */
 function cloneData(data) {
   return {
-    version: FORMAT_VERSION,
-    rev: data.rev || 0,
-    enabled: data.enabled === true,
-    disabledUntil: typeof data.disabledUntil === 'number' && Number.isFinite(data.disabledUntil) && data.disabledUntil > 0
-      ? data.disabledUntil
-      : 0,
-    budgetChars: data.budgetChars,
-    backupDir: typeof data.backupDir === 'string' ? data.backupDir : '',
-    activeSetId: data.activeSetId,
+    ...storeScalars(data),
     sets: (Array.isArray(data.sets) ? data.sets : []).map((set) => ({ ...set })),
     entries: (Array.isArray(data.entries) ? data.entries : []).map((entry) => ({ ...entry })),
     profiles: (Array.isArray(data.profiles) ? data.profiles : []).map((profile) => ({ ...profile })),
@@ -1290,17 +1355,19 @@ export function apply(ctx) {
 
   const state = {
     data: {
-      version: FORMAT_VERSION,
-      rev: 0,
-      enabled: true,
-      // Must match normalizeData's output shape KEY FOR KEY: the watcher's
-      // self-write detection compares JSON strings, and a missing key here
-      // made every fresh process's first save look like an external change
-      // (one redundant reload + frame).
-      disabledUntil: 0,
-      budgetChars: DEFAULT_BUDGET,
-      backupDir: '',
-      activeSetId: DEFAULT_SET_ID,
+      // Scalars come from the shared shape (see storeScalars) so this literal
+      // cannot drift from normalizeData / persistableData / snapshot. The key
+      // ORDER still matters: the watcher's self-write detection compares JSON
+      // strings, and a missing or reordered key made every fresh process's
+      // first save look like an external change (one redundant reload + frame).
+      ...storeScalars({
+        rev: 0,
+        enabled: true,
+        disabledUntil: 0,
+        budgetChars: DEFAULT_BUDGET,
+        backupDir: '',
+        activeSetId: DEFAULT_SET_ID,
+      }),
       sets: [{ id: DEFAULT_SET_ID, name: DEFAULT_SET_NAME }],
       entries: [],
       profiles: [],
@@ -1525,7 +1592,20 @@ export function apply(ctx) {
     const wanted = sectionText() !== ''
     if (wanted && sectionDispose === null) {
       try {
-        sectionDispose = systemPrompt.section({ name: SECTION_NAME, order: SECTION_ORDER, text: assembledSectionText })
+        // `interpolate: false` is load-bearing, not a hint. The host runs every
+        // section through `{{variable}}` interpolation by DEFAULT and THROWS on
+        // a reference it cannot resolve (`unknown prompt variable "{{x}}" in
+        // section "instruction-memory"`). Instruction memory is user prose — an
+        // nginx snippet (`proxy_pass {{upstream}};`), a Vue/Jinja/Handlebars
+        // template or any `{{…}}` pair — so the default would abort the whole
+        // agent turn at renderPrompt(). This section is literal text and never
+        // references a host variable, so opt out explicitly.
+        sectionDispose = systemPrompt.section({
+          name: SECTION_NAME,
+          order: SECTION_ORDER,
+          text: assembledSectionText,
+          interpolate: false,
+        })
         state.sectionRegistered = true
         state.sectionError = null
         logDebug('prompt section registered: order=' + SECTION_ORDER + ' chars=' + sectionText().length)
@@ -1576,10 +1656,12 @@ export function apply(ctx) {
   /* ---------------- external-change watching (SSE) ---------------- */
 
   /**
-   * Live subscribers to the settings page's event stream. Each entry maps a
-   * stream's `send(payload)` writer to its own teardown, so a write that fails
-   * (the socket went away) can dispose THAT stream instead of just dropping
-   * it from the broadcast set while its response stays open.
+   * Live subscribers to the settings page's event stream. Each entry is
+   * `{ cleanup, lite }`: `cleanup` is that stream's own teardown, so a write
+   * that fails (the socket went away) can dispose THAT stream instead of just
+   * dropping it from the broadcast set while its response stays open. `lite`
+   * records whether the subscriber asked for lightweight frames (see
+   * pushSnapshot).
    */
   const sseStreams = new Map()
   let storeWatcher = null
@@ -1587,14 +1669,25 @@ export function apply(ctx) {
 
   const pushSnapshot = () => {
     if (disposed || sseStreams.size === 0) return
-    const payload = 'data: ' + JSON.stringify(snapshot()) + '\n\n'
-    for (const [send, cleanup] of Array.from(sseStreams)) {
+    // A lite stream only needs to know that the store moved: it fetches the
+    // snapshot through POST `state` when the rev it sees differs from the one
+    // on screen. The full frame is built LAZILY, so a page whose subscribers
+    // are all lite (the current client) never pays for serialising the whole
+    // store — up to ~3.4 MB at the 200 × 6,000-character cap — on every save.
+    const litePayload = 'data: ' + JSON.stringify({ rev: state.data.rev || 0, lite: true }) + '\n\n'
+    let fullPayload = null
+    for (const [send, entry] of Array.from(sseStreams)) {
       try {
-        send(payload)
+        if (entry.lite === true) {
+          send(litePayload)
+        } else {
+          if (fullPayload === null) fullPayload = 'data: ' + JSON.stringify(snapshot()) + '\n\n'
+          send(fullPayload)
+        }
       } catch {
         // The response is dead: run its full teardown (end + unsubscribe),
         // not just a set delete that leaves the stream half-registered.
-        cleanup()
+        entry.cleanup()
       }
     }
   }
@@ -1674,9 +1767,9 @@ export function apply(ctx) {
   }
 
   const disposeStreams = () => {
-    for (const cleanup of Array.from(sseStreams.values())) {
+    for (const entry of Array.from(sseStreams.values())) {
       try {
-        cleanup()
+        entry.cleanup()
       } catch {
         // The response is already gone.
       }
@@ -1698,15 +1791,7 @@ export function apply(ctx) {
    */
   const persistableData = () => ({
     $schema: SCHEMA_REF,
-    version: FORMAT_VERSION,
-    rev: state.data.rev || 0,
-    enabled: state.data.enabled === true,
-    disabledUntil: typeof state.data.disabledUntil === 'number' && Number.isFinite(state.data.disabledUntil) && state.data.disabledUntil > 0
-      ? state.data.disabledUntil
-      : 0,
-    budgetChars: clampBudget(state.data.budgetChars),
-    backupDir: typeof state.data.backupDir === 'string' ? state.data.backupDir : '',
-    activeSetId: state.data.activeSetId,
+    ...storeScalars(state.data),
     sets: (Array.isArray(state.data.sets) ? state.data.sets : []).map((set) => ({ ...set })),
     entries: (Array.isArray(state.data.entries) ? state.data.entries : []).map((entry) => ({ ...entry })),
     profiles: (Array.isArray(state.data.profiles) ? state.data.profiles : []).map((profile) => ({ ...profile })),
@@ -1766,6 +1851,14 @@ export function apply(ctx) {
     const token = randomUUID()
     const deadline = Date.now() + LOCK_WAIT_MS
     for (;;) {
+      // The deadline is checked at the TOP of every attempt, not only on the
+      // "someone else holds it" branch. It used to sit below the stale-lock
+      // branch, which `continue`d after a failed unlink — so a stale marker
+      // that could not be deleted (a Windows handle without delete sharing, a
+      // read-only volume) spun here forever, and every later disk operation
+      // queued behind a save that would never return. Waiting is always
+      // bounded; proceeding unlocked is the documented fallback.
+      if (Date.now() >= deadline) return null
       try {
         await writeFile(lockPath, token, { flag: 'wx' })
         return { path: lockPath, token }
@@ -1776,10 +1869,10 @@ export function apply(ctx) {
           // Crashed-holder debris. It may be recreated by a live process
           // between this unlink and the next wx attempt — that case simply
           // loops and re-evaluates freshness instead of forcing its way in.
+          // A failed unlink must not become a tight loop, so every attempt
+          // still sleeps below.
           await unlink(lockPath).catch(() => {})
-          continue
         }
-        if (Date.now() >= deadline) return null
         await new Promise((resolve) => setTimeout(resolve, 50))
       }
     }
@@ -1845,11 +1938,14 @@ export function apply(ctx) {
   let lastHistoryText = null
 
   /**
-   * Enforce the per-device retention limit. Grouping by the device tag inside
-   * the filename is deliberate: two machines sharing one backup folder (the
-   * documented F18 setup) each keep MAX_HISTORY of their OWN snapshots, so one
-   * device can never prune the other's history away. Files that do not match
-   * our naming pattern are never touched.
+   * Enforce the per-device retention policy: the count and byte ceilings from
+   * planHistoryPrune(). Grouping by the device tag inside the filename is
+   * deliberate: two machines sharing one backup folder (the documented F18
+   * setup) each keep their OWN budget, so one device can never prune the
+   * other's history away. Files that do not match our naming pattern are never
+   * touched. Sizes come from stat() rather than a maintained index because the
+   * directory is shared with other processes and a stale index would prune the
+   * wrong files.
    */
   const pruneHistory = async (dir) => {
     const names = (await readdir(dir)).filter((name) => HISTORY_FILE_PATTERN.test(name))
@@ -1865,15 +1961,17 @@ export function apply(ctx) {
     for (const bucket of byDevice.values()) {
       // Fixed-width UTC stamp first => lexicographic order is chronological.
       bucket.sort()
-      const excess = bucket.length - MAX_HISTORY
-      for (let index = 0; index < bucket.length; index += 1) {
-        if (index < excess) {
-          await unlink(join(dir, bucket[index])).catch(() => {})
-          removed += 1
-        } else {
-          kept += 1
-        }
+      const sized = []
+      for (const name of bucket) {
+        const info = await stat(join(dir, name)).catch(() => null)
+        sized.push({ name, size: info === null ? 0 : info.size })
+      }
+      const plan = planHistoryPrune(sized)
+      for (const name of plan.remove) {
+        await unlink(join(dir, name)).catch(() => {})
+        removed += 1
       }
+      kept += plan.keep.length
     }
     return { removed, kept }
   }
@@ -2346,15 +2444,7 @@ export function apply(ctx) {
     const activeProfiles = state.data.profiles.filter((profile) => inScope(profile, scope))
     return {
       data: {
-        version: FORMAT_VERSION,
-        rev: state.data.rev || 0,
-        enabled: state.data.enabled === true,
-        disabledUntil: typeof state.data.disabledUntil === 'number' && Number.isFinite(state.data.disabledUntil) && state.data.disabledUntil > 0
-          ? state.data.disabledUntil
-          : 0,
-        budgetChars: clampBudget(state.data.budgetChars),
-        backupDir: typeof state.data.backupDir === 'string' ? state.data.backupDir : '',
-        activeSetId: state.data.activeSetId,
+        ...storeScalars(state.data),
         sets: state.data.sets.map((set) => ({ id: set.id, name: set.name })),
         entries: activeEntries.map((entry) => ({ ...entry })),
         profiles: activeProfiles.map((profile) => ({ ...profile })),
@@ -2466,6 +2556,16 @@ export function apply(ctx) {
         minBudget: MIN_BUDGET,
         maxBudget: MAX_BUDGET,
         maxEntries: MAX_ENTRIES,
+        // Per-field ceilings. The editor used to hard-code 120/6000/200 and
+        // 8/24 for tags, so changing a limit here left the panel silently
+        // disagreeing with the Host (it would accept text the loader then
+        // truncated, with a note the user could not have predicted). Every
+        // limit the panel enforces is published here instead.
+        maxTitle: MAX_TITLE,
+        maxContent: MAX_CONTENT,
+        maxWhen: MAX_WHEN,
+        maxTagsPerEntry: MAX_TAGS_PER_ENTRY,
+        maxTagLength: MAX_TAG_LENGTH,
         maxProfiles: MAX_PROFILES,
         maxProfileName: MAX_PROFILE_NAME,
         maxProfileContent: MAX_PROFILE_CONTENT,
@@ -3713,16 +3813,29 @@ export function apply(ctx) {
       return { ok: false, message: '反馈发送过于频繁，请 ' + waitSeconds + ' 秒后再试' }
     }
 
+    // Reserve the window BEFORE the awaits below. Checking the timestamp here and
+    // writing it only after a successful send straddled them: two concurrent
+    // requests both passed the check and both delivered. A send that never
+    // happens releases the reservation, so a genuine failure can be retried at
+    // once.
+    const previousSentAt = feedbackLastSentAt
+    feedbackLastSentAt = now
+    const releaseReservation = () => {
+      if (feedbackLastSentAt === now) feedbackLastSentAt = previousSentAt
+    }
+
     const configPath = feedbackConfigPath()
     let file = null
     try {
       file = await readFeedbackConfigFile(configPath)
     } catch (err) {
+      releaseReservation()
       logInfo('feedback config unusable: ' + describeError(err))
       return { ok: false, message: describeError(err) }
     }
     const readiness = feedbackConfigFrom({ env: process.env, file, configPath })
     if (!readiness.ok) {
+      releaseReservation()
       logDebug('feedback not configured: ' + readiness.message)
       return { ok: false, message: readiness.message }
     }
@@ -3741,8 +3854,10 @@ export function apply(ctx) {
     // back, so both UI text and the log keep it masked.
     const maskAddresses = (text) => {
       let out = String(text)
-      for (const address of [readiness.config.to, readiness.config.from]) {
-        if (address !== '') out = out.split(address).join('[邮箱已隐藏]')
+      // The SMTP account is masked too: a relay that differs from From/To is
+      // echoed by some servers inside their rejection text.
+      for (const address of [readiness.config.to, readiness.config.from, readiness.config.smtp.user]) {
+        if (typeof address === 'string' && address !== '') out = out.split(address).join('[邮箱已隐藏]')
       }
       return out
     }
@@ -3764,6 +3879,7 @@ export function apply(ctx) {
       // The failure reason is the only actionable thing a maintainer can act
       // on; it never contains the password (see mailer.js), and the configured
       // mailboxes are masked, so logging it is safe and makes reports debuggable.
+      releaseReservation()
       const reason = maskAddresses(describeError(err))
       logInfo('feedback send failed: ' + reason)
       return { ok: false, message: '反馈发送失败：' + reason + '（详细原因已写入宿主日志）' }
@@ -3871,6 +3987,30 @@ export function apply(ctx) {
 
   let routeDispose = null
 
+  /**
+   * Does this stream subscriber want lightweight frames?
+   *
+   * `?lite=1` is additive on purpose: a client that does not ask (every build
+   * before this one) keeps receiving the bare snapshot exactly as documented,
+   * so an old client against a new Host is unchanged. A lite subscriber gets
+   * `{ rev, lite: true }` and fetches the snapshot itself when the rev differs
+   * — which is the difference between pushing ~3.4 MB per save (maximal store)
+   * and pushing ~40 bytes.
+   *
+   * `req.url` is not part of the harness contract, so anything unparseable
+   * degrades to the legacy full frame rather than throwing.
+   */
+  const isLiteStreamRequest = (req) => {
+    const url = req !== null && typeof req === 'object' && typeof req.url === 'string' ? req.url : ''
+    const mark = url.indexOf('?')
+    if (mark === -1) return false
+    try {
+      return new URLSearchParams(url.slice(mark + 1)).get('lite') === '1'
+    } catch {
+      return false
+    }
+  }
+
   /**
    * The live-update stream for the settings page (GET + `Accept:
    * text/event-stream`, i.e. exactly what an EventSource sends).
@@ -3881,7 +4021,8 @@ export function apply(ctx) {
    * (someone navigating to the URL) keeps the old 405.
    *
    * Frames carry the bare snapshot — the same shape `state` answers with —
-   * which the Client's normalizer already accepts.
+   * which the Client's normalizer already accepts. A subscriber that appends
+   * `?lite=1` gets `{ rev, lite: true }` frames instead (see pushSnapshot).
    */
   const handleEventStream = (req, res) => {
     const verdict = requestOriginVerdict(req.headers, guardHost)
@@ -3899,6 +4040,7 @@ export function apply(ctx) {
     // Ask the browser to retry politely if the stream drops (e.g. the host was
     // restarted) instead of hammering the route.
     res.write('retry: 5000\n\n')
+    const lite = isLiteStreamRequest(req)
     const send = (payload) => { res.write(payload) }
     let closed = false
     const cleanup = () => {
@@ -3910,7 +4052,7 @@ export function apply(ctx) {
         // The response is already gone.
       }
     }
-    sseStreams.set(send, cleanup)
+    sseStreams.set(send, { cleanup, lite })
     req.on('close', cleanup)
     req.on('error', cleanup)
     res.on('error', cleanup)
@@ -3918,11 +4060,14 @@ export function apply(ctx) {
     // instead of waiting for the next change. It must wait for boot: a stream
     // opened while the initial disk read is still in flight would otherwise be
     // handed the empty boot state as its first frame, flashing "no
-    // instructions" before the loaded store arrives.
+    // instructions" before the loaded store arrives. A lite subscriber gets
+    // only the current rev and pulls the snapshot itself when it is behind.
     const sendConnectFrame = () => {
       if (closed) return
       try {
-        send('data: ' + JSON.stringify(snapshot()) + '\n\n')
+        send(lite
+          ? 'data: ' + JSON.stringify({ rev: state.data.rev || 0, lite: true }) + '\n\n'
+          : 'data: ' + JSON.stringify(snapshot()) + '\n\n')
       } catch {
         cleanup()
       }