@sema-agent/core 2.13.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/CHANGELOG.md +2345 -0
  2. package/dist/agents/cascade.d.ts +6 -1
  3. package/dist/agents/cascade.js +76 -32
  4. package/dist/agents/cumulative-stats.d.ts +29 -0
  5. package/dist/agents/cumulative-stats.js +29 -0
  6. package/dist/agents/observer.d.ts +4 -0
  7. package/dist/agents/observer.js +11 -10
  8. package/dist/agents/repair-loop.d.ts +2 -0
  9. package/dist/agents/repair-loop.js +110 -3
  10. package/dist/agents/retain-ledger.d.ts +20 -2
  11. package/dist/agents/retain-ledger.js +38 -8
  12. package/dist/agents/roster-store.d.ts +4 -0
  13. package/dist/agents/roster-store.js +20 -1
  14. package/dist/agents/subagent.js +66 -64
  15. package/dist/agents/teacher.d.ts +1 -9
  16. package/dist/agents/teacher.js +23 -24
  17. package/dist/agents/team.js +7 -4
  18. package/dist/agents/verify.d.ts +15 -2
  19. package/dist/agents/verify.js +39 -3
  20. package/dist/bin/sema-tb.js +2 -2
  21. package/dist/brain/anthropic.js +24 -21
  22. package/dist/brain/circuit-breaker.d.ts +1 -0
  23. package/dist/brain/circuit-breaker.js +26 -9
  24. package/dist/brain/degrading.js +13 -1
  25. package/dist/brain/failover.js +2 -0
  26. package/dist/brain/openai.js +24 -21
  27. package/dist/brain/repetition.js +10 -1
  28. package/dist/brain/routing.js +5 -1
  29. package/dist/brain/status-sink.js +5 -1
  30. package/dist/brain/stream-engine.js +5 -2
  31. package/dist/brain/stream-shared.d.ts +1 -0
  32. package/dist/brain/stream-shared.js +14 -0
  33. package/dist/brain/terminal-cause.d.ts +4 -0
  34. package/dist/brain/terminal-cause.js +12 -0
  35. package/dist/brain/tool-call-id.d.ts +1 -0
  36. package/dist/brain/tool-call-id.js +3 -0
  37. package/dist/brain/tool-call-repair.js +34 -15
  38. package/dist/core/auto-compaction.js +33 -32
  39. package/dist/core/background-agent-store.d.ts +22 -22
  40. package/dist/core/background-agent-store.js +19 -23
  41. package/dist/core/checkpoint-store.d.ts +7 -13
  42. package/dist/core/checkpoint-store.js +39 -15
  43. package/dist/core/consolidate-scope.js +4 -3
  44. package/dist/core/context-edit.js +3 -0
  45. package/dist/core/context-guard.js +3 -0
  46. package/dist/core/file-snapshot-store.d.ts +8 -0
  47. package/dist/core/file-snapshot-store.js +4 -3
  48. package/dist/core/git-worktree-env.d.ts +6 -1
  49. package/dist/core/git-worktree-env.js +12 -1
  50. package/dist/core/hooks.d.ts +4 -1
  51. package/dist/core/hooks.js +35 -7
  52. package/dist/core/lsp-diagnostics.js +13 -7
  53. package/dist/core/lsp-protocol.d.ts +1 -1
  54. package/dist/core/lsp-protocol.js +20 -8
  55. package/dist/core/lsp-session.d.ts +12 -2
  56. package/dist/core/lsp-session.js +114 -49
  57. package/dist/core/lsp.d.ts +13 -1
  58. package/dist/core/lsp.js +65 -9
  59. package/dist/core/mailbox-store.d.ts +7 -10
  60. package/dist/core/mcp.d.ts +7 -10
  61. package/dist/core/mcp.js +7 -3
  62. package/dist/core/memory-engine/data-plane.js +4 -0
  63. package/dist/core/memory-engine/engine.d.ts +2 -2
  64. package/dist/core/memory-engine/engine.js +96 -31
  65. package/dist/core/memory-engine/file-backend.d.ts +2 -2
  66. package/dist/core/memory-engine/file-backend.js +38 -39
  67. package/dist/core/memory-engine/layout.d.ts +8 -0
  68. package/dist/core/memory-engine/layout.js +71 -4
  69. package/dist/core/memory-engine/scope-contract.js +9 -3
  70. package/dist/core/memory-engine/types.d.ts +1 -1
  71. package/dist/core/memory.js +3 -0
  72. package/dist/core/permission-rules.d.ts +2 -1
  73. package/dist/core/permission-rules.js +24 -3
  74. package/dist/core/runner/active-skill-scope.d.ts +0 -1
  75. package/dist/core/runner/active-skill-scope.js +1 -15
  76. package/dist/core/runner/assemble-result.d.ts +3 -8
  77. package/dist/core/runner/assemble-result.js +3 -5
  78. package/dist/core/runner/compaction-call-options.d.ts +7 -0
  79. package/dist/core/runner/compaction-call-options.js +30 -0
  80. package/dist/core/runner/prepare-task.d.ts +6 -23
  81. package/dist/core/runner/prepare-task.js +31 -53
  82. package/dist/core/runner/runtask.d.ts +1 -0
  83. package/dist/core/runner/runtask.js +86 -178
  84. package/dist/core/runner/session-rule-policy.js +4 -1
  85. package/dist/core/runner/teardown-bounded.d.ts +7 -0
  86. package/dist/core/runner/teardown-bounded.js +36 -0
  87. package/dist/core/runner/tool-disclosure.d.ts +2 -5
  88. package/dist/core/runner/turn-attachments.d.ts +5 -15
  89. package/dist/core/runner/turn-attachments.js +1 -1
  90. package/dist/core/runner/usage-accounting.d.ts +6 -0
  91. package/dist/core/runner/usage-accounting.js +9 -8
  92. package/dist/core/safe-notify.d.ts +17 -0
  93. package/dist/core/safe-notify.js +59 -0
  94. package/dist/core/secret-env.js +5 -1
  95. package/dist/core/sensitive-path-policy.js +11 -4
  96. package/dist/core/store-contracts/checkpoint-store-contract.js +1 -1
  97. package/dist/core/store-contracts/session-repo-contract.js +40 -0
  98. package/dist/core/stub-env.d.ts +2 -6
  99. package/dist/core/surrogate-safe-slice.d.ts +4 -0
  100. package/dist/core/surrogate-safe-slice.js +18 -0
  101. package/dist/core/task-registry-monitor.js +15 -14
  102. package/dist/core/task-registry-shared.d.ts +22 -0
  103. package/dist/core/task-registry-shared.js +16 -0
  104. package/dist/core/task-registry.d.ts +1 -0
  105. package/dist/core/task-registry.js +40 -57
  106. package/dist/core/tool-errors.js +5 -4
  107. package/dist/core/tool-name-aliases.js +2 -1
  108. package/dist/core/tool-policy.js +72 -19
  109. package/dist/core/tool-result-store.d.ts +1 -0
  110. package/dist/core/tool-result-store.js +17 -1
  111. package/dist/core/trace.d.ts +3 -9
  112. package/dist/core/types.d.ts +16 -12
  113. package/dist/core/untrusted-egress.js +7 -5
  114. package/dist/core/workflow-journal-store.d.ts +9 -20
  115. package/dist/core/workflow-run-store.d.ts +11 -10
  116. package/dist/core/workflow-run-store.js +22 -16
  117. package/dist/engine/compaction/compaction.d.ts +1 -0
  118. package/dist/engine/compaction/compaction.js +14 -5
  119. package/dist/engine/compaction/utils.js +4 -2
  120. package/dist/engine/execution-env/node-execution-env.d.ts +2 -24
  121. package/dist/engine/harness/agent-harness.d.ts +1 -0
  122. package/dist/engine/harness/agent-harness.js +13 -33
  123. package/dist/engine/harness/types.d.ts +29 -67
  124. package/dist/engine/llm/types.d.ts +1 -1
  125. package/dist/engine/loop/agent-loop.js +11 -4
  126. package/dist/engine/loop/types.d.ts +11 -9
  127. package/dist/engine/lsp/frame-decoder.js +6 -3
  128. package/dist/engine/lsp/node-lsp-manager.d.ts +2 -0
  129. package/dist/engine/lsp/node-lsp-manager.js +103 -26
  130. package/dist/engine/lsp/stdio-lsp-transport.js +16 -7
  131. package/dist/engine/session/memory-repo.d.ts +2 -6
  132. package/dist/engine/session/session.d.ts +5 -30
  133. package/dist/index.d.ts +3 -1
  134. package/dist/index.js +2 -1
  135. package/dist/internal/harness-types.d.ts +3 -1
  136. package/dist/internal/harness.d.ts +1 -1
  137. package/dist/internal/harness.js +1 -1
  138. package/dist/orchestration/run-spec.js +7 -4
  139. package/dist/orchestration/run-workflow-tool.js +3 -35
  140. package/dist/orchestration/workflow-governance.js +43 -9
  141. package/dist/orchestration/workflow-sandbox-conformance.js +85 -51
  142. package/dist/orchestration/workflow-script-store.js +34 -4
  143. package/dist/orchestration/workflow.d.ts +3 -11
  144. package/dist/orchestration/workflow.js +164 -204
  145. package/dist/prompt-assembly/artifact-store.d.ts +2 -0
  146. package/dist/prompt-assembly/artifact-store.js +39 -24
  147. package/dist/prompt-assembly/assemble.js +38 -7
  148. package/dist/prompt-assembly/epoch.js +19 -2
  149. package/dist/prompt-assembly/event-registry.js +3 -3
  150. package/dist/prompt-assembly/explain.js +1 -1
  151. package/dist/prompt-assembly/tool-catalog.d.ts +10 -9
  152. package/dist/prompt-assembly/tool-catalog.js +16 -1
  153. package/dist/prompt-assembly/turn-snapshot.d.ts +2 -5
  154. package/dist/prompt-assembly/turn-snapshot.js +4 -1
  155. package/dist/stores/file/background-agent-store.d.ts +3 -12
  156. package/dist/stores/file/background-agent-store.js +3 -41
  157. package/dist/stores/file/checkpoint-store.js +6 -6
  158. package/dist/stores/file/fs-atomic.d.ts +3 -0
  159. package/dist/stores/file/fs-atomic.js +53 -8
  160. package/dist/stores/file/index.d.ts +1 -0
  161. package/dist/stores/file/index.js +7 -0
  162. package/dist/stores/file/mailbox-store.d.ts +2 -6
  163. package/dist/stores/file/session-policy-store.js +15 -1
  164. package/dist/stores/file/session-store.d.ts +4 -6
  165. package/dist/stores/file/session-store.js +28 -1
  166. package/dist/stores/file/workflow-journal-store.d.ts +3 -11
  167. package/dist/stores/file/workflow-run-store.d.ts +3 -7
  168. package/dist/stores/file/workflow-run-store.js +4 -17
  169. package/dist/tools/fs/bash-readonly-classifier.d.ts +2 -0
  170. package/dist/tools/fs/bash-readonly-classifier.js +73 -21
  171. package/dist/tools/fs/fs-bash.d.ts +1 -1
  172. package/dist/tools/fs/fs-bash.js +407 -91
  173. package/dist/tools/fs/fs-pdf.d.ts +2 -12
  174. package/dist/tools/fs/fs-shared.js +2 -2
  175. package/dist/tools/fs/notebook.d.ts +2 -5
  176. package/dist/tools/fs/safety.d.ts +3 -0
  177. package/dist/tools/fs/safety.js +43 -14
  178. package/dist/tools/fs/search.js +5 -5
  179. package/dist/tools/loop-tick.js +1 -1
  180. package/dist/tools/monitor.d.ts +1 -0
  181. package/dist/tools/monitor.js +57 -9
  182. package/dist/tools/worktree.d.ts +4 -7
  183. package/dist/tools/worktree.js +235 -143
  184. package/package.json +9 -2
@@ -8,6 +8,11 @@ export interface McpToolAxis {
8
8
  egress?: true;
9
9
  effect?: ToolEffect;
10
10
  }
11
+ export interface McpDroppedTool {
12
+ server: string;
13
+ tool: string;
14
+ reason: string;
15
+ }
11
16
  export interface MaterializedMcp {
12
17
  tools: AgentTool[];
13
18
  toolAxes: McpToolAxis[];
@@ -23,11 +28,7 @@ export interface MaterializedMcp {
23
28
  pendingRemovals: string[];
24
29
  };
25
30
  warnings: Error[];
26
- droppedTools: Array<{
27
- server: string;
28
- tool: string;
29
- reason: string;
30
- }>;
31
+ droppedTools: McpDroppedTool[];
31
32
  statuses: McpServerStatus[];
32
33
  refresh: (server?: string) => Promise<McpRefreshResult[]>;
33
34
  dispose: () => Promise<void>;
@@ -41,11 +42,7 @@ export interface McpRefreshResult {
41
42
  removed: string[];
42
43
  tools?: AgentTool[];
43
44
  axes?: McpToolAxis[];
44
- dropped?: Array<{
45
- server: string;
46
- tool: string;
47
- reason: string;
48
- }>;
45
+ dropped?: McpDroppedTool[];
49
46
  error?: string;
50
47
  }
51
48
  export interface McpServerStatus {
package/dist/core/mcp.js CHANGED
@@ -7,6 +7,7 @@ import { tmpdir } from "node:os";
7
7
  import { join } from "node:path";
8
8
  import { CallToolResultSchema, ElicitRequestSchema, ErrorCode, ListResourcesResultSchema, McpError } from "@modelcontextprotocol/sdk/types.js";
9
9
  import { MCP_IMAGE_MAX_BASE64, sharpImageResizer } from "./image-downsample.js";
10
+ import { sliceHeadSafe, sliceTailSafe } from "./surrogate-safe-slice.js";
10
11
  import { truncateError } from "./tool-errors.js";
11
12
  import { delimitUntrusted, inlineUntrusted, sanitizeUntrustedText } from "./untrusted-text.js";
12
13
  import { validateJsonSchemaShape } from "./runner/strict-output-schema.js";
@@ -75,8 +76,10 @@ export function truncateMcpErrorText(s) {
75
76
  let head = MCP_ERROR_HEAD_CHARS;
76
77
  let out = "";
77
78
  for (let i = 0; i < 4; i++) {
78
- const removed = s.length - head - MCP_ERROR_TAIL_CHARS;
79
- out = `${s.slice(0, head)}\n\n... [${removed} characters truncated] ...\n\n${s.slice(s.length - MCP_ERROR_TAIL_CHARS)}`;
79
+ const headText = sliceHeadSafe(s, head);
80
+ const tailText = sliceTailSafe(s, MCP_ERROR_TAIL_CHARS);
81
+ const removed = s.length - headText.length - tailText.length;
82
+ out = `${headText}\n\n... [${removed} characters truncated] ...\n\n${tailText}`;
80
83
  if (out.length <= max)
81
84
  break;
82
85
  head = Math.max(0, head - (out.length - max));
@@ -1008,7 +1011,8 @@ function buildResourceTools(resourceServers) {
1008
1011
  const res = await rs.client
1009
1012
  .listResources(undefined, { signal: watchdog.combinedSignal, timeout: timeoutMs })
1010
1013
  .catch((err) => rethrowHonestMcpError(err, { server, what, timeoutMs, writeEffect: false, signal, attributeServer: true, idle: { signal: watchdog.idleSignal, idleMs } }));
1011
- const children = res.resources.filter((r) => typeof r.uri === "string" && r.uri !== uri && r.uri.startsWith(uri));
1014
+ const dirPrefix = uri.endsWith("/") ? uri : `${uri}/`;
1015
+ const children = res.resources.filter((r) => typeof r.uri === "string" && r.uri !== uri && r.uri.startsWith(dirPrefix));
1012
1016
  return renderDirChildren(server, uri, children);
1013
1017
  }
1014
1018
  finally {
@@ -2,6 +2,7 @@ import { uuidv7 } from "../../internal/harness.js";
2
2
  import { firstSentence } from "../memory.js";
3
3
  import { computeEntryRev, parseEntryFile, serializeEntryFile } from "./frontmatter.js";
4
4
  import { scanMemoryFileName, scanMemoryWrite } from "./scan.js";
5
+ import { MEMORY_INDEX_FILENAME } from "./file-backend.js";
5
6
  import { scopeDirName } from "./layout.js";
6
7
  export const REMOTE_HARVEST_PER_FILE_BYTES = 100 * 1024;
7
8
  export const REMOTE_MASS_DELETION_FUSE_RATIO = 0.5;
@@ -97,6 +98,9 @@ export function harvestFilesToPatches(baseline, files, opts = {}) {
97
98
  out.rejections.push({ path: f.relPath, code: "outside_root", reason: "file path escapes the memory root (containment gate, fail-closed)" });
98
99
  continue;
99
100
  }
101
+ const baseName = f.relPath.split("/").pop() ?? f.relPath;
102
+ if (baseName === MEMORY_INDEX_FILENAME || !baseName.endsWith(".md") || baseName.startsWith("."))
103
+ continue;
100
104
  if (inReadonlyDir(f.relPath)) {
101
105
  if (base === undefined) {
102
106
  out.rejections.push({ path: f.relPath, code: "readonly_layer", reason: "a file inside a read-only inherited layer's directory is not writable from this session" });
@@ -56,10 +56,10 @@ export declare class MemoryEngine {
56
56
  };
57
57
  harvest(handle: MemorySessionHandle): Promise<HarvestReport>;
58
58
  private harvestCore;
59
- rebaseline(handle: MemorySessionHandle): Promise<void>;
59
+ rebaseline(handle: MemorySessionHandle, keepBaseline?: ReadonlySet<string>): Promise<void>;
60
60
  private rebuildIndex;
61
61
  private committedContentFor;
62
- private quarantineFile;
62
+ private gateDerivedIndex;
63
63
  private readonlyDirNamesUnderRoot;
64
64
  private writeIfChanged;
65
65
  private writeBackProjection;
@@ -1,12 +1,12 @@
1
1
  import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
2
- import { basename, dirname, join, relative, sep } from "node:path";
2
+ import { dirname, join, relative, sep } from "node:path";
3
3
  import { uuidv7 } from "../../internal/harness.js";
4
4
  import { MAX_MEMORY_BYTES, composeMemoryBlock, firstSentence } from "../memory.js";
5
5
  import { inlineUntrusted } from "../untrusted-text.js";
6
6
  import { formatMemoryAge } from "../memory-recall.js";
7
7
  import { computeEntryRev, parseEntryFile, serializeEntryFile } from "./frontmatter.js";
8
8
  import { DEFAULT_MAX_ENTRY_DEPTH, MEMORY_INDEX_FILENAME, scanEntryFiles } from "./file-backend.js";
9
- import { SCAN_FUSE_THRESHOLD, readIndexRevs, writeIndexRevs, bumpScanFuse, canonicalize, claimRootScope, clearScanFuse, deriveControlPlaneDir, drainMemoryAnnouncements, enqueueMemoryAnnouncement, ensureDirExists, isContainedIn, registerScope, registeredScopes, resolveMemoryEngineRoot, scopeDirFor, } from "./layout.js";
9
+ import { QUARANTINE_DIR, SCAN_FUSE_THRESHOLD, quarantineAndTombstone, readIndexRevs, writeIndexRevs, bumpScanFuse, canonicalize, claimRootScope, clearScanFuse, deriveControlPlaneDir, drainMemoryAnnouncements, enqueueMemoryAnnouncement, ensureDirExists, isContainedIn, registerScope, registeredScopes, resolveMemoryEngineRoot, scopeDirFor, } from "./layout.js";
10
10
  import { scanMemoryFileName, scanMemoryWrite, scanRemediation } from "./scan.js";
11
11
  export const MEMORY_INSTRUCTION_TEMPLATE = `# Memory
12
12
 
@@ -147,6 +147,18 @@ export class MemoryEngine {
147
147
  handle.baseIds.set(path, entry.id);
148
148
  }
149
149
  }
150
+ const indexGate = this.gateDerivedIndex(handle);
151
+ if (indexGate !== undefined) {
152
+ try {
153
+ enqueueMemoryAnnouncement(this.controlDir, {
154
+ kind: "gate",
155
+ at: this.now(),
156
+ items: [`memory index rejected and rebuilt: ${inlineUntrusted(indexGate.reason)}`],
157
+ });
158
+ }
159
+ catch {
160
+ }
161
+ }
150
162
  const indexText = this.rebuildIndex(handle, headers, { write: writeScope !== null });
151
163
  handle.indexBaselineLines = countIndexLines(indexText);
152
164
  handle.indexText = indexText;
@@ -278,6 +290,8 @@ export class MemoryEngine {
278
290
  report.rejections.push({ path: rel, code: "symlink", reason: "symlinks are not harvested into memory (containment gate)" });
279
291
  else if (kind === "depth")
280
292
  report.rejections.push({ path: rel, code: "nested_too_deep", reason: `memory files nested deeper than ${this.maxDepth} levels are not harvested` });
293
+ else if (kind === "unreadable")
294
+ report.rejections.push({ path: rel, code: "unreadable", reason: "the path could not be read (lstat/readdir failed) — its content was NOT harvested this pass" });
281
295
  },
282
296
  });
283
297
  const records = [];
@@ -364,6 +378,7 @@ export class MemoryEngine {
364
378
  basePathById.set(id, p);
365
379
  const patches = [];
366
380
  const pendingProjections = [];
381
+ const idsAddedThisHarvest = new Set();
367
382
  let processed = 0;
368
383
  for (let i = 0; i < kept.length; i++) {
369
384
  const f = kept[i];
@@ -401,7 +416,7 @@ export class MemoryEngine {
401
416
  scanFindings.push(...scanMemoryWrite(f.text));
402
417
  const finding = scanFindings[0];
403
418
  if (finding !== undefined) {
404
- const q = this.quarantineFile(f.canonical, f.text);
419
+ const q = quarantineAndTombstone(f.canonical, f.text, join(this.controlDir, QUARANTINE_DIR), this.now);
405
420
  let contained = q.removed;
406
421
  const priorId = idByBasePath.get(f.canonical);
407
422
  if (priorId !== undefined) {
@@ -465,7 +480,28 @@ export class MemoryEngine {
465
480
  patches.push({ op: "update", id: parsed.id, entry: { ...entry, id: parsed.id, rev: computeEntryRev({ id: parsed.id, frontmatter: fm, body: parsed.body }) }, ...(revByBasePath.get(renamedFrom) !== undefined ? { baseRev: revByBasePath.get(renamedFrom) } : {}) });
466
481
  continue;
467
482
  }
468
- pendingProjections.push({ path: f.canonical, entry, needed: minted || needsCompletion(parsed, fm) });
483
+ let reboundId;
484
+ if (parsed.id !== undefined && (renamedFrom !== undefined || idsAddedThisHarvest.has(parsed.id))) {
485
+ reboundId = parsed.id;
486
+ const idless = serializeEntryFile({ id: "", frontmatter: fm, body: parsed.body }).replace(/^id:[ \t]*$\r?\n/m, "");
487
+ try {
488
+ writeFileSync(f.canonical, idless, "utf8");
489
+ }
490
+ catch (err) {
491
+ report.rejections.push({
492
+ path: rel,
493
+ code: "filename",
494
+ reason: `memory write blocked: this file carries the entry id of another memory file (a copy is not a rename) and its frontmatter could not be rewritten (${err instanceof Error ? err.message : String(err)}) — remove the "id:" line so it can be committed as a new entry`,
495
+ });
496
+ continue;
497
+ }
498
+ id = uuidv7();
499
+ entry.id = id;
500
+ entry.rev = computeEntryRev(entry);
501
+ report.warnings.push(`${rel}: frontmatter id ${reboundId} already has a projection (${renamedFrom !== undefined ? relative(handle.memoryDir, renamedFrom) : "another new file in this harvest"}) — a copy is not a rename, so this file was committed as a NEW entry with a fresh id; the existing entry was left untouched`);
502
+ }
503
+ pendingProjections.push({ path: f.canonical, entry, needed: minted || reboundId !== undefined || needsCompletion(parsed, fm) });
504
+ idsAddedThisHarvest.add(entry.id);
469
505
  patches.push({ op: "add", id: entry.id, entry });
470
506
  }
471
507
  let patchReport;
@@ -503,12 +539,15 @@ export class MemoryEngine {
503
539
  const drained = this.backend.drainInboundFindings?.();
504
540
  if (drained !== undefined && drained.length > 0)
505
541
  report.inboundFindings = drained;
542
+ const indexRejection = this.gateDerivedIndex(handle);
543
+ if (indexRejection !== undefined)
544
+ report.rejections.push(indexRejection);
506
545
  const headers = await this.backend.listHeaders([...new Set([...handle.scopes, writeScope])]);
507
546
  handle.indexText = this.rebuildIndex(handle, headers, { write: true }, report.warnings);
508
- await this.rebaseline(handle);
547
+ await this.rebaseline(handle, new Set(report.degraded?.pending ?? []));
509
548
  return report;
510
549
  }
511
- async rebaseline(handle) {
550
+ async rebaseline(handle, keepBaseline = new Set()) {
512
551
  const writeScope = handle.writeScope;
513
552
  if (writeScope === null)
514
553
  return;
@@ -535,12 +574,17 @@ export class MemoryEngine {
535
574
  if (id === undefined)
536
575
  continue;
537
576
  const stub = stubs.get(path);
538
- const rev = revOfText(text, id);
577
+ const relPath = relative(handle.memoryDir, path);
578
+ const deferred = keepBaseline.has(relPath);
579
+ const priorRev = deferred ? handle.baseRevs.get(path) : undefined;
580
+ const priorId = deferred ? handle.baseIds.get(path) : undefined;
581
+ const rev = priorRev ?? revOfText(text, priorId ?? id);
582
+ const baselineId = priorId ?? id;
539
583
  next.push({
540
584
  path,
541
- relPath: relative(handle.memoryDir, path),
585
+ relPath,
542
586
  scope: writeScope,
543
- id,
587
+ id: baselineId,
544
588
  slug: f.slug,
545
589
  rev,
546
590
  stub: stub !== undefined && stub.rev === rev,
@@ -548,7 +592,7 @@ export class MemoryEngine {
548
592
  });
549
593
  fileToScope.set(path, writeScope);
550
594
  baseRevs.set(path, rev);
551
- baseIds.set(path, id);
595
+ baseIds.set(path, baselineId);
552
596
  }
553
597
  handle.materialized = next;
554
598
  handle.fileToScope = fileToScope;
@@ -650,34 +694,38 @@ export class MemoryEngine {
650
694
  return viaBackend;
651
695
  return readSafe(join(this.controlDir, "shadow", `${id}.md`));
652
696
  }
653
- quarantineFile(path, content) {
654
- const dest = join(this.controlDir, "quarantine", `${this.now()}-${basename(path)}`);
655
- let captured;
656
- let detail;
697
+ gateDerivedIndex(handle) {
698
+ if (handle.writeScope === null)
699
+ return undefined;
700
+ const indexPath = join(handle.writableRoot, MEMORY_INDEX_FILENAME);
701
+ const text = readSafe(indexPath);
702
+ if (text === undefined || text === "")
703
+ return undefined;
704
+ const finding = scanMemoryWrite(text).find((f) => f.code === "secret");
705
+ if (finding === undefined)
706
+ return undefined;
707
+ let detail = "";
657
708
  try {
658
- mkdirSync(dirname(dest), { recursive: true });
659
- writeFileSync(dest, content, "utf8");
660
- captured = dest;
709
+ const dest = join(this.controlDir, QUARANTINE_DIR, `${this.now()}-${MEMORY_INDEX_FILENAME}`);
710
+ ensureDirExists(dirname(dest));
711
+ writeFileSync(dest, text, "utf8");
661
712
  }
662
713
  catch (err) {
663
- detail = `quarantine copy failed: ${err instanceof Error ? err.message : String(err)}`;
714
+ detail = `; quarantine capture failed: ${err instanceof Error ? err.message : String(err)}`;
664
715
  }
665
- let removed = false;
716
+ let cleared = false;
666
717
  try {
667
- rmSync(path, { force: true });
668
- removed = true;
718
+ writeFileSync(indexPath, "", "utf8");
719
+ cleared = true;
669
720
  }
670
- catch (rmErr) {
671
- try {
672
- writeFileSync(path, "---\ndeleted: true\n---\n", "utf8");
673
- removed = true;
674
- detail = `${detail !== undefined ? `${detail}; ` : ""}delete failed (${rmErr instanceof Error ? rmErr.message : String(rmErr)}) — tombstoned in place`;
675
- }
676
- catch (clearErr) {
677
- detail = `${detail !== undefined ? `${detail}; ` : ""}suspect file could not be deleted or cleared: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`;
678
- }
721
+ catch (err) {
722
+ detail = `${detail}; the derived index could NOT be cleared: ${err instanceof Error ? err.message : String(err)}`;
679
723
  }
680
- return { ...(captured !== undefined ? { dest: captured } : {}), removed, ...(detail !== undefined ? { detail } : {}) };
724
+ return {
725
+ path: MEMORY_INDEX_FILENAME,
726
+ code: finding.code,
727
+ reason: `memory index blocked: ${finding.reason} (${cleared ? "index cleared and rebuilt from entry frontmatter; the offending prose was captured to quarantine" : "index still carries the offending content"})${detail}`,
728
+ };
681
729
  }
682
730
  readonlyDirNamesUnderRoot(handle) {
683
731
  const names = new Set();
@@ -722,12 +770,14 @@ export class MemoryEngine {
722
770
  chmodScopeTree(dir, dirMode, fileMode, opts = {}) {
723
771
  if (!existsSync(dir))
724
772
  return;
773
+ const sweepSkipped = [];
725
774
  const walk = (d, isRoot) => {
726
775
  let names = [];
727
776
  try {
728
777
  names = readdirSync(d);
729
778
  }
730
779
  catch {
780
+ sweepSkipped.push(d);
731
781
  return;
732
782
  }
733
783
  for (const name of names) {
@@ -739,6 +789,7 @@ export class MemoryEngine {
739
789
  st = statSync(p);
740
790
  }
741
791
  catch {
792
+ sweepSkipped.push(p);
742
793
  continue;
743
794
  }
744
795
  if (st.isDirectory())
@@ -750,6 +801,20 @@ export class MemoryEngine {
750
801
  chmodSafe(d, dirMode);
751
802
  };
752
803
  walk(dir, true);
804
+ if (sweepSkipped.length > 0) {
805
+ try {
806
+ enqueueMemoryAnnouncement(this.controlDir, {
807
+ kind: "gate",
808
+ at: this.now(),
809
+ items: sweepSkipped
810
+ .slice(0, 12)
811
+ .map((p) => `mode sweep could not read ${JSON.stringify(inlineUntrusted(relative(dir, p) || "."))} — files under it keep their previous mode`)
812
+ .concat(sweepSkipped.length > 12 ? [`…and ${sweepSkipped.length - 12} more sweepSkipped path(s)`] : []),
813
+ });
814
+ }
815
+ catch {
816
+ }
817
+ }
753
818
  }
754
819
  }
755
820
  const MAX_ANNOUNCEMENT_ITEMS = 12;
@@ -9,7 +9,7 @@ export interface ScannedEntryFile {
9
9
  export declare function scanEntryFiles(dir: string, opts?: {
10
10
  maxDepth?: number;
11
11
  exclude?: ReadonlySet<string>;
12
- onSkip?: (path: string, kind: "symlink" | "depth" | "nonmd" | "dotfile") => void;
12
+ onSkip?: (path: string, kind: "symlink" | "depth" | "nonmd" | "dotfile" | "unreadable") => void;
13
13
  }): ScannedEntryFile[];
14
14
  export declare class FileMemoryEngineBackend implements MemoryBackend {
15
15
  readonly directoryRoot: string;
@@ -26,10 +26,10 @@ export declare class FileMemoryEngineBackend implements MemoryBackend {
26
26
  drainInboundFindings(): HarvestRejection[];
27
27
  readCommittedShadow(id: string): string | undefined;
28
28
  private shadowPath;
29
- private quarantineFile;
30
29
  private loadLedger;
31
30
  private saveLedger;
32
31
  private recoverJournal;
32
+ private txnInFlight;
33
33
  private sweepStagedFiles;
34
34
  private excludedSubdirNames;
35
35
  private scopeDir;
@@ -1,11 +1,11 @@
1
- import { lstatSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, openSync, writeSync, fsyncSync, closeSync } from "node:fs";
1
+ import { lstatSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, openSync, fsyncSync, closeSync } from "node:fs";
2
2
  import { randomUUID } from "node:crypto";
3
- import { basename, dirname, join, relative } from "node:path";
3
+ import { dirname, join, relative } from "node:path";
4
4
  import { jaccardDistance, termSet } from "../memory-vector.js";
5
5
  import { MAX_MEMORY_BYTES } from "../memory.js";
6
6
  import { inlineUntrusted } from "../untrusted-text.js";
7
7
  import { computeEntryRev, entryFromFile, isValidEntryId, parseEntryFile, serializeEntryFile } from "./frontmatter.js";
8
- import { ControlPlaneCorruptError, CURSORS_FILE, atomicWriteFileSync, claimRootScope, deriveControlPlaneDir, ensureDirExists, enqueueMemoryAnnouncement, registerScope, registeredScopes, resolveMemoryEngineRoot, scopeDirName, } from "./layout.js";
8
+ import { ControlPlaneCorruptError, CURSORS_FILE, QUARANTINE_DIR, atomicWriteFileSync, quarantineAndTombstone, claimRootScope, deriveControlPlaneDir, ensureDirExists, enqueueMemoryAnnouncement, registerScope, registeredScopes, resolveMemoryEngineRoot, scopeDirName, writeAllSync, } from "./layout.js";
9
9
  import { scanMemoryFileName, scanMemoryWrite } from "./scan.js";
10
10
  export const MEMORY_INDEX_FILENAME = "MEMORY.md";
11
11
  export const DEFAULT_MAX_ENTRY_DEPTH = 3;
@@ -18,6 +18,8 @@ export function scanEntryFiles(dir, opts = {}) {
18
18
  names = readdirSync(d);
19
19
  }
20
20
  catch {
21
+ if (d !== dir)
22
+ opts.onSkip?.(d, "unreadable");
21
23
  return;
22
24
  }
23
25
  for (const name of names.sort()) {
@@ -32,6 +34,7 @@ export function scanEntryFiles(dir, opts = {}) {
32
34
  st = lstatSync(p);
33
35
  }
34
36
  catch {
37
+ opts.onSkip?.(p, "unreadable");
35
38
  continue;
36
39
  }
37
40
  if (st.isSymbolicLink()) {
@@ -65,7 +68,6 @@ const TXN_LOCK_STALE_MS = 30_000;
65
68
  const TXN_LOCK_WAIT_MS = 15_000;
66
69
  const TXN_LOCK_STEAL_GRACE_MS = 250;
67
70
  const SHADOW_DIR = "shadow";
68
- const QUARANTINE_DIR = "quarantine";
69
71
  export class FileMemoryEngineBackend {
70
72
  directoryRoot;
71
73
  controlPlaneRoot;
@@ -79,10 +81,10 @@ export class FileMemoryEngineBackend {
79
81
  this.controlPlaneRoot = opts.controlDir ?? deriveControlPlaneDir(resolveMemoryEngineRoot(), dir);
80
82
  ensureDirExists(dir);
81
83
  ensureDirExists(this.controlPlaneRoot);
82
- this.recoverJournal();
84
+ this.recoverJournal({ unlocked: true });
83
85
  }
84
86
  checkControlPlane() {
85
- this.recoverJournal();
87
+ this.recoverJournal({ unlocked: true });
86
88
  this.ledger = undefined;
87
89
  this.loadLedger();
88
90
  registeredScopes(this.controlPlaneRoot);
@@ -98,35 +100,6 @@ export class FileMemoryEngineBackend {
98
100
  shadowPath(id) {
99
101
  return join(this.controlPlaneRoot, SHADOW_DIR, `${id}.md`);
100
102
  }
101
- quarantineFile(path, content) {
102
- const dest = join(this.controlPlaneRoot, QUARANTINE_DIR, `${Date.now()}-${basename(path)}`);
103
- let captured;
104
- let detail;
105
- try {
106
- ensureDirExists(dirname(dest));
107
- writeFileSync(dest, content, "utf8");
108
- captured = dest;
109
- }
110
- catch (err) {
111
- detail = `quarantine copy failed: ${err instanceof Error ? err.message : String(err)}`;
112
- }
113
- let removed = false;
114
- try {
115
- rmSync(path, { force: true });
116
- removed = true;
117
- }
118
- catch (rmErr) {
119
- try {
120
- writeFileSync(path, "---\ndeleted: true\n---\n", "utf8");
121
- removed = true;
122
- detail = `${detail !== undefined ? `${detail}; ` : ""}delete failed (${rmErr instanceof Error ? rmErr.message : String(rmErr)}) — tombstoned in place`;
123
- }
124
- catch (clearErr) {
125
- detail = `${detail !== undefined ? `${detail}; ` : ""}suspect file could not be deleted or cleared: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`;
126
- }
127
- }
128
- return { ...(captured !== undefined ? { dest: captured } : {}), removed, ...(detail !== undefined ? { detail } : {}) };
129
- }
130
103
  loadLedger() {
131
104
  if (this.ledger)
132
105
  return this.ledger;
@@ -164,8 +137,10 @@ export class FileMemoryEngineBackend {
164
137
  return;
165
138
  atomicWriteFileSync(join(this.controlPlaneRoot, LEDGER_FILE), `${JSON.stringify(this.ledger, null, 2)}\n`);
166
139
  }
167
- recoverJournal() {
140
+ recoverJournal(opts = {}) {
168
141
  const jp = join(this.controlPlaneRoot, JOURNAL_FILE);
142
+ if (opts.unlocked && this.txnInFlight())
143
+ return;
169
144
  const raw = readSafe(jp);
170
145
  if (raw !== undefined) {
171
146
  let journal;
@@ -205,6 +180,14 @@ export class FileMemoryEngineBackend {
205
180
  }
206
181
  this.sweepStagedFiles();
207
182
  }
183
+ txnInFlight() {
184
+ try {
185
+ return this.now() - statSync(this.txnLockDir()).mtimeMs < TXN_LOCK_STALE_MS;
186
+ }
187
+ catch {
188
+ return false;
189
+ }
190
+ }
208
191
  sweepStagedFiles() {
209
192
  const walk = (d, depth) => {
210
193
  let names;
@@ -268,12 +251,14 @@ export class FileMemoryEngineBackend {
268
251
  const ledger = sync ? this.loadLedger() : undefined;
269
252
  let ledgerChanged = false;
270
253
  const adoptedExternal = [];
254
+ const skippedUnreadable = [];
271
255
  for (const f of files) {
272
256
  let text;
273
257
  try {
274
258
  text = readFileSync(f.path, "utf8");
275
259
  }
276
260
  catch {
261
+ skippedUnreadable.push(relative(this.directoryRoot, f.path));
277
262
  continue;
278
263
  }
279
264
  const parsed = parseEntryFile(text);
@@ -302,7 +287,7 @@ export class FileMemoryEngineBackend {
302
287
  }
303
288
  if (finding) {
304
289
  this.inboundFindings.push(finding);
305
- const q = this.quarantineFile(f.path, text);
290
+ const q = quarantineAndTombstone(f.path, text, join(this.controlPlaneRoot, QUARANTINE_DIR), this.now);
306
291
  const shadowText = committed !== undefined ? this.readCommittedShadow(entry.id) : undefined;
307
292
  let restoredShadow = false;
308
293
  if (shadowText !== undefined) {
@@ -368,6 +353,20 @@ export class FileMemoryEngineBackend {
368
353
  catch {
369
354
  }
370
355
  }
356
+ if (skippedUnreadable.length > 0) {
357
+ try {
358
+ enqueueMemoryAnnouncement(this.controlPlaneRoot, {
359
+ kind: "external",
360
+ at: this.now(),
361
+ items: skippedUnreadable
362
+ .slice(0, 12)
363
+ .map((rel) => `committed memory entry could not be read and is MISSING from this session's memory: ${JSON.stringify(inlineUntrusted(rel))} — check the file (permissions/fs)`)
364
+ .concat(skippedUnreadable.length > 12 ? [`…and ${skippedUnreadable.length - 12} more unreadable entr(ies)`] : []),
365
+ });
366
+ }
367
+ catch {
368
+ }
369
+ }
371
370
  if (!sync && this.batchScan !== undefined)
372
371
  this.batchScan.set(scope, entries);
373
372
  return entries;
@@ -448,7 +447,7 @@ export class FileMemoryEngineBackend {
448
447
  try {
449
448
  const fd = openSync(ownerPath, "w", 0o600);
450
449
  try {
451
- writeSync(fd, token);
450
+ writeAllSync(fd, token);
452
451
  fsyncSync(fd);
453
452
  }
454
453
  finally {
@@ -574,7 +573,7 @@ export class FileMemoryEngineBackend {
574
573
  mkdirSync(dirname(op.target), { recursive: true });
575
574
  const fd = openSync(staged, "w", 0o600);
576
575
  try {
577
- writeSync(fd, op.content);
576
+ writeAllSync(fd, op.content);
578
577
  fsyncSync(fd);
579
578
  }
580
579
  finally {
@@ -23,6 +23,13 @@ export declare function scopeDirFor(memoryDir: string, controlDir: string, scope
23
23
  export declare function canonicalize(p: string): string;
24
24
  export declare function isContainedIn(root: string, child: string): boolean;
25
25
  export declare function ensureDirExists(dir: string): void;
26
+ export declare const QUARANTINE_DIR = "quarantine";
27
+ export interface QuarantineOutcome {
28
+ dest?: string;
29
+ removed: boolean;
30
+ detail?: string;
31
+ }
32
+ export declare function quarantineAndTombstone(path: string, content: string, quarantineDir: string, now: () => number): QuarantineOutcome;
26
33
  export declare const ANNOUNCEMENTS_FILE = "announcements.json";
27
34
  export declare const MEMORY_ANNOUNCEMENTS_MAX = 20;
28
35
  export declare const SCAN_FUSE_FILE = "scan-fuse.json";
@@ -39,5 +46,6 @@ export declare function readIndexRevs(controlDir: string): Record<string, string
39
46
  export declare function writeIndexRevs(controlDir: string, revs: Record<string, string>): void;
40
47
  export declare function scanFuseCount(controlDir: string, key: string): number;
41
48
  export declare function clearScanFuse(controlDir: string, keys: Iterable<string>): void;
49
+ export declare function writeAllSync(fd: number, data: string): void;
42
50
  export declare function atomicWriteFileSync(path: string, data: string): void;
43
51
  export {};
@@ -1,7 +1,7 @@
1
- import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeSync } from "node:fs";
1
+ import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
2
2
  import { homedir } from "node:os";
3
3
  import { createHash } from "node:crypto";
4
- import { dirname, isAbsolute, join, resolve, sep } from "node:path";
4
+ import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
5
5
  const SCOPES_FILE = "scopes.json";
6
6
  export const CURSORS_FILE = "cursors.json";
7
7
  export class ControlPlaneCorruptError extends Error {
@@ -190,6 +190,52 @@ export function ensureDirExists(dir) {
190
190
  if (!existsSync(dir))
191
191
  mkdirSync(dir, { recursive: true, mode: 0o700 });
192
192
  }
193
+ export const QUARANTINE_DIR = "quarantine";
194
+ const DELETED_TOMBSTONE = "---\ndeleted: true\n---\n";
195
+ let quarantineSeq = 0;
196
+ export function quarantineAndTombstone(path, content, quarantineDir, now) {
197
+ let captured;
198
+ let detail;
199
+ try {
200
+ ensureDirExists(quarantineDir);
201
+ const base = `${now()}-${basename(path)}`;
202
+ for (let attempt = 0; captured === undefined && attempt < 10; attempt++) {
203
+ const candidate = join(quarantineDir, attempt === 0 ? base : `${base}.${attempt}`);
204
+ try {
205
+ writeFileSync(candidate, content, { encoding: "utf8", flag: "wx" });
206
+ captured = candidate;
207
+ }
208
+ catch (err) {
209
+ if (!(err instanceof Error && "code" in err && err.code === "EEXIST"))
210
+ throw err;
211
+ }
212
+ }
213
+ if (captured === undefined) {
214
+ const last = join(quarantineDir, `${base}.${process.pid}.${quarantineSeq++}`);
215
+ writeFileSync(last, content, { encoding: "utf8", flag: "wx" });
216
+ captured = last;
217
+ }
218
+ }
219
+ catch (err) {
220
+ detail = `quarantine copy failed: ${err instanceof Error ? err.message : String(err)}`;
221
+ }
222
+ let removed = false;
223
+ try {
224
+ rmSync(path, { force: true });
225
+ removed = true;
226
+ }
227
+ catch (rmErr) {
228
+ try {
229
+ writeFileSync(path, DELETED_TOMBSTONE, "utf8");
230
+ removed = true;
231
+ detail = `${detail !== undefined ? `${detail}; ` : ""}delete failed (${rmErr instanceof Error ? rmErr.message : String(rmErr)}) — tombstoned in place`;
232
+ }
233
+ catch (clearErr) {
234
+ detail = `${detail !== undefined ? `${detail}; ` : ""}suspect file could not be deleted or cleared: ${clearErr instanceof Error ? clearErr.message : String(clearErr)}`;
235
+ }
236
+ }
237
+ return { ...(captured !== undefined ? { dest: captured } : {}), removed, ...(detail !== undefined ? { detail } : {}) };
238
+ }
193
239
  export const ANNOUNCEMENTS_FILE = "announcements.json";
194
240
  export const MEMORY_ANNOUNCEMENTS_MAX = 20;
195
241
  export const SCAN_FUSE_FILE = "scan-fuse.json";
@@ -216,11 +262,22 @@ function acquireSidecarLock(lockDir, opts = {}) {
216
262
  try {
217
263
  mkdirSync(lockDir);
218
264
  const token = `${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
265
+ let fd;
219
266
  try {
220
- writeSync(openSync(join(lockDir, "owner"), "w"), token);
267
+ fd = openSync(join(lockDir, "owner"), "w");
268
+ writeAllSync(fd, token);
221
269
  }
222
270
  catch {
223
271
  }
272
+ finally {
273
+ if (fd !== undefined) {
274
+ try {
275
+ closeSync(fd);
276
+ }
277
+ catch {
278
+ }
279
+ }
280
+ }
224
281
  return token;
225
282
  }
226
283
  catch {
@@ -433,12 +490,22 @@ export function clearScanFuse(controlDir, keys) {
433
490
  return changed ? counts : undefined;
434
491
  });
435
492
  }
493
+ export function writeAllSync(fd, data) {
494
+ const buf = Buffer.from(data, "utf8");
495
+ let written = 0;
496
+ while (written < buf.length) {
497
+ const n = writeSync(fd, buf, written, buf.length - written);
498
+ if (n <= 0)
499
+ throw new Error(`writeAllSync: write made no progress (${written}/${buf.length} bytes) — refusing to leave a truncated file`);
500
+ written += n;
501
+ }
502
+ }
436
503
  export function atomicWriteFileSync(path, data) {
437
504
  ensureDirExists(dirname(path));
438
505
  const tmp = join(dirname(path), `.tmp-${process.pid}-${Math.random().toString(36).slice(2, 10)}`);
439
506
  const fd = openSync(tmp, "w", 0o600);
440
507
  try {
441
- writeSync(fd, data);
508
+ writeAllSync(fd, data);
442
509
  fsyncSync(fd);
443
510
  }
444
511
  finally {