opencode-codex-memory 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -3
- package/dist/opencode.json +1 -1
- package/dist/src/capture.d.ts +2 -13
- package/dist/src/capture.js +58 -56
- package/dist/src/codex-interop.d.ts +8 -0
- package/dist/src/codex-interop.js +22 -0
- package/dist/src/diagnostics.d.ts +26 -0
- package/dist/src/diagnostics.js +37 -0
- package/dist/src/host-client.d.ts +72 -0
- package/dist/src/host-client.js +91 -0
- package/dist/src/index.js +27 -6
- package/dist/src/lifecycle.d.ts +26 -0
- package/dist/src/lifecycle.js +45 -0
- package/dist/src/llm.d.ts +2 -0
- package/dist/src/llm.js +15 -36
- package/dist/src/phase1.js +35 -3
- package/dist/src/phase2.d.ts +1 -2
- package/dist/src/phase2.js +49 -10
- package/dist/src/ratelimit.d.ts +4 -0
- package/dist/src/ratelimit.js +31 -14
- package/dist/src/store.d.ts +31 -0
- package/dist/src/store.js +62 -0
- package/dist/tools/control.js +53 -9
- package/dist/tools/memory.js +8 -1
- package/opencode.json +1 -1
- package/package.json +4 -1
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plugin process lifecycle: shutdown flag + phase-2 abort signal shared by
|
|
3
|
+
* the entry dispose hook and the write pipeline.
|
|
4
|
+
*
|
|
5
|
+
* Opencode can reload plugins while a consolidator helper still holds write
|
|
6
|
+
* access to the memory root. dispose() sets the flag (so new pumps stop),
|
|
7
|
+
* aborts the in-flight consolidation prompt, and best-effort aborts active
|
|
8
|
+
* sub-sessions (llm.ts).
|
|
9
|
+
*/
|
|
10
|
+
let shuttingDown = false;
|
|
11
|
+
let phase2Abort = null;
|
|
12
|
+
export function isPluginShuttingDown() {
|
|
13
|
+
return shuttingDown;
|
|
14
|
+
}
|
|
15
|
+
/** Begin shutdown: no new phase work, abort any in-flight consolidator. */
|
|
16
|
+
export function beginPluginShutdown() {
|
|
17
|
+
shuttingDown = true;
|
|
18
|
+
phase2Abort?.abort();
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Test / re-boot seam: a fresh server() call clears the previous dispose.
|
|
22
|
+
* Abort any live consolidator controller before dropping the reference so a
|
|
23
|
+
* glitched boot order cannot orphan a still-running phase-2 prompt.
|
|
24
|
+
*/
|
|
25
|
+
export function resetPluginLifecycle() {
|
|
26
|
+
phase2Abort?.abort();
|
|
27
|
+
phase2Abort = null;
|
|
28
|
+
shuttingDown = false;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* AbortSignal for the current phase-2 consolidation run. Created when the job
|
|
32
|
+
* is claimed; aborted on heartbeat loss, dispose, or run end.
|
|
33
|
+
*/
|
|
34
|
+
export function beginPhase2AbortScope() {
|
|
35
|
+
phase2Abort?.abort();
|
|
36
|
+
phase2Abort = new AbortController();
|
|
37
|
+
return phase2Abort.signal;
|
|
38
|
+
}
|
|
39
|
+
export function endPhase2AbortScope() {
|
|
40
|
+
phase2Abort = null;
|
|
41
|
+
}
|
|
42
|
+
/** Abort the current consolidator from outside phase2 (dispose). */
|
|
43
|
+
export function abortPhase2Consolidation() {
|
|
44
|
+
phase2Abort?.abort();
|
|
45
|
+
}
|
package/dist/src/llm.d.ts
CHANGED
|
@@ -37,6 +37,8 @@ export interface ExtractOptions {
|
|
|
37
37
|
export declare function extractViaSubagent(sessionId: string, transcript: string, opts?: ExtractOptions): Promise<ExtractionResult | null>;
|
|
38
38
|
export declare function consolidateViaSubagent(memoryRoot: string, diffFileName: string, model?: string, signal?: AbortSignal): Promise<void>;
|
|
39
39
|
export declare function cleanupOldSubSessions(maxAgeMinutes?: number, timeoutMs?: number): Promise<void>;
|
|
40
|
+
/** Best-effort abort of every memory sub-session (plugin dispose / reload). */
|
|
41
|
+
export declare function abortActiveSubSessions(): Promise<void>;
|
|
40
42
|
export declare function fillTemplate(tmpl: string, vars: Record<string, string>): string;
|
|
41
43
|
export declare function buildConsolidationPrompt(memoryRoot: string, diffFileName: string): string;
|
|
42
44
|
/**
|
package/dist/src/llm.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { memoryRoot } from "./paths.js";
|
|
4
|
+
import { hostSessionCreate, hostSessionDeletionConfirmed, hostSessionPrompt, hostStructuredOutput, } from "./host-client.js";
|
|
4
5
|
let inputRef = null;
|
|
5
6
|
export function setPluginInput(input) {
|
|
6
7
|
inputRef = input;
|
|
@@ -38,10 +39,10 @@ async function createSession(agent, title) {
|
|
|
38
39
|
if (!input)
|
|
39
40
|
throw new Error("plugin input not initialized");
|
|
40
41
|
const directory = resolveSubSessionDirectory();
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
// directory is a query param (not body); without it the client inherits
|
|
43
|
+
// PluginInput.directory, which may be a deleted project path.
|
|
44
|
+
const res = await hostSessionCreate(input.client, {
|
|
45
|
+
directory,
|
|
45
46
|
body: {
|
|
46
47
|
title: title ?? `codex-memory-${agent}`,
|
|
47
48
|
metadata: { [SUBSESSION_METADATA_KEY]: true },
|
|
@@ -154,10 +155,8 @@ async function runPrompt(sessionId, prompt, agent, opts = {}) {
|
|
|
154
155
|
throw new SubagentCancelledError();
|
|
155
156
|
}
|
|
156
157
|
const model = opts.model ? parseModelRef(opts.model) : null;
|
|
157
|
-
const promptPromise = input.client
|
|
158
|
-
|
|
159
|
-
// `format` lives in the server's PromptInput but not the generated SDK body
|
|
160
|
-
// type yet (same OpenAPI lag as session.list scope/roots), hence the cast.
|
|
158
|
+
const promptPromise = hostSessionPrompt(input.client, {
|
|
159
|
+
sessionId,
|
|
161
160
|
body: {
|
|
162
161
|
agent,
|
|
163
162
|
...(opts.system ? { system: opts.system } : {}),
|
|
@@ -265,11 +264,11 @@ export async function extractViaSubagent(sessionId, transcript, opts = {}) {
|
|
|
265
264
|
format: { type: "json_schema", schema: EXTRACTION_SCHEMA },
|
|
266
265
|
});
|
|
267
266
|
// The captured JSON lands on AssistantMessage.structured (schema
|
|
268
|
-
// v1/session.ts; absent from the generated SDK type
|
|
267
|
+
// v1/session.ts; absent from the generated SDK type — see host-client.ts).
|
|
269
268
|
// Fall back to text parsing when structured output is unavailable (a host
|
|
270
269
|
// without the feature, or a model that emitted JSON as plain text).
|
|
271
|
-
const structured = data
|
|
272
|
-
if (structured
|
|
270
|
+
const structured = hostStructuredOutput(data);
|
|
271
|
+
if (structured) {
|
|
273
272
|
return validateExtraction(structured);
|
|
274
273
|
}
|
|
275
274
|
return parseExtraction(extractAssistantText(data));
|
|
@@ -396,7 +395,7 @@ async function deleteSession(id) {
|
|
|
396
395
|
// session is gone; otherwise retain ownership so hooks keep skipping it.
|
|
397
396
|
// codex runtime.rs drops the thread from its manager the same way: only
|
|
398
397
|
// after shutdown succeeded.
|
|
399
|
-
if (await
|
|
398
|
+
if (await hostSessionDeletionConfirmed(input.client, id, SUBSESSION_CONFIRM_TIMEOUT_MS)) {
|
|
400
399
|
activeSubSessions.delete(id);
|
|
401
400
|
}
|
|
402
401
|
return true;
|
|
@@ -409,30 +408,10 @@ async function deleteSession(id) {
|
|
|
409
408
|
clearTimeout(timer);
|
|
410
409
|
}
|
|
411
410
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
const controller = new AbortController();
|
|
417
|
-
let timer;
|
|
418
|
-
try {
|
|
419
|
-
const res = await Promise.race([
|
|
420
|
-
session.get({ path: { id }, signal: controller.signal }),
|
|
421
|
-
new Promise((_, reject) => {
|
|
422
|
-
timer = setTimeout(() => {
|
|
423
|
-
controller.abort();
|
|
424
|
-
reject(new Error(`session.get timed out after ${SUBSESSION_CONFIRM_TIMEOUT_MS}ms`));
|
|
425
|
-
}, SUBSESSION_CONFIRM_TIMEOUT_MS);
|
|
426
|
-
}),
|
|
427
|
-
]);
|
|
428
|
-
return res?.response?.status === 404;
|
|
429
|
-
}
|
|
430
|
-
catch {
|
|
431
|
-
return false;
|
|
432
|
-
}
|
|
433
|
-
finally {
|
|
434
|
-
clearTimeout(timer);
|
|
435
|
-
}
|
|
411
|
+
/** Best-effort abort of every memory sub-session (plugin dispose / reload). */
|
|
412
|
+
export async function abortActiveSubSessions() {
|
|
413
|
+
const ids = [...activeSubSessions];
|
|
414
|
+
await Promise.all(ids.map((id) => abortSession(id)));
|
|
436
415
|
}
|
|
437
416
|
// Substitute with a function so `$&`/`$'` sequences in the value are not
|
|
438
417
|
// expanded as String.replace replacement patterns.
|
package/dist/src/phase1.js
CHANGED
|
@@ -3,7 +3,9 @@ import { loadTranscript, selectEligibleSessions } from "./capture.js";
|
|
|
3
3
|
import { redact, isMemoryExcludedFragment } from "./redact.js";
|
|
4
4
|
import { stripCitations } from "./citation.js";
|
|
5
5
|
import { extractViaSubagent } from "./llm.js";
|
|
6
|
-
import { checkRateLimit } from "./ratelimit.js";
|
|
6
|
+
import { checkRateLimit, markRateLimitUsed } from "./ratelimit.js";
|
|
7
|
+
import { isPluginShuttingDown } from "./lifecycle.js";
|
|
8
|
+
import { recordDiagnostic } from "./diagnostics.js";
|
|
7
9
|
export const DEFAULT_PHASE1_OPTIONS = {
|
|
8
10
|
maxAgeDays: 10,
|
|
9
11
|
minIdleHours: 6,
|
|
@@ -20,6 +22,9 @@ const TRANSCRIPT_MAX_CHARS = 600_000;
|
|
|
20
22
|
const TRANSCRIPT_HEAD_CHARS = 300_000;
|
|
21
23
|
const TRANSCRIPT_TAIL_CHARS = 300_000;
|
|
22
24
|
export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitCheck = checkRateLimit) {
|
|
25
|
+
if (isPluginShuttingDown())
|
|
26
|
+
return;
|
|
27
|
+
// Prune first (no tokens), matching codex start.rs ordering before the gate.
|
|
23
28
|
store.pruneStage1Outputs(opts.maxUnusedDays ?? 30);
|
|
24
29
|
const rl = await rateLimitCheck("phase1");
|
|
25
30
|
if (!rl.ok) {
|
|
@@ -30,15 +35,31 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
30
35
|
if (eligible.length === 0)
|
|
31
36
|
return;
|
|
32
37
|
const claimed = store.claimStage1Jobs(eligible, opts.excludeSession, opts.maxClaimed);
|
|
33
|
-
|
|
38
|
+
// Empty claim: do not stamp the process timer (codex only metrics
|
|
39
|
+
// skipped_no_candidates and leaves the next startup free to try again).
|
|
40
|
+
if (claimed.length === 0) {
|
|
41
|
+
recordDiagnostic("info", "phase1", `no claims (eligible=${eligible.length})`);
|
|
34
42
|
return;
|
|
43
|
+
}
|
|
44
|
+
markRateLimitUsed("phase1");
|
|
45
|
+
recordDiagnostic("info", "phase1", `claimed ${claimed.length} session(s)`);
|
|
35
46
|
const sessionById = new Map(eligible.map((s) => [s.id, s]));
|
|
36
47
|
await runPool(claimed, STAGE1_CONCURRENCY, async (claim) => {
|
|
37
48
|
const sid = claim.sessionId;
|
|
49
|
+
// dispose mid-pass: release the claim without burning a retry or waiting
|
|
50
|
+
// for the 1h lease — otherwise jobs stay `running` until lease expiry.
|
|
51
|
+
if (isPluginShuttingDown()) {
|
|
52
|
+
store.releaseStage1OnShutdown(sid, claim.ownershipToken);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
38
55
|
try {
|
|
39
56
|
const session = sessionById.get(sid);
|
|
40
57
|
const sourceUpdatedAt = session?.updated_at ?? Date.now();
|
|
41
58
|
const transcript = await buildTranscript(sid);
|
|
59
|
+
if (isPluginShuttingDown()) {
|
|
60
|
+
store.releaseStage1OnShutdown(sid, claim.ownershipToken);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
42
63
|
if (!transcript.trim()) {
|
|
43
64
|
// A newly empty chat is a legitimate no-output result. An existing
|
|
44
65
|
// extraction plus an empty API success is anomalous: retry instead of
|
|
@@ -53,6 +74,10 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
53
74
|
cwd: session?.directory ?? undefined,
|
|
54
75
|
model: opts.extractModel,
|
|
55
76
|
});
|
|
77
|
+
// Finalize after a completed model call even if dispose raced, so the
|
|
78
|
+
// claim does not sit `running` until lease expiry. The mark is token +
|
|
79
|
+
// status guarded: if shutdown already released the job and a new process
|
|
80
|
+
// reclaimed it, this becomes a no-op (paid work dropped, never double-written).
|
|
56
81
|
if (!result) {
|
|
57
82
|
// Extractor judged the session not worth remembering.
|
|
58
83
|
store.markStage1SucceededNoOutput(sid, claim.ownershipToken, sourceUpdatedAt);
|
|
@@ -70,7 +95,14 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
70
95
|
});
|
|
71
96
|
}
|
|
72
97
|
catch (err) {
|
|
73
|
-
|
|
98
|
+
// Aborted mid-extract on dispose: release for immediate reclaim, do not
|
|
99
|
+
// burn a retry or impose the 1h failure backoff.
|
|
100
|
+
if (isPluginShuttingDown()) {
|
|
101
|
+
store.releaseStage1OnShutdown(sid, claim.ownershipToken);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
store.markStage1Failed(sid, claim.ownershipToken, err);
|
|
105
|
+
}
|
|
74
106
|
}
|
|
75
107
|
});
|
|
76
108
|
}
|
package/dist/src/phase2.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { MemoryStore } from "./store.js";
|
|
2
|
-
import { checkRateLimit } from "./ratelimit.js";
|
|
3
2
|
import { type CodexInteropOptions } from "./codex-interop.js";
|
|
4
3
|
export interface Phase2Options {
|
|
5
4
|
maxRaw: number;
|
|
@@ -13,6 +12,6 @@ export interface Phase2Options {
|
|
|
13
12
|
export declare const DEFAULT_PHASE2_OPTIONS: Phase2Options;
|
|
14
13
|
/** True while THIS process runs a consolidation (memory_reset refuses then). */
|
|
15
14
|
export declare function isPhase2InFlight(): boolean;
|
|
16
|
-
export declare function runPhase2(store: MemoryStore, opts?: Phase2Options
|
|
15
|
+
export declare function runPhase2(store: MemoryStore, opts?: Phase2Options): Promise<{
|
|
17
16
|
status: string;
|
|
18
17
|
}>;
|
package/dist/src/phase2.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { ensureLayout, rebuildRawMemories, writeRolloutSummaries, pruneExtensionResources, writeWorkspaceDiff, validateConsolidationArtifacts, } from "./workspace.js";
|
|
2
2
|
import { ensureBaseline, captureWorkspaceDiff, resetBaseline, DIFF_ARTIFACT } from "./git-baseline.js";
|
|
3
|
-
import { consolidateViaSubagent, SubagentShutdownError } from "./llm.js";
|
|
3
|
+
import { consolidateViaSubagent, SubagentCancelledError, SubagentShutdownError } from "./llm.js";
|
|
4
4
|
import { invalidateCache } from "./source.js";
|
|
5
5
|
import { memoryRoot } from "./paths.js";
|
|
6
|
-
import {
|
|
6
|
+
import { abortPhase2Consolidation, beginPhase2AbortScope, endPhase2AbortScope, isPluginShuttingDown, } from "./lifecycle.js";
|
|
7
7
|
import { resolveCodexInterop, syncCodexImport, exportToCodexMemory } from "./codex-interop.js";
|
|
8
8
|
export const DEFAULT_PHASE2_OPTIONS = {
|
|
9
9
|
maxRaw: 256,
|
|
@@ -27,18 +27,33 @@ let phase2InFlight = false;
|
|
|
27
27
|
export function isPhase2InFlight() {
|
|
28
28
|
return phase2InFlight;
|
|
29
29
|
}
|
|
30
|
-
|
|
30
|
+
/** Release the claim when dispose raced the prep path; return true if released. */
|
|
31
|
+
function releaseIfShuttingDown(store, ownershipToken) {
|
|
32
|
+
if (!isPluginShuttingDown())
|
|
33
|
+
return false;
|
|
34
|
+
store.releasePhase2OnShutdown(ownershipToken);
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS) {
|
|
38
|
+
if (isPluginShuttingDown())
|
|
39
|
+
return { status: "shutting_down" };
|
|
31
40
|
if (phase2InFlight)
|
|
32
41
|
return { status: "already_running" };
|
|
33
42
|
phase2InFlight = true;
|
|
34
43
|
try {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
44
|
+
// No process-local rate gate: codex serializes phase 2 only via the DB
|
|
45
|
+
// claim (cooldown / running / retry_at). Empty and cooldown skips must
|
|
46
|
+
// not delay a later real claim.
|
|
38
47
|
const claim = store.claimGlobalPhase2Job();
|
|
39
48
|
if (claim.type !== "claimed")
|
|
40
49
|
return { status: claim.type };
|
|
50
|
+
// Abort scope covers prep + consolidator so dispose during baseline/diff
|
|
51
|
+
// sets the flag that releaseIfShuttingDown / consolidator cancel observe.
|
|
52
|
+
const consolidationSignal = beginPhase2AbortScope();
|
|
41
53
|
try {
|
|
54
|
+
if (releaseIfShuttingDown(store, claim.ownershipToken)) {
|
|
55
|
+
return { status: "shutting_down" };
|
|
56
|
+
}
|
|
42
57
|
// Resolved once per claimed job (not per attempt): resolution warns on
|
|
43
58
|
// misconfiguration, and warning on every skipped attempt would be noise.
|
|
44
59
|
// Keep this inside the claimed-job try so resolution failures release
|
|
@@ -54,6 +69,9 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
54
69
|
store.markPhase2Failed(claim.ownershipToken, "git baseline failed");
|
|
55
70
|
return { status: "baseline_failed" };
|
|
56
71
|
}
|
|
72
|
+
if (releaseIfShuttingDown(store, claim.ownershipToken)) {
|
|
73
|
+
return { status: "shutting_down" };
|
|
74
|
+
}
|
|
57
75
|
const outputs = store.getPhase2InputSelection(opts.maxRaw, opts.maxUnusedDays);
|
|
58
76
|
rebuildRawMemories(outputs);
|
|
59
77
|
writeRolloutSummaries(outputs);
|
|
@@ -75,6 +93,9 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
75
93
|
}
|
|
76
94
|
}
|
|
77
95
|
const diff = await captureWorkspaceDiff();
|
|
96
|
+
if (releaseIfShuttingDown(store, claim.ownershipToken)) {
|
|
97
|
+
return { status: "shutting_down" };
|
|
98
|
+
}
|
|
78
99
|
// codex: early succeed only when there are no changes AND artifacts are
|
|
79
100
|
// already valid. Invalid/empty summary (e.g. ensureLayout's empty file)
|
|
80
101
|
// falls through so the consolidator can INIT/repair.
|
|
@@ -90,14 +111,13 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
90
111
|
writeWorkspaceDiff(diff);
|
|
91
112
|
let heartbeatLost = false;
|
|
92
113
|
let heartbeatFailure = "ownership lost";
|
|
93
|
-
const consolidationAbort = new AbortController();
|
|
94
114
|
const heartbeatOnce = () => {
|
|
95
115
|
if (heartbeatLost)
|
|
96
116
|
return false;
|
|
97
117
|
try {
|
|
98
118
|
if (!store.heartbeatPhase2Job(claim.ownershipToken)) {
|
|
99
119
|
heartbeatLost = true;
|
|
100
|
-
|
|
120
|
+
abortPhase2Consolidation();
|
|
101
121
|
return false;
|
|
102
122
|
}
|
|
103
123
|
}
|
|
@@ -108,7 +128,7 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
108
128
|
// the job while this helper still has live write access.
|
|
109
129
|
heartbeatLost = true;
|
|
110
130
|
heartbeatFailure = err;
|
|
111
|
-
|
|
131
|
+
abortPhase2Consolidation();
|
|
112
132
|
return false;
|
|
113
133
|
}
|
|
114
134
|
return true;
|
|
@@ -120,9 +140,12 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
120
140
|
store.markPhase2Failed(claim.ownershipToken, heartbeatFailure);
|
|
121
141
|
return { status: "heartbeat_lost" };
|
|
122
142
|
}
|
|
143
|
+
if (releaseIfShuttingDown(store, claim.ownershipToken)) {
|
|
144
|
+
return { status: "shutting_down" };
|
|
145
|
+
}
|
|
123
146
|
const heartbeat = setInterval(heartbeatOnce, opts.heartbeatIntervalMs ?? 90_000);
|
|
124
147
|
try {
|
|
125
|
-
await consolidateViaSubagent(memoryRoot(), DIFF_ARTIFACT, opts.consolidationModel,
|
|
148
|
+
await consolidateViaSubagent(memoryRoot(), DIFF_ARTIFACT, opts.consolidationModel, consolidationSignal);
|
|
126
149
|
}
|
|
127
150
|
catch (err) {
|
|
128
151
|
// codex phase2.rs: when the consolidation agent's shutdown fails, keep
|
|
@@ -137,6 +160,12 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
137
160
|
store.markPhase2Failed(claim.ownershipToken, heartbeatFailure);
|
|
138
161
|
return { status: "heartbeat_lost" };
|
|
139
162
|
}
|
|
163
|
+
// dispose() / beginPluginShutdown aborted the consolidator: release
|
|
164
|
+
// without the 1h failure backoff so the next boot can reclaim.
|
|
165
|
+
if (err instanceof SubagentCancelledError || isPluginShuttingDown()) {
|
|
166
|
+
store.releasePhase2OnShutdown(claim.ownershipToken);
|
|
167
|
+
return { status: "shutting_down" };
|
|
168
|
+
}
|
|
140
169
|
throw err;
|
|
141
170
|
}
|
|
142
171
|
finally {
|
|
@@ -152,6 +181,9 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
152
181
|
store.markPhase2Failed(claim.ownershipToken, heartbeatFailure);
|
|
153
182
|
return { status: "heartbeat_lost" };
|
|
154
183
|
}
|
|
184
|
+
if (releaseIfShuttingDown(store, claim.ownershipToken)) {
|
|
185
|
+
return { status: "shutting_down" };
|
|
186
|
+
}
|
|
155
187
|
// codex failed_invalid_artifacts: do not reset baseline on bad output so
|
|
156
188
|
// the next run still sees a diff / can re-INIT.
|
|
157
189
|
const artifacts = validateConsolidationArtifacts();
|
|
@@ -169,9 +201,16 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS, rateLimitC
|
|
|
169
201
|
return { status: "succeeded" };
|
|
170
202
|
}
|
|
171
203
|
catch (err) {
|
|
204
|
+
if (isPluginShuttingDown()) {
|
|
205
|
+
store.releasePhase2OnShutdown(claim.ownershipToken);
|
|
206
|
+
return { status: "shutting_down" };
|
|
207
|
+
}
|
|
172
208
|
store.markPhase2Failed(claim.ownershipToken, err);
|
|
173
209
|
return { status: "failed" };
|
|
174
210
|
}
|
|
211
|
+
finally {
|
|
212
|
+
endPhase2AbortScope();
|
|
213
|
+
}
|
|
175
214
|
}
|
|
176
215
|
finally {
|
|
177
216
|
phase2InFlight = false;
|
package/dist/src/ratelimit.d.ts
CHANGED
|
@@ -3,3 +3,7 @@ export interface RateLimitInfo {
|
|
|
3
3
|
reason?: string;
|
|
4
4
|
}
|
|
5
5
|
export declare function checkRateLimit(kind?: "phase1" | "phase2"): Promise<RateLimitInfo>;
|
|
6
|
+
/** Call after a phase-1 pass claimed at least one job (token-using work started). */
|
|
7
|
+
export declare function markRateLimitUsed(kind?: "phase1" | "phase2"): void;
|
|
8
|
+
/** Test seam: reset the process-local stamp. */
|
|
9
|
+
export declare function resetRateLimitForTest(): void;
|
package/dist/src/ratelimit.js
CHANGED
|
@@ -1,20 +1,37 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Process-local anti-stampede for phase 1 only.
|
|
3
|
+
*
|
|
4
|
+
* Codex has no wall-clock throttle: it reads live provider quota once per
|
|
5
|
+
* startup (memories/write/src/guard.rs) and fails open when unknown. Opencode
|
|
6
|
+
* does not expose provider rate limits to plugins, so this stub keeps
|
|
7
|
+
* chat.message/idle from hammering discovery + claim when many sessions go
|
|
8
|
+
* idle at once.
|
|
9
|
+
*
|
|
10
|
+
* Semantics deliberately match "do not start another token-using run too
|
|
11
|
+
* often", not "do not look often":
|
|
12
|
+
* - checkRateLimit only reads the clock (empty/no-claim passes do not stamp)
|
|
13
|
+
* - markRateLimitUsed stamps after a stage-1 claim actually succeeds
|
|
14
|
+
* - phase 2 has no process timer; the DB claim + 6h cooldown serialize it
|
|
15
|
+
* (same as codex phase2 job outcomes)
|
|
16
|
+
*/
|
|
17
|
+
let lastPhase1Work = 0;
|
|
3
18
|
const MIN_PHASE1_INTERVAL_MS = 30_000;
|
|
4
|
-
const MIN_PHASE2_INTERVAL_MS = 5 * 60 * 1000;
|
|
5
19
|
export async function checkRateLimit(kind = "phase1") {
|
|
20
|
+
// Phase 2: no process-local gate (codex relies on DB claim/cooldown only).
|
|
21
|
+
if (kind === "phase2")
|
|
22
|
+
return { ok: true };
|
|
6
23
|
const now = Date.now();
|
|
7
|
-
if (
|
|
8
|
-
|
|
9
|
-
return { ok: false, reason: "phase1 rate limit (30s)" };
|
|
10
|
-
}
|
|
11
|
-
lastPhase1 = now;
|
|
12
|
-
}
|
|
13
|
-
else {
|
|
14
|
-
if (now - lastPhase2 < MIN_PHASE2_INTERVAL_MS) {
|
|
15
|
-
return { ok: false, reason: "phase2 rate limit (5min)" };
|
|
16
|
-
}
|
|
17
|
-
lastPhase2 = now;
|
|
24
|
+
if (now - lastPhase1Work < MIN_PHASE1_INTERVAL_MS) {
|
|
25
|
+
return { ok: false, reason: "phase1 rate limit (30s since last claimed work)" };
|
|
18
26
|
}
|
|
19
27
|
return { ok: true };
|
|
20
28
|
}
|
|
29
|
+
/** Call after a phase-1 pass claimed at least one job (token-using work started). */
|
|
30
|
+
export function markRateLimitUsed(kind = "phase1") {
|
|
31
|
+
if (kind === "phase1")
|
|
32
|
+
lastPhase1Work = Date.now();
|
|
33
|
+
}
|
|
34
|
+
/** Test seam: reset the process-local stamp. */
|
|
35
|
+
export function resetRateLimitForTest() {
|
|
36
|
+
lastPhase1Work = 0;
|
|
37
|
+
}
|
package/dist/src/store.d.ts
CHANGED
|
@@ -57,6 +57,12 @@ export declare class MemoryStore {
|
|
|
57
57
|
/** Extraction succeeded but produced nothing worth keeping: finish the job and drop any stale output. */
|
|
58
58
|
markStage1SucceededNoOutput(sessionId: string, ownershipToken: string, sourceUpdatedAt: number): void;
|
|
59
59
|
markStage1Failed(sessionId: string, ownershipToken: string, error: unknown): void;
|
|
60
|
+
/**
|
|
61
|
+
* Plugin dispose/reload: release a claimed stage-1 job without burning a retry
|
|
62
|
+
* or imposing the 1h backoff. Leaves status=pending so the next process can
|
|
63
|
+
* reclaim immediately (unlike markStage1Failed). Ownership-token guarded.
|
|
64
|
+
*/
|
|
65
|
+
releaseStage1OnShutdown(sessionId: string, ownershipToken: string): void;
|
|
60
66
|
/**
|
|
61
67
|
* Enqueues global consolidation after stage-1 state changes. If phase 2 is
|
|
62
68
|
* already running, preserve its lease and advance only the input watermark.
|
|
@@ -91,6 +97,11 @@ export declare class MemoryStore {
|
|
|
91
97
|
last_success_watermark: number | null;
|
|
92
98
|
} | null;
|
|
93
99
|
markPhase2Failed(ownershipToken: string, error: unknown): void;
|
|
100
|
+
/**
|
|
101
|
+
* Plugin dispose/reload: release the global phase-2 job without retry backoff
|
|
102
|
+
* so the next process can reclaim immediately. Ownership-token guarded.
|
|
103
|
+
*/
|
|
104
|
+
releasePhase2OnShutdown(ownershipToken: string): void;
|
|
94
105
|
/**
|
|
95
106
|
* Phase 2 input set, mirroring codex get_phase2_input_selection:
|
|
96
107
|
* - excludes sessions marked disabled/polluted (their summary files then
|
|
@@ -109,6 +120,13 @@ export declare class MemoryStore {
|
|
|
109
120
|
* codex clear_memory_data deletes extracted memories and jobs but explicitly
|
|
110
121
|
* preserves per-session memory modes: a reset must not re-enable sessions
|
|
111
122
|
* the user disabled or that were marked polluted.
|
|
123
|
+
*
|
|
124
|
+
* After the wipe, leave a phase-2 cooldown marker (status=done, finished_at
|
|
125
|
+
* now, last_error NULL). Without it, the next idle/chat hook first-run-claims
|
|
126
|
+
* phase 2 on an empty DB and ensureLayout re-seeds the just-wiped root —
|
|
127
|
+
* memory_reset then looks like a no-op to the caller. Codex avoids this
|
|
128
|
+
* because reset is a client RPC outside the write-pipeline pump; the plugin
|
|
129
|
+
* surface is model-invoked mid-session, so hooks can race the wipe.
|
|
112
130
|
*/
|
|
113
131
|
clearMemoryData(): void;
|
|
114
132
|
setMemoryMode(sessionId: string, mode: "enabled" | "disabled" | "polluted"): void;
|
|
@@ -122,4 +140,17 @@ export declare class MemoryStore {
|
|
|
122
140
|
getMemoryMode(sessionId: string): "enabled" | "disabled" | "polluted" | null;
|
|
123
141
|
markPolluted(sessionId: string): void;
|
|
124
142
|
isPolluted(sessionId: string): boolean;
|
|
143
|
+
/**
|
|
144
|
+
* Stage-1 job counts + recent failures for memory_inspect. Helps diagnose
|
|
145
|
+
* "nothing is learning" without reading the raw jobs table.
|
|
146
|
+
*/
|
|
147
|
+
stage1JobSnapshot(): {
|
|
148
|
+
by_status: Record<string, number>;
|
|
149
|
+
recent_errors: {
|
|
150
|
+
session_id: string;
|
|
151
|
+
last_error: string;
|
|
152
|
+
retry_at: number | null;
|
|
153
|
+
status: string;
|
|
154
|
+
}[];
|
|
155
|
+
};
|
|
125
156
|
}
|
package/dist/src/store.js
CHANGED
|
@@ -202,6 +202,22 @@ export class MemoryStore {
|
|
|
202
202
|
WHERE kind='memory_stage1' AND job_key=? AND status='running' AND ownership_token=?`)
|
|
203
203
|
.run(message.slice(0, 4000), nowSec() + STAGE1_RETRY_DELAY_SECONDS, nowSec(), sessionId, ownershipToken);
|
|
204
204
|
}
|
|
205
|
+
/**
|
|
206
|
+
* Plugin dispose/reload: release a claimed stage-1 job without burning a retry
|
|
207
|
+
* or imposing the 1h backoff. Leaves status=pending so the next process can
|
|
208
|
+
* reclaim immediately (unlike markStage1Failed). Ownership-token guarded.
|
|
209
|
+
*/
|
|
210
|
+
releaseStage1OnShutdown(sessionId, ownershipToken) {
|
|
211
|
+
this.db
|
|
212
|
+
.prepare(`UPDATE memory_jobs SET
|
|
213
|
+
status = 'pending',
|
|
214
|
+
last_error = ?,
|
|
215
|
+
retry_at = NULL,
|
|
216
|
+
finished_at = ?,
|
|
217
|
+
lease_until = NULL
|
|
218
|
+
WHERE kind='memory_stage1' AND job_key=? AND status='running' AND ownership_token=?`)
|
|
219
|
+
.run("plugin shutting down", nowSec(), sessionId, ownershipToken);
|
|
220
|
+
}
|
|
205
221
|
/**
|
|
206
222
|
* Enqueues global consolidation after stage-1 state changes. If phase 2 is
|
|
207
223
|
* already running, preserve its lease and advance only the input watermark.
|
|
@@ -385,6 +401,21 @@ export class MemoryStore {
|
|
|
385
401
|
WHERE kind='memory_consolidate_global' AND job_key='global' AND status='running' AND ownership_token IS NULL`)
|
|
386
402
|
.run(message.slice(0, 4000), nowSec() + PHASE2_RETRY_DELAY_SECONDS, nowSec());
|
|
387
403
|
}
|
|
404
|
+
/**
|
|
405
|
+
* Plugin dispose/reload: release the global phase-2 job without retry backoff
|
|
406
|
+
* so the next process can reclaim immediately. Ownership-token guarded.
|
|
407
|
+
*/
|
|
408
|
+
releasePhase2OnShutdown(ownershipToken) {
|
|
409
|
+
this.db
|
|
410
|
+
.prepare(`UPDATE memory_jobs SET
|
|
411
|
+
status = 'pending',
|
|
412
|
+
last_error = ?,
|
|
413
|
+
retry_at = NULL,
|
|
414
|
+
finished_at = ?,
|
|
415
|
+
lease_until = NULL
|
|
416
|
+
WHERE kind='memory_consolidate_global' AND job_key='global' AND ownership_token=? AND status='running'`)
|
|
417
|
+
.run("plugin shutting down", nowSec(), ownershipToken);
|
|
418
|
+
}
|
|
388
419
|
/**
|
|
389
420
|
* Phase 2 input set, mirroring codex get_phase2_input_selection:
|
|
390
421
|
* - excludes sessions marked disabled/polluted (their summary files then
|
|
@@ -430,11 +461,23 @@ export class MemoryStore {
|
|
|
430
461
|
* codex clear_memory_data deletes extracted memories and jobs but explicitly
|
|
431
462
|
* preserves per-session memory modes: a reset must not re-enable sessions
|
|
432
463
|
* the user disabled or that were marked polluted.
|
|
464
|
+
*
|
|
465
|
+
* After the wipe, leave a phase-2 cooldown marker (status=done, finished_at
|
|
466
|
+
* now, last_error NULL). Without it, the next idle/chat hook first-run-claims
|
|
467
|
+
* phase 2 on an empty DB and ensureLayout re-seeds the just-wiped root —
|
|
468
|
+
* memory_reset then looks like a no-op to the caller. Codex avoids this
|
|
469
|
+
* because reset is a client RPC outside the write-pipeline pump; the plugin
|
|
470
|
+
* surface is model-invoked mid-session, so hooks can race the wipe.
|
|
433
471
|
*/
|
|
434
472
|
clearMemoryData() {
|
|
435
473
|
this.db.transaction(() => {
|
|
436
474
|
this.db.run("DELETE FROM memory_stage1_outputs");
|
|
437
475
|
this.db.run("DELETE FROM memory_jobs");
|
|
476
|
+
this.db
|
|
477
|
+
.prepare(`INSERT INTO memory_jobs
|
|
478
|
+
(kind, job_key, status, finished_at, last_error, retry_remaining, last_success_watermark)
|
|
479
|
+
VALUES ('memory_consolidate_global', 'global', 'done', ?, NULL, ?, 0)`)
|
|
480
|
+
.run(nowSec(), DEFAULT_RETRY_REMAINING);
|
|
438
481
|
}).immediate();
|
|
439
482
|
}
|
|
440
483
|
setMemoryMode(sessionId, mode) {
|
|
@@ -475,4 +518,23 @@ export class MemoryStore {
|
|
|
475
518
|
.get(sessionId);
|
|
476
519
|
return row?.p === 1;
|
|
477
520
|
}
|
|
521
|
+
/**
|
|
522
|
+
* Stage-1 job counts + recent failures for memory_inspect. Helps diagnose
|
|
523
|
+
* "nothing is learning" without reading the raw jobs table.
|
|
524
|
+
*/
|
|
525
|
+
stage1JobSnapshot() {
|
|
526
|
+
const rows = this.db
|
|
527
|
+
.prepare("SELECT status, COUNT(*) AS c FROM memory_jobs WHERE kind='memory_stage1' GROUP BY status")
|
|
528
|
+
.all();
|
|
529
|
+
const by_status = {};
|
|
530
|
+
for (const r of rows)
|
|
531
|
+
by_status[r.status] = r.c;
|
|
532
|
+
const recent_errors = this.db
|
|
533
|
+
.prepare(`SELECT job_key AS session_id, last_error, retry_at, status FROM memory_jobs
|
|
534
|
+
WHERE kind='memory_stage1' AND last_error IS NOT NULL
|
|
535
|
+
ORDER BY COALESCE(finished_at, started_at, 0) DESC
|
|
536
|
+
LIMIT 5`)
|
|
537
|
+
.all();
|
|
538
|
+
return { by_status, recent_errors };
|
|
539
|
+
}
|
|
478
540
|
}
|