opencode-codex-memory 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/src/capture.js +7 -28
- package/dist/src/host-client.d.ts +26 -2
- package/dist/src/host-client.js +86 -3
- package/dist/src/llm.d.ts +8 -0
- package/dist/src/llm.js +169 -51
- package/dist/src/phase1.js +17 -3
- package/dist/src/phase2.d.ts +12 -0
- package/dist/src/phase2.js +66 -6
- package/dist/src/ratelimit.d.ts +18 -2
- package/dist/src/ratelimit.js +91 -10
- package/dist/src/store.d.ts +22 -6
- package/dist/src/store.js +75 -6
- package/dist/tools/control.js +22 -2
- package/dist/tools/memory.js +37 -7
- package/package.json +1 -1
package/dist/src/phase1.js
CHANGED
|
@@ -3,7 +3,7 @@ import { loadTranscript, selectEligibleSessions } from "./capture.js";
|
|
|
3
3
|
import { redact, isMemoryExcludedFragment } from "./redact.js";
|
|
4
4
|
import { stripCitations } from "./citation.js";
|
|
5
5
|
import { extractViaSubagent, SubagentCancelledError } from "./llm.js";
|
|
6
|
-
import { checkRateLimit, markRateLimitUsed } from "./ratelimit.js";
|
|
6
|
+
import { checkRateLimit, isProviderCapacityBlocked, isProviderCapacityError, markRateLimitUsed, noteProviderCapacityExhausted, ProviderCapacityError, } from "./ratelimit.js";
|
|
7
7
|
import { isPluginShuttingDown } from "./lifecycle.js";
|
|
8
8
|
import { recordDiagnostic } from "./diagnostics.js";
|
|
9
9
|
export const DEFAULT_PHASE1_OPTIONS = {
|
|
@@ -26,7 +26,14 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
26
26
|
return;
|
|
27
27
|
// Prune first (no tokens), matching codex start.rs ordering before the gate.
|
|
28
28
|
store.pruneStage1Outputs(opts.maxUnusedDays ?? 30);
|
|
29
|
-
|
|
29
|
+
// Historical quota-exhausted jobs never get a newer watermark; reopen them
|
|
30
|
+
// before the gate so a later unblocked pass can claim them.
|
|
31
|
+
const requeued = store.requeueExhaustedProviderCapacityJobs();
|
|
32
|
+
if (requeued > 0) {
|
|
33
|
+
recordDiagnostic("info", "phase1", `requeued ${requeued} quota-exhausted job(s)`);
|
|
34
|
+
}
|
|
35
|
+
const extractModel = opts.extractModel;
|
|
36
|
+
const rl = await rateLimitCheck("phase1", extractModel);
|
|
30
37
|
if (!rl.ok) {
|
|
31
38
|
console.warn("[opencode-codex-memory] skipping phase1 due to rate limit:", rl.reason);
|
|
32
39
|
return;
|
|
@@ -54,6 +61,11 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
54
61
|
store.releaseStage1OnShutdown(sid, claim.ownershipToken);
|
|
55
62
|
return;
|
|
56
63
|
}
|
|
64
|
+
// Sibling jobs in this pass: do not call the model after quota was observed.
|
|
65
|
+
if (isProviderCapacityBlocked("phase1", extractModel)) {
|
|
66
|
+
store.markStage1Failed(sid, claim.ownershipToken, new ProviderCapacityError("provider capacity exhausted"));
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
57
69
|
try {
|
|
58
70
|
const session = sessionById.get(sid);
|
|
59
71
|
const sourceUpdatedAt = session?.updated_at ?? Date.now();
|
|
@@ -74,7 +86,7 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
74
86
|
}
|
|
75
87
|
const result = await extractViaSubagent(sid, transcript, {
|
|
76
88
|
cwd: session?.directory ?? undefined,
|
|
77
|
-
model:
|
|
89
|
+
model: extractModel,
|
|
78
90
|
});
|
|
79
91
|
// Finalize after a completed model call even if dispose raced, so the
|
|
80
92
|
// claim does not sit `running` until lease expiry. The mark is token +
|
|
@@ -103,6 +115,8 @@ export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS, rateLimitC
|
|
|
103
115
|
store.releaseStage1OnShutdown(sid, claim.ownershipToken);
|
|
104
116
|
}
|
|
105
117
|
else {
|
|
118
|
+
if (isProviderCapacityError(err))
|
|
119
|
+
noteProviderCapacityExhausted("phase1", extractModel);
|
|
106
120
|
store.markStage1Failed(sid, claim.ownershipToken, err);
|
|
107
121
|
}
|
|
108
122
|
}
|
package/dist/src/phase2.d.ts
CHANGED
|
@@ -12,6 +12,18 @@ export interface Phase2Options {
|
|
|
12
12
|
heartbeatIntervalMs?: number;
|
|
13
13
|
}
|
|
14
14
|
export declare const DEFAULT_PHASE2_OPTIONS: Phase2Options;
|
|
15
|
+
/**
|
|
16
|
+
* Codex get_phase2_input_selection re-validates each row against the live
|
|
17
|
+
* threads table. We only drop a row on a confirmed 404 (same as session.deleted);
|
|
18
|
+
* timeouts / missing get / other errors keep the row.
|
|
19
|
+
*/
|
|
20
|
+
export declare function dropGonePhase2Inputs(store: MemoryStore, selected: ReturnType<MemoryStore["getPhase2InputSelection"]>): Promise<ReturnType<MemoryStore["getPhase2InputSelection"]>>;
|
|
21
|
+
/**
|
|
22
|
+
* Codex pages ranked candidates until it has `maxRaw` rows whose threads are
|
|
23
|
+
* still live. Our thread metadata lives in the host, so confirmed-gone rows
|
|
24
|
+
* are deleted and the ranking is queried again to backfill their slots.
|
|
25
|
+
*/
|
|
26
|
+
export declare function selectLivePhase2Inputs(store: MemoryStore, maxRaw: number, maxUnusedDays: number): Promise<ReturnType<MemoryStore["getPhase2InputSelection"]>>;
|
|
15
27
|
/** True while THIS process runs a consolidation (memory_reset refuses then). */
|
|
16
28
|
export declare function isPhase2InFlight(): boolean;
|
|
17
29
|
export declare function runPhase2(store: MemoryStore, opts?: Phase2Options): Promise<{
|
package/dist/src/phase2.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { checkRateLimit, isProviderCapacityError, noteProviderCapacityExhausted } from "./ratelimit.js";
|
|
1
2
|
import { ensureLayout, rebuildRawMemories, writeRolloutSummaries, pruneExtensionResources, writeWorkspaceDiff, validateConsolidationArtifacts, } from "./workspace.js";
|
|
2
3
|
import { ensureBaseline, captureWorkspaceDiff, resetBaseline, DIFF_ARTIFACT } from "./git-baseline.js";
|
|
3
|
-
import { consolidateViaSubagent, SubagentCancelledError, SubagentShutdownError } from "./llm.js";
|
|
4
|
+
import { consolidateViaSubagent, getPluginInput, SubagentCancelledError, SubagentShutdownError, } from "./llm.js";
|
|
5
|
+
import { hostSessionLiveness } from "./host-client.js";
|
|
4
6
|
import { invalidateCache } from "./source.js";
|
|
5
7
|
import { memoryRoot } from "./paths.js";
|
|
6
8
|
import { abortPhase2Consolidation, beginPhase2AbortScope, endPhase2AbortScope, isPluginShuttingDown, } from "./lifecycle.js";
|
|
@@ -13,6 +15,59 @@ export const DEFAULT_PHASE2_OPTIONS = {
|
|
|
13
15
|
};
|
|
14
16
|
// Export runs only after a successful phase 2 (fresh, validated artifacts) and
|
|
15
17
|
// must never fail the run — Codex's workspace is best-effort foreign territory.
|
|
18
|
+
const PHASE2_LIVE_CHECK_CONCURRENCY = 8;
|
|
19
|
+
/**
|
|
20
|
+
* Codex get_phase2_input_selection re-validates each row against the live
|
|
21
|
+
* threads table. We only drop a row on a confirmed 404 (same as session.deleted);
|
|
22
|
+
* timeouts / missing get / other errors keep the row.
|
|
23
|
+
*/
|
|
24
|
+
export async function dropGonePhase2Inputs(store, selected) {
|
|
25
|
+
const client = getPluginInput()?.client;
|
|
26
|
+
if (!client || selected.length === 0)
|
|
27
|
+
return selected;
|
|
28
|
+
const kept = [];
|
|
29
|
+
for (let i = 0; i < selected.length; i += PHASE2_LIVE_CHECK_CONCURRENCY) {
|
|
30
|
+
const chunk = selected.slice(i, i + PHASE2_LIVE_CHECK_CONCURRENCY);
|
|
31
|
+
const results = await Promise.all(chunk.map(async (out) => ({ out, live: await hostSessionLiveness(client, out.session_id) })));
|
|
32
|
+
for (const { out, live } of results) {
|
|
33
|
+
if (live === "gone")
|
|
34
|
+
store.deleteSessionMemory(out.session_id);
|
|
35
|
+
else
|
|
36
|
+
kept.push(out);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return kept;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Codex pages ranked candidates until it has `maxRaw` rows whose threads are
|
|
43
|
+
* still live. Our thread metadata lives in the host, so confirmed-gone rows
|
|
44
|
+
* are deleted and the ranking is queried again to backfill their slots.
|
|
45
|
+
*/
|
|
46
|
+
export async function selectLivePhase2Inputs(store, maxRaw, maxUnusedDays) {
|
|
47
|
+
if (maxRaw <= 0)
|
|
48
|
+
return [];
|
|
49
|
+
const selected = [];
|
|
50
|
+
const seen = new Set();
|
|
51
|
+
while (selected.length < maxRaw) {
|
|
52
|
+
// Once most slots are filled, inspect one liveness chunk beyond the
|
|
53
|
+
// already-selected rows so a single gone row does not force serial probes.
|
|
54
|
+
const scanLimit = Math.max(maxRaw, selected.length + PHASE2_LIVE_CHECK_CONCURRENCY);
|
|
55
|
+
const candidates = store
|
|
56
|
+
.getPhase2InputSelection(scanLimit, maxUnusedDays)
|
|
57
|
+
.filter((output) => !seen.has(output.session_id));
|
|
58
|
+
if (candidates.length === 0)
|
|
59
|
+
break;
|
|
60
|
+
for (const output of candidates)
|
|
61
|
+
seen.add(output.session_id);
|
|
62
|
+
const live = await dropGonePhase2Inputs(store, candidates);
|
|
63
|
+
for (const output of live) {
|
|
64
|
+
selected.push(output);
|
|
65
|
+
if (selected.length >= maxRaw)
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return selected;
|
|
70
|
+
}
|
|
16
71
|
function maybeExportToCodex(interop) {
|
|
17
72
|
if (!interop?.exportEnabled)
|
|
18
73
|
return;
|
|
@@ -42,9 +97,12 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS) {
|
|
|
42
97
|
return { status: "already_running" };
|
|
43
98
|
phase2InFlight = true;
|
|
44
99
|
try {
|
|
45
|
-
// No process
|
|
46
|
-
//
|
|
47
|
-
|
|
100
|
+
// No 30s process gate: codex serializes phase 2 only via the DB claim.
|
|
101
|
+
// An observed quota stamp still skips both phases (Codex start.rs).
|
|
102
|
+
const consolidationModel = opts.consolidationModel;
|
|
103
|
+
const rl = await checkRateLimit("phase2", consolidationModel);
|
|
104
|
+
if (!rl.ok)
|
|
105
|
+
return { status: "skipped_rate_limit" };
|
|
48
106
|
const claim = store.claimGlobalPhase2Job();
|
|
49
107
|
if (claim.type !== "claimed")
|
|
50
108
|
return { status: claim.type };
|
|
@@ -73,7 +131,7 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS) {
|
|
|
73
131
|
if (releaseIfShuttingDown(store, claim.ownershipToken)) {
|
|
74
132
|
return { status: "shutting_down" };
|
|
75
133
|
}
|
|
76
|
-
const outputs = store
|
|
134
|
+
const outputs = await selectLivePhase2Inputs(store, opts.maxRaw, opts.maxUnusedDays);
|
|
77
135
|
rebuildRawMemories(outputs);
|
|
78
136
|
writeRolloutSummaries(outputs);
|
|
79
137
|
pruneExtensionResources(opts.extensionRetentionDays);
|
|
@@ -158,7 +216,7 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS) {
|
|
|
158
216
|
}
|
|
159
217
|
const heartbeat = setInterval(heartbeatOnce, opts.heartbeatIntervalMs ?? 90_000);
|
|
160
218
|
try {
|
|
161
|
-
await consolidateViaSubagent(memoryRoot(), DIFF_ARTIFACT,
|
|
219
|
+
await consolidateViaSubagent(memoryRoot(), DIFF_ARTIFACT, consolidationModel, consolidationSignal);
|
|
162
220
|
}
|
|
163
221
|
catch (err) {
|
|
164
222
|
// codex phase2.rs: when the consolidation agent's shutdown fails, keep
|
|
@@ -218,6 +276,8 @@ export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS) {
|
|
|
218
276
|
store.releasePhase2OnShutdown(claim.ownershipToken);
|
|
219
277
|
return { status: "shutting_down" };
|
|
220
278
|
}
|
|
279
|
+
if (isProviderCapacityError(err))
|
|
280
|
+
noteProviderCapacityExhausted("phase2", consolidationModel);
|
|
221
281
|
store.markPhase2Failed(claim.ownershipToken, err);
|
|
222
282
|
return { status: "failed" };
|
|
223
283
|
}
|
package/dist/src/ratelimit.d.ts
CHANGED
|
@@ -2,8 +2,24 @@ export interface RateLimitInfo {
|
|
|
2
2
|
ok: boolean;
|
|
3
3
|
reason?: string;
|
|
4
4
|
}
|
|
5
|
-
export
|
|
5
|
+
export type MemoryPhase = "phase1" | "phase2";
|
|
6
|
+
export interface ProviderCapacityBackoff {
|
|
7
|
+
scope: string;
|
|
8
|
+
retry_at: number;
|
|
9
|
+
}
|
|
10
|
+
export declare class ProviderCapacityError extends Error {
|
|
11
|
+
readonly statusCode?: number | undefined;
|
|
12
|
+
constructor(message: string, statusCode?: number | undefined);
|
|
13
|
+
}
|
|
14
|
+
export declare const PROVIDER_CAPACITY_BACKOFF_MS = 3600000;
|
|
15
|
+
export declare function providerCapacityMessage(error: unknown): string;
|
|
16
|
+
export declare function isProviderCapacityError(error: unknown): boolean;
|
|
17
|
+
export declare function activeProviderCapacityBackoffs(now?: number): ProviderCapacityBackoff[];
|
|
18
|
+
export declare function isProviderCapacityBlocked(phase: MemoryPhase, model?: string, now?: number): boolean;
|
|
19
|
+
/** Call after a quota/rate-limit failure so later passes skip claiming. */
|
|
20
|
+
export declare function noteProviderCapacityExhausted(phase: MemoryPhase, model?: string, now?: number): void;
|
|
21
|
+
export declare function checkRateLimit(kind?: MemoryPhase, model?: string): Promise<RateLimitInfo>;
|
|
6
22
|
/** Call after a phase-1 pass claimed at least one job (token-using work started). */
|
|
7
23
|
export declare function markRateLimitUsed(kind?: "phase1" | "phase2"): void;
|
|
8
|
-
/** Test seam: reset the process-local
|
|
24
|
+
/** Test seam: reset the process-local stamps. */
|
|
9
25
|
export declare function resetRateLimitForTest(): void;
|
package/dist/src/ratelimit.js
CHANGED
|
@@ -1,23 +1,103 @@
|
|
|
1
|
+
export class ProviderCapacityError extends Error {
|
|
2
|
+
statusCode;
|
|
3
|
+
constructor(message, statusCode) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.statusCode = statusCode;
|
|
6
|
+
this.name = "ProviderCapacityError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
1
9
|
/**
|
|
2
|
-
* Process-local anti-stampede for phase 1
|
|
10
|
+
* Process-local anti-stampede for phase 1, plus an observed-quota circuit
|
|
11
|
+
* breaker that stands in for Codex guard.rs.
|
|
3
12
|
*
|
|
4
13
|
* Codex has no wall-clock throttle: it reads live provider quota once per
|
|
5
14
|
* startup (memories/write/src/guard.rs) and fails open when unknown. Opencode
|
|
6
|
-
* does not expose provider rate limits to plugins, so this stub
|
|
7
|
-
* chat.message/idle from hammering discovery + claim
|
|
8
|
-
*
|
|
15
|
+
* does not expose provider rate limits to plugins, so this stub:
|
|
16
|
+
* - keeps chat.message/idle from hammering discovery + claim (30s, phase 1)
|
|
17
|
+
* - after a quota/rate-limit API error, skips further claims until the same
|
|
18
|
+
* 1h window Codex uses for job retry_at — so a quota outage cannot burn
|
|
19
|
+
* every eligible session's retry budget
|
|
9
20
|
*
|
|
10
21
|
* Semantics deliberately match "do not start another token-using run too
|
|
11
22
|
* often", not "do not look often":
|
|
12
|
-
* - checkRateLimit only reads
|
|
23
|
+
* - checkRateLimit only reads clocks (empty/no-claim passes do not stamp)
|
|
13
24
|
* - markRateLimitUsed stamps after a stage-1 claim actually succeeds
|
|
14
|
-
* -
|
|
15
|
-
*
|
|
25
|
+
* - noteProviderCapacityExhausted stamps after an observed quota error
|
|
26
|
+
* - phase 2 has no 30s timer; the DB claim + 6h cooldown serialize it.
|
|
27
|
+
* The observed-quota stamp still skips phase 2 (Codex start.rs skips both).
|
|
16
28
|
*/
|
|
17
29
|
let lastPhase1Work = 0;
|
|
30
|
+
const providerCapacityUntil = new Map();
|
|
18
31
|
const MIN_PHASE1_INTERVAL_MS = 30_000;
|
|
19
|
-
export
|
|
20
|
-
|
|
32
|
+
export const PROVIDER_CAPACITY_BACKOFF_MS = 3_600_000;
|
|
33
|
+
const PROVIDER_CAPACITY_RE = /usage limit|free usage exceeded|provider capacity exhausted|rate[\s_-]?limit|quota(?:\s+(?:exceeded|exhausted|reached))?|too many requests|\b429\b|resource_exhausted|insufficient_quota|billing.?hard.?limit/i;
|
|
34
|
+
function errorRecord(error) {
|
|
35
|
+
return error && typeof error === "object" ? error : null;
|
|
36
|
+
}
|
|
37
|
+
function providerCapacityStatusCode(error) {
|
|
38
|
+
const record = errorRecord(error);
|
|
39
|
+
const data = errorRecord(record?.data);
|
|
40
|
+
const value = record?.statusCode ?? data?.statusCode;
|
|
41
|
+
if (typeof value === "number")
|
|
42
|
+
return value;
|
|
43
|
+
if (typeof value === "string" && /^\d+$/.test(value))
|
|
44
|
+
return Number(value);
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
export function providerCapacityMessage(error) {
|
|
48
|
+
try {
|
|
49
|
+
if (typeof error === "string")
|
|
50
|
+
return error;
|
|
51
|
+
if (error instanceof Error)
|
|
52
|
+
return String(error.message ?? "unknown error");
|
|
53
|
+
const record = errorRecord(error);
|
|
54
|
+
const data = errorRecord(record?.data);
|
|
55
|
+
const message = record?.message ?? data?.message;
|
|
56
|
+
if (typeof message === "string")
|
|
57
|
+
return message;
|
|
58
|
+
return String(error ?? "unknown error");
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return "unknown error";
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export function isProviderCapacityError(error) {
|
|
65
|
+
if (error instanceof ProviderCapacityError)
|
|
66
|
+
return true;
|
|
67
|
+
if (providerCapacityStatusCode(error) === 429)
|
|
68
|
+
return true;
|
|
69
|
+
return PROVIDER_CAPACITY_RE.test(providerCapacityMessage(error));
|
|
70
|
+
}
|
|
71
|
+
function providerCapacityScope(phase, model) {
|
|
72
|
+
return model ? `model:${model}` : `phase:${phase}:default`;
|
|
73
|
+
}
|
|
74
|
+
export function activeProviderCapacityBackoffs(now = Date.now()) {
|
|
75
|
+
const active = [];
|
|
76
|
+
for (const [scope, until] of providerCapacityUntil) {
|
|
77
|
+
if (until <= now) {
|
|
78
|
+
providerCapacityUntil.delete(scope);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
active.push({ scope, retry_at: Math.floor(until / 1000) });
|
|
82
|
+
}
|
|
83
|
+
return active.sort((a, b) => a.scope.localeCompare(b.scope));
|
|
84
|
+
}
|
|
85
|
+
export function isProviderCapacityBlocked(phase, model, now = Date.now()) {
|
|
86
|
+
const until = providerCapacityUntil.get(providerCapacityScope(phase, model));
|
|
87
|
+
return until !== undefined && until > now;
|
|
88
|
+
}
|
|
89
|
+
/** Call after a quota/rate-limit failure so later passes skip claiming. */
|
|
90
|
+
export function noteProviderCapacityExhausted(phase, model, now = Date.now()) {
|
|
91
|
+
providerCapacityUntil.set(providerCapacityScope(phase, model), now + PROVIDER_CAPACITY_BACKOFF_MS);
|
|
92
|
+
}
|
|
93
|
+
export async function checkRateLimit(kind = "phase1", model) {
|
|
94
|
+
if (isProviderCapacityBlocked(kind, model)) {
|
|
95
|
+
return {
|
|
96
|
+
ok: false,
|
|
97
|
+
reason: `provider capacity exhausted for ${providerCapacityScope(kind, model)} (observed quota/rate-limit)`,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
// Phase 2: no 30s gate (codex relies on DB claim/cooldown only).
|
|
21
101
|
if (kind === "phase2")
|
|
22
102
|
return { ok: true };
|
|
23
103
|
const now = Date.now();
|
|
@@ -31,7 +111,8 @@ export function markRateLimitUsed(kind = "phase1") {
|
|
|
31
111
|
if (kind === "phase1")
|
|
32
112
|
lastPhase1Work = Date.now();
|
|
33
113
|
}
|
|
34
|
-
/** Test seam: reset the process-local
|
|
114
|
+
/** Test seam: reset the process-local stamps. */
|
|
35
115
|
export function resetRateLimitForTest() {
|
|
36
116
|
lastPhase1Work = 0;
|
|
117
|
+
providerCapacityUntil.clear();
|
|
37
118
|
}
|
package/dist/src/store.d.ts
CHANGED
|
@@ -57,6 +57,13 @@ export declare class MemoryStore {
|
|
|
57
57
|
/** Extraction succeeded but produced nothing worth keeping: finish the job and drop any stale output. */
|
|
58
58
|
markStage1SucceededNoOutput(sessionId: string, ownershipToken: string, sourceUpdatedAt: number): void;
|
|
59
59
|
markStage1Failed(sessionId: string, ownershipToken: string, error: unknown): void;
|
|
60
|
+
/**
|
|
61
|
+
* Re-open stage-1 jobs that exhausted their retry budget solely because of
|
|
62
|
+
* a quota/rate-limit error. Historical completed sessions never get a newer
|
|
63
|
+
* watermark, so without this they stay failed forever after a quota outage.
|
|
64
|
+
* Leaves retry_at alone so an active backoff still holds.
|
|
65
|
+
*/
|
|
66
|
+
requeueExhaustedProviderCapacityJobs(): number;
|
|
60
67
|
/**
|
|
61
68
|
* Plugin dispose/reload: release a claimed stage-1 job without burning a retry
|
|
62
69
|
* or imposing the 1h backoff. Leaves status=pending so the next process can
|
|
@@ -146,11 +153,20 @@ export declare class MemoryStore {
|
|
|
146
153
|
*/
|
|
147
154
|
stage1JobSnapshot(): {
|
|
148
155
|
by_status: Record<string, number>;
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
156
|
+
by_failure_class: {
|
|
157
|
+
backoff: number;
|
|
158
|
+
provider_capacity: number;
|
|
159
|
+
other_exhausted: number;
|
|
160
|
+
};
|
|
161
|
+
recent_errors: Stage1RecentError[];
|
|
155
162
|
};
|
|
156
163
|
}
|
|
164
|
+
export type Stage1FailureClass = "backoff" | "provider_capacity" | "other_exhausted";
|
|
165
|
+
export interface Stage1RecentError {
|
|
166
|
+
session_id: string;
|
|
167
|
+
last_error: string;
|
|
168
|
+
retry_at: number | null;
|
|
169
|
+
status: string;
|
|
170
|
+
retry_remaining: number;
|
|
171
|
+
failure_class: Stage1FailureClass | null;
|
|
172
|
+
}
|
package/dist/src/store.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { openDb } from "./db.js";
|
|
2
|
+
import { isProviderCapacityError } from "./ratelimit.js";
|
|
2
3
|
export const DEFAULT_RETRY_REMAINING = 3;
|
|
3
4
|
export const STAGE1_LEASE_SECONDS = 3600;
|
|
4
5
|
export const PHASE2_LEASE_SECONDS = 3600;
|
|
@@ -191,6 +192,24 @@ export class MemoryStore {
|
|
|
191
192
|
}
|
|
192
193
|
markStage1Failed(sessionId, ownershipToken, error) {
|
|
193
194
|
const message = failureMessage(error);
|
|
195
|
+
const tNow = nowSec();
|
|
196
|
+
const retryAt = tNow + STAGE1_RETRY_DELAY_SECONDS;
|
|
197
|
+
// Quota/rate-limit is transient provider capacity, not a bad transcript.
|
|
198
|
+
// Codex avoids claiming in that state via guard.rs; we cannot read quota,
|
|
199
|
+
// so keep the job pending and do not burn retry_remaining. Claim still
|
|
200
|
+
// honors retry_at, so this cannot tight-loop while quota is down.
|
|
201
|
+
if (isProviderCapacityError(error)) {
|
|
202
|
+
this.db
|
|
203
|
+
.prepare(`UPDATE memory_jobs SET
|
|
204
|
+
status = 'pending',
|
|
205
|
+
last_error = ?,
|
|
206
|
+
retry_at = ?,
|
|
207
|
+
finished_at = ?,
|
|
208
|
+
lease_until = NULL
|
|
209
|
+
WHERE kind='memory_stage1' AND job_key=? AND status='running' AND ownership_token=?`)
|
|
210
|
+
.run(message.slice(0, 4000), retryAt, tNow, sessionId, ownershipToken);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
194
213
|
this.db
|
|
195
214
|
.prepare(`UPDATE memory_jobs SET
|
|
196
215
|
status = CASE WHEN retry_remaining > 1 THEN 'pending' ELSE 'failed' END,
|
|
@@ -200,7 +219,30 @@ export class MemoryStore {
|
|
|
200
219
|
finished_at = ?,
|
|
201
220
|
lease_until = NULL
|
|
202
221
|
WHERE kind='memory_stage1' AND job_key=? AND status='running' AND ownership_token=?`)
|
|
203
|
-
.run(message.slice(0, 4000),
|
|
222
|
+
.run(message.slice(0, 4000), retryAt, tNow, sessionId, ownershipToken);
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Re-open stage-1 jobs that exhausted their retry budget solely because of
|
|
226
|
+
* a quota/rate-limit error. Historical completed sessions never get a newer
|
|
227
|
+
* watermark, so without this they stay failed forever after a quota outage.
|
|
228
|
+
* Leaves retry_at alone so an active backoff still holds.
|
|
229
|
+
*/
|
|
230
|
+
requeueExhaustedProviderCapacityJobs() {
|
|
231
|
+
const rows = this.db
|
|
232
|
+
.prepare(`SELECT job_key, last_error FROM memory_jobs
|
|
233
|
+
WHERE kind='memory_stage1' AND status='failed' AND last_error IS NOT NULL`)
|
|
234
|
+
.all();
|
|
235
|
+
let n = 0;
|
|
236
|
+
const stmt = this.db.prepare(`UPDATE memory_jobs SET status='pending', retry_remaining=?
|
|
237
|
+
WHERE kind='memory_stage1' AND job_key=? AND status='failed'`);
|
|
238
|
+
this.db.transaction(() => {
|
|
239
|
+
for (const row of rows) {
|
|
240
|
+
if (!isProviderCapacityError(row.last_error))
|
|
241
|
+
continue;
|
|
242
|
+
n += stmt.run(DEFAULT_RETRY_REMAINING, row.job_key).changes;
|
|
243
|
+
}
|
|
244
|
+
}).immediate();
|
|
245
|
+
return n;
|
|
204
246
|
}
|
|
205
247
|
/**
|
|
206
248
|
* Plugin dispose/reload: release a claimed stage-1 job without burning a retry
|
|
@@ -529,12 +571,39 @@ export class MemoryStore {
|
|
|
529
571
|
const by_status = {};
|
|
530
572
|
for (const r of rows)
|
|
531
573
|
by_status[r.status] = r.c;
|
|
532
|
-
const
|
|
533
|
-
|
|
574
|
+
const tNow = nowSec();
|
|
575
|
+
const errorRows = this.db
|
|
576
|
+
.prepare(`SELECT job_key AS session_id, last_error, retry_at, status, retry_remaining FROM memory_jobs
|
|
534
577
|
WHERE kind='memory_stage1' AND last_error IS NOT NULL
|
|
535
|
-
ORDER BY COALESCE(finished_at, started_at, 0) DESC
|
|
536
|
-
LIMIT 5`)
|
|
578
|
+
ORDER BY COALESCE(finished_at, started_at, 0) DESC`)
|
|
537
579
|
.all();
|
|
538
|
-
|
|
580
|
+
const by_failure_class = { backoff: 0, provider_capacity: 0, other_exhausted: 0 };
|
|
581
|
+
const recent_errors = [];
|
|
582
|
+
for (const row of errorRows) {
|
|
583
|
+
const failure_class = classifyStage1Failure(row, tNow);
|
|
584
|
+
if (failure_class)
|
|
585
|
+
by_failure_class[failure_class]++;
|
|
586
|
+
if (recent_errors.length < 5) {
|
|
587
|
+
recent_errors.push({
|
|
588
|
+
session_id: row.session_id,
|
|
589
|
+
last_error: row.last_error,
|
|
590
|
+
retry_at: row.retry_at,
|
|
591
|
+
status: row.status,
|
|
592
|
+
retry_remaining: row.retry_remaining,
|
|
593
|
+
failure_class,
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
return { by_status, by_failure_class, recent_errors };
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
function classifyStage1Failure(row, nowSec) {
|
|
601
|
+
if (row.status === "failed" || row.retry_remaining <= 0) {
|
|
602
|
+
return isProviderCapacityError(row.last_error) ? "provider_capacity" : "other_exhausted";
|
|
539
603
|
}
|
|
604
|
+
if (row.retry_at != null && row.retry_at > nowSec)
|
|
605
|
+
return "backoff";
|
|
606
|
+
if (isProviderCapacityError(row.last_error))
|
|
607
|
+
return "provider_capacity";
|
|
608
|
+
return null;
|
|
540
609
|
}
|
package/dist/tools/control.js
CHANGED
|
@@ -13,6 +13,7 @@ import { claudeImportStatus, resolveClaudeHome } from "../src/claude-import.js";
|
|
|
13
13
|
import { formatDiagnosticLine, getDiscoveryStatus, getRecentDiagnostics, } from "../src/diagnostics.js";
|
|
14
14
|
import { isPluginShuttingDown } from "../src/lifecycle.js";
|
|
15
15
|
import { getAgentHealth } from "../src/agent-health.js";
|
|
16
|
+
import { activeProviderCapacityBackoffs } from "../src/ratelimit.js";
|
|
16
17
|
function isSymlinkedRoot() {
|
|
17
18
|
try {
|
|
18
19
|
assertMemoryRootSafe();
|
|
@@ -200,7 +201,8 @@ function fmtWatermarkMs(ms) {
|
|
|
200
201
|
}
|
|
201
202
|
export const memory_inspect = tool({
|
|
202
203
|
description: "Inspect the current memory state. Returns: stage1_outputs count, stage-1 job status " +
|
|
203
|
-
"breakdown
|
|
204
|
+
"breakdown, failure classes (backoff / provider_capacity / other_exhausted), recent errors, " +
|
|
205
|
+
"Phase 2 job status (including last error / retry time), " +
|
|
204
206
|
"last discovery outcome, pipeline diagnostics, memory_summary token estimate " +
|
|
205
207
|
"(on-disk; injection caps at ~2500), a listing of the memories directory, the " +
|
|
206
208
|
"effective plugin options, and any configuration warnings. Use it to verify " +
|
|
@@ -244,9 +246,20 @@ export const memory_inspect = tool({
|
|
|
244
246
|
const stage1StatusParts = Object.entries(stage1Jobs.by_status)
|
|
245
247
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
246
248
|
.map(([s, c]) => `${s}=${c}`);
|
|
249
|
+
const fc = stage1Jobs.by_failure_class;
|
|
250
|
+
const failureParts = [
|
|
251
|
+
fc.backoff > 0 ? `backoff=${fc.backoff}` : "",
|
|
252
|
+
fc.provider_capacity > 0 ? `provider_capacity=${fc.provider_capacity}` : "",
|
|
253
|
+
fc.other_exhausted > 0 ? `other_exhausted=${fc.other_exhausted}` : "",
|
|
254
|
+
].filter(Boolean);
|
|
247
255
|
const stage1Lines = [
|
|
248
256
|
`stage1_jobs: ${stage1StatusParts.length > 0 ? stage1StatusParts.join(" ") : "none"}`,
|
|
249
|
-
|
|
257
|
+
`stage1_failures: ${failureParts.length > 0 ? failureParts.join(" ") : "none"}`,
|
|
258
|
+
...stage1Jobs.recent_errors.map((e) => {
|
|
259
|
+
const klass = e.failure_class ? `, ${e.failure_class}` : "";
|
|
260
|
+
const retry = e.retry_at ? ` retry_at=${fmtUnixSec(e.retry_at)}` : "";
|
|
261
|
+
return ` stage1_error ${e.session_id} (${e.status}${klass}): ${e.last_error.slice(0, 200)}${retry}`;
|
|
262
|
+
}),
|
|
250
263
|
];
|
|
251
264
|
const discovery = getDiscoveryStatus();
|
|
252
265
|
const discoveryLine = discovery
|
|
@@ -260,6 +273,10 @@ export const memory_inspect = tool({
|
|
|
260
273
|
`phase2_in_flight: ${isPhase2InFlight()}`,
|
|
261
274
|
`plugin_shutting_down: ${isPluginShuttingDown()}`,
|
|
262
275
|
];
|
|
276
|
+
const capacityBackoffs = activeProviderCapacityBackoffs();
|
|
277
|
+
const capacityLines = capacityBackoffs.length > 0
|
|
278
|
+
? capacityBackoffs.map((b) => `provider_capacity_backoff ${b.scope}: retry_at=${fmtUnixSec(b.retry_at)}`)
|
|
279
|
+
: ["provider_capacity_backoff: none"];
|
|
263
280
|
const diagnostics = getRecentDiagnostics(12);
|
|
264
281
|
const diagnosticLines = diagnostics.length > 0
|
|
265
282
|
? ["recent_events:", ...diagnostics.map((e) => ` ${formatDiagnosticLine(e)}`)]
|
|
@@ -271,6 +288,7 @@ export const memory_inspect = tool({
|
|
|
271
288
|
discoveryLine,
|
|
272
289
|
eligibilityHint,
|
|
273
290
|
...processLines,
|
|
291
|
+
...capacityLines,
|
|
274
292
|
`memory_summary_chars: ${summaryChars}`,
|
|
275
293
|
`memory_summary_tokens_est: ${summaryTokens} (on disk; injection caps at ~2500)`,
|
|
276
294
|
`memories_dir_entries: ${listing.length}`,
|
|
@@ -289,6 +307,7 @@ export const memory_inspect = tool({
|
|
|
289
307
|
metadata: {
|
|
290
308
|
stage1_count: outputs.length,
|
|
291
309
|
stage1_jobs: stage1Jobs.by_status,
|
|
310
|
+
stage1_failures: stage1Jobs.by_failure_class,
|
|
292
311
|
stage1_recent_errors: stage1Jobs.recent_errors,
|
|
293
312
|
phase2_status: phase2?.status ?? null,
|
|
294
313
|
phase2_last_error: phase2?.last_error ?? null,
|
|
@@ -296,6 +315,7 @@ export const memory_inspect = tool({
|
|
|
296
315
|
phase2_last_attempt_finished_at: phase2?.finished_at ?? null,
|
|
297
316
|
phase2_last_success_watermark: phase2?.last_success_watermark ?? null,
|
|
298
317
|
phase2_last_success_finished_at: phase2?.success_finished_at ?? null,
|
|
318
|
+
provider_capacity_backoffs: capacityBackoffs,
|
|
299
319
|
// Back-compat aliases used by earlier inspect consumers.
|
|
300
320
|
phase2_last_finished_at: phase2?.success_finished_at ?? null,
|
|
301
321
|
discovery,
|
package/dist/tools/memory.js
CHANGED
|
@@ -20,7 +20,9 @@ export const memory_read = tool({
|
|
|
20
20
|
}
|
|
21
21
|
const stat = fs.lstatSync(fullPath);
|
|
22
22
|
if (stat.isDirectory()) {
|
|
23
|
-
const entries =
|
|
23
|
+
const entries = visibleEntries(fullPath)
|
|
24
|
+
.sort((a, b) => comparePathNames(a.name, b.name))
|
|
25
|
+
.map((e) => e.name);
|
|
24
26
|
return {
|
|
25
27
|
output: `Directory ${args.path}/\n` + entries.map((e) => `- ${e}`).join("\n") + "\n(use memory_list for sorted, typed listings)",
|
|
26
28
|
metadata: { kind: "directory", entries },
|
|
@@ -194,7 +196,7 @@ function parseDateArg(value, endOfDay) {
|
|
|
194
196
|
function collectSearchFiles(start, prefix) {
|
|
195
197
|
const files = [];
|
|
196
198
|
const walk = (dir, rel) => {
|
|
197
|
-
const entries = visibleEntries(dir).sort((a, b) => a.name
|
|
199
|
+
const entries = visibleEntries(dir).sort((a, b) => comparePathNames(a.name, b.name));
|
|
198
200
|
for (const { name, isDir } of entries) {
|
|
199
201
|
const abs = path.join(dir, name);
|
|
200
202
|
const relPath = rel ? `${rel}/${name}` : name;
|
|
@@ -366,7 +368,21 @@ export const memory_search = tool({
|
|
|
366
368
|
}
|
|
367
369
|
const rangeLabel = timeFiltered ? ` in ${args.since ?? "..."}..${args.until ?? "..."}` : "";
|
|
368
370
|
if (queries.length === 0) {
|
|
369
|
-
|
|
371
|
+
let startIndex = 0;
|
|
372
|
+
if (args.cursor !== undefined) {
|
|
373
|
+
startIndex = Number.parseInt(args.cursor, 10);
|
|
374
|
+
if (!Number.isInteger(startIndex) || startIndex < 0 || String(startIndex) !== args.cursor.trim()) {
|
|
375
|
+
return { output: `memory_search error: invalid cursor "${args.cursor}" (must be a non-negative integer).` };
|
|
376
|
+
}
|
|
377
|
+
if (startIndex > files.length) {
|
|
378
|
+
return { output: `memory_search error: cursor ${startIndex} exceeds result count ${files.length}.` };
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
const endIndex = Math.min(startIndex + (args.max_results ?? SEARCH_MAX_RESULTS), files.length);
|
|
382
|
+
const page = files.slice(startIndex, endIndex);
|
|
383
|
+
const nextCursor = endIndex < files.length ? String(endIndex) : null;
|
|
384
|
+
const truncated = nextCursor !== null;
|
|
385
|
+
const listing = page.map((f) => {
|
|
370
386
|
let content = "";
|
|
371
387
|
try {
|
|
372
388
|
content = readRegularFileNoFollow(f.abs).content.toString("utf8");
|
|
@@ -375,11 +391,25 @@ export const memory_search = tool({
|
|
|
375
391
|
}
|
|
376
392
|
return `${new Date(f.ts).toISOString()} ${f.rel} — ${firstContentLine(content)}`;
|
|
377
393
|
});
|
|
378
|
-
if (
|
|
394
|
+
if (files.length === 0)
|
|
379
395
|
return { output: `No time-anchored memory files${rangeLabel}.` };
|
|
396
|
+
if (listing.length === 0) {
|
|
397
|
+
return {
|
|
398
|
+
output: `No memory files at cursor ${startIndex}${rangeLabel}.`,
|
|
399
|
+
metadata: { count: 0, next_cursor: nextCursor, truncated, since: args.since, until: args.until },
|
|
400
|
+
};
|
|
401
|
+
}
|
|
380
402
|
return {
|
|
381
|
-
output: `${listing.length} memory file(s)${rangeLabel}
|
|
382
|
-
|
|
403
|
+
output: `${listing.length} of ${files.length} memory file(s)${rangeLabel}` +
|
|
404
|
+
`${truncated ? ` (more available; pass cursor=${nextCursor})` : ""}:\n` +
|
|
405
|
+
listing.join("\n"),
|
|
406
|
+
metadata: {
|
|
407
|
+
count: listing.length,
|
|
408
|
+
next_cursor: nextCursor,
|
|
409
|
+
truncated,
|
|
410
|
+
since: args.since,
|
|
411
|
+
until: args.until,
|
|
412
|
+
},
|
|
383
413
|
};
|
|
384
414
|
}
|
|
385
415
|
const caseSensitive = args.case_sensitive ?? true;
|
|
@@ -402,7 +432,7 @@ export const memory_search = tool({
|
|
|
402
432
|
continue; // binary, like codex's InvalidData skip
|
|
403
433
|
searchFileContent(f, content.split(/\r?\n/), queries, preparedQueries, mode, args.line_count ?? 1, args.context_lines ?? 0, caseSensitive, normalized, all);
|
|
404
434
|
}
|
|
405
|
-
all.sort((a, b) => a.path
|
|
435
|
+
all.sort((a, b) => comparePathNames(a.path, b.path) || a.match_line_number - b.match_line_number);
|
|
406
436
|
let startIndex = 0;
|
|
407
437
|
if (args.cursor !== undefined) {
|
|
408
438
|
startIndex = Number.parseInt(args.cursor, 10);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-codex-memory",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Persistent memory plugin for opencode — ports codex's two-phase memory system (extraction → consolidation → injection → citation feedback)",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/src/index.js",
|