gsd-pi 2.28.0 → 2.29.0-dev.2ccf3fb

package/dist/resources/extensions/gsd/tests/prompt-ordering.test.ts

@@ -0,0 +1,296 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { reorderForCaching, analyzeCacheEfficiency } from "../prompt-ordering.js";
+
+describe("reorderForCaching", () => {
+  it("reorders static sections before dynamic sections", () => {
+    const prompt = [
+      "## Inlined Task Plan",
+      "Do the task steps here.",
+      "",
+      "## Output Template",
+      "Use this template.",
+      "",
+      "## Resume State",
+      "Resuming from checkpoint.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    const outputIdx = result.indexOf("## Output Template");
+    const taskIdx = result.indexOf("## Inlined Task Plan");
+    const resumeIdx = result.indexOf("## Resume State");
+
+    assert.ok(outputIdx < taskIdx, "Static 'Output Template' should come before dynamic 'Inlined Task Plan'");
+    assert.ok(outputIdx < resumeIdx, "Static 'Output Template' should come before dynamic 'Resume State'");
+  });
+
+  it("preserves preamble at the beginning", () => {
+    const prompt = [
+      "You are executing GSD auto-mode.",
+      "",
+      "## Output Template",
+      "Template content.",
+      "",
+      "## Inlined Task Plan",
+      "Task content.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    assert.ok(
+      result.startsWith("You are executing GSD auto-mode."),
+      "Preamble should remain at the start",
+    );
+  });
+
+  it("preserves relative order within groups", () => {
+    const prompt = [
+      "## Decisions",
+      "Decision A.",
+      "",
+      "## Requirements",
+      "Requirement B.",
+      "",
+      "## Overrides",
+      "Override C.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    const decisionsIdx = result.indexOf("## Decisions");
+    const requirementsIdx = result.indexOf("## Requirements");
+    const overridesIdx = result.indexOf("## Overrides");
+
+    assert.ok(decisionsIdx < requirementsIdx, "Decisions should come before Requirements (same group order)");
+    assert.ok(requirementsIdx < overridesIdx, "Requirements should come before Overrides (same group order)");
+  });
+
+  it("handles prompts with no headings (returns unchanged)", () => {
+    const prompt = "Just plain text with no markdown headings at all.";
+    const result = reorderForCaching(prompt);
+    assert.equal(result, prompt);
+  });
+
+  it("handles prompts with only static sections", () => {
+    const prompt = [
+      "## Output Template",
+      "Template A.",
+      "",
+      "## Executor Context Constraints",
+      "Constraints B.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    // Both are static, order preserved
+    assert.ok(result.indexOf("## Output Template") < result.indexOf("## Executor Context Constraints"));
+  });
+
+  it("handles prompts with only dynamic sections", () => {
+    const prompt = [
+      "## Inlined Task Plan",
+      "Plan A.",
+      "",
+      "## Resume State",
+      "State B.",
+      "",
+      "## Verification",
+      "Check C.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    // All dynamic, order preserved
+    const planIdx = result.indexOf("## Inlined Task Plan");
+    const resumeIdx = result.indexOf("## Resume State");
+    const verifyIdx = result.indexOf("## Verification");
+    assert.ok(planIdx < resumeIdx);
+    assert.ok(resumeIdx < verifyIdx);
+  });
+
+  it("unknown headings default to dynamic", () => {
+    const prompt = [
+      "## Output Template",
+      "Static content.",
+      "",
+      "## Some Unknown Section",
+      "Unknown content.",
+      "",
+      "## Decisions",
+      "Semi-static content.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    const staticIdx = result.indexOf("## Output Template");
+    const semiIdx = result.indexOf("## Decisions");
+    const unknownIdx = result.indexOf("## Some Unknown Section");
+
+    assert.ok(staticIdx < semiIdx, "Static before semi-static");
+    assert.ok(semiIdx < unknownIdx, "Semi-static before unknown (dynamic)");
+  });
+
+  it("sub-headings stay with their parent section", () => {
+    const prompt = [
+      "## Slice Plan Excerpt",
+      "Slice content.",
+      "### Task List",
+      "- T1.1",
+      "- T1.2",
+      "",
+      "## Inlined Task Plan",
+      "Dynamic task content.",
+    ].join("\n");
+
+    const result = reorderForCaching(prompt);
+    // The ### Task List should stay with ## Slice Plan Excerpt
+    const sliceIdx = result.indexOf("## Slice Plan Excerpt");
+    const taskListIdx = result.indexOf("### Task List");
+    const inlinedIdx = result.indexOf("## Inlined Task Plan");
+
+    assert.ok(sliceIdx < taskListIdx, "Sub-heading stays after its parent");
+    assert.ok(taskListIdx < inlinedIdx, "Sub-heading block comes before dynamic section");
+  });
+});
+
+describe("analyzeCacheEfficiency", () => {
+  it("returns correct ratios", () => {
+    const prompt = [
+      "Preamble text here.",
+      "",
+      "## Output Template",
+      "Static content here.",
+      "",
+      "## Decisions",
+      "Semi-static content.",
+      "",
+      "## Inlined Task Plan",
+      "Dynamic content here.",
+    ].join("\n");
+
+    const result = analyzeCacheEfficiency(prompt);
+
+    assert.ok(result.totalChars > 0, "totalChars should be positive");
+    assert.ok(result.staticChars > 0, "staticChars should be positive (includes preamble)");
+    assert.ok(result.semiStaticChars > 0, "semiStaticChars should be positive");
+    assert.ok(result.dynamicChars > 0, "dynamicChars should be positive");
+    assert.ok(result.cacheEfficiency > 0 && result.cacheEfficiency < 1, "efficiency should be between 0 and 1");
+    assert.equal(
+      result.totalChars,
+      result.staticChars + result.semiStaticChars + result.dynamicChars,
+      "chars should sum to total",
+    );
+  });
+
+  it("returns 1.0 efficiency for all-static prompts", () => {
+    const prompt = [
+      "## Output Template",
+      "All static.",
+      "",
+      "## Executor Context Constraints",
+      "Also static.",
+    ].join("\n");
+
+    const result = analyzeCacheEfficiency(prompt);
+    assert.equal(result.cacheEfficiency, 1.0);
+    assert.equal(result.dynamicChars, 0);
+  });
+
+  it("returns 0 efficiency for all-dynamic prompts", () => {
+    const prompt = [
+      "## Inlined Task Plan",
+      "All dynamic.",
+      "",
+      "## Resume State",
+      "Also dynamic.",
+    ].join("\n");
+
+    const result = analyzeCacheEfficiency(prompt);
+    assert.equal(result.cacheEfficiency, 0);
+    assert.equal(result.staticChars, 0);
+    assert.equal(result.semiStaticChars, 0);
+  });
+});
+
+describe("real-world prompt reordering", () => {
+  it("reorders a realistic execute-task prompt for better cache efficiency", () => {
+    // Simulate a prompt resembling buildExecuteTaskPrompt output
+    const prompt = [
+      "You are executing GSD auto-mode.",
+      "",
+      "## UNIT: Execute Task T1.2 (\"Add login\") -- Slice S1 (\"Auth\"), Milestone M1",
+      "",
+      "## Working Directory",
+      "Your working directory is `/project`.",
+      "",
+      "## Overrides",
+      "No overrides.",
+      "",
+      "## Resume State",
+      "Resuming from step 3.",
+      "",
+      "## Carry-Forward Context",
+      "Previous task noted the API uses JWT.",
+      "",
+      "## Inlined Task Plan",
+      "1. Create auth endpoint",
+      "2. Add JWT validation",
+      "3. Write tests",
+      "",
+      "## Slice Plan Excerpt",
+      "Tasks: T1.1, T1.2, T1.3",
+      "Verification: run tests",
+      "",
+      "## Decisions",
+      "Using bcrypt for password hashing.",
+      "",
+      "## Requirements",
+      "Must support OAuth2.",
+      "",
+      "## Prior Task Summaries",
+      "T1.1 completed: scaffolded auth module.",
+      "",
+      "## Backing Source Artifacts",
+      "- Slice plan: `.gsd/slices/S1.md`",
+      "",
+      "## Output Template",
+      "Use standard task summary format.",
+      "",
+      "## Verification",
+      "Run `npm test` and verify all pass.",
+    ].join("\n");
+
+    const beforeEfficiency = analyzeCacheEfficiency(prompt);
+    const reordered = reorderForCaching(prompt);
+    const afterEfficiency = analyzeCacheEfficiency(reordered);
+
+    // Efficiency score doesn't change (same content), but ordering improves cache prefix
+    assert.equal(beforeEfficiency.cacheEfficiency, afterEfficiency.cacheEfficiency);
+
+    // Verify static sections come first (after preamble + UNIT heading which is dynamic)
+    const outputTemplateIdx = reordered.indexOf("## Output Template");
+    const workingDirIdx = reordered.indexOf("## Working Directory");
+    const backingIdx = reordered.indexOf("## Backing Source Artifacts");
+
+    // Semi-static sections come after static
+    const decisionsIdx = reordered.indexOf("## Decisions");
+    const requirementsIdx = reordered.indexOf("## Requirements");
+    const sliceIdx = reordered.indexOf("## Slice Plan Excerpt");
+
+    // Dynamic sections come last
+    const taskPlanIdx = reordered.indexOf("## Inlined Task Plan");
+    const resumeIdx = reordered.indexOf("## Resume State");
+    const verifyIdx = reordered.indexOf("## Verification");
+
+    // Static before semi-static
+    assert.ok(outputTemplateIdx < decisionsIdx, "Static before semi-static");
+    assert.ok(workingDirIdx < sliceIdx, "Static before semi-static");
+    assert.ok(backingIdx < requirementsIdx, "Static before semi-static");
+
+    // Semi-static before dynamic
+    assert.ok(decisionsIdx < taskPlanIdx, "Semi-static before dynamic");
+    assert.ok(requirementsIdx < resumeIdx, "Semi-static before dynamic");
+    assert.ok(sliceIdx < verifyIdx, "Semi-static before dynamic");
+
+    // Preamble still first
+    assert.ok(
+      reordered.startsWith("You are executing GSD auto-mode."),
+      "Preamble preserved at start",
+    );
+  });
+});

package/dist/resources/extensions/gsd/tests/provider-errors.test.ts

@@ -0,0 +1,338 @@
+/**
+ * Provider error handling tests — consolidated from:
+ *   - provider-error-classify.test.ts (classifyProviderError)
+ *   - network-error-fallback.test.ts (isTransientNetworkError, getNextFallbackModel)
+ *   - agent-end-provider-error.test.ts (pauseAutoForProviderError)
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { classifyProviderError, pauseAutoForProviderError } from "../provider-error-pause.ts";
+import { getNextFallbackModel, isTransientNetworkError } from "../preferences.ts";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// ── classifyProviderError ────────────────────────────────────────────────────
+
+test("classifyProviderError detects rate limit from 429", () => {
+  const result = classifyProviderError("HTTP 429 Too Many Requests");
+  assert.ok(result.isTransient);
+  assert.ok(result.isRateLimit);
+  assert.ok(result.suggestedDelayMs > 0);
+});
+
+test("classifyProviderError detects rate limit from message", () => {
+  const result = classifyProviderError("rate limit exceeded");
+  assert.ok(result.isTransient);
+  assert.ok(result.isRateLimit);
+});
+
+test("classifyProviderError extracts reset delay from message", () => {
+  const result = classifyProviderError("rate limit exceeded, reset in 45s");
+  assert.ok(result.isRateLimit);
+  assert.equal(result.suggestedDelayMs, 45000);
+});
+
+test("classifyProviderError defaults to 60s for rate limit without reset", () => {
+  const result = classifyProviderError("429 too many requests");
+  assert.ok(result.isRateLimit);
+  assert.equal(result.suggestedDelayMs, 60_000);
+});
+
+test("classifyProviderError detects Anthropic internal server error", () => {
+  const msg = '{"type":"error","error":{"details":null,"type":"api_error","message":"Internal server error"}}';
+  const result = classifyProviderError(msg);
+  assert.ok(result.isTransient);
+  assert.ok(!result.isRateLimit);
+  assert.equal(result.suggestedDelayMs, 30_000);
+});
+
+test("classifyProviderError detects Codex server_error from extracted message", () => {
+  // After fix, mapCodexEvents extracts the nested error type and produces
+  // "Codex server_error: <message>" instead of raw JSON.
+  const msg = "Codex server_error: An error occurred while processing your request.";
+  const result = classifyProviderError(msg);
+  assert.ok(result.isTransient);
+  assert.ok(!result.isRateLimit);
+  assert.equal(result.suggestedDelayMs, 30_000);
+});
+
+test("classifyProviderError detects overloaded error", () => {
+  const result = classifyProviderError("overloaded_error: Overloaded");
+  assert.ok(result.isTransient);
+  assert.equal(result.suggestedDelayMs, 30_000);
+});
+
+test("classifyProviderError detects 503 service unavailable", () => {
+  const result = classifyProviderError("HTTP 503 Service Unavailable");
+  assert.ok(result.isTransient);
+});
+
+test("classifyProviderError detects 502 bad gateway", () => {
+  const result = classifyProviderError("HTTP 502 Bad Gateway");
+  assert.ok(result.isTransient);
+});
+
+test("classifyProviderError detects auth error as permanent", () => {
+  const result = classifyProviderError("unauthorized: invalid API key");
+  assert.ok(!result.isTransient);
+  assert.ok(!result.isRateLimit);
+});
+
+test("classifyProviderError detects billing error as permanent", () => {
+  const result = classifyProviderError("billing issue: payment required");
+  assert.ok(!result.isTransient);
+});
+
+test("classifyProviderError detects quota exceeded as permanent", () => {
+  const result = classifyProviderError("quota exceeded for this month");
+  assert.ok(!result.isTransient);
+});
+
+test("classifyProviderError treats unknown error as permanent", () => {
+  const result = classifyProviderError("something went wrong");
+  assert.ok(!result.isTransient);
+});
+
+test("classifyProviderError treats empty string as permanent", () => {
+  const result = classifyProviderError("");
+  assert.ok(!result.isTransient);
+});
+
+test("classifyProviderError: rate limit takes precedence over auth keywords", () => {
+  const result = classifyProviderError("429 unauthorized rate limit");
+  assert.ok(result.isRateLimit);
+  assert.ok(result.isTransient);
+});
+
+// ── isTransientNetworkError ──────────────────────────────────────────────────
+
+test("isTransientNetworkError detects ECONNRESET", () => {
+  assert.ok(isTransientNetworkError("fetch failed: ECONNRESET"));
+});
+
+test("isTransientNetworkError detects ETIMEDOUT", () => {
+  assert.ok(isTransientNetworkError("ETIMEDOUT: request timed out"));
+});
+
+test("isTransientNetworkError detects generic network error", () => {
+  assert.ok(isTransientNetworkError("network error"));
+});
+
+test("isTransientNetworkError detects socket hang up", () => {
+  assert.ok(isTransientNetworkError("socket hang up"));
+});
+
+test("isTransientNetworkError detects fetch failed", () => {
+  assert.ok(isTransientNetworkError("fetch failed"));
+});
+
+test("isTransientNetworkError detects connection reset", () => {
+  assert.ok(isTransientNetworkError("connection was reset by peer"));
+});
+
+test("isTransientNetworkError detects DNS errors", () => {
+  assert.ok(isTransientNetworkError("dns resolution failed"));
+});
+
+test("isTransientNetworkError rejects auth errors", () => {
+  assert.ok(!isTransientNetworkError("unauthorized: invalid API key"));
+});
+
+test("isTransientNetworkError rejects quota errors", () => {
+  assert.ok(!isTransientNetworkError("quota exceeded"));
+});
+
+test("isTransientNetworkError rejects billing errors", () => {
+  assert.ok(!isTransientNetworkError("billing issue: network payment required"));
+});
+
+test("isTransientNetworkError rejects empty string", () => {
+  assert.ok(!isTransientNetworkError(""));
+});
+
+test("isTransientNetworkError rejects non-network errors", () => {
+  assert.ok(!isTransientNetworkError("model not found"));
+});
+
+// ── getNextFallbackModel ─────────────────────────────────────────────────────
+
+test("getNextFallbackModel selects next fallback if current is a fallback", () => {
+  const modelConfig = { primary: "model-a", fallbacks: ["model-b", "model-c"] };
+  assert.equal(getNextFallbackModel("model-b", modelConfig), "model-c");
+});
+
+test("getNextFallbackModel returns undefined if fallbacks exhausted", () => {
+  const modelConfig = { primary: "model-a", fallbacks: ["model-b", "model-c"] };
+  assert.equal(getNextFallbackModel("model-c", modelConfig), undefined);
+});
+
+test("getNextFallbackModel finds current model with provider prefix", () => {
+  const modelConfig = { primary: "p/model-a", fallbacks: ["p/model-b"] };
+  assert.equal(getNextFallbackModel("model-a", modelConfig), "p/model-b");
+});
+
+test("getNextFallbackModel returns primary if current is unknown", () => {
+  const modelConfig = { primary: "model-a", fallbacks: ["model-b", "model-c"] };
+  assert.equal(getNextFallbackModel("model-x", modelConfig), "model-a");
+});
+
+test("getNextFallbackModel returns primary if current is undefined", () => {
+  const modelConfig = { primary: "model-a", fallbacks: ["model-b", "model-c"] };
+  assert.equal(getNextFallbackModel(undefined, modelConfig), "model-a");
+});
+
+// ── pauseAutoForProviderError ────────────────────────────────────────────────
+
+test("pauseAutoForProviderError warns and pauses without requiring ctx.log", async () => {
+  const notifications: Array<{ message: string; level: string }> = [];
+  let pauseCalls = 0;
+
+  await pauseAutoForProviderError(
+    { notify(message, level?) { notifications.push({ message, level: level ?? "info" }); } },
+    ": terminated",
+    async () => { pauseCalls += 1; },
+  );
+
+  assert.equal(pauseCalls, 1);
+  assert.deepEqual(notifications, [
+    { message: "Auto-mode paused due to provider error: terminated", level: "warning" },
+  ]);
+});
+
+test("pauseAutoForProviderError schedules auto-resume for rate limit errors", async () => {
+  const notifications: Array<{ message: string; level: string }> = [];
+  let pauseCalls = 0;
+  let resumeCalled = false;
+
+  const originalSetTimeout = globalThis.setTimeout;
+  const timers: Array<{ fn: () => void; delay: number }> = [];
+  globalThis.setTimeout = ((fn: () => void, delay: number) => {
+    timers.push({ fn, delay });
+    return 0 as unknown as ReturnType<typeof setTimeout>;
+  }) as typeof setTimeout;
+
+  try {
+    await pauseAutoForProviderError(
+      { notify(message, level?) { notifications.push({ message, level: level ?? "info" }); } },
+      ": rate limit exceeded",
+      async () => { pauseCalls += 1; },
+      { isRateLimit: true, retryAfterMs: 90000, resume: () => { resumeCalled = true; } },
+    );
+
+    assert.equal(pauseCalls, 1);
+    assert.equal(timers.length, 1);
+    assert.equal(timers[0].delay, 90000);
+    assert.deepEqual(notifications[0], {
+      message: "Rate limited: rate limit exceeded. Auto-resuming in 90s...",
+      level: "warning",
+    });
+
+    timers[0].fn();
+    assert.equal(resumeCalled, true);
+    assert.deepEqual(notifications[1], {
+      message: "Rate limit window elapsed. Resuming auto-mode.",
+      level: "info",
+    });
+  } finally {
+    globalThis.setTimeout = originalSetTimeout;
+  }
+});
+
+test("pauseAutoForProviderError falls back to indefinite pause when not rate limit", async () => {
+  const notifications: Array<{ message: string; level: string }> = [];
+  let pauseCalls = 0;
+
+  await pauseAutoForProviderError(
+    { notify(message, level?) { notifications.push({ message, level: level ?? "info" }); } },
+    ": connection refused",
+    async () => { pauseCalls += 1; },
+    { isRateLimit: false },
+  );
+
+  assert.equal(pauseCalls, 1);
+  assert.deepEqual(notifications, [
+    { message: "Auto-mode paused due to provider error: connection refused", level: "warning" },
+  ]);
+});
+
+// ── Escalating backoff for transient errors (#1166) ─────────────────────────
+
+test("index.ts tracks consecutive transient errors for escalating backoff", () => {
+  const indexSource = readFileSync(join(__dirname, "..", "index.ts"), "utf-8");
+
+  assert.ok(
+    indexSource.includes("consecutiveTransientErrors"),
+    "index.ts must track consecutiveTransientErrors for escalating backoff (#1166)",
+  );
+  assert.ok(
+    indexSource.includes("MAX_TRANSIENT_AUTO_RESUMES"),
+    "index.ts must define MAX_TRANSIENT_AUTO_RESUMES to cap infinite retries (#1166)",
+  );
+});
+
+test("index.ts resets consecutive transient error counter on success", () => {
+  const indexSource = readFileSync(join(__dirname, "..", "index.ts"), "utf-8");
+
+  // After successful unit completion, the counter must be reset
+  const marker = "successful unit completion";
+  const successSection = indexSource.indexOf(marker);
+  assert.ok(successSection > -1, "must have success section that clears network retries");
+  const nearbyCode = indexSource.slice(Math.max(0, successSection - 100), successSection + 200);
+  assert.ok(
+    nearbyCode.includes("consecutiveTransientErrors = 0"),
+    "consecutive transient error counter must be reset on successful unit completion (#1166)",
+  );
+});
+
+test("index.ts applies escalating delay for repeated transient errors", () => {
+  const indexSource = readFileSync(join(__dirname, "..", "index.ts"), "utf-8");
+
+  // Must contain the exponential backoff formula
+  assert.ok(
+    /retryAfterMs\s*[=*].*2\s*\*\*/.test(indexSource),
+    "index.ts must escalate retryAfterMs exponentially for consecutive transient errors (#1166)",
+  );
+});
+
+// ── Codex error extraction (#1166) ──────────────────────────────────────────
+
+test("openai-codex-responses.ts extracts nested error fields", () => {
+  const codexSource = readFileSync(
+    join(__dirname, "../../../../../packages/pi-ai/src/providers/openai-codex-responses.ts"),
+    "utf-8",
+  );
+
+  // Must access event.error.message (nested), not just event.message (top-level)
+  assert.ok(
+    codexSource.includes("errorObj?.message"),
+    "mapCodexEvents must extract message from nested event.error object (#1166)",
+  );
+  assert.ok(
+    codexSource.includes("errorObj?.type"),
+    "mapCodexEvents must extract type from nested event.error object (#1166)",
+  );
+});
+
+// ── agent-session retryable regex handles server_error (#1166) ──────────────
+
+test("agent-session retryable error regex matches server_error (underscore)", () => {
+  // This regex is extracted from _isRetryableError in agent-session.ts.
+  // It must match both "server error" (space) and "server_error" (underscore)
+  // to properly classify Codex streaming errors as retryable.
+  const retryableRegex = /overloaded|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|connection.?error|connection.?refused|other side closed|fetch failed|upstream.?connect|reset before headers|terminated|retry delay|network.?(?:is\s+)?unavailable|credentials.*expired|temporarily backed off/i;
+
+  // server_error (with underscore — Codex streaming error format)
+  assert.ok(retryableRegex.test("Codex server_error: An error occurred"));
+  // server error (with space — traditional HTTP error format)
+  assert.ok(retryableRegex.test("server error occurred"));
+  // internal_error (with underscore)
+  assert.ok(retryableRegex.test("internal_error: something went wrong"));
+  // internal error (with space)
+  assert.ok(retryableRegex.test("internal error"));
+  // non-retryable errors must not match
+  assert.ok(!retryableRegex.test("model not found"));
+});