handzon-core 0.7.0 → 0.8.0

package/src/server/verify/evaluator.ts

@@ -0,0 +1,134 @@
+import type { VerifyCheck, VerifySpec } from "../../collections.ts";
+
+/**
+ * Observed values the agent reports for one check. The set of valid
+ * fields depends on the check `kind` — extras are ignored. Server
+ * never trusts pass/fail flags from the agent; it always re-scores
+ * the observations against the spec.
+ */
+export interface CheckObservation {
+  /** file_exists / file_contains */
+  exists?: boolean;
+  /** file_contains */
+  body?: string;
+  /** shell */
+  exitCode?: number;
+  stdout?: string;
+  /** http */
+  status?: number;
+  responseBody?: string;
+}
+
+export interface VerifyVerdict {
+  passed: boolean;
+  /** Index into spec.checks of the first failing check (when passed === false). */
+  failingCheckIndex?: number;
+  /** The author's hint for the failing check, if any. */
+  hint?: string;
+  /** A short server-side explanation of *why* the check failed. */
+  reason?: string;
+}
+
+function fail(idx: number, check: VerifyCheck, reason: string): VerifyVerdict {
+  return { passed: false, failingCheckIndex: idx, hint: check.hint, reason };
+}
+
+/**
+ * Score one check's observation against its declared spec. Returns
+ * either a pass marker or a structured failure with reason + hint
+ * so the calling MCP tool can echo something useful to the agent.
+ *
+ * Pure of I/O. The agent does the side effects; the server scores.
+ */
+function evaluateOne(check: VerifyCheck, idx: number, obs: CheckObservation): VerifyVerdict {
+  switch (check.kind) {
+    case "file_exists": {
+      if (obs.exists === true) return { passed: true };
+      return fail(
+        idx,
+        check,
+        `Expected ${check.path} to exist but the agent reported it does not.`,
+      );
+    }
+    case "file_contains": {
+      if (obs.exists === false) {
+        return fail(idx, check, `File ${check.path} does not exist.`);
+      }
+      const body = obs.body ?? "";
+      if (new RegExp(check.pattern).test(body)) return { passed: true };
+      return fail(idx, check, `Expected ${check.path} to match /${check.pattern}/.`);
+    }
+    case "shell": {
+      if (typeof check.expect.exitCode === "number") {
+        if (obs.exitCode !== check.expect.exitCode) {
+          return fail(
+            idx,
+            check,
+            `Expected exit code ${check.expect.exitCode} from \`${check.run}\`, got ${obs.exitCode}.`,
+          );
+        }
+      }
+      if (check.expect.stdoutMatches) {
+        const stdout = obs.stdout ?? "";
+        if (!new RegExp(check.expect.stdoutMatches).test(stdout)) {
+          return fail(
+            idx,
+            check,
+            `Expected stdout of \`${check.run}\` to match /${check.expect.stdoutMatches}/.`,
+          );
+        }
+      }
+      return { passed: true };
+    }
+    case "http": {
+      if (typeof check.expect.status === "number") {
+        if (obs.status !== check.expect.status) {
+          return fail(
+            idx,
+            check,
+            `Expected status ${check.expect.status} from ${check.url}, got ${obs.status}.`,
+          );
+        }
+      }
+      const body = obs.responseBody ?? "";
+      if (check.expect.bodyIncludes && !body.includes(check.expect.bodyIncludes)) {
+        return fail(
+          idx,
+          check,
+          `Expected response body of ${check.url} to include "${check.expect.bodyIncludes}".`,
+        );
+      }
+      if (check.expect.bodyMatches && !new RegExp(check.expect.bodyMatches).test(body)) {
+        return fail(
+          idx,
+          check,
+          `Expected response body of ${check.url} to match /${check.expect.bodyMatches}/.`,
+        );
+      }
+      return { passed: true };
+    }
+  }
+}
+
+/**
+ * Pure-function evaluator: spec + agent-reported observations →
+ * pass/fail verdict. Short-circuits on first failure so the
+ * failingCheckIndex is the first thing wrong, not the last.
+ *
+ * `observations` must be the same length as `spec.checks` — one
+ * observation per check, in order. The agent is responsible for
+ * filling in the right shape per check kind.
+ */
+export function evaluate(spec: VerifySpec, observations: CheckObservation[]): VerifyVerdict {
+  if (observations.length !== spec.checks.length) {
+    return {
+      passed: false,
+      reason: `Expected ${spec.checks.length} observations, got ${observations.length}.`,
+    };
+  }
+  for (let i = 0; i < spec.checks.length; i++) {
+    const verdict = evaluateOne(spec.checks[i]!, i, observations[i] ?? {});
+    if (!verdict.passed) return verdict;
+  }
+  return { passed: true };
+}

package/src/types/ai.ts

@@ -12,6 +12,12 @@ export interface AiConfig {
   contextBudgetTokens: number;
   includeFutureSteps: boolean;
   tools: { suggestPlaygroundEdit: boolean };
+  /**
+   * When true, the tutorial layout renders a "Stuck on this step?"
+   * footer under every step body so authors don't have to drop a
+   * <HelpMe /> by hand. Off by default.
+   */
+  autoStepHelp?: boolean;
   disabledSkills?: string[];
   allowedDomains?: string[];
 }

package/styles/components/assist.css

@@ -0,0 +1,101 @@
+/* Family A: in-app tutor touchpoints (HelpMe, quiz wrong-answer hint,
+ * checkpoint stuck nudge, selection ask, playground fix, step footer)
+ * and Family B: bring-your-own-agent surfaces (CopyPrompt, deep-link
+ * row, copy-step button). Shared visual language so author-placed
+ * affordances feel like one family.
+ */
+
+.hz-helpme,
+.hz-assist-link,
+.hz-copy-prompt {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.4rem;
+  background: none;
+  border: var(--border-default) solid var(--color-border);
+  padding: 0.35rem 0.65rem;
+  font: inherit;
+  font-size: 0.85em;
+  color: var(--color-accent);
+  cursor: pointer;
+  text-decoration: none;
+  border-radius: 0;
+}
+
+.hz-helpme:hover,
+.hz-assist-link:hover,
+.hz-copy-prompt:hover {
+  background: var(--color-surface);
+}
+
+.hz-helpme[disabled],
+.hz-copy-prompt[disabled] {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+/* Block-level "Stuck on this step?" step footer */
+.hz-step-help {
+  margin-top: 2rem;
+  padding: 0.9rem 1.1rem;
+  background: var(--color-surface);
+  border: var(--border-default) solid var(--color-border);
+  display: flex;
+  align-items: center;
+  gap: 0.75rem;
+  font-size: 0.9em;
+}
+.hz-step-help-text {
+  color: var(--color-muted);
+  flex: 1;
+}
+
+/* Checkpoint "Stuck?" nudge: rendered inside .checkpoint when the
+ * step has been on-screen + unchecked for a while.
+ */
+.checkpoint-nudge {
+  margin-left: auto;
+  font-size: 0.8em;
+  color: var(--color-accent);
+}
+
+/* Selection-anchored floating "Ask about this" */
+.hz-selection-ask {
+  position: absolute;
+  z-index: 60;
+  background: var(--color-bg);
+  border: var(--border-default) solid var(--color-accent);
+  padding: 0.3rem 0.6rem;
+  font-size: 0.8em;
+  color: var(--color-accent);
+  cursor: pointer;
+  box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);
+}
+
+/* Deep-link / open-in row */
+.hz-openin {
+  margin-top: 1.5rem;
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.5rem;
+  align-items: center;
+}
+.hz-openin-label {
+  font-size: 0.8em;
+  color: var(--color-muted);
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  margin-right: 0.25rem;
+}
+
+/* Quiz wrong-answer help inline link */
+.quiz .hz-helpme {
+  margin-top: 0.4rem;
+}
+
+/* Playground toolbar: row sitting just under the Sandpack iframe */
+.playground-toolbar {
+  display: flex;
+  justify-content: flex-end;
+  margin-top: 0.4rem;
+}

package/styles/components/checkpoint.css

@@ -47,3 +47,32 @@
   letter-spacing: 0.06em;
   margin-left: auto;
 }
+
+/* Family D: inline failure feedback under the checkpoint when a
+ * submit_verification call from the agent reported a failed check.
+ * Styled as a subdued warning band, not a toast — failure UX is the
+ * agent's job in its own surface; this is a backstop. */
+.checkpoint-feedback {
+  flex-basis: 100%;
+  background: var(--color-bg);
+  border-left: var(--border-thick) solid var(--color-warning, #d9a200);
+  padding: 0.6rem 0.85rem;
+  margin-top: 0.5rem;
+  font-size: 0.9em;
+  color: var(--color-fg);
+}
+.checkpoint-feedback strong {
+  display: block;
+  font-family: var(--font-mono);
+  font-size: 0.8em;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: var(--color-warning, #d9a200);
+}
+.checkpoint-feedback p {
+  margin: 0.25rem 0 0;
+}
+.checkpoint-feedback-hint {
+  color: var(--color-muted);
+  font-size: 0.95em;
+}

package/styles/components.css

@@ -19,6 +19,7 @@
 /* AI assistant */
 @import "./components/chat.css";
 @import "./components/byok.css";
+@import "./components/assist.css";
 
 /* MDX components */
 @import "./components/tabs.css";