@jay-framework/aiditor 0.16.2

package/dist/index.js

@@ -0,0 +1,464 @@
+import { makeJayQuery, makeJayStream, makeJayStackComponent, phaseOutput, RenderPipeline } from "@jay-framework/fullstack-component";
+import { DEV_SERVER_SERVICE } from "@jay-framework/dev-server";
+import { readFile } from "fs/promises";
+import path, { extname } from "path";
+import { query } from "@anthropic-ai/claude-agent-sdk";
+import fs from "fs";
+import { setActionCallerOptions } from "@jay-framework/stack-client-runtime";
+const getAiditorBootstrap = makeJayQuery(
+  "aiditor.getAiditorBootstrap"
+).withHandler(async (input) => {
+  return {
+    projectDir: process.cwd(),
+    httpBaseUrl: input.origin.trim()
+  };
+});
+const getProjectInfoAction = makeJayQuery("aditor.getProjectInfo").withServices(DEV_SERVER_SERVICE).withHandler(async (input, devServer) => {
+  const routes = devServer.listRoutes();
+  return { routes, httpBaseUrl: input.jayDevUrl };
+});
+const getPageParamsAction = makeJayStream("aditor.getPageParams").withServices(DEV_SERVER_SERVICE).withHandler(async function* (input, devServer) {
+  for await (const batch of devServer.loadRouteParams(input.route)) {
+    yield batch;
+  }
+});
+const listFreezesAction = makeJayQuery("aditor.listFreezes").withServices(DEV_SERVER_SERVICE).withHandler(async (input, devServer) => {
+  const store = devServer.freezeStore;
+  if (!store) return { freezes: [] };
+  const freezes = await store.list(input.route);
+  return {
+    freezes: freezes.map((f) => ({
+      id: f.id,
+      name: f.name,
+      route: f.route,
+      createdAt: f.createdAt
+    }))
+  };
+});
+const IMAGE_EXTS = /* @__PURE__ */ new Set([
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".webp",
+  ".svg",
+  ".bmp",
+  ".ico"
+]);
+const readFileAction = makeJayQuery("aiditor.readFile").withHandler(
+  async (input) => {
+    const ext = extname(input.filePath).toLowerCase();
+    const isImage = IMAGE_EXTS.has(ext);
+    const buf = await readFile(input.filePath);
+    if (isImage) {
+      const mime = ext === ".svg" ? "image/svg+xml" : `image/${ext.slice(1).replace("jpg", "jpeg")}`;
+      return {
+        type: "image",
+        mime,
+        data: buf.toString("base64"),
+        filePath: input.filePath
+      };
+    }
+    return {
+      type: "text",
+      data: buf.toString("utf-8"),
+      filePath: input.filePath
+    };
+  }
+);
+function formatToolDetail(name, input) {
+  switch (name) {
+    case "Read":
+      return String(input.file_path ?? "");
+    case "Edit": {
+      const file = String(input.file_path ?? "");
+      const old_str = String(input.old_string ?? "");
+      const new_str = String(input.new_string ?? "");
+      const preview = old_str.length > 60 ? old_str.slice(0, 60) + "…" : old_str;
+      const replaceAll = input.replace_all ? " (all)" : "";
+      return `${file}${replaceAll}
+  - ${JSON.stringify(preview)} → ${JSON.stringify(new_str.length > 60 ? new_str.slice(0, 60) + "…" : new_str)}`;
+    }
+    case "Write":
+      return String(input.file_path ?? "");
+    case "Glob":
+      return `${input.pattern ?? ""}${input.path ? ` in ${input.path}` : ""}`;
+    case "Grep":
+      return `${input.pattern ?? ""}${input.path ? ` in ${input.path}` : ""}`;
+    case "Bash":
+      return String(input.command ?? "");
+    default:
+      return String(
+        input.file_path ?? input.pattern ?? input.command ?? input.query ?? ""
+      );
+  }
+}
+function* transformSDKMessage(message) {
+  if (message.type === "assistant") {
+    const content = message.message?.content;
+    if (!Array.isArray(content)) return;
+    for (const block of content) {
+      if (block.type === "thinking" && block.thinking) {
+        yield { type: "thinking", text: block.thinking };
+      } else if (block.type === "text" && block.text) {
+        yield { type: "chunk", text: block.text };
+      }
+      if (block.name) {
+        const detail = block.input ? formatToolDetail(block.name, block.input) : "";
+        yield { type: "tool", name: block.name, text: detail };
+      }
+    }
+  } else if (message.type === "result") {
+    const msg = message;
+    yield {
+      type: "result",
+      result: msg.result ?? "",
+      cost: msg.total_cost_usd ?? 0,
+      duration: msg.duration_ms ?? 0
+    };
+  }
+}
+function buildNonVisualPrompt(config, notes, imagePath) {
+  const lines = [
+    `Project directory: ${config.projectDir}`,
+    null,
+    "",
+    "Task:",
+    notes
+  ];
+  return lines.filter((l) => l !== null).join("\n");
+}
+const ANNOTATION_TARGETING_RULES = `Annotation targeting (follow strictly):
+- The only authoritative edit target is the element indicated by the annotation marker (dot, arrow, or highlighted region). Do not choose another element because it is colorful, prominent, or a common control (e.g. buttons, links) unless the marker clearly overlaps that element.
+- If several elements could match, prefer the text or content under/near the marker (e.g. a heading) over unrelated controls elsewhere on the page.
+- If you cannot confidently map the marker to a single target in source, do not change a plausible but unmarked control. Briefly say what is ambiguous and list candidate elements or ask for clarification.`;
+function parseNotesField(notes) {
+  const empty = () => ({
+    structured: null,
+    structuredVideo: null,
+    plain: notes
+  });
+  const t = notes.trim();
+  if (!t.startsWith("{")) {
+    return empty();
+  }
+  try {
+    const j = JSON.parse(t);
+    if (j.version === 2 && j.taskKind === "video" && Array.isArray(j.annotations) && j.annotations.every(
+      (a) => a && typeof a === "object" && typeof a.id === "string" && typeof a.instruction === "string" && typeof a.mode === "string" && typeof a.timeSec === "number" && Number.isFinite(a.timeSec)
+    )) {
+      let previewNavLog;
+      if (Array.isArray(j.previewNavLog) && j.previewNavLog.length > 0) {
+        const parsed = [];
+        for (const e of j.previewNavLog) {
+          if (!e || typeof e !== "object") continue;
+          const o = e;
+          if (typeof o.atRecordingSec === "number" && Number.isFinite(o.atRecordingSec) && typeof o.href === "string" && typeof o.locationPath === "string") {
+            parsed.push({
+              atRecordingSec: o.atRecordingSec,
+              href: o.href,
+              locationPath: o.locationPath
+            });
+          }
+        }
+        if (parsed.length > 0) previewNavLog = parsed;
+      }
+      return {
+        structured: null,
+        structuredVideo: {
+          annotations: j.annotations,
+          previewNavLog
+        },
+        plain: ""
+      };
+    }
+    if (j.version === 1 && Array.isArray(j.annotations) && j.annotations.length > 0 && j.annotations.every(
+      (a) => a && typeof a === "object" && typeof a.id === "string" && typeof a.instruction === "string" && typeof a.mode === "string"
+    )) {
+      return {
+        structured: { annotations: j.annotations },
+        structuredVideo: null,
+        plain: ""
+      };
+    }
+  } catch {
+  }
+  return empty();
+}
+function buildVisualPromptTriple(config, pageRoute, renderedUrl, triple, parsed, attachmentPathsByAnnotationId) {
+  const preamble = [
+    `Project directory: ${config.projectDir}`,
+    "",
+    `Implement the requested changes on page route ${pageRoute}. The captures correspond to this rendered URL: ${renderedUrl}.`,
+    "",
+    "Multimodal reference images (use together):",
+    `- Screenshot 1 — Full annotations (markers and on-image messages): ${triple.full}`,
+    `- Screenshot 2 — Numbered markers only (use with Pin list; ignore overlay text from Screenshot 1 for saliency): ${triple.markersOnly}`,
+    `- Screenshot 3 — Clean page (no markers; use to see underlying layout and align pin positions from Screenshot 2): ${triple.clean}`,
+    "",
+    "How to use these screenshots:",
+    "- Use Screenshot 1 for what the user wrote on the page.",
+    "- Use Screenshot 2 to decide which UI region each Pin N refers to.",
+    "- Use Screenshot 3 to understand unmarked typography and structure when overlays obscure content.",
+    "",
+    ANNOTATION_TARGETING_RULES,
+    ""
+  ];
+  const body = [];
+  if (parsed.structured) {
+    body.push("Annotations (apply in order):", "");
+    const sorted = [...parsed.structured.annotations].sort(
+      (a, b) => Number.parseInt(a.id, 10) - Number.parseInt(b.id, 10)
+    );
+    for (const ann of sorted) {
+      const pin = ann.id;
+      const paths = attachmentPathsByAnnotationId.get(pin) ?? [];
+      body.push(`Annotation ${pin} (${ann.mode})`);
+      body.push(`Instruction: ${ann.instruction.trim()}`);
+      if (paths.length > 0) {
+        body.push(
+          "Attachments (read these files; they belong to this pin only):"
+        );
+        for (const p of paths) {
+          body.push(`- ${p}`);
+        }
+      } else {
+        body.push("Attachments: (none)");
+      }
+      body.push("");
+    }
+  } else {
+    body.push("Instructions:", parsed.plain.trim());
+  }
+  return [...preamble, ...body].join("\n");
+}
+function pinOrder(annotations) {
+  return [...annotations].sort((a, b) => {
+    if (a.timeSec !== b.timeSec) return a.timeSec - b.timeSec;
+    return a.id.localeCompare(b.id, void 0, { sensitivity: "base" });
+  });
+}
+function buildVisualPromptVideo(config, pageRoute, renderedUrl, videoPath, frames, parsed, attachmentPathsByPin) {
+  const sv = parsed.structuredVideo;
+  const navLines = [];
+  if (sv?.previewNavLog && sv.previewNavLog.length > 0) {
+    navLines.push(
+      "Preview URL history during recording (live iframe navigations while capturing; SPA route changes appear here):",
+      ...sv.previewNavLog.map(
+        (e) => `- t≈${e.atRecordingSec.toFixed(2)}s → ${e.href} (path: ${e.locationPath})`
+      ),
+      ""
+    );
+  }
+  const preamble = [
+    `Project directory: ${config.projectDir}`,
+    "",
+    `Implement the requested changes on page route ${pageRoute}. The recording corresponds to this rendered URL: ${renderedUrl}.`,
+    "",
+    `Original screen recording (no overlays): ${videoPath}`,
+    "",
+    ...navLines,
+    ANNOTATION_TARGETING_RULES,
+    ""
+  ];
+  if (!sv || sv.annotations.length === 0) {
+    return [
+      ...preamble,
+      "No per-frame stills were submitted (video-only task). Use the recording to reproduce the flow.",
+      ""
+    ].join("\n");
+  }
+  const pinById = /* @__PURE__ */ new Map();
+  for (const [i, a] of pinOrder(sv.annotations).entries()) {
+    pinById.set(a.id, String(i + 1));
+  }
+  const body = ["Per-moment captures (triple screenshots):", ""];
+  for (const fr of frames) {
+    body.push(
+      `Moment ${fr.frameIndex + 1} — t=${fr.timeSec.toFixed(2)}s`,
+      `- Full (markers + on-image messages): ${fr.full}`,
+      `- Markers only (numbered pins): ${fr.markersOnly}`,
+      `- Clean (no markers): ${fr.clean}`,
+      ""
+    );
+    const atT = sv.annotations.filter((a) => a.timeSec === fr.timeSec);
+    body.push("Pins anchored at this timestamp:");
+    for (const ann of pinOrder(atT)) {
+      const pinNum = pinById.get(ann.id) ?? "?";
+      const paths = attachmentPathsByPin.get(pinNum) ?? [];
+      body.push(`- Pin ${pinNum} (id ${ann.id}) — ${ann.mode}`);
+      body.push(`  Instruction: ${ann.instruction.trim()}`);
+      if (ann.previewUrlAtTime) {
+        body.push(
+          `  Preview URL at this pin (~${ann.timeSec.toFixed(2)}s in the recording): ${ann.previewUrlAtTime}`
+        );
+      }
+      if (ann.pagePathAtTime) {
+        body.push(`  Page path at this pin: ${ann.pagePathAtTime}`);
+      }
+      if (paths.length > 0) {
+        body.push(
+          "  Attachments (read these files; they belong to this pin only):"
+        );
+        for (const p of paths) body.push(`  - ${p}`);
+      } else {
+        body.push("  Attachments: (none)");
+      }
+      body.push("");
+    }
+  }
+  return [...preamble, ...body].join("\n");
+}
+function persistFile(file, destDir, fileName) {
+  fs.mkdirSync(destDir, { recursive: true });
+  const dest = path.join(destDir, fileName ?? file.name);
+  fs.copyFileSync(file.path, dest);
+  return dest;
+}
+const submitTaskAction = makeJayStream("aiditor.submitTask").withServices(DEV_SERVER_SERVICE).withFiles({ maxFileSize: 2e7, maxFiles: 300 }).withHandler(async function* (input, devServer) {
+  const projectDir = process.cwd();
+  const buildFolder = devServer.buildFolder ?? path.join(projectDir, "build");
+  const taskId = Math.random().toString(36).slice(2, 10);
+  const taskDir = path.join(buildFolder, "aditor", taskId);
+  fs.mkdirSync(taskDir, { recursive: true });
+  yield { type: "status", message: "Preparing task..." };
+  const config = { projectDir };
+  const parsed = parseNotesField(input.notes);
+  const isVideo = input.visualTask === "5";
+  const isVisual = !!input.visualTask;
+  let promptContent;
+  if (isVideo) {
+    let videoPath = "";
+    if (input.video) {
+      const videoDir = path.join(taskDir, "video");
+      videoPath = persistFile(input.video, videoDir, "recording.webm");
+    }
+    const extra = input.extraFiles ?? {};
+    const frames = [];
+    for (let i = 0; ; i++) {
+      const full = extra[`frame_${i}_full`];
+      const markers = extra[`frame_${i}_markers_only`];
+      const clean = extra[`frame_${i}_clean`];
+      if (!full) break;
+      const frameDir = path.join(taskDir, "frames", String(i));
+      const timeSec = parsed.structuredVideo?.annotations.find(
+        (a, idx) => idx === i || a.timeSec !== void 0
+      )?.timeSec ?? i;
+      frames.push({
+        timeSec,
+        frameIndex: i,
+        full: persistFile(full, frameDir, "full.png"),
+        markersOnly: markers ? persistFile(markers, frameDir, "markers-only.png") : "",
+        clean: clean ? persistFile(clean, frameDir, "clean.png") : ""
+      });
+    }
+    const attachmentPathsByPin = /* @__PURE__ */ new Map();
+    for (const [key, file] of Object.entries(extra)) {
+      if (!key.startsWith("attachment_")) continue;
+      const pinId = key.split("_")[1];
+      const attDir = path.join(taskDir, `annotation-${pinId}`);
+      const filePath = persistFile(file, attDir);
+      const existing = attachmentPathsByPin.get(pinId) ?? [];
+      existing.push(filePath);
+      attachmentPathsByPin.set(pinId, existing);
+    }
+    promptContent = buildVisualPromptVideo(
+      config,
+      input.pageRoute ?? "/",
+      input.renderedUrl ?? "",
+      videoPath,
+      frames,
+      parsed,
+      attachmentPathsByPin
+    );
+  } else if (isVisual && input.screenshot_full) {
+    const ssDir = path.join(taskDir, "screenshots");
+    const triple = {
+      full: persistFile(input.screenshot_full, ssDir, "full.png"),
+      markersOnly: input.screenshot_markers_only ? persistFile(
+        input.screenshot_markers_only,
+        ssDir,
+        "markers-only.png"
+      ) : "",
+      clean: input.screenshot_clean ? persistFile(input.screenshot_clean, ssDir, "clean.png") : ""
+    };
+    const extraTriple = input.extraFiles ?? {};
+    const attachmentPathsByAnnotationId = /* @__PURE__ */ new Map();
+    for (const [key, file] of Object.entries(extraTriple)) {
+      if (!key.startsWith("attachment_")) continue;
+      const pinId = key.split("_")[1];
+      const attDir = path.join(taskDir, `annotation-${pinId}`);
+      const filePath = persistFile(file, attDir);
+      const existing = attachmentPathsByAnnotationId.get(pinId) ?? [];
+      existing.push(filePath);
+      attachmentPathsByAnnotationId.set(pinId, existing);
+    }
+    promptContent = buildVisualPromptTriple(
+      config,
+      input.pageRoute ?? "/",
+      input.renderedUrl ?? "",
+      triple,
+      parsed,
+      attachmentPathsByAnnotationId
+    );
+  } else {
+    promptContent = buildNonVisualPrompt(config, input.notes);
+  }
+  yield { type: "status", message: "Running agent..." };
+  let systemPrompt = "";
+  const agentKitPath = path.join(
+    projectDir,
+    "agent-kit",
+    "designer",
+    "INSTRUCTIONS.md"
+  );
+  if (fs.existsSync(agentKitPath)) {
+    systemPrompt = fs.readFileSync(agentKitPath, "utf-8");
+  }
+  try {
+    for await (const message of query({
+      prompt: promptContent,
+      options: {
+        cwd: projectDir,
+        tools: ["Read", "Edit", "Write", "Bash", "Glob", "Grep"],
+        allowedTools: ["Read", "Edit", "Write", "Bash", "Glob", "Grep"],
+        maxTurns: 30,
+        persistSession: false,
+        systemPrompt
+      }
+    })) {
+      for (const chunk of transformSDKMessage(message)) {
+        yield chunk;
+      }
+    }
+  } catch (err) {
+    yield {
+      type: "error",
+      message: err instanceof Error ? err.message : String(err)
+    };
+  }
+  yield { type: "done" };
+});
+setActionCallerOptions({ timeout: 12e4 });
+const page = makeJayStackComponent().withProps();
+const aiditorShell = makeJayStackComponent().withProps().withSlowlyRender(async () => {
+  return phaseOutput({}, { headline: "AIditor" });
+}).withFastRender(async (_props, carryForward) => {
+  const Pipeline = RenderPipeline.for();
+  return Pipeline.ok({}).toPhaseOutput(() => ({
+    viewState: {
+      headline: `${carryForward.headline}`
+    },
+    carryForward
+  }));
+});
+export {
+  page as aiditorPage,
+  aiditorShell,
+  getAiditorBootstrap,
+  getPageParamsAction,
+  getProjectInfoAction,
+  listFreezesAction,
+  readFileAction,
+  submitTaskAction
+};

package/dist/pages/aiditor/page.css

@@ -0,0 +1 @@
+/* AIditor page: styles live inline in page.jay-html for POC parity with aditor-poc. */

package/dist/pages/aiditor/page.jay-contract

@@ -0,0 +1,12 @@
+name: Page
+tags:
+  # Mirrors aiditor-shell.jay-contract (local plugin). Inlined so static validation sees shell.headline.
+  - tag: shell
+    type: sub-contract
+    phase: fast
+    tags:
+      - tag: headline
+        type: data
+        dataType: string
+        phase: fast
+        description: Visible title for the AIditor POC shell

package/dist/pages/aiditor/page.jay-contract.d.ts

@@ -0,0 +1,24 @@
+import {JayContract} from "@jay-framework/runtime";
+
+
+export interface ShellOfPageViewState {
+  headline: string
+}
+
+export interface PageViewState {
+  shell: ShellOfPageViewState
+}
+
+export type PageSlowViewState = {};
+
+export type PageFastViewState = {
+    shell: PageViewState['shell'];
+};
+
+export type PageInteractiveViewState = {};
+
+export interface PageRefs {}
+
+export interface PageRepeatedRefs {}
+
+export type PageContract = JayContract<PageViewState, PageRefs, PageSlowViewState, PageFastViewState, PageInteractiveViewState>