@circuitwall/jarela 1.3.0 → 1.4.0

package/.next/standalone/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@circuitwall/jarela",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Jarela — local chat interface for LangGraph agents (multi-provider, single-process, SQLite-backed).",
   "license": "Apache-2.0",
   "author": "Andrew Ge Wu",
@@ -100,6 +100,7 @@
     "test:live:isolated:full": "node scripts/live-test-isolated.mjs --llm",
     "test:e2e": "playwright test",
     "test:e2e:ui": "playwright test --ui",
+    "promo:record": "node scripts/promo-record.mjs",
     "release:docker": "node scripts/release-docker.mjs",
     "release:docker:dry": "node scripts/release-docker.mjs --dry-run"
   },

package/CHANGELOG.md

@@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.4.0] - 2026-06-08
+
+### Added
+
+- **Browser-extension element screenshot.** The page-capture flow now
+  ships a cropped PNG of the picked element alongside the text. The
+  content script grabs the visible viewport via
+  `chrome.tabs.captureVisibleTab` (loopback only, via the service worker)
+  and crops it to the element's bounding rect through `OffscreenCanvas`
+  at `devicePixelRatio`. The server validates the base64 payload (≤ 4 MB
+  encoded), persists the user message as a multipart `ContentPart[]` of
+  `[text, image]` so the bubble renders the picture inline, and
+  forwards the image part to the silent observer turn so vision-capable
+  agents see it on the immediate follow-up run. Falls back cleanly to
+  text-only capture if the snapshot is denied. See
+  [`docs/api.md`](./docs/api.md#post-apiv1page-capture) for the updated
+  request schema.
+- **Promo video recorder.** `npm run promo:record` (via
+  [`scripts/promo-record.mjs`](./scripts/promo-record.mjs)) drives your
+  real local install in a 9:16 vertical PWA viewport and records a
+  dark-theme `.webm` of the tap-to-unlock intro, agent picker, a
+  human-paced chat turn, and a tour of every side panel. First run
+  saves auth state to `promo/.storage.json` and reuses it thereafter.
+
 ## [1.3.0] - 2026-06-08
 
 Two new agent capabilities and a hardening pass on tool wall-clocks.

package/README.md

@@ -1,29 +1,31 @@
-
-
-
-<p align="center">
-  <img src="./public/logo-mark-transparent.png" alt="Jarela" width="160" />
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="./public/icon-512.png" />
+    <img src="./public/icon-512-light.png" alt="Jarela" width="140" />
+  </picture>
 </p>
 
 <h1 align="center">Jarela</h1>
 
 <p align="center">
-  <b>A local-first, browser-based GUI for orchestrating multi-provider LLM agents.</b><br/>
-  <sub>Next.js 16 + LangGraph + SQLite. PWA-installable. No cloud backend, no telemetry.</sub>
+  <b>A local-first, browser-based GUI for orchestrating multi-provider LLM agents.</b>
+</p>
+<p align="center">
+  <sub>Next.js 16 &middot; LangGraph &middot; SQLite &middot; PWA-installable &middot; no cloud backend, no telemetry</sub>
 </p>
 
 <p align="center">
-  <a href="#quick-start">Quick start</a> ·
-  <a href="#configuration-guide-home--work">Config guide</a> ·
-  <a href="#supported-platforms">Platforms</a> ·
-  <a href="#features">Features</a> ·
-  <a href="#productivity-stacks-google--microsoft-at-parity">Google + Microsoft</a> ·
-  <a href="#built-in-toolbelt">Tools</a> ·
-  <a href="#providers">Providers</a> ·
-  <a href="#connections">Connections</a> ·
-  <a href="./docs/EXTENDING.md">Extending</a> ·
-  <a href="./docs/ARCHITECTURE.md">Architecture</a> ·
-  <a href="./CONTRIBUTING.md">Contributing</a> ·
+  <a href="#quick-start">Quick start</a> &middot;
+  <a href="#configuration-guide-home--work">Config guide</a> &middot;
+  <a href="#supported-platforms">Platforms</a> &middot;
+  <a href="#features">Features</a> &middot;
+  <a href="#productivity-stacks-google--microsoft-at-parity">Google + Microsoft</a> &middot;
+  <a href="#built-in-toolbelt">Tools</a> &middot;
+  <a href="#providers">Providers</a> &middot;
+  <a href="#connections">Connections</a> &middot;
+  <a href="./docs/EXTENDING.md">Extending</a> &middot;
+  <a href="./docs/ARCHITECTURE.md">Architecture</a> &middot;
+  <a href="./CONTRIBUTING.md">Contributing</a> &middot;
   <a href="#documentation">Docs</a>
 </p>
 
@@ -51,15 +53,16 @@
   </a>
 </p>
 
----
-
 <p align="center">
-  <video src="https://github.com/user-attachments/assets/0f33f8d3-07bb-4850-9fcc-cfc97036f180" controls width="640" muted>
-    Your browser doesn't support embedded video.
-    <a href="https://github.com/user-attachments/assets/0f33f8d3-07bb-4850-9fcc-cfc97036f180">Download the clip</a>.
+  <video src="./docs/assets/jarela-promo.webm" poster="./docs/assets/jarela-promo-poster.jpg" controls muted playsinline width="320">
+    <img src="./docs/assets/jarela-promo-poster.jpg" alt="Jarela promo &mdash; PIN unlock, agent picker, chat, panel tour" width="320" />
   </video>
+  <br/>
+  <sub><i>Tap-to-unlock &rarr; agent picker &rarr; human-paced chat &rarr; full panel tour</i> &middot; <a href="./docs/assets/jarela-promo.webm">Download .webm</a></sub>
 </p>
 
+---
+
 ## Quick start
 
 Get to a working local agent in under 10 minutes:
@@ -239,9 +242,12 @@ create an Outlook Calendar invite in the same turn.
   **Memory**, **Documents**, **Profile**, **Bridges**, **Scheduled tasks**,
   and **Pending approvals**.
 - **Browser extension** ([`browser-extension/`](./browser-extension)) —
-  Chrome MV3, click an element on any page and POST it to your local
-  Jarela as a new user message (ADR-0018). Loopback only; toolbar icon
-  greys out when Jarela isn't running.
+  Chrome MV3, click an element on any page and POST it (with a cropped
+  PNG of the picked element) to your local Jarela as a new user message
+  (ADR-0018). The screenshot is rendered inline in the chat bubble and
+  forwarded to vision-capable agents on the silent observer turn that
+  fires immediately after the capture. Loopback only; toolbar icon greys
+  out when Jarela isn't running.
 
 ### Operational
 
@@ -914,6 +920,25 @@ on every push and PR: `lint + tsc --noEmit + next build`, then the same
 live integration suite against the production server output. The build
 badge at the top of this README links straight to the latest run.
 
+## Recording a promo video
+
+[scripts/promo-record.mjs](./scripts/promo-record.mjs) drives your real
+local install (default `http://localhost:4312`) inside a 540&times;960
+vertical (9:16) PWA viewport and records a `.webm` of a five-scene tour
+in dark theme: a simulated PIN unlock, agent picker, a human-paced chat
+turn, every side panel, and a closing pose.
+
+```bash
+npm run dev            # in one terminal
+npm run promo:record   # in another — output lands in ./promo/
+```
+
+The first run opens a headed Chromium so you can manually unlock the
+install if needed; the resulting auth state is saved to
+`promo/.storage.json` and reused on every subsequent run. Override the
+target with `JARELA_PROMO_URL`, the chat line with `JARELA_PROMO_MSG`,
+or skip the actual send with `JARELA_PROMO_SKIP_CHAT=1`.
+
 ## Security
 
 - **CSRF / origin guard** ([lib/auth/access.ts](./lib/auth/access.ts))

package/components/chat/InputBar.tsx

@@ -73,6 +73,12 @@ function fileToContentPart(file: File): Promise<ContentPart> {
   });
 }
 
+function attachmentKey(a: ContentPart, i: number): string {
+  if (a.type === "text") return `text:${i}:${a.text.length}`;
+  const name = a.type === "file" ? a.name : "";
+  return `${a.type}:${a.media_type}:${name}:${a.data.length}:${a.data.slice(0, 16)}`;
+}
+
 export function InputBar({ attachments, onAttachmentsChange, onSubmit, onQueue, onStop, streaming, disabled, placeholder, voiceEnabled, agentId, onVoiceTranscript }: Props) {
   // Text state is intentionally LOCAL. Lifting it to ChatView would re-render
   // the entire message list (every MessageBubble + ReactMarkdown pass) on
@@ -268,7 +274,10 @@ export function InputBar({ attachments, onAttachmentsChange, onSubmit, onQueue,
       {attachments.length > 0 && (
         <div className="flex flex-wrap gap-2 mb-2">
           {attachments.map((a, i) => (
-            <div key={i} className="relative group shrink-0">
+            // Content-derived key — using the index reused DOM nodes when
+            // earlier attachments were removed, flashing the wrong preview
+            // (and the wrong filename) into the slot of the survivor.
+            <div key={attachmentKey(a, i)} className="relative group shrink-0">
               {a.type === "image" ? (
                 // eslint-disable-next-line @next/next/no-img-element
                 <img

package/lib/api/page-capture.test.ts

@@ -307,3 +307,61 @@ describe("handlePageCapture — response shape", () => {
     });
   });
 });
+
+describe("handlePageCapture — screenshot attachment", () => {
+  // 1x1 transparent PNG, base64-encoded (no data: prefix).
+  const tinyPng =
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+
+  it("rejects screenshot with invalid base64", async () => {
+    const res = await handlePageCapture(makeReq({ ...validBody, screenshot: "not base64!!" }));
+    expect(res.status).toBe(400);
+    expect(addMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("rejects screenshot exceeding the size cap", async () => {
+    const huge = "A".repeat(4_000_001);
+    const res = await handlePageCapture(makeReq({ ...validBody, screenshot: huge }));
+    expect(res.status).toBe(400);
+  });
+
+  it("persists user message as a JSON ContentPart[] with text + image when screenshot is present", async () => {
+    const res = await handlePageCapture(makeReq({ ...validBody, screenshot: tinyPng }));
+    expect(res.status).toBe(200);
+    const stored = addMessageMock.mock.calls[0][2] as string;
+    const parsed = JSON.parse(stored) as Array<{ type: string; text?: string; media_type?: string; data?: string }>;
+    expect(Array.isArray(parsed)).toBe(true);
+    expect(parsed).toHaveLength(2);
+    expect(parsed[0]).toMatchObject({ type: "text" });
+    expect(parsed[0].text).toContain("Captured from");
+    expect(parsed[0].text).toContain("Screenshot attached.");
+    expect(parsed[1]).toEqual({ type: "image", media_type: "image/png", data: tinyPng });
+  });
+
+  it("forwards the screenshot as a vision attachment to the silent observer run", async () => {
+    await handlePageCapture(makeReq({ ...validBody, screenshot: tinyPng }));
+    expect(runAgentTurnMock).toHaveBeenCalledWith(expect.objectContaining({
+      attachments: [{ type: "image", media_type: "image/png", data: tinyPng }],
+    }));
+  });
+
+  it("honors a custom screenshotMediaType", async () => {
+    await handlePageCapture(makeReq({ ...validBody, screenshot: tinyPng, screenshotMediaType: "image/jpeg" }));
+    const stored = addMessageMock.mock.calls[0][2] as string;
+    const parsed = JSON.parse(stored) as Array<{ type: string; media_type?: string }>;
+    expect(parsed[1].media_type).toBe("image/jpeg");
+  });
+
+  it("keeps the legacy string-content path when no screenshot is sent", async () => {
+    await handlePageCapture(makeReq(validBody));
+    const stored = addMessageMock.mock.calls[0][2] as string;
+    // Not JSON-parseable as an array — it's the legacy plaintext body.
+    expect(() => JSON.parse(stored)).toThrow();
+    expect(stored).toContain("Captured from");
+    expect(stored).not.toContain("Screenshot attached.");
+    expect(runAgentTurnMock).toHaveBeenCalledWith(expect.objectContaining({
+      attachments: undefined,
+    }));
+  });
+});
+

package/lib/api/page-capture.ts

@@ -13,12 +13,18 @@ import {
 } from "@/lib/stores/agent-configs";
 import { publish } from "@/lib/notifications/bus";
 import { runAgentTurn } from "@/lib/agents/agent-turn";
+import type { ContentPart } from "@/lib/tools/types";
 
 // 100KB UTF-8 cap on captured text. The LLM context window is the real
 // constraint; this cap exists to keep a runaway "<body>" pick from
 // trashing the conversation. See ADR-0018.
 export const MAX_TEXT_BYTES = 100_000;
 
+// Hard cap on the inline element screenshot (base64 chars). 4 MB of
+// base64 ≈ 3 MB decoded — generous for a single cropped element while
+// still bounding the SQLite row and the LLM vision payload.
+export const MAX_SCREENSHOT_B64 = 4_000_000;
+
 // Preamble prepended to the LLM call for the silent observer run.
 // The captured content is already persisted in the DB — this wrapper
 // instructs the agent to observe without replying, matching bridge
@@ -37,6 +43,13 @@ const Body = z.object({
   tagName: z.string().max(64).optional(),
   text: z.string(),
   capturedAt: z.string().datetime(),
+  // Optional base64-encoded PNG of just the picked element (no data: URL
+  // prefix). The content script crops `chrome.tabs.captureVisibleTab`
+  // to the element bounding box before sending. When present, it is
+  // attached to the persisted user message as an image ContentPart so
+  // the chat UI renders it inline and vision-capable agents can see it.
+  screenshot: z.string().regex(/^[A-Za-z0-9+/=]+$/).max(MAX_SCREENSHOT_B64).optional(),
+  screenshotMediaType: z.string().regex(/^image\/[a-z0-9.+-]+$/).max(64).optional(),
 });
 
 function truncateUtf8(s: string, maxBytes: number): { text: string; truncated: boolean; originalBytes: number } {
@@ -102,12 +115,14 @@ function composeBody(args: {
   text: string;
   truncated: boolean;
   originalBytes: number;
+  hasScreenshot?: boolean;
 }): string {
   const heading = args.title
     ? `📎 Captured from [${args.title}](${args.url})`
     : `📎 Captured from <${args.url}>`;
   const lines = [heading];
   if (args.selector) lines.push(`Element: \`${args.selector}\``);
+  if (args.hasScreenshot) lines.push("Screenshot attached.");
   if (args.truncated) {
     lines.push(`> ⚠ Truncated to ${MAX_TEXT_BYTES.toLocaleString()} bytes (original was ${args.originalBytes.toLocaleString()} bytes)`);
   }
@@ -158,9 +173,23 @@ export async function handlePageCapture(req: Request): Promise<Response> {
     text,
     truncated,
     originalBytes,
+    hasScreenshot: Boolean(input.screenshot),
   });
 
-  const msg = addMessage(thread_id, "user", messageBody, undefined, "page_capture");
+  // When a screenshot is included, persist the user turn as a multipart
+  // ContentPart[] (text + image) — that's the same shape the chat UI and
+  // agent runner expect for inline images, so the picture renders in the
+  // bubble on reload and vision-capable models can see it on the silent
+  // observer turn. Without a screenshot we keep the legacy string body
+  // to avoid touching messages that never had an image.
+  const screenshotPart: ContentPart | null = input.screenshot
+    ? { type: "image", media_type: input.screenshotMediaType ?? "image/png", data: input.screenshot }
+    : null;
+  const storedContent: string = screenshotPart
+    ? JSON.stringify([{ type: "text", text: messageBody }, screenshotPart] satisfies ContentPart[])
+    : messageBody;
+
+  const msg = addMessage(thread_id, "user", storedContent, undefined, "page_capture");
 
   // Fire a silent observer run so the agent ingests the captured context
   // without being forced to reply — matching bridge silent/observer mode.
@@ -170,6 +199,7 @@ export async function handlePageCapture(req: Request): Promise<Response> {
     thread_id,
     queue_source: "extension",
     message: `${SILENT_CAPTURE_PREAMBLE}\n\n${messageBody}`,
+    attachments: screenshotPart ? [screenshotPart] : undefined,
     user_category: "page_capture",
     assistant_category: "page_capture",
     silent: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@circuitwall/jarela",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Jarela — local chat interface for LangGraph agents (multi-provider, single-process, SQLite-backed).",
   "license": "Apache-2.0",
   "author": "Andrew Ge Wu",
@@ -100,6 +100,7 @@
     "test:live:isolated:full": "node scripts/live-test-isolated.mjs --llm",
     "test:e2e": "playwright test",
     "test:e2e:ui": "playwright test --ui",
+    "promo:record": "node scripts/promo-record.mjs",
     "release:docker": "node scripts/release-docker.mjs",
     "release:docker:dry": "node scripts/release-docker.mjs --dry-run"
   },