@anynotate/protocol 0.3.0 → 0.4.0

package/PROTOCOL.md

@@ -4,7 +4,7 @@ The bridge is an HTTP server on `127.0.0.1`, port `47291` by default (`ANYNOTATE
 
 ## Handshake
 
-`POST /health` (or `GET`) → `{ "ok": true, "bridgeVersion": "0.3.0", "protocol": { "version": 2, "min": 1 } }`.
+`POST /health` (or `GET`) → `{ "ok": true, "bridgeVersion": "0.5.1", "protocol": { "version": 2, "min": 1 } }`.
 
 A client supports a range of protocol versions (the extension: `min 2, max 2`) and calls `compatibility(clientRange, response.protocol)`:
 
@@ -78,6 +78,28 @@ Each annotation may carry an `intent` telling the agent what the user wants. `IN
 
 The v1 values `question`, `bug` and `note` are still accepted and appear in the heading as before, with no directive line. `change` was also a v1 value; it keeps its name and now gets the directive line. No `intent` means no directive either.
 
+### Region notes
+
+On pages that draw into a `<canvas>` (online documents, design tools, maps), there is no text or element under the user's pointer worth selecting, so the user draws a rectangle instead. Such a note is an `element` annotation whose `element` is the node under the rectangle (usually the `<canvas>`) and which carries an optional `region`:
+
+```ts
+region?: { x: number; y: number; w: number; h: number }
+```
+
+The rectangle is in CSS pixels, relative to the viewport at capture time; add `viewport.scrollY` for the page offset. `w` and `h` are non-negative. `box` still describes the whole element, and `crops/A<n>.png` shows just the rectangle. `README.md` adds a line under the note's heading, such as `Region: 420×180 at (310, 96) on <canvas#board> · see crops/A3.png`.
+
+`region` is only valid on `kind: "element"`; a `text` annotation with a `region` is rejected. It is an additive field: a bridge that predates it parses the annotation as a plain element note and drops `region`, so the agent still gets the comment, element and crop.
+
+### Notes inside frames
+
+A note made on text or an element inside a same-origin `<iframe>` carries the frames on its anchor:
+
+```ts
+anchor.frames?: string[]
+```
+
+Each entry is a CSS selector for an `<iframe>`, outermost first, resolved in the document of the frame before it (the first in the top page). `css`, `hosts`, `path` and `quote` then refer to the innermost frame's document, while `box` and `region` stay in the top page's viewport. `README.md` names the frames after the note's selector, such as ``selector: `p#inner` inside iframe `iframe#card` ``. It is additive: a bridge that predates it drops `frames`, and the agent gets a selector without the frame it lives in.
+
 ## Evolution
 
 - Additive changes — optional fields, new endpoints — keep `PROTOCOL_VERSION`. Both sides ignore fields they don't know.
@@ -88,6 +110,9 @@ The v1 values `question`, `bug` and `note` are still accepted and appear in the
 
 - **v1** — handshake, token-only access, native-host token delivery.
 - **v2** — intents `explain`, `change`, `approve` (legacy `question`, `bug`, `note` still accepted). `PROTOCOL_MIN` stays `1`: a v2 bridge accepts v1 clients.
+  - Optional `region` on element annotations (additive, `@anynotate/protocol` 0.4.0, bridge 0.5.1).
+  - Optional `offscreen: true` on any annotation whose target was not on screen at send time (`box` and crop are from when it was last seen); `README.md` adds `(off-screen when sent)` to its heading (additive, same releases).
+  - Optional `anchor.frames` on notes made inside same-origin iframes; `README.md` names the frames after the selector (additive, same releases).
 
 ## Threat model
 

package/dist/fixtures.d.ts

@@ -42,6 +42,7 @@ export declare const responseSchemas: {
                 pull: "pull";
             }>>;
             session: z.ZodOptional<z.ZodString>;
+            agent: z.ZodOptional<z.ZodString>;
             summary: z.ZodOptional<z.ZodString>;
             note: z.ZodOptional<z.ZodString>;
         }, z.core.$strip>;
@@ -88,6 +89,7 @@ export declare const responseSchemas: {
                     path: z.ZodArray<z.ZodString>;
                     near: z.ZodString;
                     hosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                    frames: z.ZodOptional<z.ZodArray<z.ZodString>>;
                 }, z.core.$strip>;
                 element: z.ZodOptional<z.ZodObject<{
                     tag: z.ZodString;
@@ -110,6 +112,13 @@ export declare const responseSchemas: {
                     scrollY: z.ZodNumber;
                 }, z.core.$strip>;
                 crop: z.ZodString;
+                region: z.ZodOptional<z.ZodObject<{
+                    x: z.ZodNumber;
+                    y: z.ZodNumber;
+                    w: z.ZodNumber;
+                    h: z.ZodNumber;
+                }, z.core.$strip>>;
+                offscreen: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strip>>;
             id: z.ZodString;
             files: z.ZodObject<{
@@ -132,6 +141,7 @@ export declare const responseSchemas: {
                 pull: "pull";
             }>>;
             session: z.ZodOptional<z.ZodString>;
+            agent: z.ZodOptional<z.ZodString>;
             summary: z.ZodOptional<z.ZodString>;
             note: z.ZodOptional<z.ZodString>;
         }, z.core.$strip>>;

package/dist/fixtures.js

@@ -19,13 +19,21 @@ export const exampleBundleInput = {
             box: { x: 10, y: 20, w: 100, h: 18 },
             viewport: { w: 1280, h: 800, dpr: 2, scrollY: 0 },
             crop: "crops/A1.png",
+        }, {
+            id: "A2", kind: "element", comment: "make this bowl bigger", intent: "change",
+            anchor: { css: "canvas#plating", path: ["main", "figure"], near: "Serving" },
+            element: { tag: "canvas", text: "", html: '<canvas id="plating" width="800" height="600"></canvas>', attrs: { id: "plating" } },
+            box: { x: 40, y: 300, w: 800, h: 600 },
+            viewport: { w: 1280, h: 800, dpr: 2, scrollY: 0 },
+            crop: "crops/A2.png",
+            region: { x: 310, y: 420, w: 220, h: 140 },
         }],
 };
 export const examplePage = "# Tomato soup\n\nstir in ⟦A1⟧200 ml cream before serving\n";
 export const responseSchemas = {
     HealthResponse, SessionsResponse, SendResponse, BundleStatusResponse, OkResponse, ErrorResponse,
 };
-const health = { ok: true, bridgeVersion: "0.3.0", protocol: { version: 2, min: 1 } };
+const health = { ok: true, bridgeVersion: "0.5.1", protocol: { version: 2, min: 1 } };
 const queued = { state: "queued", at: "2026-09-24T15:32:01.000Z" };
 const exampleBundle = { ...exampleBundleInput, id: EXAMPLE_BUNDLE_ID, files: { page: "page.md", screenshot: "screenshot.png" } };
 export const conformance = [

package/dist/index.d.ts

@@ -6,6 +6,14 @@ export declare const INTENTS: readonly ["explain", "change", "approve"];
 export declare const LEGACY_INTENTS: readonly ["question", "bug", "note"];
 /** The intent clients should send; parsed annotations may also hold a LEGACY_INTENTS value. */
 export type Intent = (typeof INTENTS)[number];
+/** A rectangle the user drew, in CSS px relative to the viewport at capture time (add `viewport.scrollY` for page y). */
+export declare const Region: z.ZodObject<{
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    w: z.ZodNumber;
+    h: z.ZodNumber;
+}, z.core.$strip>;
+export type Region = z.infer<typeof Region>;
 export declare const Annotation: z.ZodObject<{
     id: z.ZodString;
     kind: z.ZodEnum<{
@@ -35,6 +43,7 @@ export declare const Annotation: z.ZodObject<{
         path: z.ZodArray<z.ZodString>;
         near: z.ZodString;
         hosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        frames: z.ZodOptional<z.ZodArray<z.ZodString>>;
     }, z.core.$strip>;
     element: z.ZodOptional<z.ZodObject<{
         tag: z.ZodString;
@@ -57,6 +66,13 @@ export declare const Annotation: z.ZodObject<{
         scrollY: z.ZodNumber;
     }, z.core.$strip>;
     crop: z.ZodString;
+    region: z.ZodOptional<z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+        w: z.ZodNumber;
+        h: z.ZodNumber;
+    }, z.core.$strip>>;
+    offscreen: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
 export type Annotation = z.infer<typeof Annotation>;
 export declare const Target: z.ZodObject<{
@@ -107,6 +123,7 @@ export declare const BundleInput: z.ZodObject<{
             path: z.ZodArray<z.ZodString>;
             near: z.ZodString;
             hosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            frames: z.ZodOptional<z.ZodArray<z.ZodString>>;
         }, z.core.$strip>;
         element: z.ZodOptional<z.ZodObject<{
             tag: z.ZodString;
@@ -129,6 +146,13 @@ export declare const BundleInput: z.ZodObject<{
             scrollY: z.ZodNumber;
         }, z.core.$strip>;
         crop: z.ZodString;
+        region: z.ZodOptional<z.ZodObject<{
+            x: z.ZodNumber;
+            y: z.ZodNumber;
+            w: z.ZodNumber;
+            h: z.ZodNumber;
+        }, z.core.$strip>>;
+        offscreen: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
 export type BundleInput = z.infer<typeof BundleInput>;
@@ -174,6 +198,7 @@ export declare const Bundle: z.ZodObject<{
             path: z.ZodArray<z.ZodString>;
             near: z.ZodString;
             hosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            frames: z.ZodOptional<z.ZodArray<z.ZodString>>;
         }, z.core.$strip>;
         element: z.ZodOptional<z.ZodObject<{
             tag: z.ZodString;
@@ -196,6 +221,13 @@ export declare const Bundle: z.ZodObject<{
             scrollY: z.ZodNumber;
         }, z.core.$strip>;
         crop: z.ZodString;
+        region: z.ZodOptional<z.ZodObject<{
+            x: z.ZodNumber;
+            y: z.ZodNumber;
+            w: z.ZodNumber;
+            h: z.ZodNumber;
+        }, z.core.$strip>>;
+        offscreen: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strip>>;
     id: z.ZodString;
     files: z.ZodObject<{
@@ -226,6 +258,7 @@ export declare const Status: z.ZodObject<{
         pull: "pull";
     }>>;
     session: z.ZodOptional<z.ZodString>;
+    agent: z.ZodOptional<z.ZodString>;
     summary: z.ZodOptional<z.ZodString>;
     note: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
@@ -285,6 +318,7 @@ export declare const SendResponse: z.ZodObject<{
             pull: "pull";
         }>>;
         session: z.ZodOptional<z.ZodString>;
+        agent: z.ZodOptional<z.ZodString>;
         summary: z.ZodOptional<z.ZodString>;
         note: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>;
@@ -332,6 +366,7 @@ export declare const BundleStatusResponse: z.ZodObject<{
                 path: z.ZodArray<z.ZodString>;
                 near: z.ZodString;
                 hosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                frames: z.ZodOptional<z.ZodArray<z.ZodString>>;
             }, z.core.$strip>;
             element: z.ZodOptional<z.ZodObject<{
                 tag: z.ZodString;
@@ -354,6 +389,13 @@ export declare const BundleStatusResponse: z.ZodObject<{
                 scrollY: z.ZodNumber;
             }, z.core.$strip>;
             crop: z.ZodString;
+            region: z.ZodOptional<z.ZodObject<{
+                x: z.ZodNumber;
+                y: z.ZodNumber;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>>;
+            offscreen: z.ZodOptional<z.ZodBoolean>;
         }, z.core.$strip>>;
         id: z.ZodString;
         files: z.ZodObject<{
@@ -376,6 +418,7 @@ export declare const BundleStatusResponse: z.ZodObject<{
             pull: "pull";
         }>>;
         session: z.ZodOptional<z.ZodString>;
+        agent: z.ZodOptional<z.ZodString>;
         summary: z.ZodOptional<z.ZodString>;
         note: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>;

package/dist/index.js

@@ -4,6 +4,8 @@ export const Agent = z.string().regex(AGENT_NAME);
 export const INTENTS = ["explain", "change", "approve"];
 export const LEGACY_INTENTS = ["question", "bug", "note"];
 const Box = z.object({ x: z.number(), y: z.number(), w: z.number(), h: z.number() });
+/** A rectangle the user drew, in CSS px relative to the viewport at capture time (add `viewport.scrollY` for page y). */
+export const Region = z.object({ x: z.number(), y: z.number(), w: z.number().nonnegative(), h: z.number().nonnegative() });
 export const Annotation = z.object({
     id: z.string().regex(/^A\d+$/),
     // v1 kinds only; "region" | "draw" are reserved for later tools.
@@ -17,6 +19,8 @@ export const Annotation = z.object({
         path: z.array(z.string()),
         near: z.string(),
         hosts: z.array(z.string()).optional(),
+        /** CSS selectors of the iframes holding the note, outermost first, each resolved in the previous frame's document; absent for the top document. `css`, `hosts` and `path` are then inside the innermost frame, while `box` and `region` stay in the top page's viewport. */
+        frames: z.array(z.string()).optional(),
     }),
     element: z.object({
         tag: z.string(),
@@ -29,7 +33,10 @@ export const Annotation = z.object({
     box: Box,
     viewport: z.object({ w: z.number(), h: z.number(), dpr: z.number(), scrollY: z.number() }),
     crop: z.string().regex(/^crops\/A\d+\.png$/),
-});
+    region: Region.optional(),
+    /** True when the note's target was not on screen at send time (scrolled out of a virtual list, another tab); `box` and the crop are from when it was last seen. */
+    offscreen: z.boolean().optional(),
+}).refine((a) => !a.region || a.kind === "element", { message: "region is only allowed on element annotations", path: ["region"] });
 export const Target = z.object({
     agent: Agent,
     sessionId: z.string().optional(),
@@ -58,6 +65,7 @@ export const Status = z.object({
     at: z.string(),
     via: Via.optional(),
     session: z.string().optional(),
+    agent: Agent.optional(),
     summary: z.string().optional(),
     note: z.string().optional(),
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anynotate/protocol",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Wire types and zod schemas shared by Anynotate clients and the Anynotate bridge.",
   "license": "Apache-2.0",
   "author": "Kun Xu",

package/src/fixtures.ts

@@ -24,6 +24,14 @@ export const exampleBundleInput: BundleInput = {
     box: { x: 10, y: 20, w: 100, h: 18 },
     viewport: { w: 1280, h: 800, dpr: 2, scrollY: 0 },
     crop: "crops/A1.png",
+  }, {
+    id: "A2", kind: "element", comment: "make this bowl bigger", intent: "change",
+    anchor: { css: "canvas#plating", path: ["main", "figure"], near: "Serving" },
+    element: { tag: "canvas", text: "", html: '<canvas id="plating" width="800" height="600"></canvas>', attrs: { id: "plating" } },
+    box: { x: 40, y: 300, w: 800, h: 600 },
+    viewport: { w: 1280, h: 800, dpr: 2, scrollY: 0 },
+    crop: "crops/A2.png",
+    region: { x: 310, y: 420, w: 220, h: 140 },
   }],
 };
 export const examplePage = "# Tomato soup\n\nstir in ⟦A1⟧200 ml cream before serving\n";
@@ -49,7 +57,7 @@ export type Exchange = {
   response: { schema: SchemaName; example: unknown };
 };
 
-const health = { ok: true, bridgeVersion: "0.3.0", protocol: { version: 2, min: 1 } };
+const health = { ok: true, bridgeVersion: "0.5.1", protocol: { version: 2, min: 1 } };
 const queued = { state: "queued", at: "2026-09-24T15:32:01.000Z" };
 const exampleBundle = { ...exampleBundleInput, id: EXAMPLE_BUNDLE_ID, files: { page: "page.md", screenshot: "screenshot.png" } };
 

package/src/index.ts

@@ -11,6 +11,10 @@ export type Intent = (typeof INTENTS)[number];
 
 const Box = z.object({ x: z.number(), y: z.number(), w: z.number(), h: z.number() });
 
+/** A rectangle the user drew, in CSS px relative to the viewport at capture time (add `viewport.scrollY` for page y). */
+export const Region = z.object({ x: z.number(), y: z.number(), w: z.number().nonnegative(), h: z.number().nonnegative() });
+export type Region = z.infer<typeof Region>;
+
 export const Annotation = z.object({
   id: z.string().regex(/^A\d+$/),
   // v1 kinds only; "region" | "draw" are reserved for later tools.
@@ -24,6 +28,8 @@ export const Annotation = z.object({
     path: z.array(z.string()),
     near: z.string(),
     hosts: z.array(z.string()).optional(),
+    /** CSS selectors of the iframes holding the note, outermost first, each resolved in the previous frame's document; absent for the top document. `css`, `hosts` and `path` are then inside the innermost frame, while `box` and `region` stay in the top page's viewport. */
+    frames: z.array(z.string()).optional(),
   }),
   element: z.object({
     tag: z.string(),
@@ -36,7 +42,10 @@ export const Annotation = z.object({
   box: Box,
   viewport: z.object({ w: z.number(), h: z.number(), dpr: z.number(), scrollY: z.number() }),
   crop: z.string().regex(/^crops\/A\d+\.png$/),
-});
+  region: Region.optional(),
+  /** True when the note's target was not on screen at send time (scrolled out of a virtual list, another tab); `box` and the crop are from when it was last seen. */
+  offscreen: z.boolean().optional(),
+}).refine((a) => !a.region || a.kind === "element", { message: "region is only allowed on element annotations", path: ["region"] });
 export type Annotation = z.infer<typeof Annotation>;
 
 export const Target = z.object({
@@ -76,6 +85,7 @@ export const Status = z.object({
   at: z.string(),
   via: Via.optional(),
   session: z.string().optional(),
+  agent: Agent.optional(),
   summary: z.string().optional(),
   note: z.string().optional(),
 });