@popmelt.com/core 0.14.2 → 0.16.0

package/README.md

@@ -29,6 +29,13 @@ From inside your project, give your coding agent this prompt:
 
 Your agent can detect the framework, add the provider and integration, and preserve your existing development command. Restart the development server when it is finished.
 
+To add local object recognition to an existing Popmelt project, use:
+
+> Add `@popmelt.com/see` to this Popmelt Core project
+
+The integration installs See and adds it to the existing Core provider's
+`extensions` array without replacing Core.
+
 ## Use Popmelt
 
 1. Run your project locally and open it in a browser
@@ -50,13 +57,13 @@ Double-click visible text to edit it directly. Popmelt records the text change s
 
 Steer previews code-native design changes directly on the page. Point to an element, choose a family, narrow to a property, and choose a value. Click the target or press Enter at the value level to stage the preview.
 
-| Keys | Family |
-| --- | --- |
-| `P` / `M` | Padding / margin |
-| `B` / `R` | Borders / radii |
-| `F` / `G` | Flexbox / grid |
-| `FO` / `BG` | Type / background color |
-| `T` / `O` | Transform / opacity and blend mode |
+| Keys        | Family                             |
+| ----------- | ---------------------------------- |
+| `P` / `M`   | Padding / margin                   |
+| `B` / `R`   | Borders / radii                    |
+| `F` / `G`   | Flexbox / grid                     |
+| `FO` / `BG` | Type / background color            |
+| `T` / `O`   | Transform / opacity and blend mode |
 
 The omnidial can be driven entirely from the keyboard: `Q` and `E` move between properties, `A` and `D` or the arrow keys move between values, `W` or Enter goes deeper or applies, and `S` or Escape goes back. Press Space to search families, properties, colors, and values by name.
 
@@ -79,15 +86,18 @@ Archiving hides a thread from the active canvas without deleting its stored conv
 
 ### Pointer gestures
 
-| Gesture | Action |
-| --- | --- |
-| Shake the pointer quickly | Switch between Chat and Steer |
-| Hold Command/Control while pointing | Pass through the default target to the deepest element under the pointer |
-| Double-click text in Chat | Edit the text directly |
-| Click the toolbar annotation counter | Cycle through annotation bundles, including other routes |
-| Scroll the annotation counter | Change the active annotation color |
-| Hold the annotation counter | Reset the active color |
-| Right-click Popmelt's canvas, then right-click again | Pass through to the page's native context menu |
+| Gesture                                            | Action                                                                   |
+| -------------------------------------------------- | ------------------------------------------------------------------------ |
+| Shake the pointer quickly                          | Switch between Chat and Steer                                            |
+| Hold Command/Control while pointing                | Pass through the default target to the deepest element under the pointer |
+| Double-click text in Chat                          | Edit the text directly                                                   |
+| Click the toolbar annotation counter               | Cycle through annotation bundles, including other routes                 |
+| Scroll the annotation counter                      | Change the active annotation color                                       |
+| Hold the annotation counter                        | Reset the active color                                                   |
+| Right-click Popmelt's canvas                       | Enter persistent page passthrough while keeping Popmelt overlays visible |
+| Drag, scroll, pinch, or right-click in passthrough | Interact with the page and open its native context menu                  |
+| Click a valid target in passthrough                | Apply the active Comment or Steer action without leaving passthrough     |
+| Press Escape or choose another tool                | Exit passthrough                                                         |
 
 ## Ambient Popmelt in Codex
 
@@ -233,9 +243,66 @@ If the toolbar opens but cannot reach an AI, check that:
 - The framework plugin or config wrapper is installed
 - The development server was restarted after installation
 
+## Image annotation
+
+Manual image annotation is enabled by default in `PopmeltProvider`, without a
+vision dependency or model download.
+
+```tsx
+<PopmeltProvider>{children}</PopmeltProvider>
+
+// Optional: disable image annotation.
+<PopmeltProvider imageAnnotation={false}>{children}</PopmeltProvider>
+
+// Optional: limit image discovery (all img elements are included by default).
+<PopmeltProvider imageAnnotation={{ selector: 'img[data-popmelt-media]' }}>
+  {children}
+</PopmeltProvider>
+```
+
+In Chat, drag over an image to create a named object. Double-click an object to
+rename or resize it, single-click it to comment, and right-click its label to
+delete it. Saved targets stay aligned as the image resizes.
+
+Image targets use browser-local storage by default. Supply
+`imageAnnotation.store` when the project needs another persistence layer. Add a
+stable `data-popmelt-media-id` to images whose URL may change or whose source
+appears more than once:
+
+```tsx
+<img data-popmelt-media-id="hero-product" src={imageUrl} alt="Product" />
+```
+
+No recognition or model download occurs unless See is installed and enabled.
+
+## Add local object recognition with See
+
+[`@popmelt.com/see`](https://www.npmjs.com/package/@popmelt.com/see) adds
+browser-local object detection to the existing image-annotation experience. It
+requires Core 0.16.0 or newer.
+
+Install it:
+
+```bash
+npm install @popmelt.com/see
+```
+
+Then add the extension to the existing Core provider:
+
+```tsx
+import { PopmeltProvider } from '@popmelt.com/core';
+import { see } from '@popmelt.com/see';
+
+<PopmeltProvider extensions={[see]}>{children}</PopmeltProvider>;
+```
+
+Recognition loads only when it is needed. Other extensions can be included in
+the same array. Model selection, self-hosting, and recognition settings are
+documented in the See package README.
+
 ## Data and privacy
 
-Popmelt stores its annotations, screenshots, threads, UI preferences, and Imprint locally under `.popmelt/`. Unsent annotation drafts and preview state also use browser storage.
+Popmelt stores its annotations, screenshots, threads, UI preferences, and Imprint locally under `.popmelt/`. Unsent annotation drafts, preview state, and the default image-object store use origin-scoped browser storage. Supply a project-backed `imageAnnotation.store` when image targets must survive browser-site-data removal or move between origins.
 
 When you send a thread, the context needed to complete that work is passed to your selected AI provider and is subject to that provider's privacy policy.
 
@@ -243,19 +310,19 @@ Popmelt is designed for local development. As with any AI-assisted coding, your
 
 ## Useful shortcuts
 
-| Shortcut | Action |
-| --- | --- |
-| Double-tap Command or Control | Open or collapse Popmelt |
-| `C` | Enter Chat mode |
-| `S` | Enter Steer mode |
-| `I` | Enter Imprint mode |
-| Command/Control+Enter | Send all pending work |
-| Command/Control+C | Capture a screenshot for pending work when no text is selected |
-| Command/Control+Z | Undo a pending change |
-| Command/Control+Shift+Z | Redo a pending change |
-| Backspace/Delete | Archive the open or selected historical thread |
-| Command/Control+Backspace/Delete | Clear all annotations and pending changes |
-| Escape | Go back, cancel, close, discard, or deselect in the current context |
+| Shortcut                         | Action                                                              |
+| -------------------------------- | ------------------------------------------------------------------- |
+| Double-tap Command or Control    | Open or collapse Popmelt                                            |
+| `C`                              | Enter Chat mode                                                     |
+| `S`                              | Enter Steer mode                                                    |
+| `I`                              | Enter Imprint mode                                                  |
+| Command/Control+Enter            | Send all pending work                                               |
+| Command/Control+C                | Capture a screenshot for pending work when no text is selected      |
+| Command/Control+Z                | Undo a pending change                                               |
+| Command/Control+Shift+Z          | Redo a pending change                                               |
+| Backspace/Delete                 | Archive the open or selected historical thread                      |
+| Command/Control+Backspace/Delete | Clear all annotations and pending changes                           |
+| Escape                           | Go back, cancel, close, discard, or deselect in the current context |
 
 Steer additionally owns the unmodified letter, arrow, Enter, Escape, and Space keys described above while it is active. Popmelt leaves these shortcuts alone while you are typing in an input, textarea, or editable text surface.
 

package/codex-plugin/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "popmelt-core",
-  "version": "0.14.2",
+  "version": "0.16.0",
   "description": "Ambient project memory and shared understanding for Codex",
   "author": {
     "name": "Popmelt",

package/dist/adapter-three.d.mts

@@ -0,0 +1,115 @@
+import { Camera, Object3D, Raycaster, WebGLRenderer } from 'three';
+
+/** Coordinates refer to the original image, not its responsive DOM rectangle. */
+type ImageObjectContext = {
+    src: string;
+    alt: string;
+    width: number;
+    height: number;
+    box: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    normalizedBox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+};
+
+type PopmeltSubjectBounds = {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+};
+type PopmeltSubject = {
+    key: string;
+    label: string;
+    bounds: PopmeltSubjectBounds;
+    /** Viewport bounds of the renderer surface that owns this subject. */
+    surfaceBounds?: PopmeltSubjectBounds;
+    context?: string;
+    data?: Record<string, string>;
+    /** Live owning surface; never serialized or included in model context. */
+    sourceElement?: Element;
+    imageObject?: ImageObjectContext;
+};
+type PopmeltSurfaceSnapshot = {
+    element: HTMLCanvasElement;
+    dataUrl: string;
+};
+type PopmeltSubjectCreationSession = {
+    key: string;
+    update(clientPoint: {
+        x: number;
+        y: number;
+    }): PopmeltSubject | null;
+    commit(): PopmeltSubject | null;
+    cancel(): void;
+};
+/**
+ * Minimal renderer boundary for non-DOM subjects. The adapter owns hit testing,
+ * durable keys, and viewport projection; Popmelt owns annotations and threads.
+ */
+type PopmeltSubjectAdapter = {
+    id: string;
+    /** Surface ownership; an explicit image adapter replaces Core's default. */
+    surfaceKind?: 'image';
+    hitTest(clientPoint: {
+        x: number;
+        y: number;
+    }): PopmeltSubject | null;
+    resolve(key: string): PopmeltSubject | null;
+    /** Optional clean subject crop, without Popmelt's overlay. */
+    captureSubject?(key: string): Blob | null | Promise<Blob | null>;
+    /** Optional bitmap capture for renderer surfaces that DOM cloning cannot read. */
+    captureSurfaces?(): readonly PopmeltSurfaceSnapshot[] | Promise<readonly PopmeltSurfaceSnapshot[]>;
+    /** Begin a context-aware subject-creation gesture at a viewport point. */
+    beginCreate?(clientPoint: {
+        x: number;
+        y: number;
+    }): PopmeltSubjectCreationSession | null;
+    /** Rename a subject without changing its durable key. */
+    renameSubject?(key: string, label: string): PopmeltSubject | null;
+    /** Replace a subject's viewport bounds, projected back into adapter space. */
+    updateSubjectBounds?(key: string, bounds: PopmeltSubjectBounds): PopmeltSubject | null;
+    /** Remove a user-created subject or hide an adapter-owned subject. */
+    removeSubject?(key: string): void;
+};
+
+type ThreeSubjectAdapterOptions = {
+    canvas: HTMLCanvasElement;
+    camera: Camera;
+    scene: Object3D;
+    raycaster?: Raycaster;
+    /** Select a meaningful parent from the raw intersected render object. */
+    select?: (object: Object3D) => Object3D | null;
+    /** Override durable identity. Explicit application IDs are preferred. */
+    key?: (object: Object3D) => string | null | undefined;
+    label?: (object: Object3D) => string;
+    /** Add application-specific provenance or source hints to the model context. */
+    describe?: (object: Object3D) => Record<string, string | undefined>;
+    /** Capture the renderer's current frame for DOM-cloned screenshots. */
+    captureFrame?: () => string | null | Promise<string | null>;
+};
+/**
+ * Read the renderer's visible frame without requiring preserveDrawingBuffer.
+ * Rendering and reading synchronously preserves the application's real output
+ * pipeline while the previous render target and XR state are restored afterward.
+ */
+declare function captureThreeRendererFrame(renderer: WebGLRenderer, scene: Object3D, camera: Camera): string | null;
+/** Stable while the scene's semantic hierarchy remains stable across reloads. */
+declare function threeObjectPath(object: Object3D): string;
+/**
+ * Collapses glTF material primitives such as `Object649_normal_0` onto their
+ * named parent `Object649`. Apps opt into this policy through `select` so raw
+ * mesh targeting remains available when material-level annotation is useful.
+ */
+declare function selectThreeNamedPrimitiveParent(object: Object3D): Object3D;
+declare function createThreeSubjectAdapter(options: ThreeSubjectAdapterOptions): PopmeltSubjectAdapter;
+
+export { type ThreeSubjectAdapterOptions, captureThreeRendererFrame, createThreeSubjectAdapter, selectThreeNamedPrimitiveParent, threeObjectPath };

package/dist/adapter-three.mjs

@@ -0,0 +1 @@
+import{Box3 as y,Raycaster as x,Vector2 as O,Vector3 as j}from"three";function b(e){return[e.x,e.y,e.z].map(n=>Number(n.toFixed(4))).join(", ")}function h(e){let n=e.material;return n&&(Array.isArray(n)?n:[n]).map(r=>r.name?.trim()||r.type).filter(Boolean).join(", ")||void 0}function M(e,n,c){let s=e.domElement.width,r=e.domElement.height;if(s<=0||r<=0)return null;let l=e.getRenderTarget(),t=e.xr.enabled;try{return e.xr.enabled=!1,e.setRenderTarget(null),e.render(n,c),e.domElement.toDataURL("image/png")}catch{return null}finally{e.setRenderTarget(l),e.xr.enabled=t}}function D(e){let s=(e.parent?.children??[]).filter(t=>t.type===e.type&&t.name===e.name),r=Math.max(0,s.indexOf(e)),l=e.name.trim()||e.type;return`${encodeURIComponent(l)}[${r}]`}function f(e){let n=[],c=e;for(;c?.parent;)n.push(D(c)),c=c.parent;return n.reverse().join("/")}function v(e){let n=e.userData.popmeltId;return typeof n=="string"&&n.trim()?n.trim():f(e)}function T(e){let n=e.userData.popmeltLabel;return typeof n=="string"&&n.trim()?n.trim():e.name.trim()||e.type}function R(e){let n=e;for(;n.parent;){let c=n.name.trim(),s=c.match(/^(.*)_(?:alpha|normal|material)(?:_\d+)+$/i),r=n.parent;for(;r.parent&&!r.name.trim();)r=r.parent;let l=r.name.trim();if(!l||!c.startsWith(`${l}_`)&&!s)break;n=r}return n}function P(e,n,c){let s=new y().setFromObject(e,!0);if(s.isEmpty())return null;let r=c.getBoundingClientRect(),l=[];for(let p of[s.min.x,s.max.x])for(let d of[s.min.y,s.max.y])for(let g of[s.min.z,s.max.z])l.push(new j(p,d,g).project(n));let t=l.map(p=>r.left+(p.x+1)/2*r.width),a=l.map(p=>r.top+(1-p.y)/2*r.height),i=Math.max(r.left,Math.min(...t)),o=Math.min(r.right,Math.max(...t)),m=Math.max(r.top,Math.min(...a)),u=Math.min(r.bottom,Math.max(...a));return o<=i||u<=m?null:{left:i,top:m,width:o-i,height:u-m}}function C(e){let n=e.raycaster??new x,c=new O,s=t=>e.key?.(t)??v(t),r=(t,a)=>{let i=s(t);if(!i)return null;let o=P(t,e.camera,e.canvas);return o?{key:i,label:e.label?.(t)??T(t),bounds:o,context:f(t),data:{renderer:"Three.js",objectType:t.type,...t.name?{objectName:t.name}:{},scenePath:f(t),...a&&a!==t?{renderObjectName:a.name.trim()||a.type,renderObjectType:a.type,renderScenePath:f(a)}:{},position:b(t.position),rotation:b(t.rotation),scale:b(t.scale),visible:String(t.visible),...h(t)?{material:h(t)}:{},...Object.fromEntries(Object.entries(e.describe?.(t)??{}).filter(m=>typeof m[1]=="string"&&m[1].length>0))}}:null},l=t=>{let a=null;return e.scene.traverse(i=>{!a&&s(i)===t&&(a=i)}),a};return{id:"three",hitTest({x:t,y:a}){let i=e.canvas.getBoundingClientRect();if(t<i.left||t>i.right||a<i.top||a>i.bottom)return null;c.set((t-i.left)/i.width*2-1,-((a-i.top)/i.height)*2+1),n.setFromCamera(c,e.camera);for(let o of n.intersectObject(e.scene,!0)){let m=e.select?e.select(o.object):R(o.object);if(!m||!m.visible)continue;let u=r(m,o.object);if(u)return u}return null},resolve(t){let a=l(t);return a?r(a):null},...e.captureFrame?{async captureSurfaces(){let t=await e.captureFrame?.();return t?[{element:e.canvas,dataUrl:t}]:[]}}:{}}}export{M as captureThreeRendererFrame,C as createThreeSubjectAdapter,R as selectThreeNamedPrimitiveParent,f as threeObjectPath};